diff --git a/crates/engine/src/settings/session_settings.rs b/crates/engine/src/settings/session_settings.rs index cc25f35ea..9a76dd84a 100644 --- a/crates/engine/src/settings/session_settings.rs +++ b/crates/engine/src/settings/session_settings.rs @@ -122,6 +122,11 @@ impl Session { let layers: std::sync::Arc = self.layer_cache.clone(); accountant.register(&layers); accountant.set_budget(b.bytes); + // (what one file's decoded planes may take follows it once the RAM is read, at least the + // fixed limit; until then the host's reader asks when a request is over that, #3749) + if b.installed.is_some() { + effectcraft_render::memory::set_decode_limit(Some(b.bytes)); + } // The process's measured memory as well (#3301: memory nobody accounts for, such as GPU // staging, effect buffers and the allocator's fragmentation): over the purge limit the // caches give way by what it is over, over the hard limit also the frames a playing diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index 295476df0..159156a24 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -117,9 +117,19 @@ pub fn session() -> Session { // the settings, and the caches have no limits of their own below it. Without a memory // reading yet, the budget for unknown RAM. s.apply_cache_budgets(); + // The CLI and the MCP server read no memory: a request for a file's planes over the fixed + // limit asks for it once (#3749). + effectcraft_engine::render::memory::set_decode_limit_source(Some(installed_memory_budget)); s } +/// The memory budget of the installed RAM with the default settings: what one file's decoded +/// planes may take when a request is over the fixed limit and the session read no memory. +fn installed_memory_budget() -> Option { + let memory = effectcraft_engine::sysinfo::memory()?; + Some(effectcraft_engine::prefs::Prefs::default().memory_budget(Some(memory), false).bytes) +} + /// External effect helpers run as child processes: native builds only. #[cfg(not(target_arch = "wasm32"))] fn helper_hooks() -> Option { diff --git a/crates/media/src/exr_index.rs b/crates/media/src/exr_index.rs index 988c3b370..425077c0a 100644 --- a/crates/media/src/exr_index.rs +++ b/crates/media/src/exr_index.rs @@ -150,7 +150,8 @@ const MAX_DATA_PIXELS: usize = 4 * MAX_DISPLAY_PIXELS; /// blocks of at most `craft-exr`'s default ratio of bytes per byte of their chunk (four times /// that for DWA, plus 1 MiB): a hostile header declaring a block of gigabytes over a chunk of a /// few bytes is refused, by every read and by [`safe_to_decode`]. A read's planes are bounded by -/// its own cap ([`crate::exr_read::MAX_DECODE_BYTES`]). +/// its own cap ([`crate::exr_read::MAX_DECODE_BYTES`], or more with more memory: +/// [`effectcraft_render::memory::decode_limit`]). fn limits() -> Limits { let mut limits = Limits::none(); limits.max_pixels = MAX_DATA_PIXELS as u64; diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index bf261491b..8b6daf167 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -434,14 +434,15 @@ pub(crate) fn set_cap(path: &str, bytes: usize) { CAPS.lock().unwrap_or_else(PoisonError::into_inner).push((path.to_string(), bytes)); } -/// The most bytes of planes one read of `path` may allocate. +/// The most bytes of planes one read of `path` may allocate by the caller's own limit (none: the +/// system's, [`effectcraft_render::memory::decode_limit`]). fn cap_of(path: &str) -> usize { #[cfg(test)] if let Some((_, c)) = CAPS.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(p, _)| p == path) { return *c; } let _ = path; - crate::exr_read::MAX_DECODE_BYTES + usize::MAX } impl PlaneStore { diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index 02f99a2c1..febc926ba 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -1580,6 +1580,41 @@ fn a_failed_picture_leaves_the_other_channels_readable() { assert!(rest.aux.get("Z").is_some()); } +/// #3749: a read over the fixed floor is allowed up to what the system's memory gives (the +/// session's figure), never under the floor, and refused with a reason beyond it. The floor is +/// small here: the real one is 8 GiB. +#[test] +fn a_read_over_the_floor_is_allowed_up_to_the_systems_memory() { + let bytes = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8)); + let mut src = std::io::Cursor::new(bytes.as_slice()); + let index = crate::exr_index::build(&mut src, FileStamp::Virtual).unwrap(); + // Four channels of 8×8 f32 are 1024 bytes; the floor is 512. + let request = ReadRequest { floor: 512, ..ReadRequest::full(vec![0, 1, 2, 3]) }; + let read = |src: &mut std::io::Cursor<&[u8]>| crate::exr_read::read_planes(src, &index, &request).0; + effectcraft_render::memory::set_decode_limit(None); + effectcraft_render::memory::set_decode_limit_source(None); + assert!(matches!(read(&mut src), Err(crate::exr_read::ReadError::TooLarge(_))), "over the floor, memory unknown"); + effectcraft_render::memory::set_decode_limit(Some(2048)); + assert_eq!(read(&mut src).map(|p| p.len()).ok(), Some(4), "the system's memory allows it"); + effectcraft_render::memory::set_decode_limit(Some(800)); + assert!(matches!(read(&mut src), Err(crate::exr_read::ReadError::TooLarge(_))), "still more than the system's memory"); + effectcraft_render::memory::set_decode_limit(None); +} + +/// #3749: a plane the system cannot give is not an aborted process: `craft-exr` asks for the +/// planes' memory first and reports what it can't get, a read that failed just now (asked again, +/// not remembered); a plane over the read's limit (the decode limit handed to it) is a refusal +/// for size with `craft-exr`'s reason. +#[test] +fn a_plane_the_system_cannot_give_is_refused_not_allocated() { + use crate::exr_read::{ReadError, read_error}; + use craft_exr::seek::Error; + let gone = read_error(Error::OutOfMemory("4096 bytes of planes".into())); + assert!(matches!(&gone, ReadError::Io(why) if why.contains("4096 bytes of planes")), "{gone:?}"); + let over = read_error(Error::LimitExceeded("the requested rows need 4096 bytes".into())); + assert!(matches!(&over, ReadError::TooLarge(why) if why.contains("4096 bytes")), "{over:?}"); +} + /// A two-rank Cryptomatte file of `size` with a picture: "cube" and "sphere" meet along a /// diagonal whose pixels hold both IDs (rank 00 the nearer, 55-75 %, rank 01 the other one with /// the rest), IDs and coverage stored as 32-bit floats. diff --git a/crates/media/src/exr_read.rs b/crates/media/src/exr_read.rs index ca30f6994..5f9151049 100644 --- a/crates/media/src/exr_read.rs +++ b/crates/media/src/exr_read.rs @@ -178,14 +178,17 @@ pub(crate) struct ReadRequest { pub(crate) rows: Option>, /// Every 2^level-th row and column of the display window (0: full resolution). pub(crate) level: u32, - /// The most bytes of planes the read may allocate. + /// The most bytes of planes the read may allocate (the caller's own limit; `usize::MAX`: only + /// the system's, [`effectcraft_render::memory::decode_limit`]). pub(crate) max_bytes: usize, + /// The fixed limit the system's memory may raise ([`MAX_DECODE_BYTES`]). + pub(crate) floor: usize, } impl ReadRequest { /// Every row of `channels` at full resolution. pub(crate) fn full(channels: Vec) -> ReadRequest { - ReadRequest { channels, rows: None, level: 0, max_bytes: MAX_DECODE_BYTES } + ReadRequest { channels, rows: None, level: 0, max_bytes: usize::MAX, floor: MAX_DECODE_BYTES } } } @@ -214,8 +217,10 @@ pub(crate) enum ReadError { Cancelled, } -/// The most bytes of planes one read allocates (every requested channel is a display-window -/// plane of f32): a header can ask for far more than the file holds. +/// The most bytes of planes one read allocates at least (every requested channel is a +/// display-window plane of f32): a header can ask for far more than the file holds, and a file's +/// size on disk says nothing about it. With more memory in the machine the limit is higher +/// ([`effectcraft_render::memory::decode_limit`], #3749). #[cfg(target_pointer_width = "64")] pub(crate) const MAX_DECODE_BYTES: usize = 8 << 30; #[cfg(not(target_pointer_width = "64"))] @@ -256,14 +261,19 @@ pub(crate) fn read_planes(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRe } } +fn gb(bytes: usize) -> f64 { + bytes as f64 / f64::from(1u32 << 30) +} + fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<(Vec>, ReadStats), ReadError> { let (w, h) = index.size(); let (ow, oh) = seek::level_size((w, h), req.level.min(16)); let n = ow.checked_mul(oh).ok_or_else(|| ReadError::Unreadable("display window".into()))?; let need = n.saturating_mul(req.channels.len()).saturating_mul(std::mem::size_of::()); - let most = req.max_bytes.min(MAX_DECODE_BYTES); + // (the limit is decided here and handed to `craft-exr` as the read's `max_bytes`: the planes + // it reserves, fallibly, and the largest block it decompresses follow it) + let most = req.max_bytes.min(effectcraft_render::memory::decode_limit(req.floor, need)); if need > most { - let gb = |b: usize| b as f64 / f64::from(1u32 << 30); return Err(ReadError::TooLarge(format!( "its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", w, diff --git a/crates/render/src/memory.rs b/crates/render/src/memory.rs index 7859be071..9c2cab435 100644 --- a/crates/render/src/memory.rs +++ b/crates/render/src/memory.rs @@ -419,6 +419,44 @@ impl Accountant { } } +/// What the system's memory allows one request for a file's decoded planes (an OpenEXR's +/// channels) to take; 0: not known yet ([`decode_limit`]). +static DECODE_LIMIT: AtomicUsize = AtomicUsize::new(0); + +/// Reads that limit from the system when nothing set it ([`set_decode_limit_source`]). +static DECODE_SOURCE: Mutex Option>> = Mutex::new(None); + +/// Set what the system's memory allows one request for a file's decoded planes to take +/// (`None`: unknown again). The session sets it with the memory budget once the installed RAM +/// is read. +pub fn set_decode_limit(bytes: Option) { + DECODE_LIMIT.store(bytes.unwrap_or(0), Ordering::Relaxed); +} + +/// Where [`decode_limit`] reads the limit from while nothing set it: the CLI and the MCP server +/// read no memory otherwise. The reading may take a second, so it is only asked for when a +/// request is over the fixed floor. +pub fn set_decode_limit_source(source: Option Option>) { + *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner) = source; +} + +/// The most bytes one request of `need` bytes for a file's decoded planes may take: `floor` +/// (the fixed limit), or what the system's memory allows when that is more. The system is +/// asked only when `need` is over `floor` (and once: the answer is kept, also "unknown", which +/// leaves `floor`). The browser keeps `floor`. +pub fn decode_limit(floor: usize, need: usize) -> usize { + if need <= floor || cfg!(target_arch = "wasm32") { + return floor; + } + let mut known = DECODE_LIMIT.load(Ordering::Relaxed); + if known == 0 { + let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner); + known = source.and_then(|read| read()).unwrap_or(floor); + DECODE_LIMIT.store(known, Ordering::Relaxed); + } + floor.max(known) +} + #[cfg(test)] mod tests { use super::*; @@ -831,4 +869,48 @@ mod tests { acc.set_process_memory(process_now, 1_000, 2_000); assert!(preview.held() <= 200, "{}", preview.held()); } + + static ASKED: AtomicUsize = AtomicUsize::new(0); + + fn ram_says() -> Option { + ASKED.fetch_add(1, Ordering::SeqCst); + Some(300) + } + + fn ram_unknown() -> Option { + ASKED.fetch_add(1, Ordering::SeqCst); + None + } + + /// #3749: a request for a file's planes may take the fixed floor, or the system's memory + /// when that is more: the system is asked only for a request over the floor, once. + #[test] + fn a_request_over_the_floor_is_limited_by_the_systems_memory() { + let _serial = PROCESS_SERIAL.lock().unwrap_or_else(PoisonError::into_inner); + set_decode_limit(None); + set_decode_limit_source(Some(ram_says)); + ASKED.store(0, Ordering::SeqCst); + // Under the floor: nobody is asked. + assert_eq!(decode_limit(100, 50), 100); + assert_eq!(ASKED.load(Ordering::SeqCst), 0, "a request under the floor reads no memory"); + // Over it: the system's 300, asked once. + assert_eq!(decode_limit(100, 200), 300); + assert_eq!(decode_limit(100, 250), 300); + assert_eq!(ASKED.load(Ordering::SeqCst), 1, "asked once, then kept"); + // A smaller figure than the floor never lowers it. + set_decode_limit(Some(40)); + assert_eq!(decode_limit(100, 200), 100); + // The session's figure replaces the asked one. + set_decode_limit(Some(900)); + assert_eq!(decode_limit(100, 200), 900); + // Unknown memory leaves the floor, and is not asked again. + set_decode_limit(None); + set_decode_limit_source(Some(ram_unknown)); + ASKED.store(0, Ordering::SeqCst); + assert_eq!(decode_limit(100, 200), 100); + assert_eq!(decode_limit(100, 200), 100); + assert_eq!(ASKED.load(Ordering::SeqCst), 1, "an unknown reading is kept too"); + set_decode_limit(None); + set_decode_limit_source(None); + } }