From 11604b3379b2ff10a11b4d67c22ca4a0aa55dfd4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 21:49:18 +0100 Subject: [PATCH 1/5] OpenEXR: a file's channels may take what the machine's RAM allows, not a fixed 8 GiB (#3749) --- .../engine/src/settings/session_settings.rs | 5 ++ crates/host/src/lib.rs | 10 +++ crates/media/src/exr_planes.rs | 5 +- crates/media/src/exr_planes/tests.rs | 29 +++++++ crates/media/src/exr_read.rs | 33 ++++++-- crates/render/src/memory.rs | 82 +++++++++++++++++++ 6 files changed, 155 insertions(+), 9 deletions(-) diff --git a/crates/engine/src/settings/session_settings.rs b/crates/engine/src/settings/session_settings.rs index cc25f35ea..9a76dd84a 100644 --- a/crates/engine/src/settings/session_settings.rs +++ b/crates/engine/src/settings/session_settings.rs @@ -122,6 +122,11 @@ impl Session { let layers: std::sync::Arc = self.layer_cache.clone(); accountant.register(&layers); accountant.set_budget(b.bytes); + // (what one file's decoded planes may take follows it once the RAM is read, at least the + // fixed limit; until then the host's reader asks when a request is over that, #3749) + if b.installed.is_some() { + effectcraft_render::memory::set_decode_limit(Some(b.bytes)); + } // The process's measured memory as well (#3301: memory nobody accounts for, such as GPU // staging, effect buffers and the allocator's fragmentation): over the purge limit the // caches give way by what it is over, over the hard limit also the frames a playing diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index 295476df0..159156a24 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -117,9 +117,19 @@ pub fn session() -> Session { // the settings, and the caches have no limits of their own below it. Without a memory // reading yet, the budget for unknown RAM. s.apply_cache_budgets(); + // The CLI and the MCP server read no memory: a request for a file's planes over the fixed + // limit asks for it once (#3749). + effectcraft_engine::render::memory::set_decode_limit_source(Some(installed_memory_budget)); s } +/// The memory budget of the installed RAM with the default settings: what one file's decoded +/// planes may take when a request is over the fixed limit and the session read no memory. +fn installed_memory_budget() -> Option { + let memory = effectcraft_engine::sysinfo::memory()?; + Some(effectcraft_engine::prefs::Prefs::default().memory_budget(Some(memory), false).bytes) +} + /// External effect helpers run as child processes: native builds only. #[cfg(not(target_arch = "wasm32"))] fn helper_hooks() -> Option { diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index 3e294c30b..0792647f2 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -434,14 +434,15 @@ pub(crate) fn set_cap(path: &str, bytes: usize) { CAPS.lock().unwrap_or_else(PoisonError::into_inner).push((path.to_string(), bytes)); } -/// The most bytes of planes one read of `path` may allocate. +/// The most bytes of planes one read of `path` may allocate by the caller's own limit (none: the +/// system's, [`effectcraft_render::memory::decode_limit`]). fn cap_of(path: &str) -> usize { #[cfg(test)] if let Some((_, c)) = CAPS.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(p, _)| p == path) { return *c; } let _ = path; - crate::exr_read::MAX_DECODE_BYTES + usize::MAX } impl PlaneStore { diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index 77affea98..05f3c76a2 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -1434,6 +1434,35 @@ fn a_failed_picture_leaves_the_other_channels_readable() { assert!(rest.aux.get("Z").is_some()); } +/// #3749: a read over the fixed floor is allowed up to what the system's memory gives (the +/// session's figure), never under the floor, and refused with a reason beyond it. The floor is +/// small here: the real one is 8 GiB. +#[test] +fn a_read_over_the_floor_is_allowed_up_to_the_systems_memory() { + let bytes = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8)); + let mut src = std::io::Cursor::new(bytes.as_slice()); + let index = crate::exr_index::build(&mut src, bytes.len() as u64, FileStamp::Virtual).unwrap(); + // Four channels of 8×8 f32 are 1024 bytes; the floor is 512. + let request = ReadRequest { floor: 512, ..ReadRequest::full(vec![0, 1, 2, 3]) }; + let read = |src: &mut std::io::Cursor<&[u8]>| crate::exr_read::read_planes(src, &index, &request).0; + effectcraft_render::memory::set_decode_limit(None); + effectcraft_render::memory::set_decode_limit_source(None); + assert!(matches!(read(&mut src), Err(crate::exr_read::ReadError::TooLarge(_))), "over the floor, memory unknown"); + effectcraft_render::memory::set_decode_limit(Some(2048)); + assert_eq!(read(&mut src).map(|p| p.len()).ok(), Some(4), "the system's memory allows it"); + effectcraft_render::memory::set_decode_limit(Some(800)); + assert!(matches!(read(&mut src), Err(crate::exr_read::ReadError::TooLarge(_))), "still more than the system's memory"); + effectcraft_render::memory::set_decode_limit(None); +} + +/// #3749: a plane the system cannot give is a refusal, not an aborted process (the allocation +/// is asked for first). +#[test] +fn a_plane_the_system_cannot_give_is_refused_not_allocated() { + assert_eq!(crate::exr_read::plane(3, 1.5), Some(vec![1.5; 3])); + assert!(crate::exr_read::plane(usize::MAX / 2, 0.0).is_none()); +} + /// A two-rank Cryptomatte file of `size` with a picture: "cube" and "sphere" meet along a /// diagonal whose pixels hold both IDs (rank 00 the nearer, 55-75 %, rank 01 the other one with /// the rest), IDs and coverage stored as 32-bit floats. diff --git a/crates/media/src/exr_read.rs b/crates/media/src/exr_read.rs index dcad343f8..d0ff02b29 100644 --- a/crates/media/src/exr_read.rs +++ b/crates/media/src/exr_read.rs @@ -183,14 +183,17 @@ pub(crate) struct ReadRequest { pub(crate) rows: Option>, /// Every 2^level-th row and column of the display window (0: full resolution). pub(crate) level: u32, - /// The most bytes of planes the read may allocate. + /// The most bytes of planes the read may allocate (the caller's own limit; `usize::MAX`: only + /// the system's, [`effectcraft_render::memory::decode_limit`]). pub(crate) max_bytes: usize, + /// The fixed limit the system's memory may raise ([`MAX_DECODE_BYTES`]). + pub(crate) floor: usize, } impl ReadRequest { /// Every row of `channels` at full resolution. pub(crate) fn full(channels: Vec) -> ReadRequest { - ReadRequest { channels, rows: None, level: 0, max_bytes: MAX_DECODE_BYTES } + ReadRequest { channels, rows: None, level: 0, max_bytes: usize::MAX, floor: MAX_DECODE_BYTES } } } @@ -219,8 +222,10 @@ pub(crate) enum ReadError { Cancelled, } -/// The most bytes of planes one read allocates (every requested channel is a display-window -/// plane of f32): a header can ask for far more than the file holds. +/// The most bytes of planes one read allocates at least (every requested channel is a +/// display-window plane of f32): a header can ask for far more than the file holds, and a file's +/// size on disk says nothing about it. With more memory in the machine the limit is higher +/// ([`effectcraft_render::memory::decode_limit`], #3749). #[cfg(target_pointer_width = "64")] pub(crate) const MAX_DECODE_BYTES: usize = 8 << 30; #[cfg(not(target_pointer_width = "64"))] @@ -264,6 +269,19 @@ pub(crate) fn read_planes(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRe (planes, stats) } +fn gb(bytes: usize) -> f64 { + bytes as f64 / f64::from(1u32 << 30) +} + +/// A plane of `n` samples of `fill`; `None` when the system cannot give that memory (an +/// allocation that fails aborts the process, so it is asked for first). +pub(crate) fn plane(n: usize, fill: f32) -> Option> { + let mut samples: Vec = Vec::new(); + samples.try_reserve_exact(n).ok()?; + samples.resize(n, fill); + Some(samples) +} + fn read_into(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest, stats: &mut ReadStats) -> Result>, ReadError> { let level = req.level.min(16); let stride = 1i64 << level; @@ -271,9 +289,8 @@ fn read_into(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest, stats: let (ow, oh) = level_size((w, h), level); let n = ow.checked_mul(oh).ok_or_else(|| ReadError::Unreadable("display window".into()))?; let need = n.saturating_mul(req.channels.len()).saturating_mul(std::mem::size_of::()); - let most = req.max_bytes.min(MAX_DECODE_BYTES); + let most = req.max_bytes.min(effectcraft_render::memory::decode_limit(req.floor, need)); if need > most { - let gb = |b: usize| b as f64 / f64::from(1u32 << 30); return Err(ReadError::TooLarge(format!( "its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", w, @@ -290,7 +307,9 @@ fn read_into(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest, stats: if let Some(s) = slots.get_mut(c.part).and_then(|p| p.get_mut(c.channel)) { *s = Some(slot); } - planes.push(vec![c.fill; n]); + planes.push(plane(n, c.fill).ok_or_else(|| { + ReadError::TooLarge(format!("its {}×{} OpenEXR channels need {:.1} GB of memory, and the system cannot provide that", w, h, gb(need))) + })?); } let rows = req.rows.clone().unwrap_or(0..h as i64); let (r0, r1) = (rows.start.max(0), rows.end.min(h as i64)); diff --git a/crates/render/src/memory.rs b/crates/render/src/memory.rs index 7859be071..9c2cab435 100644 --- a/crates/render/src/memory.rs +++ b/crates/render/src/memory.rs @@ -419,6 +419,44 @@ impl Accountant { } } +/// What the system's memory allows one request for a file's decoded planes (an OpenEXR's +/// channels) to take; 0: not known yet ([`decode_limit`]). +static DECODE_LIMIT: AtomicUsize = AtomicUsize::new(0); + +/// Reads that limit from the system when nothing set it ([`set_decode_limit_source`]). +static DECODE_SOURCE: Mutex Option>> = Mutex::new(None); + +/// Set what the system's memory allows one request for a file's decoded planes to take +/// (`None`: unknown again). The session sets it with the memory budget once the installed RAM +/// is read. +pub fn set_decode_limit(bytes: Option) { + DECODE_LIMIT.store(bytes.unwrap_or(0), Ordering::Relaxed); +} + +/// Where [`decode_limit`] reads the limit from while nothing set it: the CLI and the MCP server +/// read no memory otherwise. The reading may take a second, so it is only asked for when a +/// request is over the fixed floor. +pub fn set_decode_limit_source(source: Option Option>) { + *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner) = source; +} + +/// The most bytes one request of `need` bytes for a file's decoded planes may take: `floor` +/// (the fixed limit), or what the system's memory allows when that is more. The system is +/// asked only when `need` is over `floor` (and once: the answer is kept, also "unknown", which +/// leaves `floor`). The browser keeps `floor`. +pub fn decode_limit(floor: usize, need: usize) -> usize { + if need <= floor || cfg!(target_arch = "wasm32") { + return floor; + } + let mut known = DECODE_LIMIT.load(Ordering::Relaxed); + if known == 0 { + let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner); + known = source.and_then(|read| read()).unwrap_or(floor); + DECODE_LIMIT.store(known, Ordering::Relaxed); + } + floor.max(known) +} + #[cfg(test)] mod tests { use super::*; @@ -831,4 +869,48 @@ mod tests { acc.set_process_memory(process_now, 1_000, 2_000); assert!(preview.held() <= 200, "{}", preview.held()); } + + static ASKED: AtomicUsize = AtomicUsize::new(0); + + fn ram_says() -> Option { + ASKED.fetch_add(1, Ordering::SeqCst); + Some(300) + } + + fn ram_unknown() -> Option { + ASKED.fetch_add(1, Ordering::SeqCst); + None + } + + /// #3749: a request for a file's planes may take the fixed floor, or the system's memory + /// when that is more: the system is asked only for a request over the floor, once. + #[test] + fn a_request_over_the_floor_is_limited_by_the_systems_memory() { + let _serial = PROCESS_SERIAL.lock().unwrap_or_else(PoisonError::into_inner); + set_decode_limit(None); + set_decode_limit_source(Some(ram_says)); + ASKED.store(0, Ordering::SeqCst); + // Under the floor: nobody is asked. + assert_eq!(decode_limit(100, 50), 100); + assert_eq!(ASKED.load(Ordering::SeqCst), 0, "a request under the floor reads no memory"); + // Over it: the system's 300, asked once. + assert_eq!(decode_limit(100, 200), 300); + assert_eq!(decode_limit(100, 250), 300); + assert_eq!(ASKED.load(Ordering::SeqCst), 1, "asked once, then kept"); + // A smaller figure than the floor never lowers it. + set_decode_limit(Some(40)); + assert_eq!(decode_limit(100, 200), 100); + // The session's figure replaces the asked one. + set_decode_limit(Some(900)); + assert_eq!(decode_limit(100, 200), 900); + // Unknown memory leaves the floor, and is not asked again. + set_decode_limit(None); + set_decode_limit_source(Some(ram_unknown)); + ASKED.store(0, Ordering::SeqCst); + assert_eq!(decode_limit(100, 200), 100); + assert_eq!(decode_limit(100, 200), 100); + assert_eq!(ASKED.load(Ordering::SeqCst), 1, "an unknown reading is kept too"); + set_decode_limit(None); + set_decode_limit_source(None); + } } From 71f7181e24d697fe32410073c8b5d23eab926c01 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 21:52:34 +0100 Subject: [PATCH 2/5] OpenEXR limit: a refusal for size is decided anew when the limit changes; a failed reading is asked twice (#3749 review) --- crates/media/src/exr_planes.rs | 17 +++++-- crates/media/src/exr_planes/tests.rs | 30 ++++++++++++ crates/render/src/memory.rs | 71 +++++++++++++++++++++++----- 3 files changed, 100 insertions(+), 18 deletions(-) diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index 0792647f2..9ffb89079 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -183,6 +183,10 @@ struct Flight { /// When the decode was asked for ([`effectcraft_render::memory::tick`]): what it took is the /// planes' cost to the memory accountant. asked: u64, + /// The decode limit's generation when the decode was asked for + /// ([`effectcraft_render::memory::decode_limit_generation`]): a refusal for its size holds + /// only while the limit stays as it was. + limit_generation: u64, } /// [`Flight::phase`] values. @@ -219,6 +223,7 @@ impl Flight { since, hints: filmcraft_media::cancel::Context::current(), asked: effectcraft_render::memory::tick(), + limit_generation: effectcraft_render::memory::decode_limit_generation(), }) } @@ -276,8 +281,9 @@ struct Entry { flight: Option>, /// Requests that failed for good (unreadable, too large), the index they were made with, and /// why: a request of that index (the file as it was) asking for at - /// least as much fails the same way until the file is forgotten. - failed: Vec<(Arc, Want, Refused)>, + /// least as much fails the same way until the file is forgotten, or the decode limit changed + /// (the last number: its generation when the request was made). + failed: Vec<(Arc, Want, Refused, u64)>, bytes: usize, /// When it was last asked for or finished decoding ([`effectcraft_render::memory::tick`]). used: u64, @@ -522,7 +528,8 @@ impl PlaneStore { return Step::Done(Ok(b)); } // (a failure of the file as it was before: the file changed since, it is read again) - if let Some((_, _, why)) = e.failed.iter().find(|(of, failed, _)| Arc::ptr_eq(of, index) && asks_at_least(index, want, failed)) { + let limit = effectcraft_render::memory::decode_limit_generation(); + if let Some((_, _, why, _)) = e.failed.iter().find(|(of, failed, _, made_at)| Arc::ptr_eq(of, index) && *made_at == limit && asks_at_least(index, want, failed)) { return Step::Done(Err(why.clone())); } if let Some(flight) = e.flight.clone() { @@ -671,11 +678,11 @@ impl PlaneStore { (fs.index.clone(), fs.want.clone()) }; // (failures of another version of the file no longer apply) - e.failed.retain(|(of, _, _)| Arc::ptr_eq(of, &index)); + e.failed.retain(|(of, _, _, made_at)| Arc::ptr_eq(of, &index) && *made_at == flight.limit_generation); if e.failed.len() >= FAILED_KEPT { e.failed.remove(0); } - e.failed.push((index, want, why.clone())); + e.failed.push((index, want, why.clone(), flight.limit_generation)); } // (the file changed: what is held is of the old one) Err(Refused::Changed) => { diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index 05f3c76a2..1dcde8aae 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -1434,11 +1434,16 @@ fn a_failed_picture_leaves_the_other_channels_readable() { assert!(rest.aux.get("Z").is_some()); } +/// The tests that move the process-wide decode limit take turns (it also dates the remembered +/// refusals). +static LIMIT_TESTS: Mutex<()> = Mutex::new(()); + /// #3749: a read over the fixed floor is allowed up to what the system's memory gives (the /// session's figure), never under the floor, and refused with a reason beyond it. The floor is /// small here: the real one is 8 GiB. #[test] fn a_read_over_the_floor_is_allowed_up_to_the_systems_memory() { + let _only = LIMIT_TESTS.lock().unwrap_or_else(PoisonError::into_inner); let bytes = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8)); let mut src = std::io::Cursor::new(bytes.as_slice()); let index = crate::exr_index::build(&mut src, bytes.len() as u64, FileStamp::Virtual).unwrap(); @@ -1455,6 +1460,31 @@ fn a_read_over_the_floor_is_allowed_up_to_the_systems_memory() { effectcraft_render::memory::set_decode_limit(None); } +/// #3749: a refusal for the request's size is remembered only while the decode limit stays as it +/// was: when the system's memory figure arrives or changes, the request is decided anew. +#[test] +fn a_refusal_for_size_is_decided_anew_when_the_decode_limit_changes() { + let _only = LIMIT_TESTS.lock().unwrap_or_else(PoisonError::into_inner); + let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8)).into(); + let path = "/3749-refusal-generation.exr"; + // Room for one plane of 8×8 f32: four are refused. + set_cap(path, 8 * 8 * 4); + let store = PlaneStore::new(CACHE_BUDGET); + let source = ExrSource::Bytes(bytes); + let index = crate::exr_index::get(path, &source).unwrap(); + let want = Want { all: false, ids: (0..4).collect() }; + let refused = || matches!(store.get(path, &source, &index, &want, Use::Other), Err(Refused::TooLarge(_))); + assert!(refused()); + assert!(refused()); + assert_eq!(decodes_of(path).len(), 1, "the second refusal came from memory"); + // (figures under the fixed floor: only the change counts, no other test's limit moves) + effectcraft_render::memory::set_decode_limit(Some(1000)); + effectcraft_render::memory::set_decode_limit(Some(2000)); + assert!(refused(), "still over the caller's own limit"); + assert_eq!(decodes_of(path).len(), 2, "decided anew after the limit changed"); + effectcraft_render::memory::set_decode_limit(None); +} + /// #3749: a plane the system cannot give is a refusal, not an aborted process (the allocation /// is asked for first). #[test] diff --git a/crates/render/src/memory.rs b/crates/render/src/memory.rs index 9c2cab435..ae8d02104 100644 --- a/crates/render/src/memory.rs +++ b/crates/render/src/memory.rs @@ -423,14 +423,40 @@ impl Accountant { /// channels) to take; 0: not known yet ([`decode_limit`]). static DECODE_LIMIT: AtomicUsize = AtomicUsize::new(0); +/// Counts the changes of that limit: a refusal made under an older one is out of date +/// ([`decode_limit_generation`]). +static DECODE_GENERATION: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// How often the system was asked for the limit without an answer ([`decode_limit`]). +static DECODE_UNANSWERED: AtomicUsize = AtomicUsize::new(0); + /// Reads that limit from the system when nothing set it ([`set_decode_limit_source`]). static DECODE_SOURCE: Mutex Option>> = Mutex::new(None); +/// One reading of the system at a time: the second asker takes the first one's answer. +static DECODE_ASKING: Mutex<()> = Mutex::new(()); + +/// The system is asked this often at most for the limit when it does not answer. +const DECODE_ASKS: usize = 2; + +fn store_decode_limit(bytes: usize) { + if DECODE_LIMIT.swap(bytes, Ordering::Relaxed) != bytes { + DECODE_GENERATION.fetch_add(1, Ordering::Relaxed); + } +} + /// Set what the system's memory allows one request for a file's decoded planes to take /// (`None`: unknown again). The session sets it with the memory budget once the installed RAM /// is read. pub fn set_decode_limit(bytes: Option) { - DECODE_LIMIT.store(bytes.unwrap_or(0), Ordering::Relaxed); + store_decode_limit(bytes.unwrap_or(0)); + DECODE_UNANSWERED.store(0, Ordering::Relaxed); +} + +/// Changes whenever [`decode_limit`] gives another answer: a request refused for its size is +/// asked again after that. +pub fn decode_limit_generation() -> u64 { + DECODE_GENERATION.load(Ordering::Relaxed) } /// Where [`decode_limit`] reads the limit from while nothing set it: the CLI and the MCP server @@ -438,23 +464,31 @@ pub fn set_decode_limit(bytes: Option) { /// request is over the fixed floor. pub fn set_decode_limit_source(source: Option Option>) { *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner) = source; + DECODE_UNANSWERED.store(0, Ordering::Relaxed); } /// The most bytes one request of `need` bytes for a file's decoded planes may take: `floor` /// (the fixed limit), or what the system's memory allows when that is more. The system is -/// asked only when `need` is over `floor` (and once: the answer is kept, also "unknown", which -/// leaves `floor`). The browser keeps `floor`. +/// asked only when `need` is over `floor`, and the answer is kept; when it gives none (a reading +/// that failed) it is asked once more later, then `floor` stays. The browser keeps `floor`. pub fn decode_limit(floor: usize, need: usize) -> usize { if need <= floor || cfg!(target_arch = "wasm32") { return floor; } - let mut known = DECODE_LIMIT.load(Ordering::Relaxed); - if known == 0 { - let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner); - known = source.and_then(|read| read()).unwrap_or(floor); - DECODE_LIMIT.store(known, Ordering::Relaxed); + if DECODE_LIMIT.load(Ordering::Relaxed) == 0 && DECODE_UNANSWERED.load(Ordering::Relaxed) < DECODE_ASKS { + let _asking = DECODE_ASKING.lock().unwrap_or_else(PoisonError::into_inner); + // (another thread may have just read it) + if DECODE_LIMIT.load(Ordering::Relaxed) == 0 { + let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner); + match source.and_then(|read| read()) { + Some(bytes) => store_decode_limit(bytes), + None => { + DECODE_UNANSWERED.fetch_add(1, Ordering::Relaxed); + } + } + } } - floor.max(known) + floor.max(DECODE_LIMIT.load(Ordering::Relaxed)) } #[cfg(test)] @@ -903,13 +937,24 @@ mod tests { // The session's figure replaces the asked one. set_decode_limit(Some(900)); assert_eq!(decode_limit(100, 200), 900); - // Unknown memory leaves the floor, and is not asked again. + // Unknown memory leaves the floor; a reading that failed once is tried once more, then + // not asked again (and what is not known is never kept as the floor). set_decode_limit(None); set_decode_limit_source(Some(ram_unknown)); ASKED.store(0, Ordering::SeqCst); - assert_eq!(decode_limit(100, 200), 100); - assert_eq!(decode_limit(100, 200), 100); - assert_eq!(ASKED.load(Ordering::SeqCst), 1, "an unknown reading is kept too"); + for _ in 0..4 { + assert_eq!(decode_limit(100, 200), 100); + } + assert_eq!(ASKED.load(Ordering::SeqCst), 2, "a failed reading is tried twice"); + // A source that answers later is not shut out by the floor kept earlier. + set_decode_limit_source(Some(ram_says)); + assert_eq!(decode_limit(7, 200), 300, "a limit is not tied to the floor of an earlier request"); + // The generation tells that the limit changed. + let before = decode_limit_generation(); + set_decode_limit(Some(300)); + assert_eq!(decode_limit_generation(), before, "the same figure is no change"); + set_decode_limit(Some(500)); + assert_ne!(decode_limit_generation(), before); set_decode_limit(None); set_decode_limit_source(None); } From f13cb15aeaf49bc8d1e8b808b443c0993350fd6b Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 21:53:52 +0100 Subject: [PATCH 3/5] OpenEXR limit: only a refusal for size depends on the limit's generation; sequentially consistent counters (#3749 review) --- crates/media/src/exr_planes.rs | 5 +++-- crates/render/src/memory.rs | 16 +++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index 9ffb89079..7f00c956b 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -529,7 +529,7 @@ impl PlaneStore { } // (a failure of the file as it was before: the file changed since, it is read again) let limit = effectcraft_render::memory::decode_limit_generation(); - if let Some((_, _, why, _)) = e.failed.iter().find(|(of, failed, _, made_at)| Arc::ptr_eq(of, index) && *made_at == limit && asks_at_least(index, want, failed)) { + if let Some((_, _, why, _)) = e.failed.iter().find(|(of, failed, _, made_at)| Arc::ptr_eq(of, index) && (!matches!(why, Refused::TooLarge(_)) || *made_at == limit) && asks_at_least(index, want, failed)) { return Step::Done(Err(why.clone())); } if let Some(flight) = e.flight.clone() { @@ -678,7 +678,8 @@ impl PlaneStore { (fs.index.clone(), fs.want.clone()) }; // (failures of another version of the file no longer apply) - e.failed.retain(|(of, _, _, made_at)| Arc::ptr_eq(of, &index) && *made_at == flight.limit_generation); + // (a refusal for the size counts under the limit it was made under only) + e.failed.retain(|(of, _, old, made_at)| Arc::ptr_eq(of, &index) && (!matches!(old, Refused::TooLarge(_)) || *made_at == flight.limit_generation)); if e.failed.len() >= FAILED_KEPT { e.failed.remove(0); } diff --git a/crates/render/src/memory.rs b/crates/render/src/memory.rs index ae8d02104..d1600551b 100644 --- a/crates/render/src/memory.rs +++ b/crates/render/src/memory.rs @@ -440,8 +440,10 @@ static DECODE_ASKING: Mutex<()> = Mutex::new(()); const DECODE_ASKS: usize = 2; fn store_decode_limit(bytes: usize) { - if DECODE_LIMIT.swap(bytes, Ordering::Relaxed) != bytes { - DECODE_GENERATION.fetch_add(1, Ordering::Relaxed); + // (the limit before its generation, both sequentially consistent: whoever sees the new + // generation sees the new limit) + if DECODE_LIMIT.swap(bytes, Ordering::SeqCst) != bytes { + DECODE_GENERATION.fetch_add(1, Ordering::SeqCst); } } @@ -456,7 +458,7 @@ pub fn set_decode_limit(bytes: Option) { /// Changes whenever [`decode_limit`] gives another answer: a request refused for its size is /// asked again after that. pub fn decode_limit_generation() -> u64 { - DECODE_GENERATION.load(Ordering::Relaxed) + DECODE_GENERATION.load(Ordering::SeqCst) } /// Where [`decode_limit`] reads the limit from while nothing set it: the CLI and the MCP server @@ -475,10 +477,10 @@ pub fn decode_limit(floor: usize, need: usize) -> usize { if need <= floor || cfg!(target_arch = "wasm32") { return floor; } - if DECODE_LIMIT.load(Ordering::Relaxed) == 0 && DECODE_UNANSWERED.load(Ordering::Relaxed) < DECODE_ASKS { + if DECODE_LIMIT.load(Ordering::SeqCst) == 0 { let _asking = DECODE_ASKING.lock().unwrap_or_else(PoisonError::into_inner); - // (another thread may have just read it) - if DECODE_LIMIT.load(Ordering::Relaxed) == 0 { + // (another thread may have just read it, or have used up the asks) + if DECODE_LIMIT.load(Ordering::SeqCst) == 0 && DECODE_UNANSWERED.load(Ordering::Relaxed) < DECODE_ASKS { let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner); match source.and_then(|read| read()) { Some(bytes) => store_decode_limit(bytes), @@ -488,7 +490,7 @@ pub fn decode_limit(floor: usize, need: usize) -> usize { } } } - floor.max(DECODE_LIMIT.load(Ordering::Relaxed)) + floor.max(DECODE_LIMIT.load(Ordering::SeqCst)) } #[cfg(test)] From 3b869b83615ef52db1e718e6dfc840fbd8db1dfc Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 22:03:56 +0100 Subject: [PATCH 4/5] OpenEXR limit: formatting and the refusal's name in the lookup (#3749) --- crates/media/src/exr_planes.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index 7f00c956b..edfea4ea8 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -529,7 +529,9 @@ impl PlaneStore { } // (a failure of the file as it was before: the file changed since, it is read again) let limit = effectcraft_render::memory::decode_limit_generation(); - if let Some((_, _, why, _)) = e.failed.iter().find(|(of, failed, _, made_at)| Arc::ptr_eq(of, index) && (!matches!(why, Refused::TooLarge(_)) || *made_at == limit) && asks_at_least(index, want, failed)) { + if let Some((_, _, why, _)) = e.failed.iter().find(|(of, failed, refusal, made_at)| { + Arc::ptr_eq(of, index) && (!matches!(refusal, Refused::TooLarge(_)) || *made_at == limit) && asks_at_least(index, want, failed) + }) { return Step::Done(Err(why.clone())); } if let Some(flight) = e.flight.clone() { @@ -679,7 +681,8 @@ impl PlaneStore { }; // (failures of another version of the file no longer apply) // (a refusal for the size counts under the limit it was made under only) - e.failed.retain(|(of, _, old, made_at)| Arc::ptr_eq(of, &index) && (!matches!(old, Refused::TooLarge(_)) || *made_at == flight.limit_generation)); + e.failed + .retain(|(of, _, old, made_at)| Arc::ptr_eq(of, &index) && (!matches!(old, Refused::TooLarge(_)) || *made_at == flight.limit_generation)); if e.failed.len() >= FAILED_KEPT { e.failed.remove(0); } From a998f04903177e66f5d3a4e56a3c79a823d2041e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 16 Nov 2026 17:49:20 +0100 Subject: [PATCH 5/5] OpenEXR limit: the planes a request ends with count together, held and from the disk cache too (#3749) The limit applied to the planes read from the file just now, so planes held in memory or handed back by the disk cache were never counted and repeated requests could pass it. It now applies to what the request ends with, asked before anything is read or allocated. When the held planes don't fit with the missing ones they are let go and the request is read on its own; only a request that doesn't fit alone is refused. --- crates/media/src/exr_planes.rs | 58 ++++++++++++++------- crates/media/src/exr_planes/tests.rs | 78 +++++++++++++++++++++++++++- crates/media/src/exr_read.rs | 25 ++++++--- 3 files changed, 133 insertions(+), 28 deletions(-) diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index edfea4ea8..584ea76c9 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -907,6 +907,26 @@ fn run(path: &str, source: &ExrSource, flight: &Arc, base: Option Refused { + match e { + ReadError::Cancelled => Refused::Cancelled, + // (a read that failed just now: the next request reads again) + ReadError::Io(why) => { + log::warn!("media: {path}: {why}"); + Refused::Stopped + } + ReadError::TooLarge(why) => { + log::warn!("media: {path}: {why}"); + Refused::TooLarge(why) + } + ReadError::Unreadable(why) => { + log::warn!("media: {path}: {why}"); + Refused::Unreadable + } + } +} + /// Decode what `flight` asks for and `base` doesn't hold, and make the union of both. fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option>, opened: Option) -> Result, Refused> { let _decoding = Decoding::new(); @@ -947,10 +967,25 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option = wanted.into_iter().filter(|i| *i < index.channels.len()).collect(); - let base = base.filter(|b| Arc::ptr_eq(&b.index, &index)); - let missing: Vec = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect(); - let names: Vec = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect(); + let mut base = base.filter(|b| Arc::ptr_eq(&b.index, &index)); + let mut missing: Vec = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect(); let (w, h) = index.size(); + // The planes the request ends with (those held plus those missing, whether they come from the + // disk cache or the file) must fit the limit, asked before any of them is allocated (#3749). + // The planes held are only kept for later requests: when they don't fit with the missing + // ones, they are let go and the request is read on its own. + if !missing.is_empty() { + let held = base.as_ref().map_or(0, |b| b.ids.len()); + let together = crate::exr_read::check_total(w, h, held.saturating_add(missing.len()), cap_of(path)); + if together.is_err() && base.is_some() { + base = None; + missing = wanted.clone(); + } + if base.is_none() { + crate::exr_read::check_total(w, h, missing.len(), cap_of(path)).map_err(|e| refused(path, e))?; + } + } + let names: Vec = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect(); // The disk tier first (#3271): what an earlier session or a decode RAM let go of kept, one // entry per channel (any request is assembled from them, whatever was asked before). // (a file naming two channels alike can't tell their entries apart: not kept) @@ -984,22 +1019,7 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option> = read.map_err(|e| match e { - ReadError::Cancelled => Refused::Cancelled, - // (a read that failed just now: the next request reads again) - ReadError::Io(why) => { - log::warn!("media: {path}: {why}"); - Refused::Stopped - } - ReadError::TooLarge(why) => { - log::warn!("media: {path}: {why}"); - Refused::TooLarge(why) - } - ReadError::Unreadable(why) => { - log::warn!("media: {path}: {why}"); - Refused::Unreadable - } - })?; + let read: Vec> = read.map_err(|e| refused(path, e))?; let mut read = read.into_iter(); let planes: Vec> = kept.iter_mut().map(|k| k.take().or_else(|| read.next()).unwrap_or_default()).collect(); // The union, in file order: the planes held and the ones just read. diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index 1dcde8aae..f1f35dc5c 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -1473,18 +1473,92 @@ fn a_refusal_for_size_is_decided_anew_when_the_decode_limit_changes() { let source = ExrSource::Bytes(bytes); let index = crate::exr_index::get(path, &source).unwrap(); let want = Want { all: false, ids: (0..4).collect() }; + // (the refusal is made before anything is read: the decodes are counted as they begin) + let begun = Arc::new(AtomicU64::new(0)); + { + let begun = begun.clone(); + set_before_read( + path, + Some(Arc::new(move |_: &str| { + begun.fetch_add(1, Ordering::SeqCst); + })), + ); + } let refused = || matches!(store.get(path, &source, &index, &want, Use::Other), Err(Refused::TooLarge(_))); assert!(refused()); assert!(refused()); - assert_eq!(decodes_of(path).len(), 1, "the second refusal came from memory"); + assert_eq!(begun.load(Ordering::SeqCst), 1, "the second refusal came from memory"); // (figures under the fixed floor: only the change counts, no other test's limit moves) effectcraft_render::memory::set_decode_limit(Some(1000)); effectcraft_render::memory::set_decode_limit(Some(2000)); assert!(refused(), "still over the caller's own limit"); - assert_eq!(decodes_of(path).len(), 2, "decided anew after the limit changed"); + assert_eq!(begun.load(Ordering::SeqCst), 2, "decided anew after the limit changed"); + set_before_read(path, None); effectcraft_render::memory::set_decode_limit(None); } +/// A file of an 8×8 picture and a depth plane: five planes of 256 bytes. +fn picture_and_depth() -> Vec { + write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", true), ("Z", false)], (8, 8), Compression::ZIP1)], (8, 8)) +} + +/// #3749 (#3301 review): the limit is for the planes a request ends with, not for the ones read +/// just now: channels added to the planes held count with them. When together they are over +/// the limit the planes held are let go and the request is read on its own (it fits alone); a +/// request that doesn't fit alone is refused, whatever is held. +#[test] +fn channels_added_to_the_planes_held_count_toward_the_limit() { + let path = "/3749-held-and-added.exr"; + // Room for four planes: the picture fits, one more with it is five. + set_cap(path, 4 * 8 * 8 * 4); + let store = PlaneStore::new(CACHE_BUDGET); + let source = ExrSource::Bytes(picture_and_depth().into()); + let index = crate::exr_index::get(path, &source).unwrap(); + let picture = Want { all: false, ids: index.picture_ids().unwrap().into_iter().collect() }; + store.get(path, &source, &index, &picture, Use::Other).expect("the picture fits"); + let rest = Want { all: true, ids: Default::default() }; + let depth = store.get(path, &source, &index, &rest, Use::Other).expect("the depth plane fits alone"); + assert!(depth.aux.get("Z").is_some()); + assert!(depth.aux.get("R").is_none(), "the picture's planes were let go: four held and one more are five"); + // The picture again: it fits alone, the depth plane is let go in turn. + let again = store.get(path, &source, &index, &picture, Use::Other).expect("the picture fits alone"); + assert!(again.aux.get("R").is_some() && again.aux.get("Z").is_none()); + // Everything at once is five planes: refused, however it is asked. + let everything = Want { all: true, ids: (0..5).collect() }; + assert!(matches!(store.get(path, &source, &index, &everything, Use::Other).err(), Some(Refused::TooLarge(_)))); +} + +/// #3749 (#3301 review): planes the disk cache gives back count toward the limit of a request +/// like the ones read from the file. +#[test] +fn planes_from_the_disk_cache_count_toward_the_limit() { + use effectcraft_render::disk_cache::DiskCache; + let dir = std::env::temp_dir().join(format!("ec-3749-disk-union-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("union_0001.exr"); + std::fs::write(&file, picture_and_depth()).unwrap(); + let dc = DiskCache::open(dir.join("cache"), 1 << 30).unwrap(); + dc.set_read_us_per_mb(0); + use_disk_for(&dir.to_string_lossy()); + set_disk_cache(Some(&dc)); + let path = file.to_string_lossy().to_string(); + let source = ExrSource::File(file.clone()); + let index = crate::exr_index::get(&path, &source).unwrap(); + let all = Want { all: true, ids: (0..5).collect() }; + let first = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).is_ok(); + dc.flush(); + // All five planes are in the disk cache (a store without them in RAM reads them back)... + let warm = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).is_ok(); + let from_disk = decodes_of(&path).last().is_some_and(|d| d.from_disk); + // ...and a request may use four. + set_cap(&path, 4 * 8 * 8 * 4); + let refused = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).err(); + set_disk_cache(None); + let _ = std::fs::remove_dir_all(&dir); + assert!(first && warm && from_disk, "decoded without a limit, then read back from the disk cache"); + assert!(matches!(refused, Some(Refused::TooLarge(_))), "{refused:?}"); +} + /// #3749: a plane the system cannot give is a refusal, not an aborted process (the allocation /// is asked for first). #[test] diff --git a/crates/media/src/exr_read.rs b/crates/media/src/exr_read.rs index d0ff02b29..9bdc7c048 100644 --- a/crates/media/src/exr_read.rs +++ b/crates/media/src/exr_read.rs @@ -282,6 +282,23 @@ pub(crate) fn plane(n: usize, fill: f32) -> Option> { Some(samples) } +fn too_large(w: usize, h: usize, need: usize, most: usize) -> ReadError { + ReadError::TooLarge(format!("its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", w, h, gb(need), gb(most))) +} + +/// Refuse a request whose planes together, those already in hand (held in memory or read back +/// from the disk cache) and those still to read, `planes` of a `w` × `h` display window, need +/// more memory than one file may use: the limit is for what a file's request ends with, not +/// for the part of it read just now ([`effectcraft_render::memory::decode_limit`], #3749). +pub(crate) fn check_total(w: usize, h: usize, planes: usize, max_bytes: usize) -> Result<(), ReadError> { + let need = w.saturating_mul(h).saturating_mul(planes).saturating_mul(std::mem::size_of::()); + let most = max_bytes.min(effectcraft_render::memory::decode_limit(MAX_DECODE_BYTES, need)); + if need > most { + return Err(too_large(w, h, need, most)); + } + Ok(()) +} + fn read_into(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest, stats: &mut ReadStats) -> Result>, ReadError> { let level = req.level.min(16); let stride = 1i64 << level; @@ -291,13 +308,7 @@ fn read_into(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest, stats: let need = n.saturating_mul(req.channels.len()).saturating_mul(std::mem::size_of::()); let most = req.max_bytes.min(effectcraft_render::memory::decode_limit(req.floor, need)); if need > most { - return Err(ReadError::TooLarge(format!( - "its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", - w, - h, - gb(need), - gb(most) - ))); + return Err(too_large(w, h, need, most)); } // Per part: the plane slot of each of its channels (`None`: not requested). let mut slots: Vec>> = index.meta.headers.iter().map(|h| vec![None; h.channels.list.len()]).collect();