From e6a3c32357951b6d032f6f4dfe42815a8d2c0e25 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 17 Nov 2026 17:05:03 +0100 Subject: [PATCH] OpenEXR limit: planes let go are gone before the request is read, a claimed decode without memory isn't read again, and the oversized-header test keeps the fixed limit (#3750 review) --- crates/media/src/exr_channels.rs | 5 +- crates/media/src/exr_planes.rs | 42 +++++++++++---- crates/media/src/exr_planes/tests.rs | 78 ++++++++++++++++++++++++++-- crates/media/src/exr_read.rs | 5 ++ 4 files changed, 115 insertions(+), 15 deletions(-) diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index 422502c5f..5ef1b5e8d 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -296,12 +296,15 @@ mod tests { #[test] fn a_header_asking_for_too_much_memory_is_refused() { use exr::prelude::*; + // (another test raises the process-wide decode limit far above the request: not now) + let _only = crate::exr_read::LIMIT_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let list: Vec> = (0..40).map(|i| AnyChannel::new(format!("pass{i:02}.Y").as_str(), FlatSamples::F32(vec![0.5]))).collect(); let layer = Layer::new((1, 1), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())); let bytes = file(vec![layer], IntegerBounds::new((0, 0), (16384, 16384))); let mut src = std::io::Cursor::new(bytes.as_slice()); let index = crate::exr_index::build(&mut src, FileStamp::Virtual).unwrap(); - let all = ReadRequest::full((0..index.channels.len()).collect()); + // (the fixed limit, whatever the process-wide one) + let all = ReadRequest { max_bytes: crate::exr_read::MAX_DECODE_BYTES, ..ReadRequest::full((0..index.channels.len()).collect()) }; match read_planes(&mut src, &index, &all).0 { Err(crate::exr_read::ReadError::TooLarge(why)) => assert!(why.contains("16384×16384") && why.contains("GB"), "{why}"), Err(e) => panic!("readable: {e:?}"), diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index c65d2e060..6527f91d8 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -162,6 +162,10 @@ pub(crate) enum Refused { /// The decode stopped (it panicked, or the file couldn't be opened just then): not /// remembered, the next request decodes again. Stopped, + /// The planes held don't fit the limit with the ones missing, but the request fits alone: + /// the store lets the held ones go first, then the request is read on its own (so the held + /// and the new planes are never in memory together). Ask again. + LetGo, } /// A decode in progress: what it reads (until it starts reading) and its outcome. @@ -555,7 +559,11 @@ impl PlaneStore { // A decode thread doesn't wait (see `wait`), but takes over a decode still queued: // the planes are decoded once, here, and handed to the request that queued them. if may_not_wait() && flight.claim() { - return Step::Done(decode_claimed(path, source, &flight, index, want)); + return match decode_claimed(path, source, &flight, index, want) { + // (the store lets the held planes go when the asker ends the decode) + Err(Refused::LetGo) => Step::Again { retry: true }, + r => Step::Done(r), + }; } return match self.wait(&flight) { Waited::Got(Ok(b)) if b.covers(index, want) => { @@ -564,7 +572,7 @@ impl PlaneStore { } // Not all it asks for: ask again (the next decode reads what is missing). Waited::Got(Ok(_)) => Step::Again { retry: false }, - Waited::Got(Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped)) => Step::Again { retry: true }, + Waited::Got(Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped | Refused::LetGo)) => Step::Again { retry: true }, // Too large with what other requests added to it: this request alone may fit. Waited::Got(Err(Refused::TooLarge(_))) if flight.asks_beyond(index, want) => Step::Again { retry: true }, Waited::Got(Err(e)) => Step::Done(Err(e)), @@ -590,7 +598,7 @@ impl PlaneStore { crate::reads::note(b.file.clone()); Step::Done(Ok(b)) } - Ok(_) | Err(Refused::Forgotten | Refused::Stopped) => Step::Again { retry: true }, + Ok(_) | Err(Refused::Forgotten | Refused::Stopped | Refused::LetGo) => Step::Again { retry: true }, // (as a waiter: this request alone may fit) Err(Refused::TooLarge(_)) if flight.asks_beyond(index, want) => Step::Again { retry: true }, Err(e) => Step::Done(Err(e)), @@ -723,9 +731,20 @@ impl PlaneStore { e.bytes = 0; e.cost_us = 0; } + // (the planes held go first, then the request is read on its own) + Err(Refused::LetGo) => { + if flight.base.as_ref().is_some_and(|base| e.batch.as_ref().is_some_and(|b| Arc::ptr_eq(b, base))) { + e.batch = None; + e.bytes = 0; + e.cost_us = 0; + } + } // (memory refused just now: another request may get it) Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped | Refused::NoMemory(_)) => {} } + if let (Err(Refused::LetGo), Some(base)) = (&outcome, &flight.base) { + st.held.retain(|(p, h)| !(p == path && Arc::ptr_eq(h, base))); + } let grown = st.entries.iter().find(|e| e.path == path).map_or(0, |e| e.bytes.saturating_sub(had)); Self::drop_oldest(&mut st, self.budget(), CACHE_FILES, Some(path)); (outcome, grown) @@ -919,6 +938,9 @@ fn decode_claimed(path: &str, source: &ExrSource, flight: &Arc, index: & crate::reads::note(b.file.clone()); Ok(b) } + // (memory the system refused just now is not asked for again at once, see `NoMemory`; + // planes held to let go first are let go by the store, not read beside in a copy) + Err(e @ (Refused::NoMemory(_) | Refused::LetGo)) => Err(e), _ => copy(path, source, index, want), } } @@ -1077,19 +1099,20 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option = wanted.into_iter().filter(|i| *i < index.channels.len()).collect(); - let mut base = base.filter(|b| Arc::ptr_eq(&b.index, &index)); - let mut missing: Vec = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect(); + let base = base.filter(|b| Arc::ptr_eq(&b.index, &index)); + let missing: Vec = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect(); let (w, h) = index.size(); // The planes the request ends with (those held plus those missing, whether they come from the // disk cache or the file) must fit the limit, asked before any of them is allocated (#3749). // The planes held are only kept for later requests: when they don't fit with the missing - // ones, they are let go and the request is read on its own. + // ones, they are let go and the request is read on its own. The store lets them go first + // (`LetGo`), so they are not in memory while the request is read. if !missing.is_empty() { let held = base.as_ref().map_or(0, |b| b.ids.len()); let together = crate::exr_read::check_total(w, h, held.saturating_add(missing.len()), cap_of(path)); if together.is_err() && base.is_some() { - base = None; - missing = wanted.clone(); + crate::exr_read::check_total(w, h, wanted.len(), cap_of(path)).map_err(|e| refused(path, e))?; + return Err(Refused::LetGo); } if base.is_none() { crate::exr_read::check_total(w, h, missing.len(), cap_of(path)).map_err(|e| refused(path, e))?; @@ -1339,7 +1362,8 @@ pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option } // (never the whole file's decode: the memory isn't there for it either) Err(Refused::TooLarge(why) | Refused::NoMemory(why)) => return Picture::Done(Err(why)), - Err(Refused::Cancelled | Refused::Forgotten) => return Picture::Cancelled, + // (`LetGo` is asked again inside the store; never here) + Err(Refused::Cancelled | Refused::Forgotten | Refused::LetGo) => return Picture::Cancelled, // (the whole file's decode says why) Err(Refused::Stopped | Refused::Unreadable) => return Picture::Legacy, }; diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index 164246f19..a123112ef 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -12,7 +12,7 @@ use exr::prelude::{ }; use super::*; -use crate::exr_read::ExrSource; +use crate::exr_read::{ExrSource, LIMIT_TESTS}; /// A deterministic sample for pixel `i` of channel `c` (an `A` channel gets 0..=1). fn value(i: usize, c: usize, alpha: bool) -> f32 { @@ -1716,10 +1716,6 @@ fn a_failed_picture_leaves_the_other_channels_readable() { assert!(rest.aux.get("Z").is_some()); } -/// The tests that move the process-wide decode limit take turns (it also dates the remembered -/// refusals). -static LIMIT_TESTS: Mutex<()> = Mutex::new(()); - /// #3749: a read over the fixed floor is allowed up to what the system's memory gives (the /// session's figure), never under the floor, and refused with a reason beyond it. The floor is /// small here: the real one is 8 GiB. @@ -1810,6 +1806,37 @@ fn channels_added_to_the_planes_held_count_toward_the_limit() { assert!(matches!(store.get(path, &source, &index, &everything, Use::Other).err(), Some(Refused::TooLarge(_)))); } +/// #3750 review: when the planes held are let go because they don't fit the limit with the ones +/// a request adds, they are let go before the request is read: the held planes and the new ones +/// are never in memory together (the store, the decode and its flight all let go of them). Also +/// for a picture's planes held for its effects. +#[test] +fn planes_let_go_are_gone_before_the_request_is_read() { + let path = "/3750-let-go-before-read.exr"; + // Room for four planes: the picture fits, one more with it is five. + set_cap(path, 4 * 8 * 8 * 4); + let store = PlaneStore::new(CACHE_BUDGET); + let source = ExrSource::Bytes(picture_and_depth().into()); + let index = crate::exr_index::get(path, &source).unwrap(); + let picture = Want { all: false, ids: index.picture_ids().into_iter().collect() }; + let held = Arc::downgrade(&store.get(path, &source, &index, &picture, Use::Picture).expect("the picture fits")); + assert!(held.upgrade().is_some() && store.is_held(path), "kept for the effects"); + // Each decode that reads notes whether the picture's planes are still in memory. + let alive = Arc::new(Mutex::new(Vec::new())); + { + let (alive, held) = (alive.clone(), held.clone()); + let hook: Hook = Arc::new(move |_: &str| alive.lock().unwrap().push(held.upgrade().is_some())); + set_before_read(path, Some(hook)); + } + let rest = Want { all: true, ids: Default::default() }; + let depth = store.get(path, &source, &index, &rest, Use::Other).expect("the depth plane fits alone"); + set_before_read(path, None); + assert!(depth.aux.get("Z").is_some() && depth.aux.get("R").is_none()); + let alive = alive.lock().unwrap().clone(); + assert_eq!(alive.last(), Some(&false), "the picture's planes were gone when the depth plane was read: {alive:?}"); + assert_eq!(decodes_of(path).last().map(|d| d.channels.len()), Some(1), "the depth plane alone was read"); +} + /// #3749 (#3301 review): planes the disk cache gives back count toward the limit of a request /// like the ones read from the file. #[test] @@ -1935,6 +1962,47 @@ fn memory_the_system_cannot_give_is_neither_retried_nor_remembered() { set_read_error(path, None); } +/// #3750 review: a decode thread that took over a queued decode the system has no memory for +/// hands the refusal to the request that queued it and answers its own request with it too: the +/// file is read once, not again at once for a copy. +#[test] +fn a_claimed_decode_without_memory_is_not_read_again() { + let path = "/3750-claimed-no-memory.exr"; + let source = ExrSource::Bytes(picture_and_depth().into()); + let store = Arc::new(PlaneStore::new(CACHE_BUDGET)); + let index = crate::exr_index::get(path, &source).unwrap(); + let want = Want { all: true, ids: (0..5).collect() }; + let asked = no_memory_when(path, |_| true); + let threads = crate::pool::decode_threads().unwrap(); + let (tx, rx) = std::sync::mpsc::channel(); + // When the asker queues its decode: a decode thread asks for the same planes. + { + let (store, source, index, want, tx) = (store.clone(), source.clone(), index.clone(), want.clone(), tx.clone()); + let hook: Hook = Arc::new(move |_: &str| { + let (store, source, index, want, tx) = (store.clone(), source.clone(), index.clone(), want.clone(), tx.clone()); + threads.spawn(move || { + let _ = tx.send(store.get(path, &source, &index, &want, Use::Other).err()); + }); + }); + QUEUING.lock().unwrap().push((path.to_string(), hook)); + } + // The queued job waits until that decode thread claimed it. + let hold: Hook = Arc::new(|p: &str| { + let claims = CLAIMS.lock().unwrap(); + let _ = CLAIM_SIGNAL.wait_timeout_while(claims, std::time::Duration::from_secs(10), |c| !c.iter().any(|c| c == p)); + }); + STARTING.lock().unwrap().push((path.to_string(), hold)); + let queued = store.get(path, &source, &index, &want, Use::Other).err(); + let claimed = rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap(); + QUEUING.lock().unwrap().retain(|(p, _)| p != path); + STARTING.lock().unwrap().retain(|(p, _)| p != path); + set_read_error(path, None); + assert!(CLAIMS.lock().unwrap().iter().any(|p| p == path), "claimed"); + assert!(matches!(&claimed, Some(Refused::NoMemory(_))), "{claimed:?}"); + assert!(matches!(&queued, Some(Refused::NoMemory(_))), "{queued:?}"); + assert_eq!(asked.load(Ordering::SeqCst), 1, "read once, not again for a copy"); +} + /// #3749 review: a picture whose channels the system has no memory for just now is read alone /// without the effects' channels, and refused with the reason when even that doesn't fit; never /// handed to the whole file's decode (which would ask for more memory, infallibly). diff --git a/crates/media/src/exr_read.rs b/crates/media/src/exr_read.rs index 60175806e..71a3d4b82 100644 --- a/crates/media/src/exr_read.rs +++ b/crates/media/src/exr_read.rs @@ -229,6 +229,11 @@ pub(crate) const MAX_DECODE_BYTES: usize = 8 << 30; #[cfg(not(target_pointer_width = "64"))] pub(crate) const MAX_DECODE_BYTES: usize = 512 << 20; +/// The tests that move the process-wide decode limit, or rely on it, take turns (it also dates +/// the remembered refusals). +#[cfg(test)] +pub(crate) static LIMIT_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + /// What a failed `craft-exr` read means here. Memory the system refused just now is /// [`ReadError::OutOfMemory`]: the plane store doesn't remember it (another request may /// succeed), but doesn't read it again at once nor fall back to decoding the whole file