OpenEXR limit: planes let go are gone before the request is read, a claimed decode without memory isn't read again, and the oversized-header test keeps the fixed limit (#3750 review)

This commit is contained in:
vangel76 committed 2026-11-17 17:05:03 +01:00
1 parent 79fdc7b21b
commit e6a3c32357
4 files changed
+115 -15

No files matched your search

+4 -1
View File
@@ -296,12 +296,15 @@ mod tests {
#[test]
fn a_header_asking_for_too_much_memory_is_refused() {
use exr::prelude::*;
// (another test raises the process-wide decode limit far above the request: not now)
let _only = crate::exr_read::LIMIT_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let list: Vec<AnyChannel<FlatSamples>> = (0..40).map(|i| AnyChannel::new(format!("pass{i:02}.Y").as_str(), FlatSamples::F32(vec![0.5]))).collect();
let layer = Layer::new((1, 1), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into()));
let bytes = file(vec![layer], IntegerBounds::new((0, 0), (16384, 16384)));
let mut src = std::io::Cursor::new(bytes.as_slice());
let index = crate::exr_index::build(&mut src, FileStamp::Virtual).unwrap();
let all = ReadRequest::full((0..index.channels.len()).collect());
// (the fixed limit, whatever the process-wide one)
let all = ReadRequest { max_bytes: crate::exr_read::MAX_DECODE_BYTES, ..ReadRequest::full((0..index.channels.len()).collect()) };
match read_planes(&mut src, &index, &all).0 {
Err(crate::exr_read::ReadError::TooLarge(why)) => assert!(why.contains("16384×16384") && why.contains("GB"), "{why}"),
Err(e) => panic!("readable: {e:?}"),
+33 -9
View File
@@ -162,6 +162,10 @@ pub(crate) enum Refused {
/// The decode stopped (it panicked, or the file couldn't be opened just then): not
/// remembered, the next request decodes again.
Stopped,
/// The planes held don't fit the limit with the ones missing, but the request fits alone:
/// the store lets the held ones go first, then the request is read on its own (so the held
/// and the new planes are never in memory together). Ask again.
LetGo,
}
/// A decode in progress: what it reads (until it starts reading) and its outcome.
@@ -555,7 +559,11 @@ impl PlaneStore {
// A decode thread doesn't wait (see `wait`), but takes over a decode still queued:
// the planes are decoded once, here, and handed to the request that queued them.
if may_not_wait() && flight.claim() {
return Step::Done(decode_claimed(path, source, &flight, index, want));
return match decode_claimed(path, source, &flight, index, want) {
// (the store lets the held planes go when the asker ends the decode)
Err(Refused::LetGo) => Step::Again { retry: true },
r => Step::Done(r),
};
}
return match self.wait(&flight) {
Waited::Got(Ok(b)) if b.covers(index, want) => {
@@ -564,7 +572,7 @@ impl PlaneStore {
}
// Not all it asks for: ask again (the next decode reads what is missing).
Waited::Got(Ok(_)) => Step::Again { retry: false },
Waited::Got(Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped)) => Step::Again { retry: true },
Waited::Got(Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped | Refused::LetGo)) => Step::Again { retry: true },
// Too large with what other requests added to it: this request alone may fit.
Waited::Got(Err(Refused::TooLarge(_))) if flight.asks_beyond(index, want) => Step::Again { retry: true },
Waited::Got(Err(e)) => Step::Done(Err(e)),
@@ -590,7 +598,7 @@ impl PlaneStore {
crate::reads::note(b.file.clone());
Step::Done(Ok(b))
}
Ok(_) | Err(Refused::Forgotten | Refused::Stopped) => Step::Again { retry: true },
Ok(_) | Err(Refused::Forgotten | Refused::Stopped | Refused::LetGo) => Step::Again { retry: true },
// (as a waiter: this request alone may fit)
Err(Refused::TooLarge(_)) if flight.asks_beyond(index, want) => Step::Again { retry: true },
Err(e) => Step::Done(Err(e)),
@@ -723,9 +731,20 @@ impl PlaneStore {
e.bytes = 0;
e.cost_us = 0;
}
// (the planes held go first, then the request is read on its own)
Err(Refused::LetGo) => {
if flight.base.as_ref().is_some_and(|base| e.batch.as_ref().is_some_and(|b| Arc::ptr_eq(b, base))) {
e.batch = None;
e.bytes = 0;
e.cost_us = 0;
}
}
// (memory refused just now: another request may get it)
Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped | Refused::NoMemory(_)) => {}
}
if let (Err(Refused::LetGo), Some(base)) = (&outcome, &flight.base) {
st.held.retain(|(p, h)| !(p == path && Arc::ptr_eq(h, base)));
}
let grown = st.entries.iter().find(|e| e.path == path).map_or(0, |e| e.bytes.saturating_sub(had));
Self::drop_oldest(&mut st, self.budget(), CACHE_FILES, Some(path));
(outcome, grown)
@@ -919,6 +938,9 @@ fn decode_claimed(path: &str, source: &ExrSource, flight: &Arc<Flight>, index: &
crate::reads::note(b.file.clone());
Ok(b)
}
// (memory the system refused just now is not asked for again at once, see `NoMemory`;
// planes held to let go first are let go by the store, not read beside in a copy)
Err(e @ (Refused::NoMemory(_) | Refused::LetGo)) => Err(e),
_ => copy(path, source, index, want),
}
}
@@ -1077,19 +1099,20 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
want.ids.iter().copied().collect()
};
let wanted: Vec<usize> = wanted.into_iter().filter(|i| *i < index.channels.len()).collect();
let mut base = base.filter(|b| Arc::ptr_eq(&b.index, &index));
let mut missing: Vec<usize> = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect();
let base = base.filter(|b| Arc::ptr_eq(&b.index, &index));
let missing: Vec<usize> = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect();
let (w, h) = index.size();
// The planes the request ends with (those held plus those missing, whether they come from the
// disk cache or the file) must fit the limit, asked before any of them is allocated (#3749).
// The planes held are only kept for later requests: when they don't fit with the missing
// ones, they are let go and the request is read on its own.
// ones, they are let go and the request is read on its own. The store lets them go first
// (`LetGo`), so they are not in memory while the request is read.
if !missing.is_empty() {
let held = base.as_ref().map_or(0, |b| b.ids.len());
let together = crate::exr_read::check_total(w, h, held.saturating_add(missing.len()), cap_of(path));
if together.is_err() && base.is_some() {
base = None;
missing = wanted.clone();
crate::exr_read::check_total(w, h, wanted.len(), cap_of(path)).map_err(|e| refused(path, e))?;
return Err(Refused::LetGo);
}
if base.is_none() {
crate::exr_read::check_total(w, h, missing.len(), cap_of(path)).map_err(|e| refused(path, e))?;
@@ -1339,7 +1362,8 @@ pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option
}
// (never the whole file's decode: the memory isn't there for it either)
Err(Refused::TooLarge(why) | Refused::NoMemory(why)) => return Picture::Done(Err(why)),
Err(Refused::Cancelled | Refused::Forgotten) => return Picture::Cancelled,
// (`LetGo` is asked again inside the store; never here)
Err(Refused::Cancelled | Refused::Forgotten | Refused::LetGo) => return Picture::Cancelled,
// (the whole file's decode says why)
Err(Refused::Stopped | Refused::Unreadable) => return Picture::Legacy,
};
+73 -5
View File
@@ -12,7 +12,7 @@ use exr::prelude::{
};
use super::*;
use crate::exr_read::ExrSource;
use crate::exr_read::{ExrSource, LIMIT_TESTS};
/// A deterministic sample for pixel `i` of channel `c` (an `A` channel gets 0..=1).
fn value(i: usize, c: usize, alpha: bool) -> f32 {
@@ -1716,10 +1716,6 @@ fn a_failed_picture_leaves_the_other_channels_readable() {
assert!(rest.aux.get("Z").is_some());
}
/// The tests that move the process-wide decode limit take turns (it also dates the remembered
/// refusals).
static LIMIT_TESTS: Mutex<()> = Mutex::new(());
/// #3749: a read over the fixed floor is allowed up to what the system's memory gives (the
/// session's figure), never under the floor, and refused with a reason beyond it. The floor is
/// small here: the real one is 8 GiB.
@@ -1810,6 +1806,37 @@ fn channels_added_to_the_planes_held_count_toward_the_limit() {
assert!(matches!(store.get(path, &source, &index, &everything, Use::Other).err(), Some(Refused::TooLarge(_))));
}
/// #3750 review: when the planes held are let go because they don't fit the limit with the ones
/// a request adds, they are let go before the request is read: the held planes and the new ones
/// are never in memory together (the store, the decode and its flight all let go of them). Also
/// for a picture's planes held for its effects.
#[test]
fn planes_let_go_are_gone_before_the_request_is_read() {
let path = "/3750-let-go-before-read.exr";
// Room for four planes: the picture fits, one more with it is five.
set_cap(path, 4 * 8 * 8 * 4);
let store = PlaneStore::new(CACHE_BUDGET);
let source = ExrSource::Bytes(picture_and_depth().into());
let index = crate::exr_index::get(path, &source).unwrap();
let picture = Want { all: false, ids: index.picture_ids().into_iter().collect() };
let held = Arc::downgrade(&store.get(path, &source, &index, &picture, Use::Picture).expect("the picture fits"));
assert!(held.upgrade().is_some() && store.is_held(path), "kept for the effects");
// Each decode that reads notes whether the picture's planes are still in memory.
let alive = Arc::new(Mutex::new(Vec::new()));
{
let (alive, held) = (alive.clone(), held.clone());
let hook: Hook = Arc::new(move |_: &str| alive.lock().unwrap().push(held.upgrade().is_some()));
set_before_read(path, Some(hook));
}
let rest = Want { all: true, ids: Default::default() };
let depth = store.get(path, &source, &index, &rest, Use::Other).expect("the depth plane fits alone");
set_before_read(path, None);
assert!(depth.aux.get("Z").is_some() && depth.aux.get("R").is_none());
let alive = alive.lock().unwrap().clone();
assert_eq!(alive.last(), Some(&false), "the picture's planes were gone when the depth plane was read: {alive:?}");
assert_eq!(decodes_of(path).last().map(|d| d.channels.len()), Some(1), "the depth plane alone was read");
}
/// #3749 (#3301 review): planes the disk cache gives back count toward the limit of a request
/// like the ones read from the file.
#[test]
@@ -1935,6 +1962,47 @@ fn memory_the_system_cannot_give_is_neither_retried_nor_remembered() {
set_read_error(path, None);
}
/// #3750 review: a decode thread that took over a queued decode the system has no memory for
/// hands the refusal to the request that queued it and answers its own request with it too: the
/// file is read once, not again at once for a copy.
#[test]
fn a_claimed_decode_without_memory_is_not_read_again() {
let path = "/3750-claimed-no-memory.exr";
let source = ExrSource::Bytes(picture_and_depth().into());
let store = Arc::new(PlaneStore::new(CACHE_BUDGET));
let index = crate::exr_index::get(path, &source).unwrap();
let want = Want { all: true, ids: (0..5).collect() };
let asked = no_memory_when(path, |_| true);
let threads = crate::pool::decode_threads().unwrap();
let (tx, rx) = std::sync::mpsc::channel();
// When the asker queues its decode: a decode thread asks for the same planes.
{
let (store, source, index, want, tx) = (store.clone(), source.clone(), index.clone(), want.clone(), tx.clone());
let hook: Hook = Arc::new(move |_: &str| {
let (store, source, index, want, tx) = (store.clone(), source.clone(), index.clone(), want.clone(), tx.clone());
threads.spawn(move || {
let _ = tx.send(store.get(path, &source, &index, &want, Use::Other).err());
});
});
QUEUING.lock().unwrap().push((path.to_string(), hook));
}
// The queued job waits until that decode thread claimed it.
let hold: Hook = Arc::new(|p: &str| {
let claims = CLAIMS.lock().unwrap();
let _ = CLAIM_SIGNAL.wait_timeout_while(claims, std::time::Duration::from_secs(10), |c| !c.iter().any(|c| c == p));
});
STARTING.lock().unwrap().push((path.to_string(), hold));
let queued = store.get(path, &source, &index, &want, Use::Other).err();
let claimed = rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap();
QUEUING.lock().unwrap().retain(|(p, _)| p != path);
STARTING.lock().unwrap().retain(|(p, _)| p != path);
set_read_error(path, None);
assert!(CLAIMS.lock().unwrap().iter().any(|p| p == path), "claimed");
assert!(matches!(&claimed, Some(Refused::NoMemory(_))), "{claimed:?}");
assert!(matches!(&queued, Some(Refused::NoMemory(_))), "{queued:?}");
assert_eq!(asked.load(Ordering::SeqCst), 1, "read once, not again for a copy");
}
/// #3749 review: a picture whose channels the system has no memory for just now is read alone
/// without the effects' channels, and refused with the reason when even that doesn't fit; never
/// handed to the whole file's decode (which would ask for more memory, infallibly).
+5
View File
@@ -229,6 +229,11 @@ pub(crate) const MAX_DECODE_BYTES: usize = 8 << 30;
#[cfg(not(target_pointer_width = "64"))]
pub(crate) const MAX_DECODE_BYTES: usize = 512 << 20;
/// The tests that move the process-wide decode limit, or rely on it, take turns (it also dates
/// the remembered refusals).
#[cfg(test)]
pub(crate) static LIMIT_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(());
/// What a failed `craft-exr` read means here. Memory the system refused just now is
/// [`ReadError::OutOfMemory`]: the plane store doesn't remember it (another request may
/// succeed), but doesn't read it again at once nor fall back to decoding the whole file