mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
OpenEXR limit: planes let go are gone before the request is read, a claimed decode without memory isn't read again, and the oversized-header test keeps the fixed limit (#3750 review)
This commit is contained in:
1 parent
79fdc7b21b
commit
e6a3c32357
4 files changed
+115
-15
No files matched your search
@@ -296,12 +296,15 @@ mod tests {
|
||||
#[test]
|
||||
fn a_header_asking_for_too_much_memory_is_refused() {
|
||||
use exr::prelude::*;
|
||||
// (another test raises the process-wide decode limit far above the request: not now)
|
||||
let _only = crate::exr_read::LIMIT_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let list: Vec<AnyChannel<FlatSamples>> = (0..40).map(|i| AnyChannel::new(format!("pass{i:02}.Y").as_str(), FlatSamples::F32(vec![0.5]))).collect();
|
||||
let layer = Layer::new((1, 1), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into()));
|
||||
let bytes = file(vec![layer], IntegerBounds::new((0, 0), (16384, 16384)));
|
||||
let mut src = std::io::Cursor::new(bytes.as_slice());
|
||||
let index = crate::exr_index::build(&mut src, FileStamp::Virtual).unwrap();
|
||||
let all = ReadRequest::full((0..index.channels.len()).collect());
|
||||
// (the fixed limit, whatever the process-wide one)
|
||||
let all = ReadRequest { max_bytes: crate::exr_read::MAX_DECODE_BYTES, ..ReadRequest::full((0..index.channels.len()).collect()) };
|
||||
match read_planes(&mut src, &index, &all).0 {
|
||||
Err(crate::exr_read::ReadError::TooLarge(why)) => assert!(why.contains("16384×16384") && why.contains("GB"), "{why}"),
|
||||
Err(e) => panic!("readable: {e:?}"),
|
||||
|
||||
@@ -162,6 +162,10 @@ pub(crate) enum Refused {
|
||||
/// The decode stopped (it panicked, or the file couldn't be opened just then): not
|
||||
/// remembered, the next request decodes again.
|
||||
Stopped,
|
||||
/// The planes held don't fit the limit with the ones missing, but the request fits alone:
|
||||
/// the store lets the held ones go first, then the request is read on its own (so the held
|
||||
/// and the new planes are never in memory together). Ask again.
|
||||
LetGo,
|
||||
}
|
||||
|
||||
/// A decode in progress: what it reads (until it starts reading) and its outcome.
|
||||
@@ -555,7 +559,11 @@ impl PlaneStore {
|
||||
// A decode thread doesn't wait (see `wait`), but takes over a decode still queued:
|
||||
// the planes are decoded once, here, and handed to the request that queued them.
|
||||
if may_not_wait() && flight.claim() {
|
||||
return Step::Done(decode_claimed(path, source, &flight, index, want));
|
||||
return match decode_claimed(path, source, &flight, index, want) {
|
||||
// (the store lets the held planes go when the asker ends the decode)
|
||||
Err(Refused::LetGo) => Step::Again { retry: true },
|
||||
r => Step::Done(r),
|
||||
};
|
||||
}
|
||||
return match self.wait(&flight) {
|
||||
Waited::Got(Ok(b)) if b.covers(index, want) => {
|
||||
@@ -564,7 +572,7 @@ impl PlaneStore {
|
||||
}
|
||||
// Not all it asks for: ask again (the next decode reads what is missing).
|
||||
Waited::Got(Ok(_)) => Step::Again { retry: false },
|
||||
Waited::Got(Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped)) => Step::Again { retry: true },
|
||||
Waited::Got(Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped | Refused::LetGo)) => Step::Again { retry: true },
|
||||
// Too large with what other requests added to it: this request alone may fit.
|
||||
Waited::Got(Err(Refused::TooLarge(_))) if flight.asks_beyond(index, want) => Step::Again { retry: true },
|
||||
Waited::Got(Err(e)) => Step::Done(Err(e)),
|
||||
@@ -590,7 +598,7 @@ impl PlaneStore {
|
||||
crate::reads::note(b.file.clone());
|
||||
Step::Done(Ok(b))
|
||||
}
|
||||
Ok(_) | Err(Refused::Forgotten | Refused::Stopped) => Step::Again { retry: true },
|
||||
Ok(_) | Err(Refused::Forgotten | Refused::Stopped | Refused::LetGo) => Step::Again { retry: true },
|
||||
// (as a waiter: this request alone may fit)
|
||||
Err(Refused::TooLarge(_)) if flight.asks_beyond(index, want) => Step::Again { retry: true },
|
||||
Err(e) => Step::Done(Err(e)),
|
||||
@@ -723,9 +731,20 @@ impl PlaneStore {
|
||||
e.bytes = 0;
|
||||
e.cost_us = 0;
|
||||
}
|
||||
// (the planes held go first, then the request is read on its own)
|
||||
Err(Refused::LetGo) => {
|
||||
if flight.base.as_ref().is_some_and(|base| e.batch.as_ref().is_some_and(|b| Arc::ptr_eq(b, base))) {
|
||||
e.batch = None;
|
||||
e.bytes = 0;
|
||||
e.cost_us = 0;
|
||||
}
|
||||
}
|
||||
// (memory refused just now: another request may get it)
|
||||
Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped | Refused::NoMemory(_)) => {}
|
||||
}
|
||||
if let (Err(Refused::LetGo), Some(base)) = (&outcome, &flight.base) {
|
||||
st.held.retain(|(p, h)| !(p == path && Arc::ptr_eq(h, base)));
|
||||
}
|
||||
let grown = st.entries.iter().find(|e| e.path == path).map_or(0, |e| e.bytes.saturating_sub(had));
|
||||
Self::drop_oldest(&mut st, self.budget(), CACHE_FILES, Some(path));
|
||||
(outcome, grown)
|
||||
@@ -919,6 +938,9 @@ fn decode_claimed(path: &str, source: &ExrSource, flight: &Arc<Flight>, index: &
|
||||
crate::reads::note(b.file.clone());
|
||||
Ok(b)
|
||||
}
|
||||
// (memory the system refused just now is not asked for again at once, see `NoMemory`;
|
||||
// planes held to let go first are let go by the store, not read beside in a copy)
|
||||
Err(e @ (Refused::NoMemory(_) | Refused::LetGo)) => Err(e),
|
||||
_ => copy(path, source, index, want),
|
||||
}
|
||||
}
|
||||
@@ -1077,19 +1099,20 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
want.ids.iter().copied().collect()
|
||||
};
|
||||
let wanted: Vec<usize> = wanted.into_iter().filter(|i| *i < index.channels.len()).collect();
|
||||
let mut base = base.filter(|b| Arc::ptr_eq(&b.index, &index));
|
||||
let mut missing: Vec<usize> = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect();
|
||||
let base = base.filter(|b| Arc::ptr_eq(&b.index, &index));
|
||||
let missing: Vec<usize> = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect();
|
||||
let (w, h) = index.size();
|
||||
// The planes the request ends with (those held plus those missing, whether they come from the
|
||||
// disk cache or the file) must fit the limit, asked before any of them is allocated (#3749).
|
||||
// The planes held are only kept for later requests: when they don't fit with the missing
|
||||
// ones, they are let go and the request is read on its own.
|
||||
// ones, they are let go and the request is read on its own. The store lets them go first
|
||||
// (`LetGo`), so they are not in memory while the request is read.
|
||||
if !missing.is_empty() {
|
||||
let held = base.as_ref().map_or(0, |b| b.ids.len());
|
||||
let together = crate::exr_read::check_total(w, h, held.saturating_add(missing.len()), cap_of(path));
|
||||
if together.is_err() && base.is_some() {
|
||||
base = None;
|
||||
missing = wanted.clone();
|
||||
crate::exr_read::check_total(w, h, wanted.len(), cap_of(path)).map_err(|e| refused(path, e))?;
|
||||
return Err(Refused::LetGo);
|
||||
}
|
||||
if base.is_none() {
|
||||
crate::exr_read::check_total(w, h, missing.len(), cap_of(path)).map_err(|e| refused(path, e))?;
|
||||
@@ -1339,7 +1362,8 @@ pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option
|
||||
}
|
||||
// (never the whole file's decode: the memory isn't there for it either)
|
||||
Err(Refused::TooLarge(why) | Refused::NoMemory(why)) => return Picture::Done(Err(why)),
|
||||
Err(Refused::Cancelled | Refused::Forgotten) => return Picture::Cancelled,
|
||||
// (`LetGo` is asked again inside the store; never here)
|
||||
Err(Refused::Cancelled | Refused::Forgotten | Refused::LetGo) => return Picture::Cancelled,
|
||||
// (the whole file's decode says why)
|
||||
Err(Refused::Stopped | Refused::Unreadable) => return Picture::Legacy,
|
||||
};
|
||||
|
||||
@@ -12,7 +12,7 @@ use exr::prelude::{
|
||||
};
|
||||
|
||||
use super::*;
|
||||
use crate::exr_read::ExrSource;
|
||||
use crate::exr_read::{ExrSource, LIMIT_TESTS};
|
||||
|
||||
/// A deterministic sample for pixel `i` of channel `c` (an `A` channel gets 0..=1).
|
||||
fn value(i: usize, c: usize, alpha: bool) -> f32 {
|
||||
@@ -1716,10 +1716,6 @@ fn a_failed_picture_leaves_the_other_channels_readable() {
|
||||
assert!(rest.aux.get("Z").is_some());
|
||||
}
|
||||
|
||||
/// The tests that move the process-wide decode limit take turns (it also dates the remembered
|
||||
/// refusals).
|
||||
static LIMIT_TESTS: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// #3749: a read over the fixed floor is allowed up to what the system's memory gives (the
|
||||
/// session's figure), never under the floor, and refused with a reason beyond it. The floor is
|
||||
/// small here: the real one is 8 GiB.
|
||||
@@ -1810,6 +1806,37 @@ fn channels_added_to_the_planes_held_count_toward_the_limit() {
|
||||
assert!(matches!(store.get(path, &source, &index, &everything, Use::Other).err(), Some(Refused::TooLarge(_))));
|
||||
}
|
||||
|
||||
/// #3750 review: when the planes held are let go because they don't fit the limit with the ones
|
||||
/// a request adds, they are let go before the request is read: the held planes and the new ones
|
||||
/// are never in memory together (the store, the decode and its flight all let go of them). Also
|
||||
/// for a picture's planes held for its effects.
|
||||
#[test]
|
||||
fn planes_let_go_are_gone_before_the_request_is_read() {
|
||||
let path = "/3750-let-go-before-read.exr";
|
||||
// Room for four planes: the picture fits, one more with it is five.
|
||||
set_cap(path, 4 * 8 * 8 * 4);
|
||||
let store = PlaneStore::new(CACHE_BUDGET);
|
||||
let source = ExrSource::Bytes(picture_and_depth().into());
|
||||
let index = crate::exr_index::get(path, &source).unwrap();
|
||||
let picture = Want { all: false, ids: index.picture_ids().into_iter().collect() };
|
||||
let held = Arc::downgrade(&store.get(path, &source, &index, &picture, Use::Picture).expect("the picture fits"));
|
||||
assert!(held.upgrade().is_some() && store.is_held(path), "kept for the effects");
|
||||
// Each decode that reads notes whether the picture's planes are still in memory.
|
||||
let alive = Arc::new(Mutex::new(Vec::new()));
|
||||
{
|
||||
let (alive, held) = (alive.clone(), held.clone());
|
||||
let hook: Hook = Arc::new(move |_: &str| alive.lock().unwrap().push(held.upgrade().is_some()));
|
||||
set_before_read(path, Some(hook));
|
||||
}
|
||||
let rest = Want { all: true, ids: Default::default() };
|
||||
let depth = store.get(path, &source, &index, &rest, Use::Other).expect("the depth plane fits alone");
|
||||
set_before_read(path, None);
|
||||
assert!(depth.aux.get("Z").is_some() && depth.aux.get("R").is_none());
|
||||
let alive = alive.lock().unwrap().clone();
|
||||
assert_eq!(alive.last(), Some(&false), "the picture's planes were gone when the depth plane was read: {alive:?}");
|
||||
assert_eq!(decodes_of(path).last().map(|d| d.channels.len()), Some(1), "the depth plane alone was read");
|
||||
}
|
||||
|
||||
/// #3749 (#3301 review): planes the disk cache gives back count toward the limit of a request
|
||||
/// like the ones read from the file.
|
||||
#[test]
|
||||
@@ -1935,6 +1962,47 @@ fn memory_the_system_cannot_give_is_neither_retried_nor_remembered() {
|
||||
set_read_error(path, None);
|
||||
}
|
||||
|
||||
/// #3750 review: a decode thread that took over a queued decode the system has no memory for
|
||||
/// hands the refusal to the request that queued it and answers its own request with it too: the
|
||||
/// file is read once, not again at once for a copy.
|
||||
#[test]
|
||||
fn a_claimed_decode_without_memory_is_not_read_again() {
|
||||
let path = "/3750-claimed-no-memory.exr";
|
||||
let source = ExrSource::Bytes(picture_and_depth().into());
|
||||
let store = Arc::new(PlaneStore::new(CACHE_BUDGET));
|
||||
let index = crate::exr_index::get(path, &source).unwrap();
|
||||
let want = Want { all: true, ids: (0..5).collect() };
|
||||
let asked = no_memory_when(path, |_| true);
|
||||
let threads = crate::pool::decode_threads().unwrap();
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
// When the asker queues its decode: a decode thread asks for the same planes.
|
||||
{
|
||||
let (store, source, index, want, tx) = (store.clone(), source.clone(), index.clone(), want.clone(), tx.clone());
|
||||
let hook: Hook = Arc::new(move |_: &str| {
|
||||
let (store, source, index, want, tx) = (store.clone(), source.clone(), index.clone(), want.clone(), tx.clone());
|
||||
threads.spawn(move || {
|
||||
let _ = tx.send(store.get(path, &source, &index, &want, Use::Other).err());
|
||||
});
|
||||
});
|
||||
QUEUING.lock().unwrap().push((path.to_string(), hook));
|
||||
}
|
||||
// The queued job waits until that decode thread claimed it.
|
||||
let hold: Hook = Arc::new(|p: &str| {
|
||||
let claims = CLAIMS.lock().unwrap();
|
||||
let _ = CLAIM_SIGNAL.wait_timeout_while(claims, std::time::Duration::from_secs(10), |c| !c.iter().any(|c| c == p));
|
||||
});
|
||||
STARTING.lock().unwrap().push((path.to_string(), hold));
|
||||
let queued = store.get(path, &source, &index, &want, Use::Other).err();
|
||||
let claimed = rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap();
|
||||
QUEUING.lock().unwrap().retain(|(p, _)| p != path);
|
||||
STARTING.lock().unwrap().retain(|(p, _)| p != path);
|
||||
set_read_error(path, None);
|
||||
assert!(CLAIMS.lock().unwrap().iter().any(|p| p == path), "claimed");
|
||||
assert!(matches!(&claimed, Some(Refused::NoMemory(_))), "{claimed:?}");
|
||||
assert!(matches!(&queued, Some(Refused::NoMemory(_))), "{queued:?}");
|
||||
assert_eq!(asked.load(Ordering::SeqCst), 1, "read once, not again for a copy");
|
||||
}
|
||||
|
||||
/// #3749 review: a picture whose channels the system has no memory for just now is read alone
|
||||
/// without the effects' channels, and refused with the reason when even that doesn't fit; never
|
||||
/// handed to the whole file's decode (which would ask for more memory, infallibly).
|
||||
|
||||
@@ -229,6 +229,11 @@ pub(crate) const MAX_DECODE_BYTES: usize = 8 << 30;
|
||||
#[cfg(not(target_pointer_width = "64"))]
|
||||
pub(crate) const MAX_DECODE_BYTES: usize = 512 << 20;
|
||||
|
||||
/// The tests that move the process-wide decode limit, or rely on it, take turns (it also dates
|
||||
/// the remembered refusals).
|
||||
#[cfg(test)]
|
||||
pub(crate) static LIMIT_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
/// What a failed `craft-exr` read means here. Memory the system refused just now is
|
||||
/// [`ReadError::OutOfMemory`]: the plane store doesn't remember it (another request may
|
||||
/// succeed), but doesn't read it again at once nor fall back to decoding the whole file
|
||||
|
||||
Reference in new issue
Block a user