mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
OpenEXR limit: memory the system refuses is neither remembered nor read again at once, and never handed to the whole file's decode (#3749 review)
craft-exr's OutOfMemory is now a refusal of its own (ReadError::OutOfMemory, Refused::NoMemory) instead of a failed read: the store doesn't remember it, doesn't retry it or read a copy, and a picture without memory for its effects' channels is read alone, else refused with the reason, as for a file too large; never the image crate's decode of the whole file. The planes held, copied into a request's new set, are reserved fallibly too; a copy for the disk cache that can't be had is not written. The tests that set the process-wide disk cache take turns, and a test shows that an unreadable request stays refused when the decode limit changes.
This commit is contained in:
1 parent
f08a2290c2
commit
c0c53353fe
3 files changed
+191
-19
No files matched your search
@@ -150,6 +150,9 @@ pub(crate) enum Refused {
|
||||
Unreadable,
|
||||
/// More planes than one file may allocate (the reason; remembered).
|
||||
TooLarge(String),
|
||||
/// The system refused the memory the planes need just now (the reason): not remembered,
|
||||
/// but not asked again at once either, nor read as a whole file another way (#3749).
|
||||
NoMemory(String),
|
||||
/// The request was called off.
|
||||
Cancelled,
|
||||
/// The file changed since it was indexed: index it again and ask again.
|
||||
@@ -430,8 +433,9 @@ pub(crate) fn set_before_read(path: &str, hook: Option<Hook>) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Planes of `path` may take at most this many bytes in a read (tests: a small cap in place of
|
||||
/// [`crate::exr_read::MAX_DECODE_BYTES`]).
|
||||
/// Planes of `path` may take at most this many bytes in a read (tests: a cap of the caller's own,
|
||||
/// below the system's limit; outside tests there is none, `usize::MAX`, and only
|
||||
/// [`effectcraft_render::memory::decode_limit`] applies).
|
||||
#[cfg(test)]
|
||||
static CAPS: Mutex<Vec<(String, usize)>> = Mutex::new(Vec::new());
|
||||
|
||||
@@ -657,7 +661,7 @@ impl PlaneStore {
|
||||
let clock = effectcraft_render::memory::tick();
|
||||
let Some(e) = st.entries.iter_mut().find(|e| e.path == path && e.flight.as_ref().is_some_and(|f| Arc::ptr_eq(f, flight))) else {
|
||||
let outcome = match outcome {
|
||||
Err(e @ (Refused::Cancelled | Refused::Stopped)) => Err(e),
|
||||
Err(e @ (Refused::Cancelled | Refused::Stopped | Refused::NoMemory(_))) => Err(e),
|
||||
_ => Err(Refused::Forgotten),
|
||||
};
|
||||
return (outcome, 0);
|
||||
@@ -694,7 +698,8 @@ impl PlaneStore {
|
||||
e.bytes = 0;
|
||||
e.cost_us = 0;
|
||||
}
|
||||
Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped) => {}
|
||||
// (memory refused just now: another request may get it)
|
||||
Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped | Refused::NoMemory(_)) => {}
|
||||
}
|
||||
let grown = st.entries.iter().find(|e| e.path == path).map_or(0, |e| e.bytes.saturating_sub(had));
|
||||
Self::drop_oldest(&mut st, self.budget(), CACHE_FILES, Some(path));
|
||||
@@ -920,6 +925,10 @@ fn refused(path: &str, e: ReadError) -> Refused {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::TooLarge(why)
|
||||
}
|
||||
ReadError::OutOfMemory(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::NoMemory(why)
|
||||
}
|
||||
ReadError::Unreadable(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::Unreadable
|
||||
@@ -927,6 +936,53 @@ fn refused(path: &str, e: ReadError) -> Refused {
|
||||
}
|
||||
}
|
||||
|
||||
/// A copy of `plane`, or `None` when the system can't give the memory for it (an allocation that
|
||||
/// fails aborts the process, so it is asked for first).
|
||||
fn copy_plane(plane: &[f32]) -> Option<Vec<f32>> {
|
||||
let mut copy: Vec<f32> = Vec::new();
|
||||
copy.try_reserve_exact(plane.len()).ok()?;
|
||||
copy.extend_from_slice(plane);
|
||||
Some(copy)
|
||||
}
|
||||
|
||||
/// Why `planes` planes of a `w` × `h` display window couldn't be had (to show).
|
||||
fn no_memory_for(w: usize, h: usize, planes: usize) -> String {
|
||||
let bytes = w.saturating_mul(h).saturating_mul(planes).saturating_mul(std::mem::size_of::<f32>());
|
||||
format!("its {w}×{h} OpenEXR channels need {:.1} GB more memory than the system can give just now", bytes as f64 / f64::from(1u32 << 30))
|
||||
}
|
||||
|
||||
/// Fails the reads of a path as a test says, for the channels read ([`set_read_error`]).
|
||||
#[cfg(test)]
|
||||
pub(crate) type ForcedError = Arc<dyn Fn(&[usize]) -> Option<ReadError> + Send + Sync>;
|
||||
#[cfg(test)]
|
||||
static READ_ERRORS: Mutex<Vec<(String, ForcedError)>> = Mutex::new(Vec::new());
|
||||
|
||||
/// Make reads of `path` fail as `error` says for the channels read, instead of reading (`None`:
|
||||
/// read again; tests: memory the system refuses). Keyed by path, so tests running at the same
|
||||
/// time don't see each other's.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_read_error(path: &str, error: Option<ForcedError>) {
|
||||
let mut errors = READ_ERRORS.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
errors.retain(|(p, _)| p != path);
|
||||
if let Some(e) = error {
|
||||
errors.push((path.to_string(), e));
|
||||
}
|
||||
}
|
||||
|
||||
/// How a test makes the read of `ids` of `path` fail ([`set_read_error`]); never outside tests.
|
||||
fn forced_read_error(path: &str, ids: &[usize]) -> Option<ReadError> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let error = READ_ERRORS.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(p, _)| p == path).map(|(_, e)| e.clone());
|
||||
error.and_then(|e| e(ids))
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
let _ = (path, ids);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode what `flight` asks for and `base` doesn't hold, and make the union of both.
|
||||
fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc<Batch>>, opened: Option<Opened>) -> Result<Arc<Batch>, Refused> {
|
||||
let _decoding = Decoding::new();
|
||||
@@ -1010,6 +1066,8 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
let read_from = effectcraft_render::memory::tick();
|
||||
let (read, stats) = if to_read.is_empty() {
|
||||
(Ok(vec![]), Default::default())
|
||||
} else if let Some(e) = forced_read_error(path, &to_read) {
|
||||
(Err(e), Default::default())
|
||||
} else {
|
||||
let request = ReadRequest { max_bytes: cap_of(path), ..ReadRequest::full(to_read.clone()) };
|
||||
read_planes(&mut *opened.reader, &index, &request)
|
||||
@@ -1027,7 +1085,9 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
if let Some(b) = &base {
|
||||
for (at, id) in b.ids.iter().enumerate() {
|
||||
if let Some((_, p)) = b.aux.channels.get(at) {
|
||||
new.push((*id, p.clone()));
|
||||
// (the planes held are copied into the new set: memory the system may refuse)
|
||||
let copy = copy_plane(p).ok_or_else(|| refused(path, ReadError::OutOfMemory(no_memory_for(w, h, b.ids.len()))))?;
|
||||
new.push((*id, copy));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1056,7 +1116,8 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
.par_iter()
|
||||
.filter_map(|id| {
|
||||
let (name, plane) = batch.aux.channels.get(batch.ids.binary_search(id).ok()?)?;
|
||||
Some((key(name), name.clone(), plane.clone()))
|
||||
// (a copy the system can't give now is not kept on disk: the planes are in hand)
|
||||
Some((key(name), name.clone(), copy_plane(plane)?))
|
||||
})
|
||||
.collect();
|
||||
// (only planes that cost more to decode again than to read back, #3301)
|
||||
@@ -1204,11 +1265,13 @@ pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option
|
||||
Ok(b) => b,
|
||||
Err(Refused::Changed) => continue,
|
||||
// The channels with it are too many: the picture alone (the effects ask for theirs).
|
||||
Err(Refused::TooLarge(_)) if extra.is_some() => {
|
||||
// (also when the system has no memory for them just now, #3749)
|
||||
Err(Refused::TooLarge(_) | Refused::NoMemory(_)) if extra.is_some() => {
|
||||
extra = None;
|
||||
continue;
|
||||
}
|
||||
Err(Refused::TooLarge(why)) => return Picture::Done(Err(why)),
|
||||
// (never the whole file's decode: the memory isn't there for it either)
|
||||
Err(Refused::TooLarge(why) | Refused::NoMemory(why)) => return Picture::Done(Err(why)),
|
||||
Err(Refused::Cancelled | Refused::Forgotten) => return Picture::Cancelled,
|
||||
// (the whole file's decode says why)
|
||||
Err(Refused::Stopped | Refused::Unreadable) => return Picture::Legacy,
|
||||
@@ -1290,7 +1353,10 @@ pub(crate) fn picture_rows(path: &str, source: &ExrSource, op: AlphaOp, rows: (i
|
||||
return picture(path, source, op, None);
|
||||
}
|
||||
let request = ReadRequest { rows: Some(rows.0..rows.1), max_bytes: cap_of(path), ..ReadRequest::full(ids.clone()) };
|
||||
let (planes, stats) = read_planes(&mut *opened.reader, &index, &request);
|
||||
let (planes, stats) = match forced_read_error(path, &ids) {
|
||||
Some(e) => (Err(e), Default::default()),
|
||||
None => read_planes(&mut *opened.reader, &index, &request),
|
||||
};
|
||||
let names: Vec<String> = ids.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect();
|
||||
#[cfg(test)]
|
||||
DECODES.lock().unwrap_or_else(PoisonError::into_inner).push((path.to_string(), DecodeRecord { channels: names.clone(), stats, from_disk: false }));
|
||||
@@ -1299,7 +1365,7 @@ pub(crate) fn picture_rows(path: &str, source: &ExrSource, op: AlphaOp, rows: (i
|
||||
let planes = match planes {
|
||||
Ok(p) => p,
|
||||
Err(ReadError::Cancelled) => return Picture::Cancelled,
|
||||
Err(ReadError::TooLarge(why)) => return Picture::Done(Err(why)),
|
||||
Err(ReadError::TooLarge(why) | ReadError::OutOfMemory(why)) => return Picture::Done(Err(why)),
|
||||
Err(_) => return Picture::Legacy,
|
||||
};
|
||||
let (w, h) = index.size();
|
||||
|
||||
@@ -1075,8 +1075,12 @@ fn a_new_frame_opens_its_file_once() {
|
||||
/// them go they are read back from there, bit for bit (Cryptomatte IDs included), whatever was
|
||||
/// asked for before (the picture first, then the channels, then all of it after an eviction);
|
||||
/// Reload Footage drops them.
|
||||
/// The tests that set the process-wide disk cache take turns (one would switch off another's).
|
||||
static DISK_TESTS: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[test]
|
||||
fn decoded_planes_are_kept_on_disk_and_read_back_exactly() {
|
||||
let _disk = DISK_TESTS.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
use effectcraft_render::disk_cache::{DiskCache, Kind};
|
||||
let dir = std::env::temp_dir().join(format!("ec-3271-disk-planes-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
@@ -1549,7 +1553,7 @@ fn a_failure_is_forgotten_once_the_file_changes() {
|
||||
#[test]
|
||||
fn an_allocation_failure_is_not_remembered() {
|
||||
use crate::exr_read::{ReadError, read_error};
|
||||
assert_eq!(read_error(craft_exr::seek::Error::OutOfMemory("planes".into())), ReadError::Io("planes".into()));
|
||||
assert!(matches!(read_error(craft_exr::seek::Error::OutOfMemory("planes".into())), ReadError::OutOfMemory(why) if why.contains("planes")));
|
||||
assert!(matches!(read_error(craft_exr::seek::Error::LimitExceeded("planes".into())), ReadError::TooLarge(_)));
|
||||
// A read that failed just now (here: the file can't be opened) is a stop, and the next
|
||||
// request for the same channels reads them.
|
||||
@@ -1678,6 +1682,7 @@ fn channels_added_to_the_planes_held_count_toward_the_limit() {
|
||||
/// like the ones read from the file.
|
||||
#[test]
|
||||
fn planes_from_the_disk_cache_count_toward_the_limit() {
|
||||
let _disk = DISK_TESTS.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
use effectcraft_render::disk_cache::DiskCache;
|
||||
let dir = std::env::temp_dir().join(format!("ec-3749-disk-union-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
@@ -1706,19 +1711,115 @@ fn planes_from_the_disk_cache_count_toward_the_limit() {
|
||||
}
|
||||
|
||||
/// #3749: a plane the system cannot give is not an aborted process: `craft-exr` asks for the
|
||||
/// planes' memory first and reports what it can't get, a read that failed just now (asked again,
|
||||
/// not remembered); a plane over the read's limit (the decode limit handed to it) is a refusal
|
||||
/// for size with `craft-exr`'s reason.
|
||||
/// planes' memory first and reports what it can't get, memory refused just now (not remembered,
|
||||
/// not read again at once); a plane over the read's limit (the decode limit handed to it) is a
|
||||
/// refusal for size with `craft-exr`'s reason.
|
||||
#[test]
|
||||
fn a_plane_the_system_cannot_give_is_refused_not_allocated() {
|
||||
use crate::exr_read::{ReadError, read_error};
|
||||
use craft_exr::seek::Error;
|
||||
let gone = read_error(Error::OutOfMemory("4096 bytes of planes".into()));
|
||||
assert!(matches!(&gone, ReadError::Io(why) if why.contains("4096 bytes of planes")), "{gone:?}");
|
||||
assert!(matches!(&gone, ReadError::OutOfMemory(why) if why.contains("4096 bytes of planes")), "{gone:?}");
|
||||
let over = read_error(Error::LimitExceeded("the requested rows need 4096 bytes".into()));
|
||||
assert!(matches!(&over, ReadError::TooLarge(why) if why.contains("4096 bytes")), "{over:?}");
|
||||
}
|
||||
|
||||
/// Reads of `path` fail for lack of memory when `fails` says so for the channels read; the reads
|
||||
/// asked for are counted.
|
||||
fn no_memory_when(path: &str, fails: impl Fn(&[usize]) -> bool + Send + Sync + 'static) -> Arc<AtomicU64> {
|
||||
let asked = Arc::new(AtomicU64::new(0));
|
||||
let counted = asked.clone();
|
||||
set_read_error(
|
||||
path,
|
||||
Some(Arc::new(move |ids: &[usize]| {
|
||||
counted.fetch_add(1, Ordering::SeqCst);
|
||||
fails(ids).then(|| crate::exr_read::ReadError::OutOfMemory("no memory just now".into()))
|
||||
})),
|
||||
);
|
||||
asked
|
||||
}
|
||||
|
||||
/// #3749 review: memory the system refuses during a read is neither read again at once (the
|
||||
/// store's retries, a copy) nor remembered: the request is refused with the reason, and the next
|
||||
/// request reads again.
|
||||
#[test]
|
||||
fn memory_the_system_cannot_give_is_neither_retried_nor_remembered() {
|
||||
let path = "/3749-no-memory.exr";
|
||||
let source = ExrSource::Bytes(picture_and_depth().into());
|
||||
let store = PlaneStore::new(CACHE_BUDGET);
|
||||
let index = crate::exr_index::get(path, &source).unwrap();
|
||||
let want = Want { all: true, ids: (0..5).collect() };
|
||||
let asked = no_memory_when(path, |_| true);
|
||||
let refused = store.get(path, &source, &index, &want, Use::Other).err();
|
||||
assert!(matches!(&refused, Some(Refused::NoMemory(why)) if why.contains("no memory just now")), "{refused:?}");
|
||||
assert_eq!(asked.load(Ordering::SeqCst), 1, "read once, not again at once");
|
||||
let asked = no_memory_when(path, |_| false);
|
||||
assert!(store.get(path, &source, &index, &want, Use::Other).is_ok(), "not remembered: the memory is there now");
|
||||
assert_eq!(asked.load(Ordering::SeqCst), 1);
|
||||
set_read_error(path, None);
|
||||
}
|
||||
|
||||
/// #3749 review: a picture whose channels the system has no memory for just now is read alone
|
||||
/// without the effects' channels, and refused with the reason when even that doesn't fit; never
|
||||
/// handed to the whole file's decode (which would ask for more memory, infallibly).
|
||||
#[test]
|
||||
fn a_picture_without_memory_is_read_alone_or_refused_never_decoded_whole() {
|
||||
let path = "/3749-no-memory-picture.exr";
|
||||
let source = ExrSource::Bytes(picture_and_depth().into());
|
||||
let op = AlphaOp::of(&Footage::default(), FloatCurve::Linear);
|
||||
// With the depth plane it is five planes, more than the system gives: the picture alone.
|
||||
no_memory_when(path, |ids| ids.len() > 4);
|
||||
let Picture::Done(Ok(img)) = picture(path, &source, op, Some(&AuxNeed::All)) else { panic!("the picture alone") };
|
||||
assert_eq!((img.width, img.height), (8, 8));
|
||||
// Not even the picture: refused with the reason, also for some of its rows.
|
||||
no_memory_when(path, |_| true);
|
||||
match picture(path, &source, op, None) {
|
||||
Picture::Done(Err(why)) => assert!(why.contains("no memory just now"), "{why}"),
|
||||
_ => panic!("refused with the reason"),
|
||||
}
|
||||
match picture_rows(path, &source, op, (0, 4)) {
|
||||
Picture::Done(Err(why)) => assert!(why.contains("no memory just now"), "{why}"),
|
||||
_ => panic!("rows refused with the reason"),
|
||||
}
|
||||
set_read_error(path, None);
|
||||
}
|
||||
|
||||
/// #3749 review: only a refusal for size is decided anew when the decode limit changes; an
|
||||
/// unreadable request stays refused, also when another request fails under the new limit.
|
||||
#[test]
|
||||
fn an_unreadable_request_stays_refused_when_the_decode_limit_changes() {
|
||||
let _only = LIMIT_TESTS.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let path = "/3749-unreadable-generation.exr";
|
||||
let source = ExrSource::Bytes(picture_and_depth().into());
|
||||
let store = PlaneStore::new(CACHE_BUDGET);
|
||||
let index = crate::exr_index::get(path, &source).unwrap();
|
||||
let asked = Arc::new(AtomicU64::new(0));
|
||||
{
|
||||
let counted = asked.clone();
|
||||
set_read_error(
|
||||
path,
|
||||
Some(Arc::new(move |_: &[usize]| {
|
||||
counted.fetch_add(1, Ordering::SeqCst);
|
||||
Some(crate::exr_read::ReadError::Unreadable("damaged".into()))
|
||||
})),
|
||||
);
|
||||
}
|
||||
let red = Want { all: false, ids: [0].into_iter().collect() };
|
||||
let unreadable = |want: &Want| matches!(store.get(path, &source, &index, want, Use::Other), Err(Refused::Unreadable));
|
||||
assert!(unreadable(&red));
|
||||
// (figures under the fixed floor: only the change counts, no other test's limit moves)
|
||||
effectcraft_render::memory::set_decode_limit(Some(1000));
|
||||
effectcraft_render::memory::set_decode_limit(Some(2000));
|
||||
assert!(unreadable(&red));
|
||||
assert_eq!(asked.load(Ordering::SeqCst), 1, "remembered across the change");
|
||||
// Another request fails under the new limit: the first refusal is still remembered.
|
||||
assert!(unreadable(&Want { all: false, ids: [4].into_iter().collect() }));
|
||||
assert!(unreadable(&red));
|
||||
assert_eq!(asked.load(Ordering::SeqCst), 2, "only the other request was read");
|
||||
effectcraft_render::memory::set_decode_limit(None);
|
||||
set_read_error(path, None);
|
||||
}
|
||||
|
||||
/// A two-rank Cryptomatte file of `size` with a picture: "cube" and "sphere" meet along a
|
||||
/// diagonal whose pixels hold both IDs (rank 00 the nearer, 55-75 %, rank 01 the other one with
|
||||
/// the rest), IDs and coverage stored as 32-bit floats.
|
||||
|
||||
@@ -213,6 +213,9 @@ pub(crate) enum ReadError {
|
||||
Io(String),
|
||||
/// The planes need more memory than one read may allocate (the reason, to show).
|
||||
TooLarge(String),
|
||||
/// The system refused the memory the planes need just now (the reason, to show): not a file
|
||||
/// too large, but not to be asked again at once either (#3749).
|
||||
OutOfMemory(String),
|
||||
/// The request was called off.
|
||||
Cancelled,
|
||||
}
|
||||
@@ -226,13 +229,15 @@ pub(crate) const MAX_DECODE_BYTES: usize = 8 << 30;
|
||||
#[cfg(not(target_pointer_width = "64"))]
|
||||
pub(crate) const MAX_DECODE_BYTES: usize = 512 << 20;
|
||||
|
||||
/// What a failed `craft-exr` read means here. Memory the system refused just now is a read that
|
||||
/// failed ([`ReadError::Io`]: the plane store doesn't remember it, another read may succeed),
|
||||
/// not a file too large (remembered until the file changes).
|
||||
/// What a failed `craft-exr` read means here. Memory the system refused just now is
|
||||
/// [`ReadError::OutOfMemory`]: the plane store doesn't remember it (another request may
|
||||
/// succeed), but doesn't read it again at once nor fall back to decoding the whole file
|
||||
/// either; a file too large is remembered until the file or the limit changes.
|
||||
pub(crate) fn read_error(e: seek::Error) -> ReadError {
|
||||
match e {
|
||||
seek::Error::Cancelled => ReadError::Cancelled,
|
||||
seek::Error::Io(why) | seek::Error::OutOfMemory(why) => ReadError::Io(why),
|
||||
seek::Error::Io(why) => ReadError::Io(why),
|
||||
seek::Error::OutOfMemory(why) => ReadError::OutOfMemory(format!("its OpenEXR channels need more memory than the system can give just now ({why})")),
|
||||
seek::Error::LimitExceeded(why) => ReadError::TooLarge(format!("its OpenEXR channels need more memory than one file may use ({why})")),
|
||||
seek::Error::Malformed(why) | seek::Error::Unsupported(why) | seek::Error::UnsupportedCompression(why) => ReadError::Unreadable(why),
|
||||
other => ReadError::Unreadable(other.to_string()),
|
||||
|
||||
Reference in new issue
Block a user