diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index a7a3c62cc..917dab4e3 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -907,6 +907,26 @@ fn run(path: &str, source: &ExrSource, flight: &Arc, base: Option Refused { + match e { + ReadError::Cancelled => Refused::Cancelled, + // (a read that failed just now: the next request reads again) + ReadError::Io(why) => { + log::warn!("media: {path}: {why}"); + Refused::Stopped + } + ReadError::TooLarge(why) => { + log::warn!("media: {path}: {why}"); + Refused::TooLarge(why) + } + ReadError::Unreadable(why) => { + log::warn!("media: {path}: {why}"); + Refused::Unreadable + } + } +} + /// Decode what `flight` asks for and `base` doesn't hold, and make the union of both. fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option>, opened: Option) -> Result, Refused> { let _decoding = Decoding::new(); @@ -947,10 +967,25 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option = wanted.into_iter().filter(|i| *i < index.channels.len()).collect(); - let base = base.filter(|b| Arc::ptr_eq(&b.index, &index)); - let missing: Vec = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect(); - let names: Vec = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect(); + let mut base = base.filter(|b| Arc::ptr_eq(&b.index, &index)); + let mut missing: Vec = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect(); let (w, h) = index.size(); + // The planes the request ends with (those held plus those missing, whether they come from the + // disk cache or the file) must fit the limit, asked before any of them is allocated (#3749). + // The planes held are only kept for later requests: when they don't fit with the missing + // ones, they are let go and the request is read on its own. + if !missing.is_empty() { + let held = base.as_ref().map_or(0, |b| b.ids.len()); + let together = crate::exr_read::check_total(w, h, held.saturating_add(missing.len()), cap_of(path)); + if together.is_err() && base.is_some() { + base = None; + missing = wanted.clone(); + } + if base.is_none() { + crate::exr_read::check_total(w, h, missing.len(), cap_of(path)).map_err(|e| refused(path, e))?; + } + } + let names: Vec = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect(); // The disk tier first (#3271): what an earlier session or a decode RAM let go of kept, one // entry per channel (any request is assembled from them, whatever was asked before). // (a file naming two channels alike can't tell their entries apart: not kept) @@ -984,22 +1019,7 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option> = read.map_err(|e| match e { - ReadError::Cancelled => Refused::Cancelled, - // (a read that failed just now: the next request reads again) - ReadError::Io(why) => { - log::warn!("media: {path}: {why}"); - Refused::Stopped - } - ReadError::TooLarge(why) => { - log::warn!("media: {path}: {why}"); - Refused::TooLarge(why) - } - ReadError::Unreadable(why) => { - log::warn!("media: {path}: {why}"); - Refused::Unreadable - } - })?; + let read: Vec> = read.map_err(|e| refused(path, e))?; let mut read = read.into_iter(); let planes: Vec> = kept.iter_mut().map(|k| k.take().or_else(|| read.next()).unwrap_or_default()).collect(); // The union, in file order: the planes held and the ones just read. diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index 76b02d4cb..313302a4a 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -1619,18 +1619,92 @@ fn a_refusal_for_size_is_decided_anew_when_the_decode_limit_changes() { let source = ExrSource::Bytes(bytes); let index = crate::exr_index::get(path, &source).unwrap(); let want = Want { all: false, ids: (0..4).collect() }; + // (the refusal is made before anything is read: the decodes are counted as they begin) + let begun = Arc::new(AtomicU64::new(0)); + { + let begun = begun.clone(); + set_before_read( + path, + Some(Arc::new(move |_: &str| { + begun.fetch_add(1, Ordering::SeqCst); + })), + ); + } let refused = || matches!(store.get(path, &source, &index, &want, Use::Other), Err(Refused::TooLarge(_))); assert!(refused()); assert!(refused()); - assert_eq!(decodes_of(path).len(), 1, "the second refusal came from memory"); + assert_eq!(begun.load(Ordering::SeqCst), 1, "the second refusal came from memory"); // (figures under the fixed floor: only the change counts, no other test's limit moves) effectcraft_render::memory::set_decode_limit(Some(1000)); effectcraft_render::memory::set_decode_limit(Some(2000)); assert!(refused(), "still over the caller's own limit"); - assert_eq!(decodes_of(path).len(), 2, "decided anew after the limit changed"); + assert_eq!(begun.load(Ordering::SeqCst), 2, "decided anew after the limit changed"); + set_before_read(path, None); effectcraft_render::memory::set_decode_limit(None); } +/// A file of an 8×8 picture and a depth plane: five planes of 256 bytes. +fn picture_and_depth() -> Vec { + write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", true), ("Z", false)], (8, 8), Compression::ZIP1)], (8, 8)) +} + +/// #3749 (#3301 review): the limit is for the planes a request ends with, not for the ones read +/// just now: channels added to the planes held count with them. When together they are over +/// the limit the planes held are let go and the request is read on its own (it fits alone); a +/// request that doesn't fit alone is refused, whatever is held. +#[test] +fn channels_added_to_the_planes_held_count_toward_the_limit() { + let path = "/3749-held-and-added.exr"; + // Room for four planes: the picture fits, one more with it is five. + set_cap(path, 4 * 8 * 8 * 4); + let store = PlaneStore::new(CACHE_BUDGET); + let source = ExrSource::Bytes(picture_and_depth().into()); + let index = crate::exr_index::get(path, &source).unwrap(); + let picture = Want { all: false, ids: index.picture_ids().into_iter().collect() }; + store.get(path, &source, &index, &picture, Use::Other).expect("the picture fits"); + let rest = Want { all: true, ids: Default::default() }; + let depth = store.get(path, &source, &index, &rest, Use::Other).expect("the depth plane fits alone"); + assert!(depth.aux.get("Z").is_some()); + assert!(depth.aux.get("R").is_none(), "the picture's planes were let go: four held and one more are five"); + // The picture again: it fits alone, the depth plane is let go in turn. + let again = store.get(path, &source, &index, &picture, Use::Other).expect("the picture fits alone"); + assert!(again.aux.get("R").is_some() && again.aux.get("Z").is_none()); + // Everything at once is five planes: refused, however it is asked. + let everything = Want { all: true, ids: (0..5).collect() }; + assert!(matches!(store.get(path, &source, &index, &everything, Use::Other).err(), Some(Refused::TooLarge(_)))); +} + +/// #3749 (#3301 review): planes the disk cache gives back count toward the limit of a request +/// like the ones read from the file. +#[test] +fn planes_from_the_disk_cache_count_toward_the_limit() { + use effectcraft_render::disk_cache::DiskCache; + let dir = std::env::temp_dir().join(format!("ec-3749-disk-union-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("union_0001.exr"); + std::fs::write(&file, picture_and_depth()).unwrap(); + let dc = DiskCache::open(dir.join("cache"), 1 << 30).unwrap(); + dc.set_read_us_per_mb(0); + use_disk_for(&dir.to_string_lossy()); + set_disk_cache(Some(&dc)); + let path = file.to_string_lossy().to_string(); + let source = ExrSource::File(file.clone()); + let index = crate::exr_index::get(&path, &source).unwrap(); + let all = Want { all: true, ids: (0..5).collect() }; + let first = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).is_ok(); + dc.flush(); + // All five planes are in the disk cache (a store without them in RAM reads them back)... + let warm = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).is_ok(); + let from_disk = decodes_of(&path).last().is_some_and(|d| d.from_disk); + // ...and a request may use four. + set_cap(&path, 4 * 8 * 8 * 4); + let refused = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).err(); + set_disk_cache(None); + let _ = std::fs::remove_dir_all(&dir); + assert!(first && warm && from_disk, "decoded without a limit, then read back from the disk cache"); + assert!(matches!(refused, Some(Refused::TooLarge(_))), "{refused:?}"); +} + /// #3749: a plane the system cannot give is not an aborted process: `craft-exr` asks for the /// planes' memory first and reports what it can't get, a read that failed just now (asked again, /// not remembered); a plane over the read's limit (the decode limit handed to it) is a refusal diff --git a/crates/media/src/exr_read.rs b/crates/media/src/exr_read.rs index 5f9151049..5624fc2d0 100644 --- a/crates/media/src/exr_read.rs +++ b/crates/media/src/exr_read.rs @@ -265,6 +265,23 @@ fn gb(bytes: usize) -> f64 { bytes as f64 / f64::from(1u32 << 30) } +fn too_large(w: usize, h: usize, need: usize, most: usize) -> ReadError { + ReadError::TooLarge(format!("its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", w, h, gb(need), gb(most))) +} + +/// Refuse a request whose planes together, those already in hand (held in memory or read back +/// from the disk cache) and those still to read, `planes` of a `w` × `h` display window, need +/// more memory than one file may use: the limit is for what a file's request ends with, not +/// for the part of it read just now ([`effectcraft_render::memory::decode_limit`], #3749). +pub(crate) fn check_total(w: usize, h: usize, planes: usize, max_bytes: usize) -> Result<(), ReadError> { + let need = w.saturating_mul(h).saturating_mul(planes).saturating_mul(std::mem::size_of::()); + let most = max_bytes.min(effectcraft_render::memory::decode_limit(MAX_DECODE_BYTES, need)); + if need > most { + return Err(too_large(w, h, need, most)); + } + Ok(()) +} + fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<(Vec>, ReadStats), ReadError> { let (w, h) = index.size(); let (ow, oh) = seek::level_size((w, h), req.level.min(16)); @@ -274,13 +291,7 @@ fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<( // it reserves, fallibly, and the largest block it decompresses follow it) let most = req.max_bytes.min(effectcraft_render::memory::decode_limit(req.floor, need)); if need > most { - return Err(ReadError::TooLarge(format!( - "its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", - w, - h, - gb(need), - gb(most) - ))); + return Err(too_large(w, h, need, most)); } let mut channels = Vec::with_capacity(req.channels.len()); let mut fill = Vec::with_capacity(req.channels.len());