From f08a2290c223f50797a01d0dae1e79da0272d1d3 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 16 Nov 2026 17:49:20 +0100 Subject: [PATCH] OpenEXR limit: the planes a request ends with count together, held and from the disk cache too (#3749) The limit applied to the planes read from the file just now, so planes held in memory or handed back by the disk cache were never counted and repeated requests could pass it. It now applies to what the request ends with, asked before anything is read or allocated. When the held planes don't fit with the missing ones they are let go and the request is read on its own; only a request that doesn't fit alone is refused. --- crates/media/src/exr_planes.rs | 58 ++++++++++++++------- crates/media/src/exr_planes/tests.rs | 78 +++++++++++++++++++++++++++- crates/media/src/exr_read.rs | 25 ++++++--- 3 files changed, 133 insertions(+), 28 deletions(-) diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index a7a3c62cc..917dab4e3 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -907,6 +907,26 @@ fn run(path: &str, source: &ExrSource, flight: &Arc, base: Option Refused { + match e { + ReadError::Cancelled => Refused::Cancelled, + // (a read that failed just now: the next request reads again) + ReadError::Io(why) => { + log::warn!("media: {path}: {why}"); + Refused::Stopped + } + ReadError::TooLarge(why) => { + log::warn!("media: {path}: {why}"); + Refused::TooLarge(why) + } + ReadError::Unreadable(why) => { + log::warn!("media: {path}: {why}"); + Refused::Unreadable + } + } +} + /// Decode what `flight` asks for and `base` doesn't hold, and make the union of both. fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option>, opened: Option) -> Result, Refused> { let _decoding = Decoding::new(); @@ -947,10 +967,25 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option = wanted.into_iter().filter(|i| *i < index.channels.len()).collect(); - let base = base.filter(|b| Arc::ptr_eq(&b.index, &index)); - let missing: Vec = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect(); - let names: Vec = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect(); + let mut base = base.filter(|b| Arc::ptr_eq(&b.index, &index)); + let mut missing: Vec = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect(); let (w, h) = index.size(); + // The planes the request ends with (those held plus those missing, whether they come from the + // disk cache or the file) must fit the limit, asked before any of them is allocated (#3749). + // The planes held are only kept for later requests: when they don't fit with the missing + // ones, they are let go and the request is read on its own. + if !missing.is_empty() { + let held = base.as_ref().map_or(0, |b| b.ids.len()); + let together = crate::exr_read::check_total(w, h, held.saturating_add(missing.len()), cap_of(path)); + if together.is_err() && base.is_some() { + base = None; + missing = wanted.clone(); + } + if base.is_none() { + crate::exr_read::check_total(w, h, missing.len(), cap_of(path)).map_err(|e| refused(path, e))?; + } + } + let names: Vec = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect(); // The disk tier first (#3271): what an earlier session or a decode RAM let go of kept, one // entry per channel (any request is assembled from them, whatever was asked before). // (a file naming two channels alike can't tell their entries apart: not kept) @@ -984,22 +1019,7 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option> = read.map_err(|e| match e { - ReadError::Cancelled => Refused::Cancelled, - // (a read that failed just now: the next request reads again) - ReadError::Io(why) => { - log::warn!("media: {path}: {why}"); - Refused::Stopped - } - ReadError::TooLarge(why) => { - log::warn!("media: {path}: {why}"); - Refused::TooLarge(why) - } - ReadError::Unreadable(why) => { - log::warn!("media: {path}: {why}"); - Refused::Unreadable - } - })?; + let read: Vec> = read.map_err(|e| refused(path, e))?; let mut read = read.into_iter(); let planes: Vec> = kept.iter_mut().map(|k| k.take().or_else(|| read.next()).unwrap_or_default()).collect(); // The union, in file order: the planes held and the ones just read. diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index 76b02d4cb..313302a4a 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -1619,18 +1619,92 @@ fn a_refusal_for_size_is_decided_anew_when_the_decode_limit_changes() { let source = ExrSource::Bytes(bytes); let index = crate::exr_index::get(path, &source).unwrap(); let want = Want { all: false, ids: (0..4).collect() }; + // (the refusal is made before anything is read: the decodes are counted as they begin) + let begun = Arc::new(AtomicU64::new(0)); + { + let begun = begun.clone(); + set_before_read( + path, + Some(Arc::new(move |_: &str| { + begun.fetch_add(1, Ordering::SeqCst); + })), + ); + } let refused = || matches!(store.get(path, &source, &index, &want, Use::Other), Err(Refused::TooLarge(_))); assert!(refused()); assert!(refused()); - assert_eq!(decodes_of(path).len(), 1, "the second refusal came from memory"); + assert_eq!(begun.load(Ordering::SeqCst), 1, "the second refusal came from memory"); // (figures under the fixed floor: only the change counts, no other test's limit moves) effectcraft_render::memory::set_decode_limit(Some(1000)); effectcraft_render::memory::set_decode_limit(Some(2000)); assert!(refused(), "still over the caller's own limit"); - assert_eq!(decodes_of(path).len(), 2, "decided anew after the limit changed"); + assert_eq!(begun.load(Ordering::SeqCst), 2, "decided anew after the limit changed"); + set_before_read(path, None); effectcraft_render::memory::set_decode_limit(None); } +/// A file of an 8×8 picture and a depth plane: five planes of 256 bytes. +fn picture_and_depth() -> Vec { + write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", true), ("Z", false)], (8, 8), Compression::ZIP1)], (8, 8)) +} + +/// #3749 (#3301 review): the limit is for the planes a request ends with, not for the ones read +/// just now: channels added to the planes held count with them. When together they are over +/// the limit the planes held are let go and the request is read on its own (it fits alone); a +/// request that doesn't fit alone is refused, whatever is held. +#[test] +fn channels_added_to_the_planes_held_count_toward_the_limit() { + let path = "/3749-held-and-added.exr"; + // Room for four planes: the picture fits, one more with it is five. + set_cap(path, 4 * 8 * 8 * 4); + let store = PlaneStore::new(CACHE_BUDGET); + let source = ExrSource::Bytes(picture_and_depth().into()); + let index = crate::exr_index::get(path, &source).unwrap(); + let picture = Want { all: false, ids: index.picture_ids().into_iter().collect() }; + store.get(path, &source, &index, &picture, Use::Other).expect("the picture fits"); + let rest = Want { all: true, ids: Default::default() }; + let depth = store.get(path, &source, &index, &rest, Use::Other).expect("the depth plane fits alone"); + assert!(depth.aux.get("Z").is_some()); + assert!(depth.aux.get("R").is_none(), "the picture's planes were let go: four held and one more are five"); + // The picture again: it fits alone, the depth plane is let go in turn. + let again = store.get(path, &source, &index, &picture, Use::Other).expect("the picture fits alone"); + assert!(again.aux.get("R").is_some() && again.aux.get("Z").is_none()); + // Everything at once is five planes: refused, however it is asked. + let everything = Want { all: true, ids: (0..5).collect() }; + assert!(matches!(store.get(path, &source, &index, &everything, Use::Other).err(), Some(Refused::TooLarge(_)))); +} + +/// #3749 (#3301 review): planes the disk cache gives back count toward the limit of a request +/// like the ones read from the file. +#[test] +fn planes_from_the_disk_cache_count_toward_the_limit() { + use effectcraft_render::disk_cache::DiskCache; + let dir = std::env::temp_dir().join(format!("ec-3749-disk-union-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("union_0001.exr"); + std::fs::write(&file, picture_and_depth()).unwrap(); + let dc = DiskCache::open(dir.join("cache"), 1 << 30).unwrap(); + dc.set_read_us_per_mb(0); + use_disk_for(&dir.to_string_lossy()); + set_disk_cache(Some(&dc)); + let path = file.to_string_lossy().to_string(); + let source = ExrSource::File(file.clone()); + let index = crate::exr_index::get(&path, &source).unwrap(); + let all = Want { all: true, ids: (0..5).collect() }; + let first = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).is_ok(); + dc.flush(); + // All five planes are in the disk cache (a store without them in RAM reads them back)... + let warm = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).is_ok(); + let from_disk = decodes_of(&path).last().is_some_and(|d| d.from_disk); + // ...and a request may use four. + set_cap(&path, 4 * 8 * 8 * 4); + let refused = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).err(); + set_disk_cache(None); + let _ = std::fs::remove_dir_all(&dir); + assert!(first && warm && from_disk, "decoded without a limit, then read back from the disk cache"); + assert!(matches!(refused, Some(Refused::TooLarge(_))), "{refused:?}"); +} + /// #3749: a plane the system cannot give is not an aborted process: `craft-exr` asks for the /// planes' memory first and reports what it can't get, a read that failed just now (asked again, /// not remembered); a plane over the read's limit (the decode limit handed to it) is a refusal diff --git a/crates/media/src/exr_read.rs b/crates/media/src/exr_read.rs index 5f9151049..5624fc2d0 100644 --- a/crates/media/src/exr_read.rs +++ b/crates/media/src/exr_read.rs @@ -265,6 +265,23 @@ fn gb(bytes: usize) -> f64 { bytes as f64 / f64::from(1u32 << 30) } +fn too_large(w: usize, h: usize, need: usize, most: usize) -> ReadError { + ReadError::TooLarge(format!("its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", w, h, gb(need), gb(most))) +} + +/// Refuse a request whose planes together, those already in hand (held in memory or read back +/// from the disk cache) and those still to read, `planes` of a `w` × `h` display window, need +/// more memory than one file may use: the limit is for what a file's request ends with, not +/// for the part of it read just now ([`effectcraft_render::memory::decode_limit`], #3749). +pub(crate) fn check_total(w: usize, h: usize, planes: usize, max_bytes: usize) -> Result<(), ReadError> { + let need = w.saturating_mul(h).saturating_mul(planes).saturating_mul(std::mem::size_of::()); + let most = max_bytes.min(effectcraft_render::memory::decode_limit(MAX_DECODE_BYTES, need)); + if need > most { + return Err(too_large(w, h, need, most)); + } + Ok(()) +} + fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<(Vec>, ReadStats), ReadError> { let (w, h) = index.size(); let (ow, oh) = seek::level_size((w, h), req.level.min(16)); @@ -274,13 +291,7 @@ fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<( // it reserves, fallibly, and the largest block it decompresses follow it) let most = req.max_bytes.min(effectcraft_render::memory::decode_limit(req.floor, need)); if need > most { - return Err(ReadError::TooLarge(format!( - "its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", - w, - h, - gb(need), - gb(most) - ))); + return Err(too_large(w, h, need, most)); } let mut channels = Vec::with_capacity(req.channels.len()); let mut fill = Vec::with_capacity(req.channels.len());