mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
OpenEXR limit: the planes a request ends with count together, held and from the disk cache too (#3749)
The limit applied to the planes read from the file just now, so planes held in memory or handed back by the disk cache were never counted and repeated requests could pass it. It now applies to what the request ends with, asked before anything is read or allocated. When the held planes don't fit with the missing ones they are let go and the request is read on its own; only a request that doesn't fit alone is refused.
This commit is contained in:
1 parent
a7e7cb6b6a
commit
f08a2290c2
3 files changed
+133
-28
No files matched your search
@@ -907,6 +907,26 @@ fn run(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc<Ba
|
||||
decode(path, source, flight, base, opened)
|
||||
}
|
||||
|
||||
/// What a failed read of `path` means for the request.
|
||||
fn refused(path: &str, e: ReadError) -> Refused {
|
||||
match e {
|
||||
ReadError::Cancelled => Refused::Cancelled,
|
||||
// (a read that failed just now: the next request reads again)
|
||||
ReadError::Io(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::Stopped
|
||||
}
|
||||
ReadError::TooLarge(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::TooLarge(why)
|
||||
}
|
||||
ReadError::Unreadable(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::Unreadable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode what `flight` asks for and `base` doesn't hold, and make the union of both.
|
||||
fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc<Batch>>, opened: Option<Opened>) -> Result<Arc<Batch>, Refused> {
|
||||
let _decoding = Decoding::new();
|
||||
@@ -947,10 +967,25 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
want.ids.iter().copied().collect()
|
||||
};
|
||||
let wanted: Vec<usize> = wanted.into_iter().filter(|i| *i < index.channels.len()).collect();
|
||||
let base = base.filter(|b| Arc::ptr_eq(&b.index, &index));
|
||||
let missing: Vec<usize> = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect();
|
||||
let names: Vec<String> = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect();
|
||||
let mut base = base.filter(|b| Arc::ptr_eq(&b.index, &index));
|
||||
let mut missing: Vec<usize> = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect();
|
||||
let (w, h) = index.size();
|
||||
// The planes the request ends with (those held plus those missing, whether they come from the
|
||||
// disk cache or the file) must fit the limit, asked before any of them is allocated (#3749).
|
||||
// The planes held are only kept for later requests: when they don't fit with the missing
|
||||
// ones, they are let go and the request is read on its own.
|
||||
if !missing.is_empty() {
|
||||
let held = base.as_ref().map_or(0, |b| b.ids.len());
|
||||
let together = crate::exr_read::check_total(w, h, held.saturating_add(missing.len()), cap_of(path));
|
||||
if together.is_err() && base.is_some() {
|
||||
base = None;
|
||||
missing = wanted.clone();
|
||||
}
|
||||
if base.is_none() {
|
||||
crate::exr_read::check_total(w, h, missing.len(), cap_of(path)).map_err(|e| refused(path, e))?;
|
||||
}
|
||||
}
|
||||
let names: Vec<String> = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect();
|
||||
// The disk tier first (#3271): what an earlier session or a decode RAM let go of kept, one
|
||||
// entry per channel (any request is assembled from them, whatever was asked before).
|
||||
// (a file naming two channels alike can't tell their entries apart: not kept)
|
||||
@@ -984,22 +1019,7 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
DECODES.lock().unwrap_or_else(PoisonError::into_inner).push((path.to_string(), DecodeRecord { channels: names.clone(), stats: stats.clone(), from_disk }));
|
||||
#[cfg(not(test))]
|
||||
let _ = (&stats, from_disk);
|
||||
let read: Vec<Vec<f32>> = read.map_err(|e| match e {
|
||||
ReadError::Cancelled => Refused::Cancelled,
|
||||
// (a read that failed just now: the next request reads again)
|
||||
ReadError::Io(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::Stopped
|
||||
}
|
||||
ReadError::TooLarge(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::TooLarge(why)
|
||||
}
|
||||
ReadError::Unreadable(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::Unreadable
|
||||
}
|
||||
})?;
|
||||
let read: Vec<Vec<f32>> = read.map_err(|e| refused(path, e))?;
|
||||
let mut read = read.into_iter();
|
||||
let planes: Vec<Vec<f32>> = kept.iter_mut().map(|k| k.take().or_else(|| read.next()).unwrap_or_default()).collect();
|
||||
// The union, in file order: the planes held and the ones just read.
|
||||
|
||||
@@ -1619,18 +1619,92 @@ fn a_refusal_for_size_is_decided_anew_when_the_decode_limit_changes() {
|
||||
let source = ExrSource::Bytes(bytes);
|
||||
let index = crate::exr_index::get(path, &source).unwrap();
|
||||
let want = Want { all: false, ids: (0..4).collect() };
|
||||
// (the refusal is made before anything is read: the decodes are counted as they begin)
|
||||
let begun = Arc::new(AtomicU64::new(0));
|
||||
{
|
||||
let begun = begun.clone();
|
||||
set_before_read(
|
||||
path,
|
||||
Some(Arc::new(move |_: &str| {
|
||||
begun.fetch_add(1, Ordering::SeqCst);
|
||||
})),
|
||||
);
|
||||
}
|
||||
let refused = || matches!(store.get(path, &source, &index, &want, Use::Other), Err(Refused::TooLarge(_)));
|
||||
assert!(refused());
|
||||
assert!(refused());
|
||||
assert_eq!(decodes_of(path).len(), 1, "the second refusal came from memory");
|
||||
assert_eq!(begun.load(Ordering::SeqCst), 1, "the second refusal came from memory");
|
||||
// (figures under the fixed floor: only the change counts, no other test's limit moves)
|
||||
effectcraft_render::memory::set_decode_limit(Some(1000));
|
||||
effectcraft_render::memory::set_decode_limit(Some(2000));
|
||||
assert!(refused(), "still over the caller's own limit");
|
||||
assert_eq!(decodes_of(path).len(), 2, "decided anew after the limit changed");
|
||||
assert_eq!(begun.load(Ordering::SeqCst), 2, "decided anew after the limit changed");
|
||||
set_before_read(path, None);
|
||||
effectcraft_render::memory::set_decode_limit(None);
|
||||
}
|
||||
|
||||
/// A file of an 8×8 picture and a depth plane: five planes of 256 bytes.
|
||||
fn picture_and_depth() -> Vec<u8> {
|
||||
write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", true), ("Z", false)], (8, 8), Compression::ZIP1)], (8, 8))
|
||||
}
|
||||
|
||||
/// #3749 (#3301 review): the limit is for the planes a request ends with, not for the ones read
|
||||
/// just now: channels added to the planes held count with them. When together they are over
|
||||
/// the limit the planes held are let go and the request is read on its own (it fits alone); a
|
||||
/// request that doesn't fit alone is refused, whatever is held.
|
||||
#[test]
|
||||
fn channels_added_to_the_planes_held_count_toward_the_limit() {
|
||||
let path = "/3749-held-and-added.exr";
|
||||
// Room for four planes: the picture fits, one more with it is five.
|
||||
set_cap(path, 4 * 8 * 8 * 4);
|
||||
let store = PlaneStore::new(CACHE_BUDGET);
|
||||
let source = ExrSource::Bytes(picture_and_depth().into());
|
||||
let index = crate::exr_index::get(path, &source).unwrap();
|
||||
let picture = Want { all: false, ids: index.picture_ids().into_iter().collect() };
|
||||
store.get(path, &source, &index, &picture, Use::Other).expect("the picture fits");
|
||||
let rest = Want { all: true, ids: Default::default() };
|
||||
let depth = store.get(path, &source, &index, &rest, Use::Other).expect("the depth plane fits alone");
|
||||
assert!(depth.aux.get("Z").is_some());
|
||||
assert!(depth.aux.get("R").is_none(), "the picture's planes were let go: four held and one more are five");
|
||||
// The picture again: it fits alone, the depth plane is let go in turn.
|
||||
let again = store.get(path, &source, &index, &picture, Use::Other).expect("the picture fits alone");
|
||||
assert!(again.aux.get("R").is_some() && again.aux.get("Z").is_none());
|
||||
// Everything at once is five planes: refused, however it is asked.
|
||||
let everything = Want { all: true, ids: (0..5).collect() };
|
||||
assert!(matches!(store.get(path, &source, &index, &everything, Use::Other).err(), Some(Refused::TooLarge(_))));
|
||||
}
|
||||
|
||||
/// #3749 (#3301 review): planes the disk cache gives back count toward the limit of a request
|
||||
/// like the ones read from the file.
|
||||
#[test]
|
||||
fn planes_from_the_disk_cache_count_toward_the_limit() {
|
||||
use effectcraft_render::disk_cache::DiskCache;
|
||||
let dir = std::env::temp_dir().join(format!("ec-3749-disk-union-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let file = dir.join("union_0001.exr");
|
||||
std::fs::write(&file, picture_and_depth()).unwrap();
|
||||
let dc = DiskCache::open(dir.join("cache"), 1 << 30).unwrap();
|
||||
dc.set_read_us_per_mb(0);
|
||||
use_disk_for(&dir.to_string_lossy());
|
||||
set_disk_cache(Some(&dc));
|
||||
let path = file.to_string_lossy().to_string();
|
||||
let source = ExrSource::File(file.clone());
|
||||
let index = crate::exr_index::get(&path, &source).unwrap();
|
||||
let all = Want { all: true, ids: (0..5).collect() };
|
||||
let first = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).is_ok();
|
||||
dc.flush();
|
||||
// All five planes are in the disk cache (a store without them in RAM reads them back)...
|
||||
let warm = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).is_ok();
|
||||
let from_disk = decodes_of(&path).last().is_some_and(|d| d.from_disk);
|
||||
// ...and a request may use four.
|
||||
set_cap(&path, 4 * 8 * 8 * 4);
|
||||
let refused = PlaneStore::new(CACHE_BUDGET).get(&path, &source, &index, &all, Use::Other).err();
|
||||
set_disk_cache(None);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
assert!(first && warm && from_disk, "decoded without a limit, then read back from the disk cache");
|
||||
assert!(matches!(refused, Some(Refused::TooLarge(_))), "{refused:?}");
|
||||
}
|
||||
|
||||
/// #3749: a plane the system cannot give is not an aborted process: `craft-exr` asks for the
|
||||
/// planes' memory first and reports what it can't get, a read that failed just now (asked again,
|
||||
/// not remembered); a plane over the read's limit (the decode limit handed to it) is a refusal
|
||||
|
||||
@@ -265,6 +265,23 @@ fn gb(bytes: usize) -> f64 {
|
||||
bytes as f64 / f64::from(1u32 << 30)
|
||||
}
|
||||
|
||||
fn too_large(w: usize, h: usize, need: usize, most: usize) -> ReadError {
|
||||
ReadError::TooLarge(format!("its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", w, h, gb(need), gb(most)))
|
||||
}
|
||||
|
||||
/// Refuse a request whose planes together, those already in hand (held in memory or read back
|
||||
/// from the disk cache) and those still to read, `planes` of a `w` × `h` display window, need
|
||||
/// more memory than one file may use: the limit is for what a file's request ends with, not
|
||||
/// for the part of it read just now ([`effectcraft_render::memory::decode_limit`], #3749).
|
||||
pub(crate) fn check_total(w: usize, h: usize, planes: usize, max_bytes: usize) -> Result<(), ReadError> {
|
||||
let need = w.saturating_mul(h).saturating_mul(planes).saturating_mul(std::mem::size_of::<f32>());
|
||||
let most = max_bytes.min(effectcraft_render::memory::decode_limit(MAX_DECODE_BYTES, need));
|
||||
if need > most {
|
||||
return Err(too_large(w, h, need, most));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<(Vec<Vec<f32>>, ReadStats), ReadError> {
|
||||
let (w, h) = index.size();
|
||||
let (ow, oh) = seek::level_size((w, h), req.level.min(16));
|
||||
@@ -274,13 +291,7 @@ fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<(
|
||||
// it reserves, fallibly, and the largest block it decompresses follow it)
|
||||
let most = req.max_bytes.min(effectcraft_render::memory::decode_limit(req.floor, need));
|
||||
if need > most {
|
||||
return Err(ReadError::TooLarge(format!(
|
||||
"its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use",
|
||||
w,
|
||||
h,
|
||||
gb(need),
|
||||
gb(most)
|
||||
)));
|
||||
return Err(too_large(w, h, need, most));
|
||||
}
|
||||
let mut channels = Vec::with_capacity(req.channels.len());
|
||||
let mut fill = Vec::with_capacity(req.channels.len());
|
||||
|
||||
Reference in new issue
Block a user