OpenEXR limit: a refusal for size is decided anew when the limit changes; a failed reading is asked twice (#3749 review)

This commit is contained in:
vangel76 committed 2026-11-16 23:09:59 +01:00
1 parent b591f95ce4
commit bdf500f4ba
3 files changed
+100 -18

No files matched your search

+12 -5
View File
@@ -183,6 +183,10 @@ struct Flight {
/// When the decode was asked for ([`effectcraft_render::memory::tick`]): what it took is the
/// planes' cost to the memory accountant.
asked: u64,
/// The decode limit's generation when the decode was asked for
/// ([`effectcraft_render::memory::decode_limit_generation`]): a refusal for its size holds
/// only while the limit stays as it was.
limit_generation: u64,
}
/// [`Flight::phase`] values.
@@ -219,6 +223,7 @@ impl Flight {
since,
hints: filmcraft_media::cancel::Context::current(),
asked: effectcraft_render::memory::tick(),
limit_generation: effectcraft_render::memory::decode_limit_generation(),
})
}
@@ -276,8 +281,9 @@ struct Entry {
flight: Option<Arc<Flight>>,
/// Requests that failed for good (unreadable, too large), the index they were made with, and
/// why: a request of that index (the file as it was) asking for at
/// least as much fails the same way until the file is forgotten.
failed: Vec<(Arc<ExrIndex>, Want, Refused)>,
/// least as much fails the same way until the file is forgotten, or the decode limit changed
/// (the last number: its generation when the request was made).
failed: Vec<(Arc<ExrIndex>, Want, Refused, u64)>,
bytes: usize,
/// When it was last asked for or finished decoding ([`effectcraft_render::memory::tick`]).
used: u64,
@@ -522,7 +528,8 @@ impl PlaneStore {
return Step::Done(Ok(b));
}
// (a failure of the file as it was before: the file changed since, it is read again)
if let Some((_, _, why)) = e.failed.iter().find(|(of, failed, _)| Arc::ptr_eq(of, index) && asks_at_least(index, want, failed)) {
let limit = effectcraft_render::memory::decode_limit_generation();
if let Some((_, _, why, _)) = e.failed.iter().find(|(of, failed, _, made_at)| Arc::ptr_eq(of, index) && *made_at == limit && asks_at_least(index, want, failed)) {
return Step::Done(Err(why.clone()));
}
if let Some(flight) = e.flight.clone() {
@@ -671,11 +678,11 @@ impl PlaneStore {
(fs.index.clone(), fs.want.clone())
};
// (failures of another version of the file no longer apply)
e.failed.retain(|(of, _, _)| Arc::ptr_eq(of, &index));
e.failed.retain(|(of, _, _, made_at)| Arc::ptr_eq(of, &index) && *made_at == flight.limit_generation);
if e.failed.len() >= FAILED_KEPT {
e.failed.remove(0);
}
e.failed.push((index, want, why.clone()));
e.failed.push((index, want, why.clone(), flight.limit_generation));
}
// (the file changed: what is held is of the old one)
Err(Refused::Changed) => {
+30
View File
@@ -1580,11 +1580,16 @@ fn a_failed_picture_leaves_the_other_channels_readable() {
assert!(rest.aux.get("Z").is_some());
}
/// The tests that move the process-wide decode limit take turns (it also dates the remembered
/// refusals).
static LIMIT_TESTS: Mutex<()> = Mutex::new(());
/// #3749: a read over the fixed floor is allowed up to what the system's memory gives (the
/// session's figure), never under the floor, and refused with a reason beyond it. The floor is
/// small here: the real one is 8 GiB.
#[test]
fn a_read_over_the_floor_is_allowed_up_to_the_systems_memory() {
let _only = LIMIT_TESTS.lock().unwrap_or_else(PoisonError::into_inner);
let bytes = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8));
let mut src = std::io::Cursor::new(bytes.as_slice());
let index = crate::exr_index::build(&mut src, FileStamp::Virtual).unwrap();
@@ -1601,6 +1606,31 @@ fn a_read_over_the_floor_is_allowed_up_to_the_systems_memory() {
effectcraft_render::memory::set_decode_limit(None);
}
/// #3749: a refusal for the request's size is remembered only while the decode limit stays as it
/// was: when the system's memory figure arrives or changes, the request is decided anew.
#[test]
fn a_refusal_for_size_is_decided_anew_when_the_decode_limit_changes() {
let _only = LIMIT_TESTS.lock().unwrap_or_else(PoisonError::into_inner);
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8)).into();
let path = "/3749-refusal-generation.exr";
// Room for one plane of 8×8 f32: four are refused.
set_cap(path, 8 * 8 * 4);
let store = PlaneStore::new(CACHE_BUDGET);
let source = ExrSource::Bytes(bytes);
let index = crate::exr_index::get(path, &source).unwrap();
let want = Want { all: false, ids: (0..4).collect() };
let refused = || matches!(store.get(path, &source, &index, &want, Use::Other), Err(Refused::TooLarge(_)));
assert!(refused());
assert!(refused());
assert_eq!(decodes_of(path).len(), 1, "the second refusal came from memory");
// (figures under the fixed floor: only the change counts, no other test's limit moves)
effectcraft_render::memory::set_decode_limit(Some(1000));
effectcraft_render::memory::set_decode_limit(Some(2000));
assert!(refused(), "still over the caller's own limit");
assert_eq!(decodes_of(path).len(), 2, "decided anew after the limit changed");
effectcraft_render::memory::set_decode_limit(None);
}
/// #3749: a plane the system cannot give is not an aborted process: `craft-exr` asks for the
/// planes' memory first and reports what it can't get, a read that failed just now (asked again,
/// not remembered); a plane over the read's limit (the decode limit handed to it) is a refusal
+58 -13
View File
@@ -423,14 +423,40 @@ impl Accountant {
/// channels) to take; 0: not known yet ([`decode_limit`]).
static DECODE_LIMIT: AtomicUsize = AtomicUsize::new(0);
/// Counts the changes of that limit: a refusal made under an older one is out of date
/// ([`decode_limit_generation`]).
static DECODE_GENERATION: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
/// How often the system was asked for the limit without an answer ([`decode_limit`]).
static DECODE_UNANSWERED: AtomicUsize = AtomicUsize::new(0);
/// Reads that limit from the system when nothing set it ([`set_decode_limit_source`]).
static DECODE_SOURCE: Mutex<Option<fn() -> Option<usize>>> = Mutex::new(None);
/// One reading of the system at a time: the second asker takes the first one's answer.
static DECODE_ASKING: Mutex<()> = Mutex::new(());
/// The system is asked this often at most for the limit when it does not answer.
const DECODE_ASKS: usize = 2;
fn store_decode_limit(bytes: usize) {
if DECODE_LIMIT.swap(bytes, Ordering::Relaxed) != bytes {
DECODE_GENERATION.fetch_add(1, Ordering::Relaxed);
}
}
/// Set what the system's memory allows one request for a file's decoded planes to take
/// (`None`: unknown again). The session sets it with the memory budget once the installed RAM
/// is read.
pub fn set_decode_limit(bytes: Option<usize>) {
DECODE_LIMIT.store(bytes.unwrap_or(0), Ordering::Relaxed);
store_decode_limit(bytes.unwrap_or(0));
DECODE_UNANSWERED.store(0, Ordering::Relaxed);
}
/// Changes whenever [`decode_limit`] gives another answer: a request refused for its size is
/// asked again after that.
pub fn decode_limit_generation() -> u64 {
DECODE_GENERATION.load(Ordering::Relaxed)
}
/// Where [`decode_limit`] reads the limit from while nothing set it: the CLI and the MCP server
@@ -438,23 +464,31 @@ pub fn set_decode_limit(bytes: Option<usize>) {
/// request is over the fixed floor.
pub fn set_decode_limit_source(source: Option<fn() -> Option<usize>>) {
*DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner) = source;
DECODE_UNANSWERED.store(0, Ordering::Relaxed);
}
/// The most bytes one request of `need` bytes for a file's decoded planes may take: `floor`
/// (the fixed limit), or what the system's memory allows when that is more. The system is
/// asked only when `need` is over `floor` (and once: the answer is kept, also "unknown", which
/// leaves `floor`). The browser keeps `floor`.
/// asked only when `need` is over `floor`, and the answer is kept; when it gives none (a reading
/// that failed) it is asked once more later, then `floor` stays. The browser keeps `floor`.
pub fn decode_limit(floor: usize, need: usize) -> usize {
if need <= floor || cfg!(target_arch = "wasm32") {
return floor;
}
let mut known = DECODE_LIMIT.load(Ordering::Relaxed);
if known == 0 {
let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner);
known = source.and_then(|read| read()).unwrap_or(floor);
DECODE_LIMIT.store(known, Ordering::Relaxed);
if DECODE_LIMIT.load(Ordering::Relaxed) == 0 && DECODE_UNANSWERED.load(Ordering::Relaxed) < DECODE_ASKS {
let _asking = DECODE_ASKING.lock().unwrap_or_else(PoisonError::into_inner);
// (another thread may have just read it)
if DECODE_LIMIT.load(Ordering::Relaxed) == 0 {
let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner);
match source.and_then(|read| read()) {
Some(bytes) => store_decode_limit(bytes),
None => {
DECODE_UNANSWERED.fetch_add(1, Ordering::Relaxed);
}
}
}
}
floor.max(known)
floor.max(DECODE_LIMIT.load(Ordering::Relaxed))
}
#[cfg(test)]
@@ -903,13 +937,24 @@ mod tests {
// The session's figure replaces the asked one.
set_decode_limit(Some(900));
assert_eq!(decode_limit(100, 200), 900);
// Unknown memory leaves the floor, and is not asked again.
// Unknown memory leaves the floor; a reading that failed once is tried once more, then
// not asked again (and what is not known is never kept as the floor).
set_decode_limit(None);
set_decode_limit_source(Some(ram_unknown));
ASKED.store(0, Ordering::SeqCst);
assert_eq!(decode_limit(100, 200), 100);
assert_eq!(decode_limit(100, 200), 100);
assert_eq!(ASKED.load(Ordering::SeqCst), 1, "an unknown reading is kept too");
for _ in 0..4 {
assert_eq!(decode_limit(100, 200), 100);
}
assert_eq!(ASKED.load(Ordering::SeqCst), 2, "a failed reading is tried twice");
// A source that answers later is not shut out by the floor kept earlier.
set_decode_limit_source(Some(ram_says));
assert_eq!(decode_limit(7, 200), 300, "a limit is not tied to the floor of an earlier request");
// The generation tells that the limit changed.
let before = decode_limit_generation();
set_decode_limit(Some(300));
assert_eq!(decode_limit_generation(), before, "the same figure is no change");
set_decode_limit(Some(500));
assert_ne!(decode_limit_generation(), before);
set_decode_limit(None);
set_decode_limit_source(None);
}