mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
OpenEXR limit: a refusal for size is decided anew when the limit changes; a failed reading is asked twice (#3749 review)
This commit is contained in:
1 parent
b591f95ce4
commit
bdf500f4ba
3 files changed
+100
-18
No files matched your search
@@ -183,6 +183,10 @@ struct Flight {
|
||||
/// When the decode was asked for ([`effectcraft_render::memory::tick`]): what it took is the
|
||||
/// planes' cost to the memory accountant.
|
||||
asked: u64,
|
||||
/// The decode limit's generation when the decode was asked for
|
||||
/// ([`effectcraft_render::memory::decode_limit_generation`]): a refusal for its size holds
|
||||
/// only while the limit stays as it was.
|
||||
limit_generation: u64,
|
||||
}
|
||||
|
||||
/// [`Flight::phase`] values.
|
||||
@@ -219,6 +223,7 @@ impl Flight {
|
||||
since,
|
||||
hints: filmcraft_media::cancel::Context::current(),
|
||||
asked: effectcraft_render::memory::tick(),
|
||||
limit_generation: effectcraft_render::memory::decode_limit_generation(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -276,8 +281,9 @@ struct Entry {
|
||||
flight: Option<Arc<Flight>>,
|
||||
/// Requests that failed for good (unreadable, too large), the index they were made with, and
|
||||
/// why: a request of that index (the file as it was) asking for at
|
||||
/// least as much fails the same way until the file is forgotten.
|
||||
failed: Vec<(Arc<ExrIndex>, Want, Refused)>,
|
||||
/// least as much fails the same way until the file is forgotten, or the decode limit changed
|
||||
/// (the last number: its generation when the request was made).
|
||||
failed: Vec<(Arc<ExrIndex>, Want, Refused, u64)>,
|
||||
bytes: usize,
|
||||
/// When it was last asked for or finished decoding ([`effectcraft_render::memory::tick`]).
|
||||
used: u64,
|
||||
@@ -522,7 +528,8 @@ impl PlaneStore {
|
||||
return Step::Done(Ok(b));
|
||||
}
|
||||
// (a failure of the file as it was before: the file changed since, it is read again)
|
||||
if let Some((_, _, why)) = e.failed.iter().find(|(of, failed, _)| Arc::ptr_eq(of, index) && asks_at_least(index, want, failed)) {
|
||||
let limit = effectcraft_render::memory::decode_limit_generation();
|
||||
if let Some((_, _, why, _)) = e.failed.iter().find(|(of, failed, _, made_at)| Arc::ptr_eq(of, index) && *made_at == limit && asks_at_least(index, want, failed)) {
|
||||
return Step::Done(Err(why.clone()));
|
||||
}
|
||||
if let Some(flight) = e.flight.clone() {
|
||||
@@ -671,11 +678,11 @@ impl PlaneStore {
|
||||
(fs.index.clone(), fs.want.clone())
|
||||
};
|
||||
// (failures of another version of the file no longer apply)
|
||||
e.failed.retain(|(of, _, _)| Arc::ptr_eq(of, &index));
|
||||
e.failed.retain(|(of, _, _, made_at)| Arc::ptr_eq(of, &index) && *made_at == flight.limit_generation);
|
||||
if e.failed.len() >= FAILED_KEPT {
|
||||
e.failed.remove(0);
|
||||
}
|
||||
e.failed.push((index, want, why.clone()));
|
||||
e.failed.push((index, want, why.clone(), flight.limit_generation));
|
||||
}
|
||||
// (the file changed: what is held is of the old one)
|
||||
Err(Refused::Changed) => {
|
||||
|
||||
@@ -1580,11 +1580,16 @@ fn a_failed_picture_leaves_the_other_channels_readable() {
|
||||
assert!(rest.aux.get("Z").is_some());
|
||||
}
|
||||
|
||||
/// The tests that move the process-wide decode limit take turns (it also dates the remembered
|
||||
/// refusals).
|
||||
static LIMIT_TESTS: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// #3749: a read over the fixed floor is allowed up to what the system's memory gives (the
|
||||
/// session's figure), never under the floor, and refused with a reason beyond it. The floor is
|
||||
/// small here: the real one is 8 GiB.
|
||||
#[test]
|
||||
fn a_read_over_the_floor_is_allowed_up_to_the_systems_memory() {
|
||||
let _only = LIMIT_TESTS.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let bytes = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8));
|
||||
let mut src = std::io::Cursor::new(bytes.as_slice());
|
||||
let index = crate::exr_index::build(&mut src, FileStamp::Virtual).unwrap();
|
||||
@@ -1601,6 +1606,31 @@ fn a_read_over_the_floor_is_allowed_up_to_the_systems_memory() {
|
||||
effectcraft_render::memory::set_decode_limit(None);
|
||||
}
|
||||
|
||||
/// #3749: a refusal for the request's size is remembered only while the decode limit stays as it
|
||||
/// was: when the system's memory figure arrives or changes, the request is decided anew.
|
||||
#[test]
|
||||
fn a_refusal_for_size_is_decided_anew_when_the_decode_limit_changes() {
|
||||
let _only = LIMIT_TESTS.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8)).into();
|
||||
let path = "/3749-refusal-generation.exr";
|
||||
// Room for one plane of 8×8 f32: four are refused.
|
||||
set_cap(path, 8 * 8 * 4);
|
||||
let store = PlaneStore::new(CACHE_BUDGET);
|
||||
let source = ExrSource::Bytes(bytes);
|
||||
let index = crate::exr_index::get(path, &source).unwrap();
|
||||
let want = Want { all: false, ids: (0..4).collect() };
|
||||
let refused = || matches!(store.get(path, &source, &index, &want, Use::Other), Err(Refused::TooLarge(_)));
|
||||
assert!(refused());
|
||||
assert!(refused());
|
||||
assert_eq!(decodes_of(path).len(), 1, "the second refusal came from memory");
|
||||
// (figures under the fixed floor: only the change counts, no other test's limit moves)
|
||||
effectcraft_render::memory::set_decode_limit(Some(1000));
|
||||
effectcraft_render::memory::set_decode_limit(Some(2000));
|
||||
assert!(refused(), "still over the caller's own limit");
|
||||
assert_eq!(decodes_of(path).len(), 2, "decided anew after the limit changed");
|
||||
effectcraft_render::memory::set_decode_limit(None);
|
||||
}
|
||||
|
||||
/// #3749: a plane the system cannot give is not an aborted process: `craft-exr` asks for the
|
||||
/// planes' memory first and reports what it can't get, a read that failed just now (asked again,
|
||||
/// not remembered); a plane over the read's limit (the decode limit handed to it) is a refusal
|
||||
|
||||
+58
-13
@@ -423,14 +423,40 @@ impl Accountant {
|
||||
/// channels) to take; 0: not known yet ([`decode_limit`]).
|
||||
static DECODE_LIMIT: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
/// Counts the changes of that limit: a refusal made under an older one is out of date
|
||||
/// ([`decode_limit_generation`]).
|
||||
static DECODE_GENERATION: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
|
||||
/// How often the system was asked for the limit without an answer ([`decode_limit`]).
|
||||
static DECODE_UNANSWERED: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
/// Reads that limit from the system when nothing set it ([`set_decode_limit_source`]).
|
||||
static DECODE_SOURCE: Mutex<Option<fn() -> Option<usize>>> = Mutex::new(None);
|
||||
|
||||
/// One reading of the system at a time: the second asker takes the first one's answer.
|
||||
static DECODE_ASKING: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// The system is asked this often at most for the limit when it does not answer.
|
||||
const DECODE_ASKS: usize = 2;
|
||||
|
||||
fn store_decode_limit(bytes: usize) {
|
||||
if DECODE_LIMIT.swap(bytes, Ordering::Relaxed) != bytes {
|
||||
DECODE_GENERATION.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Set what the system's memory allows one request for a file's decoded planes to take
|
||||
/// (`None`: unknown again). The session sets it with the memory budget once the installed RAM
|
||||
/// is read.
|
||||
pub fn set_decode_limit(bytes: Option<usize>) {
|
||||
DECODE_LIMIT.store(bytes.unwrap_or(0), Ordering::Relaxed);
|
||||
store_decode_limit(bytes.unwrap_or(0));
|
||||
DECODE_UNANSWERED.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Changes whenever [`decode_limit`] gives another answer: a request refused for its size is
|
||||
/// asked again after that.
|
||||
pub fn decode_limit_generation() -> u64 {
|
||||
DECODE_GENERATION.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Where [`decode_limit`] reads the limit from while nothing set it: the CLI and the MCP server
|
||||
@@ -438,23 +464,31 @@ pub fn set_decode_limit(bytes: Option<usize>) {
|
||||
/// request is over the fixed floor.
|
||||
pub fn set_decode_limit_source(source: Option<fn() -> Option<usize>>) {
|
||||
*DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner) = source;
|
||||
DECODE_UNANSWERED.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// The most bytes one request of `need` bytes for a file's decoded planes may take: `floor`
|
||||
/// (the fixed limit), or what the system's memory allows when that is more. The system is
|
||||
/// asked only when `need` is over `floor` (and once: the answer is kept, also "unknown", which
|
||||
/// leaves `floor`). The browser keeps `floor`.
|
||||
/// asked only when `need` is over `floor`, and the answer is kept; when it gives none (a reading
|
||||
/// that failed) it is asked once more later, then `floor` stays. The browser keeps `floor`.
|
||||
pub fn decode_limit(floor: usize, need: usize) -> usize {
|
||||
if need <= floor || cfg!(target_arch = "wasm32") {
|
||||
return floor;
|
||||
}
|
||||
let mut known = DECODE_LIMIT.load(Ordering::Relaxed);
|
||||
if known == 0 {
|
||||
let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
known = source.and_then(|read| read()).unwrap_or(floor);
|
||||
DECODE_LIMIT.store(known, Ordering::Relaxed);
|
||||
if DECODE_LIMIT.load(Ordering::Relaxed) == 0 && DECODE_UNANSWERED.load(Ordering::Relaxed) < DECODE_ASKS {
|
||||
let _asking = DECODE_ASKING.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
// (another thread may have just read it)
|
||||
if DECODE_LIMIT.load(Ordering::Relaxed) == 0 {
|
||||
let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
match source.and_then(|read| read()) {
|
||||
Some(bytes) => store_decode_limit(bytes),
|
||||
None => {
|
||||
DECODE_UNANSWERED.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
floor.max(known)
|
||||
floor.max(DECODE_LIMIT.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -903,13 +937,24 @@ mod tests {
|
||||
// The session's figure replaces the asked one.
|
||||
set_decode_limit(Some(900));
|
||||
assert_eq!(decode_limit(100, 200), 900);
|
||||
// Unknown memory leaves the floor, and is not asked again.
|
||||
// Unknown memory leaves the floor; a reading that failed once is tried once more, then
|
||||
// not asked again (and what is not known is never kept as the floor).
|
||||
set_decode_limit(None);
|
||||
set_decode_limit_source(Some(ram_unknown));
|
||||
ASKED.store(0, Ordering::SeqCst);
|
||||
assert_eq!(decode_limit(100, 200), 100);
|
||||
assert_eq!(decode_limit(100, 200), 100);
|
||||
assert_eq!(ASKED.load(Ordering::SeqCst), 1, "an unknown reading is kept too");
|
||||
for _ in 0..4 {
|
||||
assert_eq!(decode_limit(100, 200), 100);
|
||||
}
|
||||
assert_eq!(ASKED.load(Ordering::SeqCst), 2, "a failed reading is tried twice");
|
||||
// A source that answers later is not shut out by the floor kept earlier.
|
||||
set_decode_limit_source(Some(ram_says));
|
||||
assert_eq!(decode_limit(7, 200), 300, "a limit is not tied to the floor of an earlier request");
|
||||
// The generation tells that the limit changed.
|
||||
let before = decode_limit_generation();
|
||||
set_decode_limit(Some(300));
|
||||
assert_eq!(decode_limit_generation(), before, "the same figure is no change");
|
||||
set_decode_limit(Some(500));
|
||||
assert_ne!(decode_limit_generation(), before);
|
||||
set_decode_limit(None);
|
||||
set_decode_limit_source(None);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user