Caching: an oversized Cryptomatte plane stays beside held ones, disk copies are made a job at a time, and the review tests prove what they claim (#3750 review)

This commit is contained in:
vangel76 committed 2026-11-17 17:40:16 +01:00
1 parent 5d0bfe7869
commit 3494ef8552
6 files changed
+81 -33

No files matched your search

+12 -9
View File
@@ -107,20 +107,23 @@ fn plane_within(a: &Arc<AuxChannels>, layer: &str, sel: &[u32], output: CryptoOu
if bytes > max_bytes && (budget == usize::MAX || bytes > budget) {
return made;
}
c.entries.push(Entry { aux: Arc::downgrade(a), key: key.clone(), plane: made.clone(), used: memory::tick(), cost_us });
// Whether it can stay once every plane nobody holds is dropped: the planes renders hold
// stay whatever the limits, so with them it must fit the entry limit, and the byte limit
// unless it is over that on its own (kept for the common budget to trim, as above).
// Else it is handed out but not kept (and nothing is dropped for it), so the cache never
// stays over its limits once the renders let go.
let held: Vec<usize> = c.entries.iter().filter(|e| Arc::strong_count(&e.plane) > 1).map(|e| e.plane.bytes()).collect();
let held_bytes = held.iter().fold(0usize, |t, b| t.saturating_add(*b));
if held.len() >= MAX_ENTRIES || (bytes <= max_bytes && held_bytes.saturating_add(bytes) > max_bytes) {
return made;
}
c.entries.push(Entry { aux: Arc::downgrade(a), key, plane: made.clone(), used: memory::tick(), cost_us });
loop {
let total: usize = c.entries.iter().map(|e| e.plane.bytes()).fold(0, usize::saturating_add);
if c.entries.len() <= 1 || (c.entries.len() <= MAX_ENTRIES && total <= max_bytes) {
break;
}
let Some(i) = oldest(&c) else {
// Renders hold every plane kept: the new one is handed out, not kept, so the
// cache never stays over its limits once they let go.
if let Some(i) = c.entries.iter().position(|e| e.of(a) && e.key == key) {
c.entries.swap_remove(i);
}
return made;
};
let Some(i) = oldest(&c) else { break };
c.entries.swap_remove(i);
}
}
+17
View File
@@ -445,6 +445,23 @@ fn the_entry_limit_holds_while_renders_hold_every_plane() {
drop(held);
}
/// #3750 review: a plane over the cache's byte limit kept under a common budget is kept also
/// while a render holds another plane (only the entry limit, or a plane within the byte limit
/// that can't fit beside the held ones, hands a new plane out without keeping it).
#[test]
fn an_oversized_plane_is_kept_beside_a_held_one_under_a_budget() {
use crate::fx_depth::matte_cache;
use effectcraft_effects::CryptoOutput;
crate::tests::hold_gpu_lock();
let a = Arc::new(aux_scene(70, 44, 1.0));
// (each Matte plane: 70 × 44 × 4 bytes, over a 100-byte limit; the budget holds them)
let neighbour = matte_cache::plane_held_to(&a, "CryptoObject", &[crypto_hash("beta")], CryptoOutput::Matte, 100, 1 << 30);
let first = matte_cache::plane_held_to(&a, "CryptoObject", &[crypto_hash("gamma")], CryptoOutput::Matte, 100, 1 << 30);
let again = matte_cache::plane_held_to(&a, "CryptoObject", &[crypto_hash("gamma")], CryptoOutput::Matte, 100, 1 << 30);
assert!(Arc::ptr_eq(&first, &again), "kept beside the held plane");
drop(neighbour);
}
#[test]
fn extractor() {
case("ec.3d.extractor", &[("red", Value::Str("Z".into())), ("green", Value::Str("UV.U".into())), ("whitePoint", n(1000.0))]);
+33 -14
View File
@@ -1091,20 +1091,39 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
// (only planes that cost more to decode again than to read back, #3301)
let bytes = fresh.iter().map(|(_, _, p)| p.len().saturating_mul(std::mem::size_of::<f32>())).fold(0, usize::saturating_add);
if !fresh.is_empty() && d.pays_off(bytes, read_took) {
// (copies, each owned by its write: a queued write never keeps the store's set;
// a plane the system can't give the memory for is left out)
let planes: Vec<(u128, String, Vec<f32>)> = fresh
.par_iter()
.filter_map(|(k, name, plane)| {
let mut copy = Vec::new();
copy.try_reserve_exact(plane.len()).ok()?;
copy.extend_from_slice(plane);
#[cfg(test)]
DISK_COPIES.lock().unwrap_or_else(PoisonError::into_inner).push(path.to_string());
Some((*k, (*name).clone(), copy))
})
.collect();
d.put_planes(w as u32, h as u32, planes, &entry, since);
// Copied a job at a time, just before the job is queued (copies, each owned by its
// write: a queued write never keeps the store's set); once the queue refuses a
// job, the rest isn't copied. A plane the system can't give the memory for is
// left out.
let most = d.plane_job_bytes();
let mut rest = fresh.as_slice();
while !rest.is_empty() {
let (mut n, mut job_bytes) = (0usize, 0usize);
while let Some((_, _, p)) = rest.get(n) {
let b = p.len().saturating_mul(std::mem::size_of::<f32>());
if n > 0 && job_bytes.saturating_add(b) > most {
break;
}
job_bytes = job_bytes.saturating_add(b);
n += 1;
}
let (job, tail) = rest.split_at(n.min(rest.len()));
rest = tail;
let planes: Vec<(u128, String, Vec<f32>)> = job
.par_iter()
.filter_map(|(k, name, plane)| {
let mut copy = Vec::new();
copy.try_reserve_exact(plane.len()).ok()?;
copy.extend_from_slice(plane);
#[cfg(test)]
DISK_COPIES.lock().unwrap_or_else(PoisonError::into_inner).push(path.to_string());
Some((*k, (*name).clone(), copy))
})
.collect();
if !d.put_planes(w as u32, h as u32, planes, &entry, since) {
break;
}
}
}
}
}
+10 -5
View File
@@ -887,8 +887,10 @@ fn a_merged_decode_too_large_leaves_each_request_that_fits_its_planes() {
// Before the first request's decode reads: the second request joins it.
let (tx, rx) = std::sync::mpsc::channel();
let joined = Arc::new(std::sync::atomic::AtomicBool::new(false));
let merged_seen = Arc::new(std::sync::atomic::AtomicBool::new(false));
{
let (store, source, index, second, joined) = (store.clone(), source.clone(), index.clone(), second.clone(), joined.clone());
let (store, source, index, second, joined, merged_seen) =
(store.clone(), source.clone(), index.clone(), second.clone(), joined.clone(), merged_seen.clone());
let hook: Hook = Arc::new(move |_: &str| {
if joined.swap(true, Ordering::SeqCst) {
return;
@@ -907,6 +909,7 @@ fn a_merged_decode_too_large_leaves_each_request_that_fits_its_planes() {
.and_then(|e| e.flight.clone())
.is_some_and(|f| second.ids.iter().all(|id| f.state().want.ids.contains(id)));
if merged {
merged_seen.store(true, Ordering::SeqCst);
break;
}
std::thread::sleep(std::time::Duration::from_millis(2));
@@ -918,7 +921,7 @@ fn a_merged_decode_too_large_leaves_each_request_that_fits_its_planes() {
let got_second = rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap();
set_before_read(path, None);
assert!(joined.load(Ordering::SeqCst));
assert!(decodes_of(path).first().is_some_and(|d| d.channels.len() == 14), "merged first: {:?}", decodes_of(path));
assert!(merged_seen.load(Ordering::SeqCst), "the second request joined the first one's decode");
let got_first = got_first.expect("the picture alone fits");
let got_second = got_second.expect("the passes alone fit");
assert!(picture.iter().all(|id| got_first.contains(id)), "{got_first:?}");
@@ -1172,7 +1175,7 @@ fn decoded_planes_are_kept_on_disk_and_read_back_exactly() {
set_disk_cache(None);
let copies = DISK_COPIES.lock().unwrap().iter().filter(|p| **p == g.path).count();
assert_eq!(copies, 0, "no plane copied for a cache it can't go in");
assert!(DISK_COPIES.lock().unwrap().iter().any(|p| *p == f.path), "copied while it could");
assert!(DISK_COPIES.lock().unwrap().contains(&f.path), "copied while it could");
let from_disk: Vec<bool> = decodes_of(&f.path).iter().map(|d| d.from_disk).collect();
assert_eq!(from_disk, [false, false, true, false]);
let _ = std::fs::remove_dir_all(&dir);
@@ -1589,7 +1592,8 @@ fn the_picture_planes_are_not_kept_beside_the_picture() {
/// system can't give the memory for the copy, the set stays as it was (nothing aborts).
#[test]
fn a_shared_set_stays_whole_when_its_copy_finds_no_memory() {
let bytes: Arc<[u8]> = write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", true), ("Z", false)], (16, 8), Compression::ZIP1)], (16, 8)).into();
let bytes: Arc<[u8]> =
write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", true), ("Z", false)], (16, 8), Compression::ZIP1)], (16, 8)).into();
let path = "/3750-slim-no-memory.exr";
let source = ExrSource::Bytes(bytes);
let store = PlaneStore::new(CACHE_BUDGET);
@@ -1647,7 +1651,8 @@ fn a_frame_of_another_size_is_read_whole_under_a_region() {
let before = decodes_of(path).len();
let rows = Renderer::new(&p, &fresh(), opts).comp_frame(cid, Tick::ZERO);
assert_eq!(bits(&rows), bits(&whole));
assert!(decodes_of(path).get(before..).unwrap_or_default().iter().all(|d| d.stats.lines == SIZE.1), "read whole");
let after = decodes_of(path).get(before..).unwrap_or_default().to_vec();
assert!(!after.is_empty() && after.iter().all(|d| d.stats.lines == SIZE.1), "read whole: {after:?}");
}
/// #3271 review (#3301): a remembered failure belongs to the file as it was. A file that was
+1 -3
View File
@@ -1667,9 +1667,7 @@ impl Inner {
use crate::exr_planes::Picture;
let extra = lock(&self.aux_footage).plan(&footage.path);
let picture = match loc.key.rows {
Some(rows) if extra.is_none() => {
crate::exr_planes::picture_rows(path, &self.exr_source(path), op, rows, (footage.width, footage.height))
}
Some(rows) if extra.is_none() => crate::exr_planes::picture_rows(path, &self.exr_source(path), op, rows, (footage.width, footage.height)),
_ => crate::exr_planes::picture(path, &self.exr_source(path), op, extra.as_ref()),
};
match picture {
+8 -2
View File
@@ -1408,6 +1408,12 @@ impl DiskCache {
self.get_checked(Kind::Plane, key, FORMAT_PLANES_FILES, None, |p| decode_plane(p, (width, height))).map(|(plane, _)| plane)
}
/// The most plane bytes one queued write of decoded footage channels holds: a caller copying
/// planes for [`DiskCache::put_planes`] copies this much at a time.
pub fn plane_job_bytes(&self) -> usize {
PLANE_JOB_BYTES
}
/// Queue decoded footage channels of `width` x `height` (each [`plane_key`], name and plane)
/// for writing, with the source file they were read from; encoded on the writer thread.
/// Planes stored already, of another size or larger than the cache may hold are left out;
@@ -1865,10 +1871,10 @@ fn encode_plane_after(mut out: Vec<u8>, w: u32, h: u32, name: &str, plane: &[f32
Some(out)
}
/// Allocations [`zeroed`] refuses on this thread: those over this many bytes (tests: memory the
/// system can't give).
#[cfg(test)]
thread_local! {
/// Allocations [`zeroed`] refuses on this thread: those over this many bytes (tests: memory
/// the system can't give).
static NO_MEMORY_OVER: std::cell::Cell<usize> = const { std::cell::Cell::new(usize::MAX) };
}