mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
Caching: an oversized Cryptomatte plane stays beside held ones, disk copies are made a job at a time, and the review tests prove what they claim (#3750 review)
This commit is contained in:
1 parent
5d0bfe7869
commit
3494ef8552
6 files changed
+81
-33
No files matched your search
@@ -107,20 +107,23 @@ fn plane_within(a: &Arc<AuxChannels>, layer: &str, sel: &[u32], output: CryptoOu
|
||||
if bytes > max_bytes && (budget == usize::MAX || bytes > budget) {
|
||||
return made;
|
||||
}
|
||||
c.entries.push(Entry { aux: Arc::downgrade(a), key: key.clone(), plane: made.clone(), used: memory::tick(), cost_us });
|
||||
// Whether it can stay once every plane nobody holds is dropped: the planes renders hold
|
||||
// stay whatever the limits, so with them it must fit the entry limit, and the byte limit
|
||||
// unless it is over that on its own (kept for the common budget to trim, as above).
|
||||
// Else it is handed out but not kept (and nothing is dropped for it), so the cache never
|
||||
// stays over its limits once the renders let go.
|
||||
let held: Vec<usize> = c.entries.iter().filter(|e| Arc::strong_count(&e.plane) > 1).map(|e| e.plane.bytes()).collect();
|
||||
let held_bytes = held.iter().fold(0usize, |t, b| t.saturating_add(*b));
|
||||
if held.len() >= MAX_ENTRIES || (bytes <= max_bytes && held_bytes.saturating_add(bytes) > max_bytes) {
|
||||
return made;
|
||||
}
|
||||
c.entries.push(Entry { aux: Arc::downgrade(a), key, plane: made.clone(), used: memory::tick(), cost_us });
|
||||
loop {
|
||||
let total: usize = c.entries.iter().map(|e| e.plane.bytes()).fold(0, usize::saturating_add);
|
||||
if c.entries.len() <= 1 || (c.entries.len() <= MAX_ENTRIES && total <= max_bytes) {
|
||||
break;
|
||||
}
|
||||
let Some(i) = oldest(&c) else {
|
||||
// Renders hold every plane kept: the new one is handed out, not kept, so the
|
||||
// cache never stays over its limits once they let go.
|
||||
if let Some(i) = c.entries.iter().position(|e| e.of(a) && e.key == key) {
|
||||
c.entries.swap_remove(i);
|
||||
}
|
||||
return made;
|
||||
};
|
||||
let Some(i) = oldest(&c) else { break };
|
||||
c.entries.swap_remove(i);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -445,6 +445,23 @@ fn the_entry_limit_holds_while_renders_hold_every_plane() {
|
||||
drop(held);
|
||||
}
|
||||
|
||||
/// #3750 review: a plane over the cache's byte limit kept under a common budget is kept also
|
||||
/// while a render holds another plane (only the entry limit, or a plane within the byte limit
|
||||
/// that can't fit beside the held ones, hands a new plane out without keeping it).
|
||||
#[test]
|
||||
fn an_oversized_plane_is_kept_beside_a_held_one_under_a_budget() {
|
||||
use crate::fx_depth::matte_cache;
|
||||
use effectcraft_effects::CryptoOutput;
|
||||
crate::tests::hold_gpu_lock();
|
||||
let a = Arc::new(aux_scene(70, 44, 1.0));
|
||||
// (each Matte plane: 70 × 44 × 4 bytes, over a 100-byte limit; the budget holds them)
|
||||
let neighbour = matte_cache::plane_held_to(&a, "CryptoObject", &[crypto_hash("beta")], CryptoOutput::Matte, 100, 1 << 30);
|
||||
let first = matte_cache::plane_held_to(&a, "CryptoObject", &[crypto_hash("gamma")], CryptoOutput::Matte, 100, 1 << 30);
|
||||
let again = matte_cache::plane_held_to(&a, "CryptoObject", &[crypto_hash("gamma")], CryptoOutput::Matte, 100, 1 << 30);
|
||||
assert!(Arc::ptr_eq(&first, &again), "kept beside the held plane");
|
||||
drop(neighbour);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor() {
|
||||
case("ec.3d.extractor", &[("red", Value::Str("Z".into())), ("green", Value::Str("UV.U".into())), ("whitePoint", n(1000.0))]);
|
||||
|
||||
@@ -1091,20 +1091,39 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
// (only planes that cost more to decode again than to read back, #3301)
|
||||
let bytes = fresh.iter().map(|(_, _, p)| p.len().saturating_mul(std::mem::size_of::<f32>())).fold(0, usize::saturating_add);
|
||||
if !fresh.is_empty() && d.pays_off(bytes, read_took) {
|
||||
// (copies, each owned by its write: a queued write never keeps the store's set;
|
||||
// a plane the system can't give the memory for is left out)
|
||||
let planes: Vec<(u128, String, Vec<f32>)> = fresh
|
||||
.par_iter()
|
||||
.filter_map(|(k, name, plane)| {
|
||||
let mut copy = Vec::new();
|
||||
copy.try_reserve_exact(plane.len()).ok()?;
|
||||
copy.extend_from_slice(plane);
|
||||
#[cfg(test)]
|
||||
DISK_COPIES.lock().unwrap_or_else(PoisonError::into_inner).push(path.to_string());
|
||||
Some((*k, (*name).clone(), copy))
|
||||
})
|
||||
.collect();
|
||||
d.put_planes(w as u32, h as u32, planes, &entry, since);
|
||||
// Copied a job at a time, just before the job is queued (copies, each owned by its
|
||||
// write: a queued write never keeps the store's set); once the queue refuses a
|
||||
// job, the rest isn't copied. A plane the system can't give the memory for is
|
||||
// left out.
|
||||
let most = d.plane_job_bytes();
|
||||
let mut rest = fresh.as_slice();
|
||||
while !rest.is_empty() {
|
||||
let (mut n, mut job_bytes) = (0usize, 0usize);
|
||||
while let Some((_, _, p)) = rest.get(n) {
|
||||
let b = p.len().saturating_mul(std::mem::size_of::<f32>());
|
||||
if n > 0 && job_bytes.saturating_add(b) > most {
|
||||
break;
|
||||
}
|
||||
job_bytes = job_bytes.saturating_add(b);
|
||||
n += 1;
|
||||
}
|
||||
let (job, tail) = rest.split_at(n.min(rest.len()));
|
||||
rest = tail;
|
||||
let planes: Vec<(u128, String, Vec<f32>)> = job
|
||||
.par_iter()
|
||||
.filter_map(|(k, name, plane)| {
|
||||
let mut copy = Vec::new();
|
||||
copy.try_reserve_exact(plane.len()).ok()?;
|
||||
copy.extend_from_slice(plane);
|
||||
#[cfg(test)]
|
||||
DISK_COPIES.lock().unwrap_or_else(PoisonError::into_inner).push(path.to_string());
|
||||
Some((*k, (*name).clone(), copy))
|
||||
})
|
||||
.collect();
|
||||
if !d.put_planes(w as u32, h as u32, planes, &entry, since) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -887,8 +887,10 @@ fn a_merged_decode_too_large_leaves_each_request_that_fits_its_planes() {
|
||||
// Before the first request's decode reads: the second request joins it.
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let joined = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let merged_seen = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
{
|
||||
let (store, source, index, second, joined) = (store.clone(), source.clone(), index.clone(), second.clone(), joined.clone());
|
||||
let (store, source, index, second, joined, merged_seen) =
|
||||
(store.clone(), source.clone(), index.clone(), second.clone(), joined.clone(), merged_seen.clone());
|
||||
let hook: Hook = Arc::new(move |_: &str| {
|
||||
if joined.swap(true, Ordering::SeqCst) {
|
||||
return;
|
||||
@@ -907,6 +909,7 @@ fn a_merged_decode_too_large_leaves_each_request_that_fits_its_planes() {
|
||||
.and_then(|e| e.flight.clone())
|
||||
.is_some_and(|f| second.ids.iter().all(|id| f.state().want.ids.contains(id)));
|
||||
if merged {
|
||||
merged_seen.store(true, Ordering::SeqCst);
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(2));
|
||||
@@ -918,7 +921,7 @@ fn a_merged_decode_too_large_leaves_each_request_that_fits_its_planes() {
|
||||
let got_second = rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap();
|
||||
set_before_read(path, None);
|
||||
assert!(joined.load(Ordering::SeqCst));
|
||||
assert!(decodes_of(path).first().is_some_and(|d| d.channels.len() == 14), "merged first: {:?}", decodes_of(path));
|
||||
assert!(merged_seen.load(Ordering::SeqCst), "the second request joined the first one's decode");
|
||||
let got_first = got_first.expect("the picture alone fits");
|
||||
let got_second = got_second.expect("the passes alone fit");
|
||||
assert!(picture.iter().all(|id| got_first.contains(id)), "{got_first:?}");
|
||||
@@ -1172,7 +1175,7 @@ fn decoded_planes_are_kept_on_disk_and_read_back_exactly() {
|
||||
set_disk_cache(None);
|
||||
let copies = DISK_COPIES.lock().unwrap().iter().filter(|p| **p == g.path).count();
|
||||
assert_eq!(copies, 0, "no plane copied for a cache it can't go in");
|
||||
assert!(DISK_COPIES.lock().unwrap().iter().any(|p| *p == f.path), "copied while it could");
|
||||
assert!(DISK_COPIES.lock().unwrap().contains(&f.path), "copied while it could");
|
||||
let from_disk: Vec<bool> = decodes_of(&f.path).iter().map(|d| d.from_disk).collect();
|
||||
assert_eq!(from_disk, [false, false, true, false]);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
@@ -1589,7 +1592,8 @@ fn the_picture_planes_are_not_kept_beside_the_picture() {
|
||||
/// system can't give the memory for the copy, the set stays as it was (nothing aborts).
|
||||
#[test]
|
||||
fn a_shared_set_stays_whole_when_its_copy_finds_no_memory() {
|
||||
let bytes: Arc<[u8]> = write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", true), ("Z", false)], (16, 8), Compression::ZIP1)], (16, 8)).into();
|
||||
let bytes: Arc<[u8]> =
|
||||
write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", true), ("Z", false)], (16, 8), Compression::ZIP1)], (16, 8)).into();
|
||||
let path = "/3750-slim-no-memory.exr";
|
||||
let source = ExrSource::Bytes(bytes);
|
||||
let store = PlaneStore::new(CACHE_BUDGET);
|
||||
@@ -1647,7 +1651,8 @@ fn a_frame_of_another_size_is_read_whole_under_a_region() {
|
||||
let before = decodes_of(path).len();
|
||||
let rows = Renderer::new(&p, &fresh(), opts).comp_frame(cid, Tick::ZERO);
|
||||
assert_eq!(bits(&rows), bits(&whole));
|
||||
assert!(decodes_of(path).get(before..).unwrap_or_default().iter().all(|d| d.stats.lines == SIZE.1), "read whole");
|
||||
let after = decodes_of(path).get(before..).unwrap_or_default().to_vec();
|
||||
assert!(!after.is_empty() && after.iter().all(|d| d.stats.lines == SIZE.1), "read whole: {after:?}");
|
||||
}
|
||||
|
||||
/// #3271 review (#3301): a remembered failure belongs to the file as it was. A file that was
|
||||
|
||||
@@ -1667,9 +1667,7 @@ impl Inner {
|
||||
use crate::exr_planes::Picture;
|
||||
let extra = lock(&self.aux_footage).plan(&footage.path);
|
||||
let picture = match loc.key.rows {
|
||||
Some(rows) if extra.is_none() => {
|
||||
crate::exr_planes::picture_rows(path, &self.exr_source(path), op, rows, (footage.width, footage.height))
|
||||
}
|
||||
Some(rows) if extra.is_none() => crate::exr_planes::picture_rows(path, &self.exr_source(path), op, rows, (footage.width, footage.height)),
|
||||
_ => crate::exr_planes::picture(path, &self.exr_source(path), op, extra.as_ref()),
|
||||
};
|
||||
match picture {
|
||||
|
||||
@@ -1408,6 +1408,12 @@ impl DiskCache {
|
||||
self.get_checked(Kind::Plane, key, FORMAT_PLANES_FILES, None, |p| decode_plane(p, (width, height))).map(|(plane, _)| plane)
|
||||
}
|
||||
|
||||
/// The most plane bytes one queued write of decoded footage channels holds: a caller copying
|
||||
/// planes for [`DiskCache::put_planes`] copies this much at a time.
|
||||
pub fn plane_job_bytes(&self) -> usize {
|
||||
PLANE_JOB_BYTES
|
||||
}
|
||||
|
||||
/// Queue decoded footage channels of `width` x `height` (each [`plane_key`], name and plane)
|
||||
/// for writing, with the source file they were read from; encoded on the writer thread.
|
||||
/// Planes stored already, of another size or larger than the cache may hold are left out;
|
||||
@@ -1865,10 +1871,10 @@ fn encode_plane_after(mut out: Vec<u8>, w: u32, h: u32, name: &str, plane: &[f32
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Allocations [`zeroed`] refuses on this thread: those over this many bytes (tests: memory the
|
||||
/// system can't give).
|
||||
#[cfg(test)]
|
||||
thread_local! {
|
||||
/// Allocations [`zeroed`] refuses on this thread: those over this many bytes (tests: memory
|
||||
/// the system can't give).
|
||||
static NO_MEMORY_OVER: std::cell::Cell<usize> = const { std::cell::Cell::new(usize::MAX) };
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user