From ac34105a57d737088a4254aafb205b4e111acd4c Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 11:42:56 +0100 Subject: [PATCH] Disk cache: RAM frames keep the drop generation their render started at Reload Footage empties the RAM preview, but an idle pass of persist_missing may already hold a frame from before the reload. It took the cache's drop generation when it ran, after the drop, so the old pixels could be written back and, with the file's size and time unchanged, served again after a restart. Each RAM frame now keeps the disk cache its render started with and that cache's drop generation then. persist_missing writes a frame only to that cache, with that generation, and skips it once one of its files was dropped since; a frame rendered without that cache is not written from there. Tests use tempfile directories (the layer drop test, the frame_files folders under target/). Co-Authored-By: Claude Opus 5.5 --- crates/render/src/cache.rs | 6 +- crates/ui-egui/src/frames.rs | 57 ++++++++---- crates/ui-egui/src/frames/tests.rs | 135 +++++++++++++++++++++-------- 3 files changed, 143 insertions(+), 55 deletions(-) diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 7b0e7fc4e..692a80576 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -920,9 +920,8 @@ mod shared_insert_tests { fn a_buffer_rendered_across_a_drop_is_not_written() { use crate::disk_cache::{DiskCache, Kind, layer_disk_key}; use crate::reads::{FileStamp, ReadFile, ReadFiles}; - let dir = std::env::temp_dir().join(format!("effectcraft-layer-drop-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let dc = DiskCache::open(dir.path(), 1 << 30).unwrap(); dc.set_min_layer_ms(0); let cache = LayerCache::default(); cache.set_disk(Some(dc.clone())); @@ -939,7 +938,6 @@ mod shared_insert_tests { assert!(!dc.contains(Kind::Layer, layer_disk_key(1)), "made of the file as it was before the drop"); assert!(dc.contains(Kind::Layer, layer_disk_key(2))); drop((cache, dc)); - let _ = std::fs::remove_dir_all(&dir); } /// A render that gets another render's buffer gets that buffer's files too (#3184): when a diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 3595026ee..14c3f32ef 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -168,6 +168,28 @@ struct Cached { /// (the browser's frames are stored by its frame workers, not from here) #[cfg_attr(target_arch = "wasm32", allow(dead_code))] files: Option>, + /// The disk cache its render started with and that cache's drop generation then: the frame + /// is written to that cache only, and only while none of its files was dropped since + /// (`None`: rendered without a disk cache, or elsewhere; never written from here). + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + since: Option, +} + +/// A disk cache and its drop generation ([`DiskCache::drop_generation`]) when a render started. +#[derive(Clone)] +struct DiskSince(std::sync::Weak, u64); + +impl DiskSince { + fn of(dc: &Arc, generation: u64) -> DiskSince { + DiskSince(Arc::downgrade(dc), generation) + } + + /// The generation, when `dc` is the cache it was taken from (the weak reference keeps that + /// cache's address from being reused by another). + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + fn for_cache(&self, dc: &Arc) -> Option { + std::ptr::eq(self.0.as_ptr(), Arc::as_ptr(dc)).then_some(self.1) + } } struct Cache { @@ -191,12 +213,13 @@ impl Cache { /// A frame without its source files (tests). #[cfg(test)] fn insert(&mut self, k: FrameKey, img: FrameImage) { - self.insert_with(k, img, None); + self.insert_with(k, img, None, None); } - /// [`Cache::insert`] with the source files the frame was made of. - fn insert_with(&mut self, k: FrameKey, img: FrameImage, files: Option>) { + /// [`Cache::insert`] with the source files the frame was made of and the disk cache's drop + /// generation when its render started. + fn insert_with(&mut self, k: FrameKey, img: FrameImage, files: Option>, since: Option) { let (sz, gpu) = (img.bytes(), img.is_gpu()); - if let Some(old) = self.map.insert(k, Cached { img, files }) { + if let Some(old) = self.map.insert(k, Cached { img, files, since }) { self.forget(&old.img); } else { self.order.push_back(k); @@ -1036,7 +1059,7 @@ impl Frames { self.request_content_key(src.project.clone(), base.revision, base.comp); return true; }; - let (missing, converting): (Vec<(u128, FrameImage, Arc)>, bool) = { + let (missing, converting): (Vec<(u128, FrameImage, Arc, u64)>, bool) = { let (Ok(c), Ok(mut queued), Ok(mut written)) = (self.cache.lock(), self.persisted.lock(), self.written.lock()) else { return false }; if queued.len() > 100_000 { queued.clear(); @@ -1053,10 +1076,13 @@ impl Frames { // (GPU frames have no CPU pixels to write; a frame that can't name its files // can't be checked when read) let files = c.files.as_ref().filter(|f| f.is_complete() && !c.img.is_gpu())?; + // (only with the generation its render started at, for this cache: a frame + // made of a file Reload Footage dropped since holds the pixels from before) + let since = c.since.as_ref().and_then(|s| s.for_cache(&dc)).filter(|s| !dc.dropped_since(files, *s))?; let dk = disk_cache::frame_key(content, k.frame, k.scale, k.view ^ oh); let tries = written.1.get(&dk).copied().unwrap_or(0); (tries < 2 && !dc.contains(disk_cache::Kind::Frame, dk) && !dc.frame_pending(dk) && queued.insert(dk)) - .then(|| (dk, c.img.clone(), files.clone())) + .then(|| (dk, c.img.clone(), files.clone(), since)) }) .collect(); (missing, !queued.is_empty()) @@ -1066,9 +1092,7 @@ impl Frames { } let (persisted, written, virtual_id) = (self.persisted.clone(), self.written.clone(), self.virtual_id.clone()); let work = move || { - // (a frame made of a file dropped while it rendered kept an incomplete list: not here) - let since = dc.drop_generation(); - for (dk, img, files) in missing { + for (dk, img, files, since) in missing { // (accepted: the disk cache tracks it until written, and it counts as a try; // refused by a full queue: retried, without counting) let accepted = put_disk_frame(&dc, dk, &img, &files, virtual_id.as_deref(), since); @@ -1481,7 +1505,7 @@ impl Frames { // (A frame kept on the GPU is not written to the disk cache: its files are not // needed.) if let Some((img, _files)) = w.render_gpu(&job) { - w.finish(&job, img, None, t0); + w.finish(&job, img, None, None, t0); continue; } } @@ -1504,7 +1528,7 @@ impl Frames { match r { Ok(f) => { let img = egui::ColorImage::from_rgba_premultiplied([f.width as usize, f.height as usize], &f.rgba); - fw.finish_key(key, urgent, FrameImage::Cpu(Arc::new(img)), None, t0); + fw.finish_key(key, urgent, FrameImage::Cpu(Arc::new(img)), None, None, t0); } Err(e) => { // Not rendered (a stale revision, a lost worker): the viewer asks again. @@ -1642,7 +1666,8 @@ impl Worker { let files = Arc::new(files); // Publish the shared frame and wake the viewer before optional RGBA conversion // and compression. Persistence still runs on this worker, using the same snapshot. - self.finish(&job, ci.clone(), Some(files.clone()), t0); + let since = disk.as_ref().map(|(dc, _, since)| DiskSince::of(dc, *since)); + self.finish(&job, ci.clone(), Some(files.clone()), since, t0); if !disk_hit && let Some((dc, k, since)) = &disk && put_disk_frame(dc, *k, &ci, &files, self.virtual_id.as_deref(), *since) @@ -1672,18 +1697,18 @@ impl Worker { /// Store a finished frame (with the source files it was made of, when known) and release its /// job. - fn finish(&self, job: &Job, img: FrameImage, files: Option>, t0: web_time::Instant) { - self.finish_key(job.key, job.urgent, img, files, t0); + fn finish(&self, job: &Job, img: FrameImage, files: Option>, since: Option, t0: web_time::Instant) { + self.finish_key(job.key, job.urgent, img, files, since, t0); } - fn finish_key(&self, key: FrameKey, urgent: bool, img: FrameImage, files: Option>, t0: web_time::Instant) { + fn finish_key(&self, key: FrameKey, urgent: bool, img: FrameImage, files: Option>, since: Option, t0: web_time::Instant) { if urgent && let Ok(mut l) = self.last.lock() { *l = t0.elapsed().as_secs_f64() * 1000.0; } if let Ok(mut c) = self.cache.lock() && (!img.is_gpu() || !self.gpu_retired.load(Ordering::Acquire)) { - c.insert_with(key, img, files); + c.insert_with(key, img, files, since); } self.release(key, urgent); } diff --git a/crates/ui-egui/src/frames/tests.rs b/crates/ui-egui/src/frames/tests.rs index 959d14de2..13fda15e8 100644 --- a/crates/ui-egui/src/frames/tests.rs +++ b/crates/ui-egui/src/frames/tests.rs @@ -211,7 +211,7 @@ fn retirement_reaches_existing_workers_and_keeps_cpu_frames_and_completion_live( let next = key(1, 1, 0); frames.inflight.lock().unwrap().insert(next); frames.queue.lock().unwrap().running.insert(next); - worker.finish_key(next, false, img(), None, web_time::Instant::now()); + worker.finish_key(next, false, img(), None, None, web_time::Instant::now()); assert!(frames.is_cached(&next), "CPU retry can still publish"); assert_eq!(frames.inflight(), 0); assert!(frames.queue.lock().unwrap().running.is_empty()); @@ -1593,6 +1593,71 @@ fn half_float_ram_frames_missing_on_disk_are_written() { ram_frames_missing_on_disk_are_written_as(true); } +/// A RAM frame whose render started before Reload Footage dropped one of its files (#3184) +/// holds the pixels from before: writing it later to the disk cache (an idle pass after the +/// drop) would bring them back, also after a restart, since the file kept its size and time. It +/// is written only with the drop generation its render started at, for the cache it started +/// with: so it is not written, nor is a frame rendered without that cache. Frames of other files, +/// and frames rendered after the drop, are written. +#[test] +fn ram_frames_from_before_a_drop_are_not_written() { + use effectcraft_engine::Session; + use effectcraft_engine::render::reads::{FileStamp, ReadFile}; + use serde_json::json; + let dir = tempfile::tempdir().unwrap(); + let disk = DiskCache::open(dir.path().join("cache"), 1 << 30).unwrap(); + let other = DiskCache::open(dir.path().join("other"), 1 << 30).unwrap(); + let mut s = Session::default(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 8, "height": 8, "duration": 1})).unwrap()["comp"].as_u64().unwrap()); + s.execute("layer.newSolid", json!({})).unwrap(); + let src = RenderSource { + project: s.project.clone(), + footage: Arc::new(effectcraft_engine::render::NoFootage), + expr: None, + layer_cache: Arc::default(), + gpu: None, + gpu_display: false, + disk: Some(disk.clone()), + }; + let f = Frames::default(); + let series = FrameKey { revision: 1, content: f.content_of(&src.project, 1, cid), comp: cid.0, opts: opts_hash(&RenderOpts::default()), ..key(0, 0, 0) }; + let read = |p: &str| Some(Arc::new([ReadFile::new(p, FileStamp::Native { len: 3, modified: Some(1) })].into_iter().collect::())); + let before = DiskSince::of(&disk, disk.drop_generation()); + { + let mut c = f.cache.lock().unwrap(); + c.insert_with(FrameKey { frame: 0, ..series }, img(), read("/a.png"), Some(before.clone())); + c.insert_with(FrameKey { frame: 1, ..series }, img(), read("/b.png"), Some(before)); + c.insert_with(FrameKey { frame: 2, ..series }, img(), read("/a.png"), None); + c.insert_with(FrameKey { frame: 3, ..series }, img(), read("/a.png"), Some(DiskSince::of(&other, other.drop_generation()))); + } + // Reload Footage of a.png. + disk.drop_files(["/a.png"]); + disk.flush(); + f.cache.lock().unwrap().insert_with(FrameKey { frame: 4, ..series }, img(), read("/a.png"), Some(DiskSince::of(&disk, disk.drop_generation()))); + let on_disk = |fr: i64| { + disk.contains( + disk_cache::Kind::Frame, + frame_disk_key(&f.content_keys, &src.project, &FrameKey { frame: fr, ..series }, opts_hash(&RenderOpts::default())), + ) + }; + // Idle passes until there is nothing left to write. + let mut done = false; + for _ in 0..1000 { + let more = f.persist_missing(&src, &series, &RenderOpts::default()); + disk.flush(); + if !more && f.persisted.lock().unwrap().is_empty() { + done = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(done, "the idle passes end"); + assert!(on_disk(1) && on_disk(4), "another file's frame, and a frame rendered after the drop"); + assert!(!on_disk(0), "made of a.png as it was before the drop"); + assert!(!on_disk(2) && !on_disk(3), "rendered without this disk cache"); + assert_eq!(disk.stats().entries, 2); +} + fn ram_frames_missing_on_disk_are_written_as(half: bool) { use effectcraft_engine::Session; use serde_json::json; @@ -1616,7 +1681,7 @@ fn ram_frames_missing_on_disk_are_written_as(half: bool) { FrameKey { revision: 1, content: f.content_of(&src.project, 1, cid), comp: cid.0, opts: opts_hash(&RenderOpts::default()) ^ 1, ..key(0, 0, 0) }; for fr in 0..10 { let frame = if half { FrameImage::CpuF16(Arc::new(F16Frame::from_image(&effectcraft_engine::render::Image::new(8, 8), SRGB))) } else { img() }; - f.cache.lock().unwrap().insert_with(FrameKey { frame: fr, ..series }, frame, Some(Arc::default())); + f.cache.lock().unwrap().insert_with(FrameKey { frame: fr, ..series }, frame, Some(Arc::default()), Some(DiskSince::of(&disk, disk.drop_generation()))); } let on_disk = |fr: i64| { disk.contains( @@ -2529,12 +2594,12 @@ mod frame_files { const RATE: FrameRate = FrameRate::FPS_30; const FRAMES: i64 = 5; - /// A fresh folder for test `name` (on the build drive, not the system temp folder). - fn folder(name: &str) -> PathBuf { - let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!("../../target/test-frame-files-{name}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - dir + /// A fresh folder for test `name` (on the build drive, not the system temp folder), deleted + /// when the guard is dropped. + fn folder(name: &str) -> tempfile::TempDir { + let target = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target"); + std::fs::create_dir_all(&target).unwrap(); + tempfile::Builder::new().prefix(&format!("test-frame-files-{name}-")).tempdir_in(target).unwrap() } /// A flat grey PNG of `level` at `path`. @@ -2671,10 +2736,11 @@ mod frame_files { /// others come from disk, also after reopening. #[test] fn a_changed_source_file_re_renders_exactly_its_frames() { - let dir = folder("correct"); - let (p, cid, files) = project(&dir); + let tmp = folder("correct"); + let dir = tmp.path(); + let (p, cid, files) = project(dir); let all: Vec = (0..FRAMES).collect(); - let first = session(&p, &dir); + let first = session(&p, dir); let shown = first.show(cid, &all); assert!(all.iter().all(|f| close(shown[*f as usize], level(*f))), "{shown:?}"); assert_eq!(first.hits(), (0, FRAMES as u64), "rendered, nothing on disk yet"); @@ -2683,7 +2749,7 @@ mod frame_files { // Frame 2's file gets new pixels (and size), frame 3's only a new time. png(Path::new(&files[2]), 250); touch(&files[3], 30); - let second = session(&p, &dir); + let second = session(&p, dir); let shown = second.show(cid, &all); assert!(close(shown[2], 250), "frame 2 shows its new file: {shown:?}"); assert!([0, 1, 3, 4].iter().all(|f| close(shown[*f as usize], level(*f))), "{shown:?}"); @@ -2692,11 +2758,10 @@ mod frame_files { assert_eq!(second.disk.stats().entries, FRAMES as usize, "the entries of 2 and 3 were replaced"); drop(second); // Reopened: every frame from disk, the replaced ones with their new pixels. - let third = session(&p, &dir); + let third = session(&p, dir); let shown = third.show(cid, &all); assert_eq!(third.hits(), (FRAMES as u64, 0)); assert!(close(shown[2], 250) && close(shown[3], level(3)), "{shown:?}"); - let _ = std::fs::remove_dir_all(&dir); } /// Reopened (a new preview cache, media pool, layer cache and disk cache on the same folder): @@ -2704,21 +2769,21 @@ mod frame_files { /// source file is opened or decoded for it. #[test] fn a_reopened_project_serves_its_frame_from_disk_without_opening_a_source_file() { - let dir = folder("reopen"); - let (p, cid, _) = project(&dir); - let first = session(&p, &dir); + let tmp = folder("reopen"); + let dir = tmp.path(); + let (p, cid, _) = project(dir); + let first = session(&p, dir); let shown = first.show(cid, &[1]); assert!(close(shown[0], level(1)), "{shown:?}"); assert!(first.pool.stats().misses > 0, "the first session decodes the frame's file: {:?}", first.pool.stats()); assert!(first.disk.contains(disk_cache::Kind::Frame, first.disk_key(cid, 1))); drop(first); - let again = session(&p, &dir); + let again = session(&p, dir); let shown = again.show(cid, &[1]); assert!(close(shown[0], level(1)), "{shown:?}"); assert_eq!(again.hits(), (1, 0), "served from disk"); assert_eq!(again.disk.file_checks(), 1, "its one file checked"); assert_eq!(again.pool.stats(), effectcraft_media::PoolStats::default(), "no source file opened or decoded"); - let _ = std::fs::remove_dir_all(&dir); } /// No scan: opening a comp of a project that also holds a large sequence, and showing a @@ -2726,16 +2791,17 @@ mod frame_files { /// stays the same when its files change, and never asks for the identity of a path). #[test] fn showing_a_frame_checks_only_the_files_it_reads() { - let dir = folder("no-scan"); - let (p, cid, files) = project(&dir); + let tmp = folder("no-scan"); + let dir = tmp.path(); + let (p, cid, files) = project(dir); let key = disk_cache::comp_content_key(&p, cid); png(Path::new(&files[1]), 99); touch(&files[2], 5); assert_eq!(disk_cache::comp_content_key(&p, cid), key, "the key looks at no file"); - let first = session(&p, &dir); + let first = session(&p, dir); first.show(cid, &[2]); drop(first); - let again = session(&p, &dir); + let again = session(&p, dir); // (the indicator's key is computed off the UI thread first) let mut blue = vec![]; for _ in 0..400 { @@ -2752,7 +2818,6 @@ mod frame_files { assert_eq!(again.hits(), (1, 0)); assert_eq!(again.disk.file_checks(), 1, "only the file frame 2 read"); assert_eq!(again.hook_calls.load(Ordering::Relaxed), 0, "no path of the unused sequence was asked about"); - let _ = std::fs::remove_dir_all(&dir); } /// The frame entry files under the cache folder. @@ -2776,22 +2841,23 @@ mod frame_files { /// cache entry never serve the frame from before; each is rendered again and stored anew. #[test] fn missing_or_unreadable_files_and_damaged_entries_never_serve_a_stale_frame() { - let dir = folder("safe"); - let (p, cid, files) = project(&dir); - let first = session(&p, &dir); + let tmp = folder("safe"); + let dir = tmp.path(); + let (p, cid, files) = project(dir); + let first = session(&p, dir); first.show(cid, &[0]); assert_eq!(first.disk.stats().entries, 1); drop(first); // Missing: not served; the frame is rendered without it. std::fs::remove_file(&files[0]).unwrap(); - let missing = session(&p, &dir); + let missing = session(&p, dir); let shown = missing.show(cid, &[0]); assert!(!close(shown[0], level(0)), "the frame from before was served: {shown:?}"); assert_eq!(missing.hits(), (0, 1)); drop(missing); // Back, with other pixels: the entry made while it was missing is not served either. png(Path::new(&files[0]), 180); - let back = session(&p, &dir); + let back = session(&p, dir); let shown = back.show(cid, &[0]); assert!(close(shown[0], 180), "{shown:?}"); assert_eq!(back.hits(), (0, 1)); @@ -2799,30 +2865,29 @@ mod frame_files { // Unreadable (a folder in its place). std::fs::remove_file(&files[0]).unwrap(); std::fs::create_dir(&files[0]).unwrap(); - let unreadable = session(&p, &dir); + let unreadable = session(&p, dir); let shown = unreadable.show(cid, &[0]); assert!(!close(shown[0], 180), "the frame from before was served: {shown:?}"); assert_eq!(unreadable.hits(), (0, 1)); drop(unreadable); std::fs::remove_dir(&files[0]).unwrap(); png(Path::new(&files[0]), 120); - let fresh = session(&p, &dir); + let fresh = session(&p, dir); assert!(close(fresh.show(cid, &[0])[0], 120)); drop(fresh); // A damaged entry: a miss, rendered, stored again, then served. - let found = entries(&dir); + let found = entries(dir); assert_eq!(found.len(), 1, "{found:?}"); let mut bytes = std::fs::read(&found[0]).unwrap(); let n = bytes.len() / 2; bytes[n] ^= 0x5a; std::fs::write(&found[0], bytes).unwrap(); - let damaged = session(&p, &dir); + let damaged = session(&p, dir); assert!(close(damaged.show(cid, &[0])[0], 120)); assert_eq!(damaged.hits(), (0, 1)); drop(damaged); - let healed = session(&p, &dir); + let healed = session(&p, dir); assert!(close(healed.show(cid, &[0])[0], 120)); assert_eq!(healed.hits(), (1, 0), "the rewritten entry is used"); - let _ = std::fs::remove_dir_all(&dir); } }