From 0bf8bf2b5e0ca0f5239298f71d00337a8930453c Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 01:11:55 +0100 Subject: [PATCH] Disk cache: Reload Footage drops the entries of its files, docs File > Reload Footage now deletes the frame and layer entries whose file lists name the reloaded item's files (DiskCache::drop_files), so a file replaced by one with the same size and modification time, which the check on hit can't tell apart, is read again. The index knows keys, not files: the writer thread reads the files section at the start of each frame and layer entry (not the pixels) after the writes queued before it, and deletes the entries that list one of the files. Until it is done, an entry made of one of them is a miss and frames aren't linked to new keys. The entries are gone for good, so they don't come back after a restart. Automatic reloads don't drop anything: they follow a changed stamp, which the check on hit catches. The browser needs nothing (its file identities are content hashes). Removed: disk_cache::seal, which steps 2 and 3 left without callers (the tests use seal_format). Updated the module, seal_format and forget_media docs, and docs/architecture.md / docs/web.md describe the final design: what is cached, keys from memory, file lists checked on hit, nothing scans the project, incomplete lists stay in memory, Reload Footage drops entries, edits keep the frames they can't change. Tests: render unit tests (same size and time passes the check but not the drop; other files' entries stay; an entry queued before the drop is dropped; a layer entry too; written again afterwards it is used; the drop survives a restart; a pending drop is a miss), and an engine test through file.reloadFootage. Closes #3184. Part of #1492. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/footage/footage_reload.rs | 14 +- crates/engine/src/tests/disk_cache.rs | 58 ++++++ crates/render/src/disk_cache.rs | 212 ++++++++++++++++++-- docs/architecture.md | 48 +++-- docs/web.md | 8 +- 5 files changed, 297 insertions(+), 43 deletions(-) diff --git a/crates/engine/src/footage/footage_reload.rs b/crates/engine/src/footage/footage_reload.rs index 4232e1d11..2744d0224 100644 --- a/crates/engine/src/footage/footage_reload.rs +++ b/crates/engine/src/footage/footage_reload.rs @@ -6,7 +6,8 @@ //! or PDF page an item shows and its interpretation (alpha, conformed frame rate, pixel aspect, //! fields, colour). A Photoshop layer is found again by name when layers were added or removed //! around it. The media layer then forgets the file and the layer cache is emptied, so the next -//! render reads the new pixels. +//! render reads the new pixels; File ▸ Reload Footage also deletes the disk cache entries made of +//! the file (#3184). //! //! The automatic part compares each footage file's size and modification time with what they //! were when the file was last seen. The desktop app keeps a [`Watcher`]: a worker thread that @@ -279,6 +280,12 @@ fn reload(s: &mut Session, ids: &[ItemId], undoable: bool) -> crate::Result<(usi let n = updates.len(); apply_reloads(s, updates, undoable)?; forget_media(s, &paths); + // An explicit reload also reads files again that kept their size and time (the check on hit + // can't tell those apart): their entries in the disk cache go (#3184). An automatic one + // only follows a changed stamp, which the check catches. + if undoable && let Some(dc) = &s.disk_cache { + dc.drop_files(paths.iter().map(String::as_str)); + } Ok((n, failed)) } @@ -317,8 +324,9 @@ fn apply_reloads(s: &mut Session, updates: Vec<(ItemId, Footage)>, undoable: boo Ok(()) } -/// Make the media layer and the layer cache let go of `paths`, so renders read them again. The -/// disk cache needs nothing: its entries check the files they were made of when read (#3184). +/// Make the media layer and the layer cache let go of `paths`, so renders read them again. Disk +/// cache entries check the files they were made of when read (#3184); File ▸ Reload Footage +/// also drops the entries of its files ([`reload`]). pub(crate) fn forget_media(s: &mut Session, paths: &BTreeSet) { for p in paths { s.footage.forget(p); diff --git a/crates/engine/src/tests/disk_cache.rs b/crates/engine/src/tests/disk_cache.rs index 8f4973ae3..fed00e852 100644 --- a/crates/engine/src/tests/disk_cache.rs +++ b/crates/engine/src/tests/disk_cache.rs @@ -339,3 +339,61 @@ fn markers_count_only_where_an_expression_can_read_them() { assert_eq!(key(&s), k0, "M itself still doesn't"); } +/// File ▸ Reload Footage drops the disk entries made of the item's files (#3184): a file +/// replaced by one with the same size and modification time is read again (the check on hit +/// can't tell), and entries of other files stay. +#[test] +fn reload_footage_drops_the_entries_of_its_files() { + use crate::render::reads::{FileStamp, ReadFile, ReadFiles}; + struct Probe; + impl crate::Importer for Probe { + fn probe(&self, path: &str) -> std::result::Result { + Ok(effectcraft_project::Footage { + path: path.into(), + kind: effectcraft_project::FootageKind::Still, + width: 1, + height: 1, + has_video: true, + ..Default::default() + }) + } + } + let dir = folder("reload"); + let mut s = Session { importer: Some(std::sync::Arc::new(Probe)), ..Default::default() }; + s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": &dir, "disk.diskCacheEnabled": true}})).unwrap(); + let src = std::path::Path::new(&dir).join("src"); + std::fs::create_dir_all(&src).unwrap(); + let mut add = |name: &str| { + let p = src.join(name); + std::fs::write(&p, b"aa").unwrap(); + let path = p.to_string_lossy().to_string(); + let f = effectcraft_project::Footage { + path: path.clone(), + kind: effectcraft_project::FootageKind::Still, + width: 1, + height: 1, + has_video: true, + ..Default::default() + }; + let id = s.edit("add", None, |p, _| Ok(p.add_item(name, effectcraft_color::Label::None, None, effectcraft_project::ItemKind::Footage(f)))).unwrap(); + (id, path) + }; + let (a, a_path) = add("a.png"); + let (_, b_path) = add("b.png"); + let dc = s.disk_cache.clone().unwrap(); + let read = |p: &str| -> ReadFiles { [ReadFile::new(p, FileStamp::native(p).unwrap())].into_iter().collect() }; + dc.put_frame_now(1, 1, 1, &[1; 4], &read(&a_path)); + dc.put_frame_now(2, 1, 1, &[2; 4], &read(&b_path)); + dc.put_layer_now(3, &effectcraft_effects::Buf { img: effectcraft_raster::Image::new(1, 1), offset: [0.0; 2], scale: 1.0 }, &read(&a_path)); + // Other bytes, same size, the old time back. + let t = std::fs::metadata(&a_path).unwrap().modified().unwrap(); + std::fs::write(&a_path, b"AA").unwrap(); + std::fs::File::options().append(true).open(&a_path).unwrap().set_modified(t).unwrap(); + assert!(dc.contains(crate::render::disk_cache::Kind::Frame, 1)); + s.execute_checked("file.reloadFootage", json!({"item": a.0})).unwrap(); + dc.flush(); + assert!(dc.get_frame(1, None).is_none(), "the reloaded file's frame renders again"); + assert!(dc.get_layer(3, None).is_none(), "and its layer buffer"); + assert!(dc.get_frame(2, None).is_some(), "another file's frame stays"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index 61f2d8240..d92505c98 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -2,16 +2,18 @@ //! and layer cache entries kept on disk between sessions, like After Effects' disk cache. //! //! * Entries are files keyed by a 128-bit content hash (`/v1//<2 hex>/<32 hex>.ecc`): -//! frames by the hash of the composition's content as it is in memory (the comp and everything -//! it uses, footage items with their settings and paths; no file is looked at) plus frame, -//! scale and view; layer buffers by the layer cache's content key (which names the footage -//! items a buffer uses with their settings and paths) and the renderer version -//! ([`layer_disk_key`]). A changed project therefore never reads stale pixels: its keys simply -//! change. +//! frames by the hash of the composition's content as it is in memory (the comp and what it can +//! draw, footage items with their settings and paths, and what its expressions can read; no +//! file is looked at, [`comp_content_key`]) plus frame, scale and view; layer buffers by the +//! layer cache's content key (which names the footage items a buffer uses with their settings +//! and paths) and the renderer version ([`layer_disk_key`]). A changed project therefore never +//! reads stale pixels: its keys simply change, and an edit elsewhere keeps them. //! * Frame and layer entries keep the source files they were made of, each with the size and //! modification time it was read with ([`EntryFiles`], #3184). An entry is used only while //! those files are unchanged: a handful of `metadata` calls when it is read, never a scan of -//! the project's files. A frame or buffer whose files can't all be named is not stored. +//! the project's files. A frame or buffer whose files can't all be named is not stored. File ▸ +//! Reload Footage deletes the entries made of the reloaded files ([`DiskCache::drop_files`]): +//! a file replaced by one of the same size and time is read again. //! * Payloads are LZ4-compressed (`lz4_flex`, MIT): frames as 8-bit premultiplied RGBA (or, for //! 32 bpc viewer frames, half-float RGBA with the bytes shuffled into planes; the entry //! header's format byte tells them apart: 2 and 3, both with their source files first), layer @@ -261,6 +263,8 @@ impl LayerPrefetch { enum Job { /// Kind, key, header format byte, payload. Write(Kind, u128, u8, Vec), + /// Delete the frame and layer entries made of these files ([`DiskCache::drop_files`]). + Drop(Vec>), } struct Queue { @@ -279,8 +283,9 @@ impl Queue { #[cfg(not(target_arch = "wasm32"))] fn pop(&mut self) -> Option { if let Some(j) = self.frames.pop_front() { - let Job::Write(_, _, _, p) = &j; - self.frame_bytes = self.frame_bytes.saturating_sub(p.len()); + if let Job::Write(_, _, _, p) = &j { + self.frame_bytes = self.frame_bytes.saturating_sub(p.len()); + } return Some(j); } self.jobs.pop_front() @@ -310,6 +315,9 @@ pub struct DiskCache { queue: Arc<(Mutex, Condvar)>, /// Layer buffers that took less than this to render are not written (ms). min_layer_ms: AtomicU64, + /// Files whose entries are being deleted ([`DiskCache::drop_files`]), each with the number + /// of drops of it pending: until then an entry made of one is a miss. + dropping: Mutex, u32>>, } /// FNV-1a 128-bit (content keys). @@ -395,6 +403,7 @@ impl DiskCache { Condvar::new(), )), min_layer_ms: AtomicU64::new(30), + dropping: Mutex::new(HashMap::new()), }); dc.rescan(); // Shrinking a large cache deletes many files: the writer thread does it. @@ -631,7 +640,7 @@ impl DiskCache { return miss(); }; self.file_checks.fetch_add(files.len() as u64, Ordering::Relaxed); - if !files.unchanged(virtual_id) { + if self.made_of_dropped(&files) || !files.unchanged(virtual_id) { self.discard(kind, key); return miss(); } @@ -752,6 +761,91 @@ impl DiskCache { } } + /// Delete the frame and layer entries made of any of `paths` (File ▸ Reload Footage: a file + /// replaced by one with the same size and modification time must be read again, which the + /// check on hit can't tell). The index knows keys, not files, so the writer thread reads the + /// files section at the start of each frame and layer entry (no pixels) and deletes the + /// entries that list one of `paths`, after the writes queued before (those are swept too); + /// until it is done, an entry made of one of them is a miss. Entries are deleted for good, + /// so they don't come back after a restart. Nothing else in the cache is touched. + pub fn drop_files(&self, paths: impl IntoIterator>>) { + let paths: Vec> = paths.into_iter().map(Into::into).collect(); + if paths.is_empty() { + return; + } + { + let mut d = self.dropping.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for p in &paths { + let n = d.entry(p.clone()).or_default(); + *n = n.saturating_add(1); + } + } + #[cfg(not(target_arch = "wasm32"))] + { + let (m, cv) = &*self.queue; + if let Ok(mut q) = m.lock() { + // (never refused for a full queue) + q.jobs.push_back(Job::Drop(paths)); + cv.notify_all(); + return; + } + } + self.sweep(&paths); + } + + /// Whether a [`DiskCache::drop_files`] is pending. + pub fn is_dropping(&self) -> bool { + !self.dropping.lock().unwrap_or_else(std::sync::PoisonError::into_inner).is_empty() + } + + /// An entry made of a file being dropped. + fn made_of_dropped(&self, files: &EntryFiles) -> bool { + let d = self.dropping.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + !d.is_empty() && files.files().iter().any(|f| d.contains_key(&f.path)) + } + + /// The writer's part of [`DiskCache::drop_files`]: delete the entries made of `paths`, then + /// end their drop. + fn sweep(&self, paths: &[Arc]) { + let set: std::collections::HashSet<&str> = paths.iter().map(|p| &**p).collect(); + for kind in [Kind::Frame, Kind::Layer] { + for key in self.keys(kind) { + if self.entry_files(kind, key).is_some_and(|f| f.files().iter().any(|f| set.contains(&*f.path))) { + self.discard(kind, key); + } + } + } + let mut d = self.dropping.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for p in paths { + if let Some(n) = d.get_mut(p) { + *n = n.saturating_sub(1); + if *n == 0 { + d.remove(p); + } + } + } + } + + /// The source files a frame or layer entry lists, read from the start of its file (the + /// whole file only when the list is longer than that). `None` when it has none to read (an + /// earlier format, gone, damaged): such an entry is never used anyway. + fn entry_files(&self, kind: Kind, key: u128) -> Option { + use std::io::Read; + const HEAD: u64 = 64 << 10; + let parse = |d: &[u8]| { + let format = *d.get(5)?; + let ours = d.get(..4)? == MAGIC && *d.get(4)? == kind.tag(); + if !ours || !matches!(format, FORMAT_RGBA8_FILES | FORMAT_F16_FILES | FORMAT_LAYER_FILES) { + return None; + } + EntryFiles::decode(d.get(8..)?).map(|(files, _)| files) + }; + let path = self.path(kind, key); + let mut head = Vec::new(); + std::fs::File::open(&path).ok()?.take(HEAD).read_to_end(&mut head).ok()?; + parse(&head).or_else(|| if head.len() as u64 == HEAD { parse(&std::fs::read(&path).ok()?) } else { None }) + } + // ------------------------------------------------------------ typed entries /// An 8-bit frame whose source files are unchanged (see [`DiskCache::get_checked`]), with @@ -840,7 +934,8 @@ impl DiskCache { /// has no links. `false` when `from` isn't cached, `to` already is, or the file system /// refused. (Entries don't contain their key, so the same file serves both.) pub fn link_frame(&self, from: u128, to: u128) -> bool { - if !self.contains(Kind::Frame, from) || self.contains(Kind::Frame, to) { + // (while files are being dropped, `from` may be made of one: it renders again instead) + if !self.contains(Kind::Frame, from) || self.contains(Kind::Frame, to) || self.is_dropping() { return false; } let (src, dst) = (self.path(Kind::Frame, from), self.path(Kind::Frame, to)); @@ -913,12 +1008,20 @@ fn writer(dc: std::sync::Weak, q: Arc<(Mutex, Condvar)>) { } }; let Some(cache) = dc.upgrade() else { return }; - let Job::Write(kind, key, format, payload) = job; - cache.write_format(kind, key, format, &payload); + let written = match job { + Job::Write(kind, key, format, payload) => { + cache.write_format(kind, key, format, &payload); + (kind == Kind::Frame).then_some(key) + } + Job::Drop(paths) => { + cache.sweep(&paths); + None + } + }; drop(cache); if let Ok(mut g) = m.lock() { g.busy = false; - if kind == Kind::Frame { + if let Some(key) = written { g.pending_frames.remove(&key); } } @@ -926,12 +1029,9 @@ fn writer(dc: std::sync::Weak, q: Arc<(Mutex, Condvar)>) { } } -/// An entry file's bytes: header (magic, kind, format 0), payload, checksum. -pub fn seal(kind: Kind, payload: &[u8]) -> Vec { - seal_format(kind, FORMAT_RGBA8, payload) -} - -/// [`seal`] with the header's format byte (frames: 0 = 8-bit, 1 = half float). +/// An entry file's bytes: header (magic, kind, the format byte: [`FORMAT_RGBA8`] for +/// segmentations, [`FORMAT_RGBA8_FILES`] / [`FORMAT_F16_FILES`] for frames, +/// [`FORMAT_LAYER_FILES`] for layer buffers), payload, checksum. fn seal_format(kind: Kind, format: u8, payload: &[u8]) -> Vec { let mut data = Vec::with_capacity(payload.len() + 16); data.extend_from_slice(MAGIC); @@ -1331,7 +1431,7 @@ mod tests { bad[20] ^= 1; assert!(read_frame_entry(&bad, Some(&one)).is_none()); // An entry of the format before #3184 (pixels without files) is never read. - assert!(read_frame_entry(&seal(Kind::Frame, &encode_frame(6, 3, &rgba)), Some(&one)).is_none()); + assert!(read_frame_entry(&seal_format(Kind::Frame, FORMAT_RGBA8, &encode_frame(6, 3, &rgba)), Some(&one)).is_none()); let key = 0x0123_4567_89ab_cdef_0011_2233_4455_6677u128; assert_eq!(parse_entry_name(&entry_name(key)), Some(key)); assert_eq!(parse_entry_name("abc.ecc"), None); @@ -1698,7 +1798,7 @@ mod tests { let e = layer_entry(&buf(2, 3, 2), &EntryFiles::of(&files, Some(&one)).unwrap()); let (b, list) = read_layer_entry(&e).unwrap(); assert!(b.img == buf(2, 3, 2).img && list.unchanged(Some(&one)) && !list.unchanged(Some(&two))); - assert!(read_layer_entry(&seal(Kind::Layer, &encode_layer(&buf(2, 3, 2)))).is_none(), "the format before #3184"); + assert!(read_layer_entry(&seal_format(Kind::Layer, FORMAT_RGBA8, &encode_layer(&buf(2, 3, 2)))).is_none(), "the format before #3184"); let mut bad = e.clone(); bad[12] ^= 1; assert!(read_layer_entry(&bad).is_none(), "damaged"); @@ -1860,4 +1960,72 @@ mod tests { assert!(!dc.contains(Kind::Frame, 8)); let _ = std::fs::remove_dir_all(&dir); } + + /// Reload Footage drops the entries made of the reloaded files (#3184): a file replaced by + /// one with the same size and modification time passes the check on hit, so only the drop + /// makes the next use a miss. Entries of other files stay; an entry written after the drop + /// is used; a layer entry and one written while the drop waits in the queue are dropped too; + /// the drop survives a restart (the files are gone). + #[test] + fn dropped_files_take_their_entries_with_them() { + use crate::reads::{FileStamp, ReadFile}; + let dir = tmpdir("drop"); + let src = dir.join("src"); + std::fs::create_dir_all(&src).unwrap(); + let file = |name: &str, bytes: &[u8]| { + let p = src.join(name); + std::fs::write(&p, bytes).unwrap(); + p.to_string_lossy().to_string() + }; + let (a, b) = (file("a.png", b"aa"), file("b.png", b"bb")); + let read = |paths: &[&str]| -> ReadFiles { paths.iter().map(|p| ReadFile::new(*p, FileStamp::native(p).unwrap())).collect() }; + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + dc.put_frame_now(1, 1, 1, &[1; 4], &read(&[&a])); + dc.put_frame_now(2, 1, 1, &[2; 4], &read(&[&b])); + dc.put_frame_now(3, 1, 1, &[3; 4], &read(&[&a, &b])); + dc.put_layer_now(4, &buf(4, 2, 2), &read(&[&a])); + // a.png gets other bytes of the same size and its old time back: the check can't tell. + let t = std::fs::metadata(&a).unwrap().modified().unwrap(); + std::fs::write(&a, b"AA").unwrap(); + std::fs::File::options().append(true).open(&a).unwrap().set_modified(t).unwrap(); + assert!(dc.get_frame(1, None).is_some(), "same size and time: still a hit"); + // A frame queued (not written yet) when the drop comes is dropped too. + assert!(dc.put_frame(5, 1, 1, &[5; 4], &read(&[&a]), None)); + dc.drop_files([a.clone()]); + dc.flush(); + assert!(!dc.is_dropping()); + for (kind, key) in [(Kind::Frame, 1), (Kind::Frame, 3), (Kind::Layer, 4), (Kind::Frame, 5)] { + assert!(!dc.contains(kind, key), "{kind:?} {key} made of a.png is gone"); + } + assert!(dc.get_frame(1, None).is_none() && dc.get_layer(4, None).is_none()); + assert_eq!(dc.get_frame(2, None).map(|f| f.0.rgba), Some(vec![2; 4]), "b.png's frame stays"); + // Written again from the reloaded file: used. + dc.put_frame_now(1, 1, 1, &[9; 4], &read(&[&a])); + assert_eq!(dc.get_frame(1, None).map(|f| f.0.rgba), Some(vec![9; 4])); + // The drop is for good: a new session on the folder finds what is left. + drop(dc); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(dc.keys(Kind::Frame).into_iter().collect::>(), [1, 2].into()); + assert!(dc.keys(Kind::Layer).is_empty()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// While a drop waits, an entry made of a dropped file is a miss (and deleted), and frames + /// aren't linked to new keys. + #[test] + fn entries_of_a_pending_drop_are_misses() { + use crate::reads::{FileStamp, ReadFile}; + let dir = tmpdir("drop-pending"); + std::fs::create_dir_all(&dir).unwrap(); + let a = dir.join("a.png"); + std::fs::write(&a, b"aa").unwrap(); + let a = a.to_string_lossy().to_string(); + let files: ReadFiles = [ReadFile::new(a.as_str(), FileStamp::native(&a).unwrap())].into_iter().collect(); + let dc = DiskCache::open(dir.join("cache"), 1 << 30).unwrap(); + dc.put_frame_now(1, 1, 1, &[1; 4], &files); + dc.dropping.lock().unwrap().insert(a.as_str().into(), 1); + assert!(!dc.link_frame(1, 2)); + assert!(dc.get_frame(1, None).is_none() && !dc.contains(Kind::Frame, 1)); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/docs/architecture.md b/docs/architecture.md index 213aaaf57..90573a1c9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -240,21 +240,39 @@ converted on demand with the HDR step and without the exposure; the Info panel s frame's float values. The browser keeps 8-bit frames for now (its frame workers send RGBA8). A persistent **disk cache** (`render::disk_cache`, Settings ▸ Media & Disk Cache) backs both -the layer cache and the viewer's RAM preview: layer buffers that were slow to render and every -preview frame are written (LZ4, atomic rename, checksummed) under 128-bit content keys — the -comp's content hash (the comp and what it uses, footage items with their settings and paths, -from memory alone) plus frame, scale and view for frames; the layer cache's content key (which -names the footage items a buffer uses with their paths) and the renderer version for layers. -Every frame and layer entry keeps the source files it was made of, with the size and -modification time they were read with, and is used only while they are unchanged: a handful of -`metadata` calls when an entry is read, never a scan of the project's files (#3184). Entries -survive restarts and never go stale; frames and buffers whose files can't all be named are not -written. Frame entries are 8-bit RGBA, or for 32 bpc viewer frames -half floats with their bytes shuffled into planes before LZ4 (the entry header's format byte -tells them apart; the render options in the key keep the two kinds apart). LRU eviction keeps it under the size limit; Edit ▸ Purge -empties it, `cache.diskStats` reports it, and the timeline draws disk-only frames in blue. In the -browser the frames live in the Origin Private File System: frame workers write them, the page -keeps the index (`DiskIndex`) and reads hits ([web.md](web.md#disk-cache)). +the layer cache and the viewer's RAM preview: layer buffers that were slow to render, every +preview frame and Roto Brush / Object Matte segmentations are written (LZ4, atomic rename, +checksummed) and survive restarts. The design (#3184): + +- **Keys from memory.** A frame's key is its comp's content key plus frame, scale, view and + render options; the content key hashes the project settings and the items the comp can draw + (its layers but those that can't change the picture, nested comps, footage with its settings + and paths, solids, Media Replacement footage, proxies: `Project::pixel_reach`). Expressions + among them add what they can read: every layer of their comps, the markers, the comps and + footage they name with literal names (`comp("Name")`, `footage("Name")`, and which items have + those names), or the whole project when they compute a name or reach items another way + (`expression_reach`). An edit to anything else keeps the key. A layer buffer's key is the + layer cache's content key (which names the footage it uses with its paths) and the renderer + version. Nothing here looks at a file, and nothing ever scans the project's files. +- **File lists checked on hit.** Every frame and layer entry keeps the source files it was + made of, with the size and modification time they were read with (the browser: size and + content hash). An entry is used only while they are unchanged: one `metadata` call per listed + file when it is read. A changed, missing or unreadable file, a damaged entry or an older + format is a miss; the entry is deleted and the frame renders again and replaces it. +- **Incomplete lists stay in memory.** A frame or buffer whose files can't all be named (a + source that doesn't say what it read, more than 4096 files) is shown but never written. +- **Reload Footage drops entries.** File ▸ Reload Footage deletes the frame and layer entries + whose lists name the item's files (the writer thread reads each entry's list, not its + pixels), so a file replaced by one with the same size and time is read again. +- **Edits keep what they can't change.** RAM frames outside an edit's dirty spans are kept + (`Project::dirty_spans`), and their disk entries are linked to the comp's new key. + +Frame entries are 8-bit RGBA, or for 32 bpc viewer frames half floats with their bytes shuffled +into planes before LZ4 (the entry header's format byte tells them apart; the render options in +the key keep the two kinds apart). LRU eviction keeps the cache under the size limit; Edit ▸ +Purge empties it, `cache.diskStats` reports it, and the timeline draws disk-only frames in blue. +In the browser the frames live in the Origin Private File System: frame workers write them, the +page keeps the index (`DiskIndex`) and reads hits ([web.md](web.md#disk-cache)). **Deferred readbacks** (`effectcraft-gpu` `deferred`): where a readback can't be waited for (WebGPU in a browser worker) a frame renders in passes. `Accelerator::frame_begin` / `pass_begin` / diff --git a/docs/web.md b/docs/web.md index 4f43b753e..666ffe37b 100644 --- a/docs/web.md +++ b/docs/web.md @@ -334,9 +334,11 @@ Settings ▸ Disk ▸ Disk Cache works in the browser, backed by the **Origin Pr (`src/diskcache.rs`): viewer frames are kept under `effectcraft-cache/v1/frames/<32 hex>.ecc` between visits, in the desktop cache's format (`disk_cache::frame_entry`: the source files the frame read with their size and content hash, then 8-bit premultiplied RGBA, LZ4, a checksum), -keyed by the same content hash (the comp and everything it uses as held in memory, frame, scale, -view, render options). The page reads an entry only while the files it lists are unchanged; a -frame whose files can't all be named is not stored. +keyed by the same content hash (from memory: the comp, everything it can draw and what its +expressions can read, frame, scale, view, render options; see +[architecture.md](architecture.md)). The page reads an entry only while the files it lists are +unchanged; a frame whose files can't all be named is not stored. Reload Footage has nothing to +drop here: the file table's identities are content hashes, so a replaced file never matches. - A render request to a frame worker carries the frame's disk key; the worker posts the frame, then writes the entry with a **synchronous access handle** into a temporary file moved into