diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index a1bd85290..b3f38baab 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -63,9 +63,10 @@ struct Inner { clock: u64, hits: u64, misses: u64, - /// When each recent miss happened (to time the render that follows it), with the store's - /// drop generation then ([`LayerStore::drop_generation`]). - missed_at: HashMap, + /// When each recent miss happened, to time the render that follows it (the drop generation + /// a render's reads started at travels with its files instead, [`ReadFiles::since`]: two + /// renders of a key may overlap). + missed_at: HashMap, } /// Hit/miss counters and memory use (for perf readouts and tests). @@ -169,18 +170,19 @@ impl LayerCache { g.misses += 1; drop(g); let store = self.persistent(); - // (before the buffer's reads: what it is made of may be dropped meanwhile) + // (before the entry is read: what it is made of may be dropped meanwhile) let since = store.as_ref().map_or(0, |s| s.drop_generation()); if let Ok(mut g) = self.inner.lock() { if g.missed_at.len() > 4096 { g.missed_at.clear(); } - g.missed_at.insert(key, (web_time::Instant::now(), since)); + g.missed_at.insert(key, web_time::Instant::now()); } - // Disk cache: a hit comes back into memory. + // Disk cache: a hit comes back into memory, read at generation `since`. let store = store?; let id = self.virtual_id(); - let (buf, files) = store.get_layer(layer_disk_key(key), id.as_deref())?; + let (buf, mut files) = store.get_layer(layer_disk_key(key), id.as_deref())?; + files.note_since(since); let files = Arc::new(files); Some(self.insert_mem(key, buf, files)) } @@ -195,6 +197,12 @@ impl LayerCache { /// the very work it waits for. `files`: the source files `buf` was made of (see /// [`crate::reads`]); a buffer whose files can't all be named stays in memory. The files /// returned are the ones the returned buffer was made of. + /// + /// The drop generation `buf`'s reads started at comes with `files` ([`ReadFiles::since`], + /// from [`LayerCache::drop_generation`] when the render began): a buffer made of a file + /// dropped since (File ▸ Reload Footage while it rendered) is kept neither on disk nor in + /// memory. A list that doesn't say counts as made before every drop. A buffer made of values + /// its key doesn't capture ([`ReadFiles::is_unkeyed`]) stays in memory. pub fn insert(&self, key: u64, buf: Arc, files: Arc) -> (Arc, Arc) { if self.gated() { return (buf, files); @@ -202,11 +210,18 @@ impl LayerCache { if let Some(kept) = self.inner.lock().ok().and_then(|g| g.map.get(&key).map(|e| (e.buf.clone(), e.files.clone()))) { return kept; } + let store = self.persistent(); + let since = files.since().unwrap_or(0); + // (before the miss's time is taken: it may time another render of the key) + if store.as_ref().is_some_and(|s| s.dropped_since(&files, since)) { + return (buf, files); + } let started = self.inner.lock().ok().and_then(|mut g| g.missed_at.remove(&key)); - if let Some(store) = self.persistent() { + if let Some(store) = store { let id = self.virtual_id(); // (an incomplete list, or a file from memory without its identity: not stored) - if let Some((t, since)) = started + if let Some(t) = started + && !files.is_unkeyed() && t.elapsed().as_secs_f64() * 1e3 >= store.min_layer_ms() as f64 && let Some(entry) = EntryFiles::of(&files, id.as_deref()) { @@ -216,6 +231,12 @@ impl LayerCache { self.insert_mem(key, buf, files) } + /// The persistent store's drop generation now ([`LayerStore::drop_generation`]), taken by a + /// render before it starts reading (`None` without a store). + pub fn drop_generation(&self) -> Option { + self.persistent().map(|s| s.drop_generation()) + } + /// See [`LayerCache::insert`]: the buffer kept under `key` and its files (`buf` and `files`, /// unless one was kept first or `buf` is too large to keep). fn insert_mem(&self, key: u64, buf: Arc, files: Arc) -> (Arc, Arc) { @@ -301,6 +322,11 @@ pub trait LayerStore: Send + Sync { fn drop_generation(&self) -> u64 { 0 } + /// Whether one of `files` was dropped after generation `since`: what was made of them is + /// from before the drop. `false` for a store that never drops. + fn dropped_since(&self, _files: &ReadFiles, _since: u64) -> bool { + false + } /// Buffers faster to render than this (ms) are not stored. fn min_layer_ms(&self) -> u64 { 0 @@ -542,6 +568,12 @@ fn time_dependent(layer: &Layer, effects: bool) -> bool { false } +/// A text layer with an Expression Selector: its amount is evaluated per character, and the +/// layer's key holds only the first character's (see [`crate::reads::ReadFiles::is_unkeyed`]). +pub(crate) fn per_character_expressions(layer: &Layer) -> bool { + matches!(layer.source, LayerSource::Text) && layer.props.sub("text").is_some_and(|t| group_has(t, "expressionSelector")) +} + fn group_has(g: &PropGroup, match_id: &str) -> bool { g.children.iter().any(|c| match c { Node::Group(sub) => sub.match_id == match_id || group_has(sub, match_id), @@ -927,24 +959,48 @@ mod shared_insert_tests { #[test] fn a_buffer_rendered_across_a_drop_is_not_written() { use crate::disk_cache::{DiskCache, Kind, layer_disk_key}; - use crate::reads::{FileStamp, ReadFile, ReadFiles}; + use crate::reads::{FileStamp, ReadFile, ReadLog}; let dir = tempfile::tempdir().unwrap(); let dc = DiskCache::open(dir.path(), 1 << 30).unwrap(); dc.set_min_layer_ms(0); let cache = LayerCache::default(); cache.set_disk(Some(dc.clone())); let buf = || Arc::new(Buf { img: Image::new(2, 2), offset: [0.0; 2], scale: 1.0 }); - let files = Arc::new([ReadFile::new("/a.png", FileStamp::Native { len: 3, modified: Some(1) })].into_iter().collect::()); + // A render's files, read after it took the drop generation (`Renderer::recorded`). + let render = || { + let log = ReadLog::starting_at(cache.drop_generation().unwrap()); + move || { + log.note(ReadFile::new("/a.png", FileStamp::Native { len: 3, modified: Some(1) })); + Arc::new(log.into_files()) + } + }; // The miss starts the render; the file is reloaded before the buffer is kept. assert!(cache.get(1).is_none()); + let a = render(); dc.drop_files(["/a.png"]); - cache.insert(1, buf(), files.clone()); - // A render after the drop. - assert!(cache.get(2).is_none()); - cache.insert(2, buf(), files); + dc.flush(); + // Another render of the same key starts after the drop, while the first one still runs + // (its miss must not lend the first render its generation). + assert!(cache.get(1).is_none()); + let b = render(); + let (stale, _) = cache.insert(1, buf(), a()); dc.flush(); assert!(!dc.contains(Kind::Layer, layer_disk_key(1)), "made of the file as it was before the drop"); + // Not kept in memory either: the second render keeps its own buffer, and writes it. + let fresh = buf(); + let (kept, _) = cache.insert(1, fresh.clone(), b()); + assert!(Arc::ptr_eq(&kept, &fresh) && !Arc::ptr_eq(&kept, &stale)); + // A render of another key after the drop. + assert!(cache.get(2).is_none()); + cache.insert(2, buf(), render()()); + dc.flush(); + assert!(dc.contains(Kind::Layer, layer_disk_key(1)), "the render after the drop"); assert!(dc.contains(Kind::Layer, layer_disk_key(2))); + // A list that doesn't say when its reads began counts as made before the drop. + assert!(cache.get(3).is_none()); + cache.insert(3, buf(), Arc::new([ReadFile::new("/a.png", FileStamp::Native { len: 3, modified: Some(1) })].into_iter().collect())); + dc.flush(); + assert!(!dc.contains(Kind::Layer, layer_disk_key(3))); drop((cache, dc)); } diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index bb200bd29..f509a5896 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -917,9 +917,12 @@ impl DiskCache { self.queue.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).drops.generation } - /// Whether one of `files` was dropped after generation `since` ([`DiskCache::drop_generation`]): - /// what a render that started then made of them is from before the drop (shown, never stored). + /// Whether one of `files` was dropped after generation `since` ([`DiskCache::drop_generation`]), + /// or after the earlier generation the list's own reads started at ([`ReadFiles::since`]: a + /// buffer made by an earlier render, from memory): what was made of them then is from before + /// the drop (shown, never stored). pub fn dropped_since(&self, files: &ReadFiles, since: u64) -> bool { + let since = files.since().map_or(since, |s| s.min(since)); self.queue.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).drops.stale(files.files().iter().map(|f| &f.path), since) } @@ -1296,6 +1299,9 @@ impl crate::cache::LayerStore for DiskCache { fn drop_generation(&self) -> u64 { DiskCache::drop_generation(self) } + fn dropped_since(&self, files: &ReadFiles, since: u64) -> bool { + DiskCache::dropped_since(self, files, since) + } fn min_layer_ms(&self) -> u64 { DiskCache::min_layer_ms(self) } diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index be3c3da8f..6c79ba7bb 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -183,7 +183,8 @@ impl<'a> Renderer<'a> { if key.is_none() || self.cache.is_none() { return (make(self), Arc::default()); } - let log = ReadLog::new(); + // (the generation before any read: a file dropped while `make` runs makes it stale) + let log = self.cache.and_then(LayerCache::drop_generation).map_or_else(ReadLog::new, ReadLog::starting_at); let out = make(&Renderer { reads: Some(&log), ..*self }); let files = log.into_files(); if let Some(own) = self.reads { @@ -870,7 +871,18 @@ impl<'a> Renderer<'a> { } // (the font files go to this render's log, which `ctx` may not carry: see // [`Renderer::with_reads`]) - LayerSource::Text => Some(self.authored(text::render(&self.with_reads(ctx), layer, self.raster_scale(ctx, layer)))), + LayerSource::Text => { + // An Expression Selector evaluates per character: its results at other characters + // than the first are not in the layer's cache key (an amount reading a footage + // item's size by name, `textIndex == 1 ? 0 : footage("ref.png").width`), so the + // buffer is kept in memory only, never on disk (the frame's key covers them). + if cache::per_character_expressions(layer) + && let Some(log) = self.reads + { + log.mark_unkeyed(); + } + Some(self.authored(text::render(&self.with_reads(ctx), layer, self.raster_scale(ctx, layer)))) + } LayerSource::Shape => layer.props.sub("contents").map(|c| self.authored(shapes::render(ctx, layer, c, self.raster_scale(ctx, layer)))), _ => None, } diff --git a/crates/render/src/reads.rs b/crates/render/src/reads.rs index 326554c5b..ed97e50fd 100644 --- a/crates/render/src/reads.rs +++ b/crates/render/src/reads.rs @@ -75,10 +75,39 @@ pub const MAX_READ_FILES: usize = 4096; /// buffer from a store that doesn't keep its files, more than [`MAX_READ_FILES`] files, or one /// file read with two identities (it changed during the render). A cache entry whose list is /// incomplete can't be checked against the files, so it must not outlive the session. -#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +/// +/// It also carries where its reads come from, which equality and hashing ignore: the drop +/// generation the earliest of them started at ([`ReadFiles::since`]), and whether a layer cache +/// key can't capture what it was made of ([`ReadFiles::is_unkeyed`]). +#[derive(Clone, Debug, Default)] pub struct ReadFiles { files: Vec, incomplete: bool, + /// The disk cache's drop generation ([`DiskCache::drop_generation`]) when the earliest read + /// behind the list started; `None` when nothing says (a list built by hand). + /// + /// [`DiskCache::drop_generation`]: crate::disk_cache::DiskCache::drop_generation + since: Option, + /// Made of values a layer's cache key doesn't capture (a text Expression Selector evaluates + /// per character): such a buffer is not stored on disk. + unkeyed: bool, +} + +impl PartialEq for ReadFiles { + /// The same files with the same identities (where the reads come from doesn't count). + fn eq(&self, other: &Self) -> bool { + self.files == other.files && self.incomplete == other.incomplete + } +} + +impl Eq for ReadFiles {} + +impl std::hash::Hash for ReadFiles { + /// Hashes what [`PartialEq`] compares. + fn hash(&self, state: &mut H) { + self.files.hash(state); + self.incomplete.hash(state); + } } impl ReadFiles { @@ -100,9 +129,13 @@ impl ReadFiles { } } - /// Note every file of `other` (and whether it is incomplete). + /// Note every file of `other` (whether it is incomplete or unkeyed, and its generation). pub fn extend(&mut self, other: &ReadFiles) { self.incomplete |= other.incomplete; + self.unkeyed |= other.unkeyed; + if let Some(g) = other.since { + self.note_since(g); + } for f in &other.files { self.add(f.clone()); } @@ -118,6 +151,28 @@ impl ReadFiles { !self.incomplete } + /// The drop generation the earliest read behind the list started at (`None`: unknown). A + /// buffer made of a file dropped after it holds that file as it was before the drop. + pub fn since(&self) -> Option { + self.since + } + + /// A read behind the list started at drop generation `generation` (the earliest one counts). + pub fn note_since(&mut self, generation: u64) { + self.since = Some(self.since.map_or(generation, |s| s.min(generation))); + } + + /// Made of values a layer cache key doesn't capture (see [`ReadFiles::is_unkeyed`]). + pub fn mark_unkeyed(&mut self) { + self.unkeyed = true; + } + + /// Made of values a layer cache key doesn't capture: kept in memory, never stored on disk as + /// a layer buffer. (A frame's key captures them: frame entries are not affected.) + pub fn is_unkeyed(&self) -> bool { + self.unkeyed + } + /// The files, sorted by path. pub fn files(&self) -> &[ReadFile] { &self.files @@ -164,6 +219,14 @@ impl ReadLog { ReadLog::default() } + /// An empty read log for reads that start at drop generation `generation` (see + /// [`ReadFiles::since`]). + pub fn starting_at(generation: u64) -> ReadLog { + let mut files = ReadFiles::default(); + files.note_since(generation); + ReadLog { files: Mutex::new(files) } + } + /// Locks the file list, recovering from a poisoned lock. fn lock(&self) -> std::sync::MutexGuard<'_, ReadFiles> { self.files.lock().unwrap_or_else(PoisonError::into_inner) @@ -176,7 +239,7 @@ impl ReadLog { /// Note every file of `files` (and whether it is incomplete). pub fn note_all(&self, files: &ReadFiles) { - if files.is_empty() && files.is_complete() { + if files.is_empty() && files.is_complete() && files.since.is_none() && !files.unkeyed { return; } self.lock().extend(files); @@ -187,6 +250,12 @@ impl ReadLog { self.lock().mark_incomplete(); } + /// What is read is made of values a layer cache key doesn't capture + /// ([`ReadFiles::is_unkeyed`]). + pub fn mark_unkeyed(&self) { + self.lock().mark_unkeyed(); + } + /// The files noted so far. pub fn files(&self) -> ReadFiles { self.lock().clone() diff --git a/crates/render/src/tests.rs b/crates/render/src/tests.rs index 325eb5d2e..80b1aa12d 100644 --- a/crates/render/src/tests.rs +++ b/crates/render/src/tests.rs @@ -545,6 +545,82 @@ fn what_expressions_read_goes_to_the_files_of_the_buffer_they_make() { assert!(files.get("/sampled.png").is_some(), "footprint: {files:?}"); } +/// A text Expression Selector whose Amount is 0 for the first character and the width of the +/// footage item named `ref.png` for the others (`textIndex == 1 ? 0 : footage("ref.png").width`). +struct WidthOfRef; + +impl crate::eval::ExprHost for WidthOfRef { + fn eval(&self, _: &crate::eval::EvalCtx, _: &effectcraft_project::Layer, _: &effectcraft_project::Property, value: &Value) -> Result { + Ok(value.clone()) + } + fn eval_text_selector( + &self, + ctx: &crate::eval::EvalCtx, + _: &effectcraft_project::Layer, + _: &effectcraft_project::Property, + index: usize, + _: usize, + _: [f64; 3], + ) -> Result<[f64; 3], String> { + let width = ctx.project.items.values().find(|i| i.name == "ref.png").and_then(|i| i.dimensions()).map_or(0.0, |d| d.0 as f64); + Ok([if index == 1 { 0.0 } else { width }; 3]) + } +} + +/// A text layer whose Expression Selector reads a footage item's size for every character but +/// the first: the layer's key holds the first character's amount only, so its buffer is not +/// stored on disk. Reopened over the same disk cache after the footage's width changed (16 to +/// 64), the frame matches one rendered without a cache. +#[test] +fn a_text_expression_selector_is_not_served_from_the_disk_cache() { + use crate::disk_cache::{DiskCache, Kind}; + use crate::{LayerCache, NoFootage, RenderOpts, Renderer}; + let project = |width: u32| { + let (mut p, cid, comp) = setup(); + let f = effectcraft_project::Footage { has_video: true, width, height: 16, ..Default::default() }; + p.add_item("ref.png", Label::None, None, ItemKind::Footage(f)); + let mut l = build::layer(&mut p, &comp, "Text", LayerSource::Text, (200, 100), None); + l.props.prop_mut("text/sourceText").unwrap().value = Value::Text(Box::new(TextDoc { text: "HH".into(), size: 60.0, ..Default::default() })); + let mut next = p.next_id; + let mut ids = Ids(&mut next); + let mut props = build::text_anim_props(&mut ids, "opacity", false); + for pr in &mut props { + pr.value = Value::Scalar(0.0); + } + let mut animator = build::text_animator(&mut ids, "Animator 1", props); + let sel = build::expression_selector(&mut ids, "Expression Selector 1"); + p.next_id = next; + animator.sub_mut("selectors").unwrap().children.push(sel.into()); + l.props.group_mut("text/animators").unwrap().children.push(animator.into()); + p.comp_mut(cid).unwrap().layers.push(l); + (p, cid) + }; + let render = |p: &Project, cid: ItemId, cache: Option<&LayerCache>| { + let mut r = Renderer::new(p, &NoFootage, RenderOpts::default()); + r.expr = Some(&WidthOfRef); + r.cache = cache; + r.comp_frame(cid, Tick::ZERO) + }; + let dir = tempfile::tempdir().unwrap(); + let dc = DiskCache::open(dir.path(), 1 << 30).unwrap(); + dc.set_min_layer_ms(0); + // First session: the 16 px wide footage. + let (p16, cid) = project(16); + let cache = LayerCache::default(); + cache.set_disk(Some(dc.clone())); + let first = render(&p16, cid, Some(&cache)); + dc.flush(); + assert!(first.data.iter().any(|px| px[3] > 0.5), "the text renders"); + // Reopened with a fresh memory cache, the footage now 64 px wide. + let (p64, cid) = project(64); + let uncached = render(&p64, cid, None); + assert_ne!(uncached.data, first.data, "the second character's opacity follows the width"); + let reopened = LayerCache::default(); + reopened.set_disk(Some(dc.clone())); + assert_eq!(render(&p64, cid, Some(&reopened)).data, uncached.data, "not the buffer from before the change"); + assert!(dc.keys(Kind::Layer).is_empty(), "the text layer's buffer is not on disk"); +} + /// A Birth Rate expression reading another layer: changing that layer changes the history but /// not the current rate (0) nor this layer's properties, which is all the layer cache key sees. /// CC Particle World layers with expressions therefore aren't layer cached. diff --git a/crates/render/src/text.rs b/crates/render/src/text.rs index efe95023f..ed369afa5 100644 --- a/crates/render/src/text.rs +++ b/crates/render/src/text.rs @@ -243,6 +243,13 @@ pub fn animator_selection(ctx: &EvalCtx, layer: &Layer, anim: &PropGroup, lay: & } "expressionSelector" => { let Some(prop) = sel.get("amount") else { continue }; + // Evaluated per character (`textIndex`): the layer's cache key holds the amount + // for the first character only, so the buffer must not outlive the session. + if prop.has_expression() + && let Some(log) = ctx.reads + { + log.mark_unkeyed(); + } let stat = ctx.v3(layer, sel, "amount", [100.0; 3]); Box::new(move |i, n, prev| { let pv = prev.map(|x| x * 100.0);