Disk cache: record the source files each frame reads

Part of #3184 (step 1): renders now report which footage files they read,
each with its identity (size and modification time) as read. Nothing uses
the list yet: images, cache keys and cache behaviour are unchanged.

- render: `reads` module with `FileStamp`, `ReadFile`, `ReadFiles` (sorted,
  deduplicated, bounded; marked incomplete when it can't vouch for every
  file) and the thread-safe `ReadLog`. `Renderer::reads` collects a frame's
  files; `FootageSource::recording` lets a source note what it reads.
  `file_stamp` uses the same identity.
- Layer cache entries keep the files their buffer was made of and report
  them on every hit; derived entries (styles, blend space, flattened) keep
  their base's. Buffers from the disk cache or the browser store don't know
  their files yet and mark the list incomplete.
- Media pool: every file read notes its identity, taken from the file it
  opens. Decoded frames, opened movies, GIFs, models and EXR channels keep
  it with the cache entry, so a frame served from memory reports its file
  without touching the disk.
- Viewer frame worker collects each frame's list and carries it to where
  the disk entry is written (step 2 stores it).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
vangel76andClaude Opus 5.5 committed 2026-11-06 22:20:40 +01:00
1 parent 69114a8294
commit 91443ef25e
20 files changed
+968 -247

No files matched your search

+8 -12
View File
@@ -224,7 +224,7 @@ impl<'a> Renderer<'a> {
LayerSource::Footage { item } => {
let item = ctx.media_item(layer, *item);
let ItemKind::Footage(f) = &self.project.item(item)?.kind else { return None };
let a = self.footage.aux(item, f, ctx.source_time(layer))?;
let a = self.read(|| self.footage.aux(item, f, ctx.source_time(layer)))?;
// Aux pixels per layer pixel (the layer is the footage at its nominal size). The
// planes are only copied when that changes the scale.
let scale = a.width as f64 / f.width as f64;
@@ -349,20 +349,16 @@ impl<'a> Renderer<'a> {
pub fn adjustment_footprint(&self, ctx: &EvalCtx, layer: &Layer) -> Option<Arc<Buf>> {
// Cached, so an accelerator sees the same buffer (and uploads it once) every frame.
let key = self.cache.and_then(|_| cache::footprint_key(ctx, layer, self.raster_scale(ctx, layer), self.opts.draft, self.mb_on(ctx, layer)));
if let (Some(c), Some(k)) = (self.cache, key)
&& let Some(b) = c.get(k)
{
if let Some((b, _)) = self.cached(key) {
return Some(b);
}
let mut foot = self.source(ctx, layer)?;
let _ = masks::apply(ctx, layer, &mut foot, self.mask_blur(ctx, layer));
let foot = Arc::new(foot);
let (foot, files) = self.recorded(key, |r| {
let mut foot = r.source(ctx, layer)?;
let _ = masks::apply(ctx, layer, &mut foot, r.mask_blur(ctx, layer));
Some(foot)
});
// The buffer the cache keeps (another worker's, if it kept one first).
let foot = match (self.cache_to_keep(), key) {
(Some(c), Some(k)) => c.insert(k, foot),
_ => foot,
};
Some(foot)
Some(self.keep(key, Arc::new(foot?), &files))
}
/// Draw one 2D layer into `canvas` on the CPU (any kind: adjustment, wireframe, styled…).
+22 -12
View File
@@ -25,6 +25,7 @@ use effectcraft_effects::Buf;
use effectcraft_project::{FootageKind, ItemKind, Layer, LayerSource, Node, PropGroup};
use crate::eval::EvalCtx;
use crate::reads::ReadFiles;
mod dependencies;
@@ -43,6 +44,8 @@ pub struct LayerCache {
struct Entry {
buf: Arc<Buf>,
/// The source files the buffer was made of (see [`crate::reads`]).
files: Arc<ReadFiles>,
bytes: usize,
last_use: u64,
}
@@ -147,16 +150,19 @@ impl LayerCache {
Some((st.clone(), salted(*salt)))
}
pub fn get(&self, key: u64) -> Option<Arc<Buf>> {
/// The buffer kept under `key` and the source files it was made of (see [`crate::reads`]).
/// A buffer from the disk cache or another store doesn't know its files: its list is
/// incomplete.
pub fn get(&self, key: u64) -> Option<(Arc<Buf>, Arc<ReadFiles>)> {
let mut g = self.inner.lock().ok()?;
g.clock += 1;
let now = g.clock;
match g.map.get_mut(&key) {
Some(e) => {
e.last_use = now;
let b = e.buf.clone();
let found = (e.buf.clone(), e.files.clone());
g.hits += 1;
Some(b)
Some(found)
}
None => {
g.misses += 1;
@@ -168,7 +174,10 @@ impl LayerCache {
// Disk cache: a hit comes back into memory.
let (dc, dk) = self.disk_key(key)?;
let buf = dc.get_layer(dk)?;
Some(self.insert_mem(key, buf))
let mut files = ReadFiles::default();
files.mark_incomplete();
let files = Arc::new(files);
Some((self.insert_mem(key, buf, files.clone()), files))
}
}
}
@@ -178,8 +187,9 @@ impl LayerCache {
/// together each rendered every layer; sharing the first buffer, the GPU compositor uploads
/// and packs it once instead of once per worker, which ran it out of memory). Renders don't
/// wait for each other: on the frame workers' shared thread pool a wait can end up under
/// the very work it waits for.
pub fn insert(&self, key: u64, buf: Arc<Buf>) -> Arc<Buf> {
/// the very work it waits for. `files`: the source files `buf` was made of (see
/// [`crate::reads`]).
pub fn insert(&self, key: u64, buf: Arc<Buf>, files: Arc<ReadFiles>) -> Arc<Buf> {
if self.gated() {
return buf;
}
@@ -193,12 +203,12 @@ impl LayerCache {
dc.put_layer(dk, &buf);
}
}
self.insert_mem(key, buf)
self.insert_mem(key, buf, files)
}
/// See [`LayerCache::insert`]: the buffer kept under `key` (`buf`, unless one was kept first
/// or `buf` is too large to keep).
fn insert_mem(&self, key: u64, buf: Arc<Buf>) -> Arc<Buf> {
fn insert_mem(&self, key: u64, buf: Arc<Buf>, files: Arc<ReadFiles>) -> Arc<Buf> {
let bytes = buf.img.data.len() * std::mem::size_of::<effectcraft_raster::Px>();
let Ok(mut g) = self.inner.lock() else { return buf };
if bytes > g.budget / 4 {
@@ -210,7 +220,7 @@ impl LayerCache {
e.last_use = now;
return e.buf.clone();
}
g.map.insert(key, Entry { buf: buf.clone(), bytes, last_use: now });
g.map.insert(key, Entry { buf: buf.clone(), files, bytes, last_use: now });
g.bytes += bytes;
if g.bytes > g.budget {
// Evict least-recently used entries down to 3/4 of the budget.
@@ -786,8 +796,8 @@ mod shared_insert_tests {
let cache = LayerCache::default();
let buf = || Arc::new(Buf { img: Image::new(4, 4), offset: [0.0; 2], scale: 1.0 });
let (a, b) = (buf(), buf());
assert!(Arc::ptr_eq(&cache.insert(3, a.clone()), &a));
assert!(Arc::ptr_eq(&cache.insert(3, b), &a), "the later render gets the first buffer");
assert!(Arc::ptr_eq(&cache.get(3).unwrap(), &a));
assert!(Arc::ptr_eq(&cache.insert(3, a.clone(), Default::default()), &a));
assert!(Arc::ptr_eq(&cache.insert(3, b, Default::default()), &a), "the later render gets the first buffer");
assert!(Arc::ptr_eq(&cache.get(3).unwrap().0, &a));
}
}
+7 -8
View File
@@ -1055,18 +1055,17 @@ fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) {
/// Size and content hash of a path that has no native metadata (the browser file table).
pub type VirtualFileId<'a> = &'a (dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync);
/// A file's size and modification time (so edited footage gets new keys). When the path has no
/// native metadata, `virtual_id` supplies a durable size and content hash instead.
/// A file's size and modification time (so edited footage gets new keys): the identity a render
/// records for the files it reads ([`crate::reads::FileStamp`]). When the path has no native
/// metadata, `virtual_id` supplies a durable size and content hash instead.
fn file_stamp(h: &mut Hash128, path: &str, virtual_id: Option<VirtualFileId<'_>>) {
if path.is_empty() {
return;
}
if let Ok(md) = std::fs::metadata(path) {
h.write_u64(md.len());
if let Ok(t) = md.modified()
&& let Ok(d) = t.duration_since(std::time::UNIX_EPOCH)
{
h.write_u64(d.as_nanos() as u64);
if let Some(crate::reads::FileStamp::Native { len, modified }) = crate::reads::FileStamp::native(path) {
h.write_u64(len);
if let Some(t) = modified {
h.write_u64(t);
}
return;
}
+17
View File
@@ -1,4 +1,5 @@
use super::{Arc, AtomicBool, Footage, Image, ItemId, ItemKind, Ordering, Tick};
use crate::reads::ReadLog;
/// How a [`FootageSource`] hands over float footage (OpenEXR, float TIFF), whose values are
/// linear light: encoded with a curve, as the compositor takes footage, or as stored. Other
@@ -92,6 +93,15 @@ pub trait FootageSource: Send + Sync {
fn called_off(&self) -> bool {
false
}
/// Run `f`, which reads from this source on this thread, noting in `reads` every source file
/// those reads use, with its identity as read (see [`crate::reads`]): the file a frame was
/// decoded from, also when it comes from the source's own memory cache (without reading the
/// file again). Sources that can't tell which files they read keep the default, which marks
/// `reads` incomplete.
fn recording(&self, reads: &ReadLog, f: &mut dyn FnMut()) {
reads.mark_incomplete();
f()
}
/// Why `footage` shows no picture although its file is there: a read of it failed because
/// no decoder handles its codec, profile, chroma format or bit depth, or the stream is damaged
/// (e.g. "H.264 (High 4:2:2) isn't supported by the built-in decoder: ..."). `None` while it
@@ -194,6 +204,9 @@ impl FootageSource for CancellableFootage<'_> {
fn called_off(&self) -> bool {
self.cancel.load(Ordering::Relaxed) || self.inner.called_off()
}
fn recording(&self, reads: &ReadLog, f: &mut dyn FnMut()) {
self.inner.recording(reads, f)
}
fn problem(&self, footage: &Footage) -> Option<String> {
self.inner.problem(footage)
}
@@ -214,4 +227,8 @@ impl FootageSource for NoFootage {
fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option<Arc<Image>> {
None
}
/// Reads no files.
fn recording(&self, _: &ReadLog, f: &mut dyn FnMut()) {
f()
}
}
+1 -1
View File
@@ -104,7 +104,7 @@ impl EffectHost for FxHost<'_, '_, '_> {
return None;
}
let st = other.layer_time(Tick::from_seconds_f64(start));
self.r.footage.audio(item, f, st, frames, rate)
self.r.read(|| self.r.footage.audio(item, f, st, frames, rate))
}
fn self_at(&self, layer_time: f64, effects: usize) -> Option<Buf> {
+18 -20
View File
@@ -37,18 +37,17 @@ impl Shutter {
impl<'a> Renderer<'a> {
/// The layer with its styles: exterior passes and body (cached per pass).
pub(crate) fn styled(&self, ctx: &EvalCtx, layer: &Layer, mut timing: Option<&mut LayerTiming>) -> Option<StyledLayer> {
let (content, ckey) = self.content_buf_timed(ctx, layer, timing.as_deref_mut())?;
let (content, ckey, files) = self.content_buf_timed(ctx, layer, timing.as_deref_mut())?;
if !styles::active(ctx, layer) {
return Some(StyledLayer { passes: vec![], body: content.clone(), content, key: ckey, ckey, plain: true });
return Some(StyledLayer { passes: vec![], body: content.clone(), content, key: ckey, ckey, plain: true, files });
}
let modes = styles::pass_modes(ctx, layer);
let key = ckey.map(|k| cache::styles_key(ctx, layer, k));
if let (Some(c), Some(k)) = (self.cache, key)
&& let Some(body) = c.get(k)
{
let passes: Option<Vec<_>> = modes.iter().enumerate().map(|(i, m)| c.get(cache::derive(k, i as u64 + 1)).map(|b| (b, *m))).collect();
if let Some((body, _)) = self.cached(key) {
let passes: Option<Vec<_>> =
modes.iter().enumerate().map(|(i, m)| self.cached(key.map(|k| cache::derive(k, i as u64 + 1))).map(|(b, _)| (b, *m))).collect();
if let Some(passes) = passes {
return Some(StyledLayer { passes, body, content, key, ckey, plain: false });
return Some(StyledLayer { passes, body, content, key, ckey, plain: false, files });
}
}
let t0 = web_time::Instant::now();
@@ -59,16 +58,12 @@ impl<'a> Renderer<'a> {
t.cached = false;
t.effects.push(("layerStyles".into(), t0.elapsed().as_secs_f64() * 1e3));
}
let mut body = Arc::new(st.body);
let mut passes: Vec<_> = st.passes.into_iter().map(|(b, m)| (Arc::new(b), m)).collect();
// The buffers the cache keeps (another worker's, if it kept them first).
if let (Some(c), Some(k)) = (self.cache_to_keep(), key) {
body = c.insert(k, body);
for (i, (b, _)) in passes.iter_mut().enumerate() {
*b = c.insert(cache::derive(k, i as u64 + 1), b.clone());
}
}
Some(StyledLayer { passes, body, content, key, ckey, plain: false })
// The buffers the cache keeps (another worker's, if it kept them first), made of the
// content's files.
let body = self.keep(key, Arc::new(st.body), &files);
let passes: Vec<_> =
st.passes.into_iter().enumerate().map(|(i, (b, m))| (self.keep(key.map(|k| cache::derive(k, i as u64 + 1)), Arc::new(b), &files), m)).collect();
Some(StyledLayer { passes, body, content, key, ckey, plain: false, files })
}
/// The styled layer's buffers in the blending space.
@@ -76,9 +71,10 @@ impl<'a> Renderer<'a> {
if self.pipe.to_blend().is_none() {
return st;
}
let body = self.to_blend(st.body, st.key);
let content = if st.plain { body.clone() } else { self.to_blend(st.content, st.ckey) };
let passes = st.passes.into_iter().enumerate().map(|(i, (b, m))| (self.to_blend(b, st.key.map(|k| cache::derive(k, i as u64 + 1))), m)).collect();
let body = self.to_blend(st.body, st.key, &st.files);
let content = if st.plain { body.clone() } else { self.to_blend(st.content, st.ckey, &st.files) };
let passes =
st.passes.into_iter().enumerate().map(|(i, (b, m))| (self.to_blend(b, st.key.map(|k| cache::derive(k, i as u64 + 1)), &st.files), m)).collect();
StyledLayer { passes, body, content, ..st }
}
@@ -582,6 +578,8 @@ pub struct StyledLayer {
ckey: Option<u64>,
/// No layer styles: composite `body` the usual way.
pub plain: bool,
/// The source files the layer's buffers were made of (see [`crate::reads`]).
pub(crate) files: Arc<crate::reads::ReadFiles>,
}
/// Where a layer buffer lands (see [`Renderer::placement`]).
+5
View File
@@ -17,6 +17,7 @@ use color::Region;
pub mod eval;
pub mod masks;
pub mod passes;
pub mod reads;
pub mod shapes;
pub mod styles;
pub mod text;
@@ -69,6 +70,10 @@ pub struct Renderer<'a> {
pub opts: RenderOpts,
/// Processed-layer cache shared across frames (see [`cache`]). `None` renders everything.
pub cache: Option<&'a LayerCache>,
/// Where the source files this render reads are noted (see [`reads`]): the frame's own,
/// from the footage source and from the buffers the layer cache serves. `None` records
/// nothing for the frame (buffers kept in the layer cache still record theirs).
pub reads: Option<&'a reads::ReadLog>,
/// Optional per-layer timing sink (`frame --bench` / perf readouts).
pub profile: Option<&'a std::sync::Mutex<Vec<LayerTiming>>>,
/// GPU backend used when [`RenderOpts::backend`] asks for it.
+97 -67
View File
@@ -3,6 +3,7 @@ use super::footage::{FootageSource, is_vector_footage};
use super::frames::{blend_frames, cineon_frame, content_bounds, field_frame, frame_blend_mode, frame_position, frame_size, interpret_pixels, roi_inside};
use super::fx_host::{FxHost, essential_overrides};
use super::opts::{Collapse3d, LayerTiming, MAX_THROUGH, RenderOpts, Through};
use super::reads::{ReadFiles, ReadLog};
use super::{
Arc, AutoKey, AutoPick, BlendMode, Buf, Comp, EffectCtx, EffectEnv, EvalCtx, Footage, FootageKind, FrameBlend, FrameRate, GroupKind, Image, ItemId,
ItemKind, Layer, LayerCache, LayerSource, Params, Project, Quality, Region, Renderer, Tick, cache, color, masks, shapes, styles, text, three_d, vec2,
@@ -108,6 +109,7 @@ impl<'a> Renderer<'a> {
expr: None,
opts,
cache: None,
reads: None,
profile: None,
accel: None,
depth: 0,
@@ -129,6 +131,52 @@ impl<'a> Renderer<'a> {
self.cache.filter(|_| !self.footage.called_off())
}
/// `read`, a read of this render's footage source, with the files it uses noted in
/// [`Renderer::reads`] (see [`FootageSource::recording`]).
pub(crate) fn read<R>(&self, read: impl FnOnce() -> Option<R>) -> Option<R> {
let Some(log) = self.reads else { return read() };
let (mut read, mut out) = (Some(read), None);
self.footage.recording(log, &mut || out = read.take().and_then(|f| f()));
out
}
/// The buffer the layer cache keeps under `key`, with the files it was made of noted in
/// [`Renderer::reads`].
pub(crate) fn cached(&self, key: Option<u64>) -> Option<(Arc<Buf>, Arc<ReadFiles>)> {
let (buf, files) = self.cache?.get(key?)?;
if let Some(log) = self.reads {
log.note_all(&files);
}
Some((buf, files))
}
/// Keep `buf`, made of `files`, under `key` (see [`Renderer::recorded`]): the buffer the cache
/// keeps (another worker's, if it kept one first), or `buf` when nothing is kept.
pub(crate) fn keep(&self, key: Option<u64>, buf: Arc<Buf>, files: &Arc<ReadFiles>) -> Arc<Buf> {
match (self.cache_to_keep(), key) {
(Some(c), Some(k)) => c.insert(k, buf, files.clone()),
_ => buf,
}
}
/// Run `make`, which makes a buffer to keep under `key` in the layer cache, on this renderer
/// with its reads noted apart: what `make` returns and the files it read (from the footage
/// source, or the files of cached buffers it used), which go to [`Renderer::reads`] too. A
/// cached buffer then reports the files it was made of whichever render uses it. Without a
/// cache or a key, `make` runs on this renderer as it is (no files: nothing is kept).
pub(crate) fn recorded<T>(&self, key: Option<u64>, make: impl FnOnce(&Renderer<'_>) -> T) -> (T, Arc<ReadFiles>) {
if key.is_none() || self.cache.is_none() {
return (make(self), Arc::default());
}
let log = ReadLog::new();
let out = make(&Renderer { reads: Some(&log), ..*self });
let files = log.into_files();
if let Some(own) = self.reads {
own.note_all(&files);
}
(out, Arc::new(files))
}
/// A layer's Quality switch, limited by the precomp layers above it when
/// [`RenderOpts::nested_switches`] is on.
pub fn quality(&self, layer: &Layer) -> Quality {
@@ -760,7 +808,7 @@ impl<'a> Renderer<'a> {
// Continuously rasterised vector footage: drawn at the on-screen scale.
if layer.switches.collapse && is_vector_footage(f) {
let k = self.raster_scale(ctx, layer);
if let Some(img) = self.footage.vector_frame(item, f, k) {
if let Some(img) = self.read(|| self.footage.vector_frame(item, f, k)) {
let mut buf = Buf { img: (*img).clone(), offset: [0.0; 2], scale: k };
if let Some(c) = self.pipe.media_in(f) {
color::convert(&mut buf.img, &c);
@@ -831,7 +879,7 @@ impl<'a> Renderer<'a> {
if f.fields != effectcraft_project::FieldOrder::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) {
let field_rate = FrameRate::new(f.frame_rate.num * 2, f.frame_rate.den);
let i = field_rate.frame_at(t).max(0);
let img = cineon_frame(self.footage.frame_with(item, f, f.frame_rate.tick_of(i / 2), None, self.pipe.float_curve(f))?, f);
let img = cineon_frame(self.read(|| self.footage.frame_with(item, f, f.frame_rate.tick_of(i / 2), None, self.pipe.float_curve(f)))?, f);
let dominant_upper = f.fields == effectcraft_project::FieldOrder::UpperFirst;
// Upper field = even lines (0, 2, …).
let parity = if (i % 2 == 0) == dominant_upper { 0 } else { 1 };
@@ -865,24 +913,22 @@ impl<'a> Renderer<'a> {
}
h.finish()
});
if let (Some(c), Some(k)) = (self.cache, key)
&& let Some(b) = c.get(k)
{
if let Some((b, _)) = self.cached(key) {
return Some(Arc::new(b.img.clone()));
}
// Log scans are converted before they blend: the conversion is not linear.
let a = self.footage.frame_with(item, f, f.frame_rate.tick_of(i), None, float).map(|a| cineon_frame(a, f));
let b = self.footage.frame_with(item, f, f.frame_rate.tick_of(i + 1), None, float).map(|b| cineon_frame(b, f));
let ((a, b), files) = self.recorded(key, |r| {
let frame = |i| r.read(|| r.footage.frame_with(item, f, f.frame_rate.tick_of(i), None, float)).map(|a| cineon_frame(a, f));
(frame(i), frame(i + 1))
});
if let (Some(a), Some(b)) = (a, b) {
let img = blend_frames(mode, &a, &b, w as f32);
if let (Some(c), Some(k)) = (self.cache_to_keep(), key) {
c.insert(k, Arc::new(Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }));
}
self.keep(key, Arc::new(Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }), &files);
return Some(Arc::new(img));
}
}
}
self.footage.frame_with(item, f, t, size, float).map(|img| cineon_frame(img, f))
self.read(|| self.footage.frame_with(item, f, t, size, float)).map(|img| cineon_frame(img, f))
}
/// Source → masks, clamped/quantised to the project depth.
@@ -902,52 +948,40 @@ impl<'a> Renderer<'a> {
/// Fully processed layer buffer (source → masks → effects → layer styles, flattened).
/// Served from the layer cache when the layer's content key matches.
pub fn layer_buf(&self, ctx: &EvalCtx, layer: &Layer) -> Option<Arc<Buf>> {
self.layer_buf_keyed(ctx, layer).map(|(b, _)| b)
self.layer_buf_keyed(ctx, layer).map(|(b, _, _)| b)
}
/// [`Self::layer_buf`] in the blending space (linear with Blend Colors Using 1.0 Gamma).
pub fn blend_layer_buf(&self, ctx: &EvalCtx, layer: &Layer) -> Option<Arc<Buf>> {
let (b, k) = self.layer_buf_keyed(ctx, layer)?;
Some(self.to_blend(b, k))
let (b, k, files) = self.layer_buf_keyed(ctx, layer)?;
Some(self.to_blend(b, k, &files))
}
/// A buffer converted to the blending space (cached under a key derived from `key`).
pub(crate) fn to_blend(&self, b: Arc<Buf>, key: Option<u64>) -> Arc<Buf> {
/// A buffer converted to the blending space (cached under a key derived from `key`, with
/// `files`, the source files `b` was made of).
pub(crate) fn to_blend(&self, b: Arc<Buf>, key: Option<u64>, files: &Arc<ReadFiles>) -> Arc<Buf> {
let Some(c) = self.pipe.to_blend() else { return b };
let key = key.map(|k| cache::derive(k, 0x1ea7));
if let (Some(cache), Some(k)) = (self.cache, key)
&& let Some(x) = cache.get(k)
{
if let Some((x, _)) = self.cached(key) {
return x;
}
let mut nb = (*b).clone();
color::convert(&mut nb.img, &c);
let nb = Arc::new(nb);
// The buffer the cache keeps (another worker's, if it kept one first).
match (self.cache_to_keep(), key) {
(Some(cache), Some(k)) => cache.insert(k, nb),
_ => nb,
}
self.keep(key, Arc::new(nb), files)
}
fn layer_buf_keyed(&self, ctx: &EvalCtx, layer: &Layer) -> Option<(Arc<Buf>, Option<u64>)> {
/// The flattened layer buffer, its cache key and the source files it was made of.
fn layer_buf_keyed(&self, ctx: &EvalCtx, layer: &Layer) -> Option<(Arc<Buf>, Option<u64>, Arc<ReadFiles>)> {
let st = self.styled(ctx, layer, None)?;
if st.passes.is_empty() {
return Some((st.body, st.key));
return Some((st.body, st.key, st.files));
}
let key = st.key.map(|k| cache::derive(k, 0xf1a7));
if let (Some(c), Some(k)) = (self.cache, key)
&& let Some(b) = c.get(k)
{
return Some((b, key));
if let Some((b, files)) = self.cached(key) {
return Some((b, key, files));
}
let flat = Arc::new(styles::Styled { passes: st.passes.iter().map(|(b, m)| ((**b).clone(), *m)).collect(), body: (*st.body).clone() }.flatten());
// The buffer the cache keeps (another worker's, if it kept one first).
let flat = match (self.cache_to_keep(), key) {
(Some(c), Some(k)) => c.insert(k, flat),
_ => flat,
};
Some((flat, key))
Some((self.keep(key, flat, &st.files), key, st.files))
}
/// A layer's source pixels at the context time, before masks, effects and the transform
@@ -983,28 +1017,29 @@ impl<'a> Renderer<'a> {
/// Layer pixels after source → masks → effects (no layer styles).
pub fn content_buf(&self, ctx: &EvalCtx, layer: &Layer) -> Option<Arc<Buf>> {
self.content_buf_timed(ctx, layer, None).map(|(b, _)| b)
self.content_buf_timed(ctx, layer, None).map(|(b, _, _)| b)
}
pub(crate) fn content_buf_timed(&self, ctx: &EvalCtx, layer: &Layer, mut timing: Option<&mut LayerTiming>) -> Option<(Arc<Buf>, Option<u64>)> {
/// [`Self::content_buf`], its cache key and the source files it was made of.
pub(crate) fn content_buf_timed(
&self,
ctx: &EvalCtx,
layer: &Layer,
mut timing: Option<&mut LayerTiming>,
) -> Option<(Arc<Buf>, Option<u64>, Arc<ReadFiles>)> {
let key = self.cache.and_then(|_| cache::layer_key(ctx, layer, self.raster_scale(ctx, layer), self.opts.draft, self.mb_on(ctx, layer)));
let key = self.content_key(ctx, layer, key);
if let (Some(c), Some(k)) = (self.cache, key)
&& let Some(b) = c.get(k)
{
if let Some((b, files)) = self.cached(key) {
if let Some(t) = timing.as_deref_mut() {
t.cached = true;
}
return Some((b, key));
return Some((b, key, files));
}
let buf = self.masked_source(ctx, layer)?;
let buf = Arc::new(self.apply_effects_timed(ctx, layer, buf, false, usize::MAX, timing.map(|t| &mut t.effects)));
// The buffer the cache keeps (another worker's, if it kept one first).
let buf = match (self.cache_to_keep(), key) {
(Some(c), Some(k)) => c.insert(k, buf),
_ => buf,
};
Some((buf, key))
let (buf, files) = self.recorded(key, |r| {
let buf = r.masked_source(ctx, layer)?;
Some(r.apply_effects_timed(ctx, layer, buf, false, usize::MAX, timing.map(|t| &mut t.effects)))
});
Some((self.keep(key, Arc::new(buf?), &files), key, files))
}
/// A layer's input at the context time: source → masks → its first `effects` effects
@@ -1014,23 +1049,18 @@ impl<'a> Renderer<'a> {
pub fn layer_input(&self, ctx: &EvalCtx, layer: &Layer, effects: usize) -> Option<Arc<Buf>> {
let key = self.cache.and_then(|_| cache::input_key(ctx, layer, self.raster_scale(ctx, layer), self.opts.draft, self.mb_on(ctx, layer), effects));
let key = self.content_key(ctx, layer, key);
if let (Some(c), Some(k)) = (self.cache, key)
&& let Some(b) = c.get(k)
{
if let Some((b, _)) = self.cached(key) {
return Some(b);
}
let mut buf = self.masked_source(ctx, layer)?;
let origin = ctx.effect_bounds(layer).1;
buf.rebase(origin);
if effects > 0 {
buf = self.effect_space_stack(ctx, layer, buf, EffectStack { adjustment: false, limit: effects, origin }, None);
}
let buf = Arc::new(buf);
// The buffer the cache keeps (another worker's, if it kept one first).
let buf = match (self.cache_to_keep(), key) {
(Some(c), Some(k)) => c.insert(k, buf),
_ => buf,
};
Some(buf)
let (buf, files) = self.recorded(key, |r| {
let mut buf = r.masked_source(ctx, layer)?;
let origin = ctx.effect_bounds(layer).1;
buf.rebase(origin);
if effects > 0 {
buf = r.effect_space_stack(ctx, layer, buf, EffectStack { adjustment: false, limit: effects, origin }, None);
}
Some(buf)
});
Some(self.keep(key, Arc::new(buf?), &files))
}
}
+230
View File
@@ -0,0 +1,230 @@
//! The source files a render read: which footage files its pixels came from, each with its
//! identity (size and modification time) as it was when it was read.
//!
//! A [`FootageSource`](crate::FootageSource) that knows which files it opens notes them while
//! [`FootageSource::recording`](crate::FootageSource::recording) runs; the renderer collects them
//! in a [`ReadLog`] ([`Renderer::reads`](crate::Renderer::reads)). Buffers in the layer cache keep
//! the files they were made of, so a frame served from memory reports the same files as the
//! render that made it, without touching the disk.
//!
//! The list is what a cache entry on disk has to check before it is used: a handful of files per
//! frame, never every file of every footage item.
use std::sync::{Arc, Mutex, PoisonError};
/// The identity of a file as a render read it (see [`file_stamp`](crate::disk_cache) for the
/// same identity in content keys).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum FileStamp {
/// A file on disk: its size in bytes and its modification time (nanoseconds since the Unix
/// epoch; `None` where the file system has none).
Native { len: u64, modified: Option<u64> },
/// Bytes given from memory instead of the file system (the browser's file table): their
/// identity is the size and content hash a
/// [`VirtualFileId`](crate::disk_cache::VirtualFileId) gives for the path.
Virtual,
/// The file could not be read (missing, not a regular file, no permission). What was made
/// without it changes once it can be read again.
Missing,
}
impl FileStamp {
/// The identity of a file from metadata the caller already has (of an open file).
pub fn of_metadata(md: &std::fs::Metadata) -> FileStamp {
let modified = md.modified().ok().and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()).map(|d| d.as_nanos() as u64);
FileStamp::Native { len: md.len(), modified }
}
/// The identity of the file at `path` now: one `metadata` call. `None` when the file system
/// can't describe it (missing, or a path served from memory).
pub fn native(path: &str) -> Option<FileStamp> {
std::fs::metadata(path).ok().map(|md| FileStamp::of_metadata(&md))
}
}
/// One file a render read.
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ReadFile {
pub path: Arc<str>,
pub stamp: FileStamp,
}
impl ReadFile {
pub fn new(path: impl Into<Arc<str>>, stamp: FileStamp) -> ReadFile {
ReadFile { path: path.into(), stamp }
}
}
/// At most this many files are listed per render or cached buffer; more mark the list
/// incomplete (a comp reading more files than this for one frame is not checked file by file).
pub const MAX_READ_FILES: usize = 4096;
/// The files a render (or one cached buffer) read, each once, sorted by path.
///
/// Incomplete when it can't say for sure: a footage source that doesn't tell what it reads, a
/// buffer from a store that doesn't keep its files, more than [`MAX_READ_FILES`] files, or one
/// file read with two identities (it changed during the render). A cache entry whose list is
/// incomplete can't be checked against the files, so it must not outlive the session.
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
pub struct ReadFiles {
files: Vec<ReadFile>,
incomplete: bool,
}
impl ReadFiles {
/// Note a file read (once per path).
pub fn add(&mut self, file: ReadFile) {
match self.files.binary_search_by(|f| f.path.cmp(&file.path)) {
Ok(i) => {
if self.files.get(i).is_some_and(|f| f.stamp != file.stamp) {
self.incomplete = true;
}
}
Err(i) => {
if self.files.len() >= MAX_READ_FILES {
self.incomplete = true;
} else {
self.files.insert(i, file);
}
}
}
}
/// Note every file of `other` (and whether it is incomplete).
pub fn extend(&mut self, other: &ReadFiles) {
self.incomplete |= other.incomplete;
for f in &other.files {
self.add(f.clone());
}
}
/// Some files read may be missing from the list (see [`ReadFiles`]).
pub fn mark_incomplete(&mut self) {
self.incomplete = true;
}
/// Whether the list names every file read.
pub fn is_complete(&self) -> bool {
!self.incomplete
}
/// The files, sorted by path.
pub fn files(&self) -> &[ReadFile] {
&self.files
}
/// The identity `path` was read with.
pub fn get(&self, path: &str) -> Option<FileStamp> {
let i = self.files.binary_search_by(|f| (*f.path).cmp(path)).ok()?;
self.files.get(i).map(|f| f.stamp)
}
pub fn len(&self) -> usize {
self.files.len()
}
pub fn is_empty(&self) -> bool {
self.files.is_empty()
}
}
impl FromIterator<ReadFile> for ReadFiles {
fn from_iter<I: IntoIterator<Item = ReadFile>>(iter: I) -> Self {
let mut out = ReadFiles::default();
for f in iter {
out.add(f);
}
out
}
}
/// Collects the files a render reads, from any thread (the layers of a frame render in
/// parallel).
#[derive(Debug, Default)]
pub struct ReadLog {
files: Mutex<ReadFiles>,
}
impl ReadLog {
pub fn new() -> ReadLog {
ReadLog::default()
}
fn lock(&self) -> std::sync::MutexGuard<'_, ReadFiles> {
self.files.lock().unwrap_or_else(PoisonError::into_inner)
}
/// Note a file read.
pub fn note(&self, file: ReadFile) {
self.lock().add(file);
}
/// Note every file of `files` (and whether it is incomplete).
pub fn note_all(&self, files: &ReadFiles) {
if files.is_empty() && files.is_complete() {
return;
}
self.lock().extend(files);
}
/// Some files read can't be listed (see [`ReadFiles`]).
pub fn mark_incomplete(&self) {
self.lock().mark_incomplete();
}
/// The files noted so far.
pub fn files(&self) -> ReadFiles {
self.lock().clone()
}
/// The files noted.
pub fn into_files(self) -> ReadFiles {
self.files.into_inner().unwrap_or_else(PoisonError::into_inner)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn file(path: &str, len: u64) -> ReadFile {
ReadFile::new(path, FileStamp::Native { len, modified: Some(7) })
}
#[test]
fn files_are_listed_once_sorted_by_path() {
let files: ReadFiles = [file("/b", 1), file("/a", 2), file("/b", 1)].into_iter().collect();
assert_eq!(files.files().iter().map(|f| &*f.path).collect::<Vec<_>>(), ["/a", "/b"]);
assert!(files.is_complete());
assert_eq!(files.get("/a"), Some(FileStamp::Native { len: 2, modified: Some(7) }));
assert_eq!(files.get("/c"), None);
}
/// A file read twice with different identities changed during the render: the list can't
/// vouch for what was made of it.
#[test]
fn a_file_read_with_two_identities_makes_the_list_incomplete() {
let files: ReadFiles = [file("/a", 1), file("/a", 2)].into_iter().collect();
assert_eq!(files.len(), 1);
assert!(!files.is_complete());
}
#[test]
fn the_list_is_bounded() {
let files: ReadFiles = (0..MAX_READ_FILES + 3).map(|i| file(&format!("/f{i:06}"), 1)).collect();
assert_eq!(files.len(), MAX_READ_FILES);
assert!(!files.is_complete(), "files past the bound make the list incomplete, never silently short");
}
#[test]
fn incompleteness_carries_over() {
let mut part = ReadFiles::default();
part.mark_incomplete();
let log = ReadLog::new();
log.note(file("/a", 1));
log.note_all(&part);
let all = log.into_files();
assert_eq!(all.len(), 1);
assert!(!all.is_complete());
}
}
+2 -2
View File
@@ -487,7 +487,7 @@ fn model_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, b: &mut Builder) {
LayerSource::Model { item } => {
let Some(it) = r.project.item(*item) else { return };
let ItemKind::Footage(f) = &it.kind else { return };
let Some(m) = r.footage.model(*item, f) else { return };
let Some(m) = r.read(|| r.footage.model(*item, f)) else { return };
let unit = geo.map_or(1.0, |g| ctx.f(layer, g, "unitScale", 1.0));
let clip = geo.map_or(0, |g| ctx.e(layer, g, "animation")) as usize;
let clip = clip.checked_sub(1).filter(|c| *c < m.animations.len());
@@ -700,7 +700,7 @@ fn card_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, b: &mut Builder) {
let world = b.outer * ctx.world_matrix(layer);
if crate::text::per_char_3d(ctx, layer) {
for (buf, m) in crate::text::per_char_planes(ctx, layer, r.opts.scale) {
let buf = r.to_blend(Arc::new(buf), None);
let buf = r.to_blend(Arc::new(buf), None, &Arc::default());
card(b, &buf, world * m, mat);
}
return;
+3 -1
View File
@@ -205,7 +205,9 @@ pub(crate) fn prepare_all<'a>(r: &Renderer, ctx: &EvalCtx<'_>, layer: &'a Layer,
if crate::text::per_char_3d(ctx, layer) {
return crate::text::per_char_planes(ctx, layer, r.opts.scale)
.into_iter()
.filter_map(|(buf, m)| prepare_with(r, ctx, layer, order, out, in_run, PlaneInput { buf: r.to_blend(Arc::new(buf), None), local: m }))
.filter_map(|(buf, m)| {
prepare_with(r, ctx, layer, order, out, in_run, PlaneInput { buf: r.to_blend(Arc::new(buf), None, &Arc::default()), local: m })
})
.collect();
}
prepare(r, ctx, layer, order, out, in_run).into_iter().collect()