Merge remote-tracking branch 'upstream/main' into text-layer-name-follow-3233

This commit is contained in:
vangel76 committed 2026-11-08 04:42:27 +01:00
commit e41d6d9d68
104 files changed
+6694 -1263

No files matched your search

+1 -1
View File
@@ -208,7 +208,7 @@ pub(crate) fn source_frame(
t: Tick,
expr: Option<&dyn ExprHost>,
) -> Option<(Arc<Image>, [f64; 2])> {
let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr, footage: None, property_overrides: &[], media_overrides: &[] };
let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr, footage: None, property_overrides: &[], media_overrides: &[], reads: None };
if let effectcraft_project::LayerSource::Footage { item } = &layer.source
&& let Some(effectcraft_project::ItemKind::Footage(f)) = project.item(*item).map(|i| &i.kind)
&& f.has_video
+35 -18
View File
@@ -20,13 +20,16 @@
//! calls [`FrameServer::resume`] for the next. Backend Auto picks CPU or GPU per comp from
//! the frames' total times ([`effectcraft_render::AutoPick`]).
//! - Disk cache: a render request may carry the frame's disk-cache key ([`FrameMsg::Render`]
//! `disk`); the transport stores the finished frame under it (the browser: in the Origin
//! Private File System) and reports [`FrameReply::Stored`].
//! `disk`); the transport stores the finished frame under it with the source files it read
//! ([`FrameServer::take_files`], `disk_cache::frame_entry`; the browser: in the Origin Private
//! File System) and reports [`FrameReply::Stored`]. A frame whose files can't all be named is
//! not stored. The page checks an entry's files before it uses it (#3184).
//! - Layer buffers on disk: with `layers`, the server's layer cache is backed by a
//! [`PrefetchStore`] ([`FrameServer::set_layer_store`]): before the request the transport
//! fetches the entries named in `prefetch` (recent misses that are on disk, chosen by the
//! page, `disk_cache::LayerPrefetch`), so the lookups in the middle of the render find them;
//! the reply lists the layer keys that missed and those served from the store
//! page, `disk_cache::LayerPrefetch`), so the lookups in the middle of the render find them
//! (each entry keeps the source files its buffer was made of and is used only while they are
//! unchanged, #3184); the reply lists the layer keys that missed and those served from the store
//! ([`FrameReply::Frame`] `layer_misses` / `layer_hits`), and slow buffers queue up for the
//! transport to write after the frame ([`PrefetchStore::take_writes`], reported as
//! [`FrameReply::Stored`] with `layer`).
@@ -37,6 +40,7 @@
use std::sync::Arc;
use effectcraft_project::{ItemId, Project};
use effectcraft_render::reads::{ReadFiles, ReadLog};
use effectcraft_render::{Accelerator, AutoKey, Backend, ExprHost, FootageSource, Image, LayerCache, LayerStore, PrefetchStore, RenderOpts, Renderer};
use effectcraft_time::Tick;
use serde::{Deserialize, Serialize};
@@ -177,7 +181,7 @@ pub enum FrameReply {
passes: u32,
#[serde(default)]
gpu: bool,
/// Layer-cache lookups that missed the store (salted keys, 32 hex digits).
/// Layer-cache lookups that missed the store (disk keys, 32 hex digits).
#[serde(default, rename = "layerMisses", skip_serializing_if = "Vec::is_empty")]
layer_misses: Vec<String>,
/// Layer buffers served from the store (fetched from disk).
@@ -260,11 +264,12 @@ pub struct FrameServer {
revision: Option<u64>,
/// The frame being rendered in passes.
pending: Option<Pending>,
/// The disk cache's layer buffers (see the module docs) and the project's salt.
/// The disk cache's layer buffers (see the module docs).
layer_store: Option<Arc<PrefetchStore>>,
salt: u64,
/// Size and content hash for footage paths with no native metadata (the browser file table).
virtual_id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>,
/// The source files the last finished frame read ([`FrameServer::take_files`]).
files: Option<ReadFiles>,
}
/// A frame whose readbacks are in flight.
@@ -284,6 +289,7 @@ struct Pending {
pub const MAX_PASSES: u32 = 24;
impl FrameServer {
/// A frame server over `footage` and the optional expression host, with an empty layer cache.
pub fn new(footage: Arc<dyn FootageSource>, expr: Option<Arc<dyn ExprHost>>) -> FrameServer {
FrameServer {
footage,
@@ -295,17 +301,30 @@ impl FrameServer {
revision: None,
pending: None,
layer_store: None,
salt: 0,
virtual_id: None,
files: None,
}
}
/// Identity of virtual footage files, used when native metadata cannot describe the path.
/// Called from `FrameServer::load`, not from the UI thread's cache lookup.
/// Identity of virtual footage files, used when native metadata cannot describe the path:
/// what the disk entries of frames and layer buffers keep for them and are checked against.
pub fn set_virtual_id(&mut self, id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>) {
self.cache.set_virtual_id(id.clone());
self.virtual_id = id;
}
/// The source files the frame of the last [`FrameReply::Frame`] read, with their identities
/// as read: what its disk entry keeps (`disk_cache::frame_entry`). Taken once.
pub fn take_files(&mut self) -> Option<ReadFiles> {
self.files.take()
}
/// The identity of footage paths with no native metadata (the browser file table), to store
/// with a frame's disk entry.
pub fn virtual_id(&self) -> Option<&Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>> {
self.virtual_id.as_ref()
}
/// The store behind the layer cache when a request asks for `layers` (the browser: the
/// disk cache's layer buffers, fetched before the request).
pub fn set_layer_store(&mut self, store: Option<Arc<PrefetchStore>>) {
@@ -316,11 +335,6 @@ impl FrameServer {
self.layer_store.as_ref()
}
/// The salt of the replica's layer keys on disk (keys are `(salt, key)`).
pub fn salt(&self) -> u64 {
self.salt
}
/// Render with `accel` when the request's backend asks for it (the worker's GPU). Its miss
/// gate keeps placeholder buffers of unfinished passes out of the layer cache.
pub fn set_accel(&mut self, accel: Option<Arc<dyn Accelerator>>) {
@@ -347,8 +361,6 @@ impl FrameServer {
fn load(&mut self, revision: u64) -> Result<(), String> {
let p = Project::from_json(&self.doc.to_string()).map_err(|e| e.to_string())?;
self.salt = effectcraft_render::disk_cache::footage_salt_with(&p, self.virtual_id.as_deref());
self.cache.set_store_salt(self.salt);
self.project = Some(Arc::new(p));
self.revision = Some(revision);
Ok(())
@@ -392,7 +404,7 @@ impl FrameServer {
}
FrameMsg::Render { id, revision, comp, time, opts, layers, .. } => {
let store = self.layer_store.clone().filter(|_| layers);
self.cache.set_store(store.map(|s| (s as Arc<dyn LayerStore>, self.salt)));
self.cache.set_store(store.map(|s| s as Arc<dyn LayerStore>));
let fail = |error: String, stale: bool| vec![(FrameReply::Failed { id, error, stale }, None)];
if self.revision != Some(revision) {
return fail(format!("the replica is at {:?}, not {revision}", self.revision), true);
@@ -432,12 +444,16 @@ impl FrameServer {
/// Run the next pass of the frame in progress; its reply once it is done.
pub fn resume(&mut self) -> Vec<(FrameReply, Option<Vec<u8>>)> {
let Some(mut pr) = self.pending.take() else { return vec![] };
self.files = None;
pr.passes += 1;
let accel = self.accel.clone().filter(|_| pr.opts.backend != Backend::Cpu);
let give_up = pr.passes > MAX_PASSES;
// (each pass renders the whole frame: the last one's files are the frame's)
let reads = ReadLog::new();
let mut r = Renderer::new(&pr.project, self.footage.as_ref(), pr.opts);
r.expr = self.expr.as_deref();
r.cache = Some(&self.cache);
r.reads = Some(&reads);
if !give_up {
r.accel = accel.as_deref();
}
@@ -451,6 +467,7 @@ impl FrameServer {
}
let ms = pr.t0.elapsed().as_secs_f64() * 1000.0;
let gpu = r.accel.is_some() && r.active_accel().is_some();
self.files = Some(reads.files());
let hex = |v: Vec<u128>| v.into_iter().map(|k| format!("{k:032x}")).collect::<Vec<_>>();
let (layer_misses, layer_hits) = match &self.layer_store {
Some(st) => (hex(st.take_misses()), hex(st.take_hits())),
+137 -1
View File
@@ -69,7 +69,8 @@ fn layer_buffers_go_through_the_disk_store() {
assert!(!layer_misses.is_empty() && layer_hits.is_empty());
let writes = store_a.take_writes();
assert!(!writes.is_empty());
assert!(writes.iter().all(|(k, _)| (*k >> 64) as u64 == a.salt()), "keys are salted per project");
// Sealed with the files each buffer was made of (none here: the demo reads no footage).
assert!(writes.iter().all(|(_, e)| effectcraft_render::disk_cache::read_layer_entry(e).is_some_and(|(_, f)| f.is_empty())));
let mut index = DiskIndex::default();
let mut disk = std::collections::HashMap::new();
for (k, e) in writes {
@@ -375,3 +376,138 @@ fn every_frame_begin_has_a_frame_end() {
assert!(matches!(w.handle(req(6, s.revision, comp, cpu))[0].0, FrameReply::Frame { id: 6, .. }));
assert_eq!(load(), (3, 3));
}
/// Footage that reads one file from memory (the browser's file table) for every frame.
struct OneVirtualFile {
/// Tell which files it reads (else every render's list is incomplete).
records: bool,
}
impl FootageSource for OneVirtualFile {
fn frame(&self, _: ItemId, _: &effectcraft_project::Footage, _: Tick) -> Option<Arc<Image>> {
Some(Arc::new(Image::filled(4, 4, [1.0, 0.5, 0.0, 1.0])))
}
fn recording(&self, reads: &ReadLog, f: &mut dyn FnMut()) {
if self.records {
reads.note(effectcraft_render::reads::ReadFile::new("/clip.png", effectcraft_render::reads::FileStamp::Virtual));
} else {
reads.mark_incomplete();
}
f()
}
}
/// The worker hands the transport the files each frame read, so its disk entry keeps them; the
/// page reads the entry back only while those files are unchanged. A frame whose files can't
/// all be named is not stored (#3184).
#[test]
fn the_worker_reports_the_files_of_each_frame_for_its_disk_entry() {
use effectcraft_project::{Comp, Footage, ItemKind, LayerSource, Project, build};
use effectcraft_render::disk_cache::{frame_entry, read_frame_entry};
use effectcraft_time::FrameRate;
let mut p = Project::default();
let f = Footage { path: "/clip.png".into(), width: 4, height: 4, duration: Tick::from_seconds_f64(1.0), has_video: true, ..Default::default() };
let fid = p.add_item("clip.png", effectcraft_color::Label::None, None, ItemKind::Footage(f));
let mut comp = Comp::new(4, 4, FrameRate::FPS_30, Tick::from_seconds_f64(1.0));
comp.layers = vec![build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (4, 4), None)].into();
let cid = p.add_item("C", effectcraft_color::Label::None, None, ItemKind::Comp(comp.into()));
let id = |mark: u64| move |path: &str| (path == "/clip.png").then_some((10, mark));
let (one, two) = (id(1), id(2));
for records in [true, false] {
let mut w = FrameServer::new(Arc::new(OneVirtualFile { records }), None);
w.set_virtual_id(Some(Arc::new(one)));
let mut m = Mirror::default();
assert!(w.handle(round(&m.sync(1, &p).unwrap())).is_empty());
assert!(w.take_files().is_none(), "no frame yet");
let out = w.handle(round(&FrameMsg::Render {
id: 1,
revision: 1,
comp: cid,
time: Tick::ZERO,
opts: Box::default(),
disk: Some(format!("{:032x}", 5u128)),
layers: false,
prefetch: vec![],
}));
let (FrameReply::Frame { width, height, .. }, Some(px)) = &out[0] else { panic!("{:?}", out[0].0) };
let files = w.take_files().expect("the frame's files");
assert!(w.take_files().is_none(), "taken once");
let vid = w.virtual_id().cloned();
let entry = frame_entry(*width, *height, px, &files, vid.as_deref());
if !records {
assert!(!files.is_complete() && entry.is_none(), "an incomplete list is not stored");
continue;
}
assert_eq!(files.files().iter().map(|f| &*f.path).collect::<Vec<_>>(), ["/clip.png"]);
let entry = entry.expect("stored");
assert_eq!(read_frame_entry(&entry, Some(&one)).map(|f| f.rgba), Some(px.clone()));
assert!(read_frame_entry(&entry, Some(&two)).is_none(), "the file was replaced: a miss");
}
}
/// No salt work at all (#3184, replacing the background salt of #3106): syncing a project with a
/// large image sequence nobody shows, patching it and rendering with the layer store never asks
/// the file hook about the sequence (the salt asked about every file at every sync). Only the
/// file the rendered buffer reads is asked about: to store its entry and, in another worker, to
/// check it before the buffer from the store is used.
#[test]
fn the_worker_asks_only_about_the_files_its_buffers_read() {
use effectcraft_project::{Comp, Footage, FootageKind, ItemKind, LayerSource, Project, build};
use effectcraft_time::FrameRate;
use std::sync::Mutex;
let mut p = Project::default();
let f = Footage { path: "/clip.png".into(), width: 4, height: 4, has_video: true, ..Default::default() };
let fid = p.add_item("clip.png", effectcraft_color::Label::None, None, ItemKind::Footage(f));
let seq = Footage {
kind: FootageKind::Sequence,
path: "/seq/f_00000.png".into(),
sequence: (0..5000).map(|i| format!("/seq/f_{i:05}.png")).collect(),
width: 4,
height: 4,
has_video: true,
..Default::default()
};
p.add_item("seq", effectcraft_color::Label::None, None, ItemKind::Footage(seq));
let mut comp = Comp::new(4, 4, FrameRate::FPS_30, Tick::from_seconds_f64(1.0));
comp.layers = vec![build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (4, 4), None)].into();
let cid = p.add_item("C", effectcraft_color::Label::None, None, ItemKind::Comp(comp.into()));
let asked: Arc<Mutex<Vec<String>>> = Arc::default();
let worker = || {
let mut w = FrameServer::new(Arc::new(OneVirtualFile { records: true }), None);
let seen = asked.clone();
w.set_virtual_id(Some(Arc::new(move |path: &str| {
seen.lock().unwrap().push(path.to_string());
(path == "/clip.png").then_some((10, 1))
})));
let store = Arc::new(PrefetchStore::new(1 << 20).with_min_ms(0));
w.set_layer_store(Some(store.clone()));
(w, store)
};
let render = |w: &mut FrameServer, revision: u64, prefetch: Vec<String>| {
let out = w.handle(round(&FrameMsg::Render { id: 1, revision, comp: cid, time: Tick::ZERO, opts: Box::default(), disk: None, layers: true, prefetch }));
let FrameReply::Frame { layer_hits, .. } = &out[0].0 else { panic!("{:?}", out[0].0) };
layer_hits.len()
};
// Worker A: synced, patched, rendered; its buffer is queued with its file.
let (mut a, store_a) = worker();
let mut m = Mirror::default();
assert!(a.handle(round(&m.sync(1, &p).unwrap())).is_empty());
let mut edited = p.clone();
edited.item_mut(cid).unwrap().name = "Renamed".into();
assert!(a.handle(round(&m.sync(2, &edited).unwrap())).is_empty());
assert!(asked.lock().unwrap().is_empty(), "syncing looks at no file");
assert_eq!(render(&mut a, 2, vec![]), 0);
let writes = store_a.take_writes();
assert!(!writes.is_empty(), "the clip's buffer is stored");
// Worker B: fetched before its frame, checked, used.
let (mut b, store_b) = worker();
let mut m = Mirror::default();
assert!(b.handle(round(&m.sync(2, &edited).unwrap())).is_empty());
for (k, e) in &writes {
assert!(store_b.provide(*k, e));
}
assert!(render(&mut b, 2, writes.iter().map(|(k, _)| format!("{k:032x}")).collect()) > 0, "served from the store");
let asked = asked.lock().unwrap();
assert!(!asked.is_empty(), "the hook names the buffer's file");
assert!(asked.iter().all(|p| p == "/clip.png"), "only the file the buffer read: {:?}", &asked[..asked.len().min(5)]);
}
+5
View File
@@ -345,6 +345,7 @@ fn warning(s: &mut Session, p: &Value) -> Result<Value> {
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
Ok(json!({"effect": uid, "warning": effectcraft_effects::warning(g, &mut |pr| ctx.value(layer, pr))}))
}
@@ -378,6 +379,7 @@ fn pick_color(s: &mut Session, p: &Value) -> Result<Value> {
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let mut r = effectcraft_render::Renderer::new(&s.project, &*s.footage, effectcraft_render::RenderOpts::default());
r.expr = s.expr.as_deref();
@@ -515,6 +517,7 @@ impl CryptoProbe {
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let params = effectcraft_effects::flatten_params(g, &mut |pr| ctx.value(layer, pr));
let aux = r.layer_aux(&ctx, layer).ok_or(NO_CRYPTO)?;
@@ -586,6 +589,7 @@ fn probe_key<'a>(s: &'a Session, p: &Value, cmd: &str) -> Result<(ProbeKey, Tick
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let frame = probe_frame(&ctx, layer, pos);
let o = s.render_opts(Default::default());
@@ -780,6 +784,7 @@ fn crypto_hit(s: &Session, p: &Value, cmd: &str) -> Result<Hit> {
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let selection = effectcraft_effects::flatten_params(g, &mut |pr| ctx.value(layer, pr)).s("selection").to_string();
let driven = prop.expr.as_ref().is_some_and(|e| e.enabled);
+1
View File
@@ -249,6 +249,7 @@ pub(crate) fn parent_fixes(s: &Session, cid: ItemId, ids: &[LayerId], parent: Op
footage: Some(s.footage.as_ref()),
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let space = |id: Option<LayerId>| -> effectcraft_geom::Mat3 {
id.and_then(|i| comp.layer(i)).filter(|l| !l.is_3d()).map(|l| ectx.layer_to_comp(l).0).unwrap_or(effectcraft_geom::Mat3::IDENTITY)
+1
View File
@@ -194,6 +194,7 @@ fn freeze(s: &mut Session, p: &Value, last: bool) -> Result<Value> {
footage: Some(s.footage.as_ref()),
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let now: Vec<(effectcraft_project::LayerId, f64)> = comp
.layers
+4
View File
@@ -73,6 +73,8 @@ struct PathInfo {
span: (f64, f64),
}
/// Reads a mask or shape path at the current time and returns its comp-space vertices, value, expression reference and layer name.
/// Errors when the property is missing, is not a path or has no vertices.
fn info(s: &Session, cid: ItemId, lid: LayerId, uid: Uid, cmd: &str) -> Result<PathInfo> {
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
@@ -87,6 +89,7 @@ fn info(s: &Session, cid: ItemId, lid: LayerId, uid: Uid, cmd: &str) -> Result<P
footage: Some(s.footage.as_ref()),
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let KV::Path(path) = ctx.value(layer, pr) else { return Err(bad(cmd, "that property is not a path")) };
if path.vertices.is_empty() {
@@ -165,6 +168,7 @@ fn pin_targets(s: &Session, p: &Value, cmd: &str) -> Result<Option<(ItemId, Vec<
footage: Some(s.footage.as_ref()),
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let mut out = vec![];
for (_, lid, uid) in pins {
+3
View File
@@ -1380,6 +1380,8 @@ fn convert_expr_to_keys(s: &mut Session, p: &Value) -> Result<Value> {
convert_one_expr(s, cid, lid, uid)
}
/// Replaces the expression on one property with keyframes sampled from its evaluated values.
/// Returns the number of properties converted.
fn convert_one_expr(s: &mut Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<Value> {
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone();
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?.clone();
@@ -1397,6 +1399,7 @@ fn convert_one_expr(s: &mut Session, cid: ItemId, lid: LayerId, uid: Uid) -> Res
footage: Some(s.footage.as_ref()),
property_overrides: &[],
media_overrides: &[],
reads: None,
};
keys.push(Keyframe::new(layer.layer_time(t), ctx.value(&layer, &pr)));
t += fd;
+11 -2
View File
@@ -39,8 +39,17 @@ pub fn puppet_eval(s: &Session, cid: ItemId, lid: LayerId, fx_uid: Uid, t: Tick)
let fx = layer.effects()?;
let index = fx.groups().position(|g| g.uid == fx_uid)?;
let g = fx.groups().nth(index)?;
let ctx =
EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None, property_overrides: &[], media_overrides: &[] };
let ctx = EvalCtx {
project: &s.project,
comp_id: cid,
comp,
time: t,
expr: s.expr.as_deref(),
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let mut r = Renderer::new(&s.project, &*s.footage, RenderOpts::default());
r.expr = s.expr.as_deref();
r.cache = Some(&s.layer_cache);
+2
View File
@@ -47,6 +47,7 @@ impl Env {
fn of(s: &Session, cid: ItemId) -> Env {
Env { time: s.time_of(cid), expr: s.expr.clone(), footage: s.footage.clone() }
}
/// The evaluation context of `comp` in `proj` at the session's current time.
fn ctx<'a>(&'a self, proj: &'a Project, cid: ItemId, comp: &'a Comp) -> EvalCtx<'a> {
EvalCtx {
project: proj,
@@ -57,6 +58,7 @@ impl Env {
footage: Some(self.footage.as_ref()),
property_overrides: &[],
media_overrides: &[],
reads: None,
}
}
}
+23 -4
View File
@@ -418,14 +418,24 @@ fn set_view_camera(s: &mut Session, p: &Value) -> Result<Value> {
Ok(json!({"view": v.id()}))
}
/// Returns the active comp's 3D view state, with the comp size, for the 3D view commands.
fn get_3d(s: &mut Session, p: &Value) -> Result<Value> {
let cid = comp_id(s, p)?;
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let (w, h) = (comp.width as f64, comp.height as f64);
let vs = s.state.views3d.get(&cid).cloned().unwrap_or_default();
let t = s.state.times.get(&cid).copied().unwrap_or(Tick::ZERO);
let ctx =
EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None, property_overrides: &[], media_overrides: &[] };
let ctx = EvalCtx {
project: &s.project,
comp_id: cid,
comp,
time: t,
expr: s.expr.as_deref(),
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let cam = vs.override_camera(w, h).unwrap_or_else(|| camera::active_camera(&ctx));
let f = cam.forward();
let lights: Vec<Value> = comp
@@ -567,8 +577,17 @@ fn auto_orient(s: &mut Session, p: &Value) -> Result<Value> {
/// through each layer's world matrix; cameras and lights contribute their position).
fn layer_points(s: &Session, cid: ItemId, ids: &[LayerId], t: Tick) -> Vec<Vec3> {
let Some(comp) = s.project.comp(cid) else { return vec![] };
let ctx =
EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None, property_overrides: &[], media_overrides: &[] };
let ctx = EvalCtx {
project: &s.project,
comp_id: cid,
comp,
time: t,
expr: s.expr.as_deref(),
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let mut out = vec![];
for l in comp.layers.iter().filter(|l| ids.contains(&l.id)) {
let m = ctx.world_matrix(l);
+18 -25
View File
@@ -215,20 +215,18 @@ impl Scanner {
}
}
/// A disk observation prepared entirely by the existing worker, including the potentially
/// expensive sequence stamps used for disk-cache keys. The UI only swaps a current snapshot.
/// A disk observation prepared entirely by the existing worker. The UI only swaps a current
/// snapshot (the disk cache needs nothing: its entries check their own files when read, #3184).
pub(crate) struct Update {
before: Arc<Project>,
after: Arc<Project>,
paths: BTreeSet<String>,
newly_missing: usize,
cache_enabled: bool,
salt: Option<(u64, u64)>,
}
/// Prepare a replacement only when availability changed. Project cloning and optional disk
/// salt computation belong on the worker; apply later rejects an outdated project snapshot.
pub(crate) fn prepare(project: &Arc<Project>, sources: &[Source], missing: &[bool], cache_enabled: bool) -> Option<Update> {
/// Prepare a replacement only when availability changed. Project cloning belongs on the
/// worker; apply later rejects an outdated project snapshot.
pub(crate) fn prepare(project: &Arc<Project>, sources: &[Source], missing: &[bool]) -> Option<Update> {
let mut after = project.clone();
let mut paths = BTreeSet::new();
let mut newly_missing = 0;
@@ -245,22 +243,17 @@ pub(crate) fn prepare(project: &Arc<Project>, sources: &[Source], missing: &[boo
if paths.is_empty() {
return None;
}
let salt = cache_enabled.then(|| (Session::footage_fingerprint(&after), crate::render::disk_cache::footage_salt(&after)));
Some(Update { before: project.clone(), after, paths, newly_missing, cache_enabled, salt })
Some(Update { before: project.clone(), after, paths, newly_missing })
}
impl Update {
/// False means a newer project or cache configuration needs a fresh observation.
/// False means a newer project needs a fresh observation.
pub fn apply(self, s: &mut Session) -> bool {
if !Arc::ptr_eq(&s.project, &self.before) || s.disk_cache.is_some() != self.cache_enabled {
if !Arc::ptr_eq(&s.project, &self.before) {
return false;
}
let dirty = s.is_dirty();
s.project = self.after;
if let Some((fingerprint, salt)) = self.salt {
s.disk_salt = Some((fingerprint, salt));
s.layer_cache.set_disk_salt(salt);
}
s.bump();
if !dirty {
s.mark_saved();
@@ -371,7 +364,7 @@ mod tests {
fn stale_observation_cannot_mark_a_replacement_missing() {
let (mut s, id) = session(Arc::new(Disk::default()));
let old = sources(&s);
let update = prepare(&s.project, &old, &[true, true], false).unwrap();
let update = prepare(&s.project, &old, &[true, true]).unwrap();
if let ItemKind::Footage(f) = &mut Arc::make_mut(&mut s.project).item_mut(id).unwrap().kind {
f.path = "/new.png".into();
}
@@ -398,9 +391,12 @@ mod tests {
assert_eq!(disk.looks.load(Ordering::Relaxed), 2, "one file per source, no scan of an unmounted folder");
}
/// An availability change does no disk-cache work (#3184): the cache stays attached and
/// no file is looked at for it, on the worker or when the UI applies the observation (it
/// used to stamp every footage file for a new layer salt).
#[test]
fn applying_prepared_cache_salt_does_not_restat_files_on_the_ui_thread() {
let dir = std::env::temp_dir().join(format!("ec-availability-salt-{}", std::process::id()));
fn applying_an_availability_change_does_no_disk_cache_work() {
let dir = std::env::temp_dir().join(format!("ec-availability-cache-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let file = dir.join("plate.png");
std::fs::write(&file, b"plate").unwrap();
@@ -413,16 +409,13 @@ mod tests {
);
let cache = crate::render::disk_cache::DiskCache::open(dir.join("cache"), 1 << 20).unwrap();
s.disk_cache = Some(cache.clone());
s.layer_cache.set_disk(Some((cache, 0)));
s.layer_cache.set_disk(Some(cache.clone()));
s.mark_saved();
let update = prepare(&s.project, &sources(&s), &[false], true).unwrap();
let expected = update.salt.unwrap().1;
// The filesystem can change again after the worker's observation. Applying it must
// retain that observation's salt, rather than scan the disk a second time on the UI.
let update = prepare(&s.project, &sources(&s), &[false]).unwrap();
std::fs::remove_file(file).unwrap();
assert_ne!(crate::render::disk_cache::footage_salt(&update.after), expected);
assert!(update.apply(&mut s));
assert_eq!(s.disk_salt.map(|(_, salt)| salt), Some(expected));
assert!(s.layer_cache.disk().is_some_and(|d| Arc::ptr_eq(&d, &cache)), "still attached");
assert_eq!(cache.file_checks(), 0, "no file looked at for the disk cache");
assert!(!s.is_dirty());
drop(s);
std::fs::remove_dir_all(dir).unwrap();
+1 -1
View File
@@ -272,7 +272,7 @@ mod tests {
let id = source.id;
let mut recovered = source.footage.clone();
recovered.path = "/new/project/plate.png".into();
let observed = super::super::availability::prepare(&s.project, std::slice::from_ref(&source), &[true], false).unwrap();
let observed = super::super::availability::prepare(&s.project, std::slice::from_ref(&source), &[true]).unwrap();
assert!(observed.apply(&mut s));
assert!(matches!(&s.project.item(id).unwrap().kind, ItemKind::Footage(f) if f.missing));
let result = apply(&mut s, vec![(source, Found::Moved(Box::new(recovered)))]);
+21 -24
View File
@@ -6,7 +6,8 @@
//! or PDF page an item shows and its interpretation (alpha, conformed frame rate, pixel aspect,
//! fields, colour). A Photoshop layer is found again by name when layers were added or removed
//! around it. The media layer then forgets the file and the layer cache is emptied, so the next
//! render reads the new pixels.
//! render reads the new pixels; File ▸ Reload Footage also deletes the disk cache entries made of
//! the file (#3184).
//!
//! The automatic part compares each footage file's size and modification time with what they
//! were when the file was last seen. The desktop app keeps a [`Watcher`]: a worker thread that
@@ -76,7 +77,6 @@ pub struct Watched {
reload: BTreeSet<String>,
sources: Arc<Vec<Source>>,
project: Option<Arc<effectcraft_project::Project>>,
cache_enabled: bool,
reload_mode: String,
stored_paths: crate::doc_path::StoredPaths,
}
@@ -85,8 +85,7 @@ impl PartialEq for Watched {
/// Compare watched inputs, excluding unrelated project edits whose latest Arc is refreshed
/// separately without waking the worker or restarting partial sequence scans.
fn eq(&self, other: &Self) -> bool {
self.cache_enabled == other.cache_enabled
&& self.stored_paths == other.stored_paths
self.stored_paths == other.stored_paths
&& self.reload_mode == other.reload_mode
&& self.reload == other.reload
&& (Arc::ptr_eq(&self.sources, &other.sources) || self.sources == other.sources)
@@ -100,7 +99,6 @@ pub fn watched(s: &Session) -> Watched {
reload: automatic_paths(s),
sources: Arc::new(if s.check_footage_on_open { availability::sources(s) } else { Vec::new() }),
project: Some(s.project.clone()),
cache_enabled: s.disk_cache.is_some(),
reload_mode: s.prefs.import.auto_reload_footage.clone(),
stored_paths: s.stored_paths.clone(),
}
@@ -139,10 +137,10 @@ pub struct Scan {
}
impl Scan {
/// Prepare availability and cache state before the session applies the observation.
/// Prepare the availability change before the session applies the observation.
/// Automatic callers construct this result on the worker; explicit commands run inline.
fn new(watched: Watched, observation: Observation, policy: ReloadPolicy) -> Self {
let update = watched.project.as_ref().and_then(|p| availability::prepare(p, &watched.sources, &observation.missing, watched.cache_enabled));
let update = watched.project.as_ref().and_then(|p| availability::prepare(p, &watched.sources, &observation.missing));
Self { watched, observation, update, policy }
}
@@ -281,7 +279,17 @@ fn reload(s: &mut Session, ids: &[ItemId], undoable: bool) -> crate::Result<(usi
}
let n = updates.len();
apply_reloads(s, updates, undoable)?;
forget(s, &paths);
forget_media(s, &paths);
// An explicit reload also reads files again that kept their size and time (the check on hit
// can't tell those apart): their entries in the disk cache go (#3184), when the cache is
// switched on if it is off now. An automatic one only follows a changed stamp, which the
// check catches.
if undoable {
match &s.disk_cache {
Some(dc) => dc.drop_files(paths.iter().map(String::as_str)),
None => s.pending_disk_drops.extend(paths),
}
}
Ok((n, failed))
}
@@ -320,14 +328,9 @@ fn apply_reloads(s: &mut Session, updates: Vec<(ItemId, Footage)>, undoable: boo
Ok(())
}
/// Make the media layer and the layer cache let go of `paths`, so renders read them again.
fn forget(s: &mut Session, paths: &BTreeSet<String>) {
forget_media(s, paths);
s.refresh_disk_salt(true);
}
/// Invalidate decoded and rendered buffers without probing the disk for a new salt.
/// Availability updates already carry their worker-computed salt; content reload refreshes it.
/// Make the media layer and the layer cache let go of `paths`, so renders read them again. Disk
/// cache entries check the files they were made of when read (#3184); File ▸ Reload Footage
/// also drops the entries of its files ([`reload`]).
pub(crate) fn forget_media(s: &mut Session, paths: &BTreeSet<String>) {
for p in paths {
s.footage.forget(p);
@@ -374,7 +377,7 @@ pub fn apply_stamps(s: &mut Session, now: Vec<(String, Option<Stamp>)>) -> crate
return Ok(json!({"files": 0, "reloaded": 0}));
}
// Changed frames of image sequences (their items keep their metadata).
forget(s, &changed);
forget_media(s, &changed);
let files = changed.len();
s.toast(if reloaded > 0 {
format!("Reloaded {reloaded} footage item(s) changed on disk")
@@ -391,7 +394,6 @@ pub(crate) fn command(s: &mut Session, _: &Value) -> crate::Result<Value> {
reload: reload_paths(s),
sources: Arc::new(availability::sources(s)),
project: Some(s.project.clone()),
cache_enabled: s.disk_cache.is_some(),
reload_mode: s.prefs.import.auto_reload_footage.clone(),
stored_paths: s.stored_paths.clone(),
};
@@ -452,12 +454,7 @@ impl Watcher {
pub fn update(&mut self, s: &Session) {
let same_sources = availability::sources_match(s, &self.paths.sources);
let same_data = self.paths.project.as_ref().is_some_and(|before| data_paths(before).eq(data_paths(&s.project)));
if same_sources
&& same_data
&& self.paths.stored_paths == s.stored_paths
&& self.paths.reload_mode == s.prefs.import.auto_reload_footage
&& self.paths.cache_enabled == s.disk_cache.is_some()
{
if same_sources && same_data && self.paths.stored_paths == s.stored_paths && self.paths.reload_mode == s.prefs.import.auto_reload_footage {
let project = Some(s.project.clone());
self.paths.project = project.clone();
self.pending.lock().unwrap_or_else(PoisonError::into_inner).paths.project = project;
+5 -12
View File
@@ -97,7 +97,7 @@ pub use error::{EngineError, Result};
pub use event::Event;
pub use fonts::{FontRow, font_features, font_menu, font_preview, font_styles, text_families};
pub use services::{FsServices, Importer, Services};
pub use session::{DiskSaltJob, upgrade_effects};
pub use session::upgrade_effects;
pub struct Session {
pub project: Arc<Project>,
@@ -171,6 +171,9 @@ pub struct Session {
pub crypto_probe: Arc<std::sync::Mutex<commands::CryptoProbeSlot>>,
/// The persistent disk cache (Settings ▸ Media & Disk Cache), when enabled.
pub disk_cache: Option<Arc<effectcraft_render::disk_cache::DiskCache>>,
/// Files File ▸ Reload Footage reloaded while the disk cache was off: their entries are
/// dropped when it is switched on ([`Session::configure_disk_cache`]), before it is used.
pub pending_disk_drops: std::collections::BTreeSet<String>,
/// Settings (Preferences).
pub prefs: prefs::Prefs,
/// Bumped whenever settings change (frontends re-apply theme, labels…).
@@ -254,14 +257,6 @@ pub struct Session {
/// Each footage file's size and modification time when last seen (Settings ▸ Import ▸
/// Reload Footage Changed on Disk, [`footage_reload`]).
pub footage_stamps: std::collections::HashMap<String, footage_reload::Stamp>,
/// The disk cache's footage salt and the footage it was taken for ([`Session::refresh_disk_salt`]).
pub disk_salt: Option<(u64, u64)>,
/// Take the disk cache's footage salt on a worker thread ([`Session::refresh_disk_salt`]):
/// frontends with an event loop (the desktop app) turn this on and call
/// [`Session::poll_disk_salt`] each frame; headless sessions take it at once.
pub disk_salt_background: bool,
/// The footage salt being taken in the background.
pub disk_salt_job: Option<DiskSaltJob>,
/// The footage files effects read the channels of, as last given to the footage source
/// ([`Session::refresh_aux_footage`]); `None` = not given yet.
pub aux_footage: Option<Vec<String>>,
@@ -356,6 +351,7 @@ impl Default for Session {
layer_cache: Arc::new(LayerCache::default()),
crypto_probe: Default::default(),
disk_cache: None,
pending_disk_drops: Default::default(),
prefs: prefs::Prefs::default(),
prefs_revision: 0,
config: None,
@@ -391,9 +387,6 @@ impl Default for Session {
check_footage_on_open: false,
stored_paths: Default::default(),
footage_stamps: Default::default(),
disk_salt: None,
disk_salt_background: false,
disk_salt_job: None,
aux_footage: None,
script_calls: ScriptCalls::default(),
}
+1 -1
View File
@@ -10,4 +10,4 @@ mod dispatch;
mod edit;
mod render;
pub use edit::{DiskSaltJob, upgrade_effects};
pub use edit::upgrade_effects;
-117
View File
@@ -97,114 +97,10 @@ impl Session {
if Arc::ptr_eq(&self.project, &self.saved_project) {
self.saved_revision = self.revision;
}
self.refresh_disk_salt(false);
self.refresh_aux_footage();
self.events.push(Event::ProjectChanged { revision: self.revision });
}
/// Salt the disk cache's layer keys with the project's footage files
/// ([`effectcraft_render::disk_cache::footage_salt`]) and attach the disk cache to the layer
/// cache with it. Taking it reads the size and time of every file, every frame of an image
/// sequence included, so it is taken again only when the footage items changed, or with
/// `force` (a project opened, footage reloaded, the disk cache attached), not at every edit:
/// with a few large sequences that cost each keystroke in a text field over half a second
/// (#1926).
///
/// With [`Session::disk_salt_background`] the files are read on a worker thread: an imported
/// or opened project with thousands of sequence frames on a network share took half a minute
/// to stamp, all of it on the UI thread. Until [`Session::poll_disk_salt`] picks the salt up,
/// renders use the memory cache only (keys under the old salt may name other files, or
/// another project's items) and this returns `None`.
pub fn refresh_disk_salt(&mut self, force: bool) -> Option<u64> {
let Some(dc) = self.disk_cache.clone() else {
self.disk_salt = None;
self.disk_salt_job = None;
return None;
};
self.poll_disk_salt();
let footage = Self::footage_fingerprint(&self.project);
if !force {
if let Some((seen, salt)) = self.disk_salt
&& seen == footage
{
return Some(salt);
}
if self.disk_salt_job.as_ref().is_some_and(|j| j.footage == footage) {
return None;
}
}
if self.disk_salt_background && !cfg!(target_arch = "wasm32") {
let project = self.project.clone();
let slot: Arc<std::sync::Mutex<Option<u64>>> = Arc::default();
let out = slot.clone();
let spawned = std::thread::Builder::new().name("ec-disk-salt".into()).spawn(move || {
let salt = effectcraft_render::disk_cache::footage_salt(&project);
if let Ok(mut s) = out.lock() {
*s = Some(salt);
}
});
// Without a thread the salt is taken here, as before.
if let Ok(thread) = spawned {
self.disk_salt = None;
self.disk_salt_job = Some(DiskSaltJob { footage, slot, thread });
self.layer_cache.set_disk(None);
return None;
}
}
self.disk_salt_job = None;
let salt = effectcraft_render::disk_cache::footage_salt(&self.project);
self.disk_salt = Some((footage, salt));
self.layer_cache.set_disk(Some((dc, salt)));
Some(salt)
}
/// Pick up a footage salt taken in the background ([`Session::refresh_disk_salt`]) and attach
/// the disk cache with it. Returns whether one is still being taken (frontends call this each
/// frame while it is).
pub fn poll_disk_salt(&mut self) -> bool {
let Some(job) = &self.disk_salt_job else { return false };
// (looked at first: a worker that has finished has left its salt in the slot)
let finished = job.thread.is_finished();
let salt = job.slot.lock().ok().and_then(|mut s| s.take());
let footage = job.footage;
match salt {
Some(salt) => {
self.disk_salt_job = None;
if let Some(dc) = self.disk_cache.clone() {
self.disk_salt = Some((footage, salt));
self.layer_cache.set_disk(Some((dc, salt)));
}
false
}
None if finished => {
// The worker stopped without a salt: the disk cache stays off until the next
// edit takes it again.
log::warn!("disk cache: the footage files could not be stamped; the disk cache is off until the next edit");
self.disk_salt_job = None;
false
}
None => true,
}
}
/// In-memory identity of the footage settings represented by a disk salt. Workers use the
/// same fingerprint when preparing availability changes without disk access on the UI.
pub(crate) fn footage_fingerprint(project: &effectcraft_project::Project) -> u64 {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
for (id, item) in &project.items {
if let effectcraft_project::ItemKind::Footage(f) = &item.kind {
id.hash(&mut h);
format!("{f:?}").hash(&mut h);
}
if let Some(proxy) = &item.proxy {
id.hash(&mut h);
format!("{proxy:?}").hash(&mut h);
}
}
h.finish()
}
/// Set a comp's Resolution. As in After Effects it is saved with the project (which becomes
/// modified) but is no undo step, and it changes no pixels: the comp's cached frames stay
/// valid ([`effectcraft_project::Comp::same_pixels`]).
@@ -322,11 +218,6 @@ impl Session {
}
self.layer_cache.clear();
self.project = Arc::new(p);
// Its files may have changed since they were last stamped.
self.disk_salt = None;
// A salt still being taken is for the old project's files: dropped, so that the next refresh
// stamps the new project anew (a matching fingerprint would otherwise keep the stale job).
self.disk_salt_job = None;
// The set of footage effects read the channels of is sent again, even when it is empty.
self.aux_footage = None;
self.history = History::default();
@@ -389,11 +280,3 @@ pub fn upgrade_effects(p: &mut Project) {
p.next_id = next;
p.schema = effectcraft_project::SCHEMA_VERSION;
}
/// A footage salt being taken on a worker thread ([`Session::refresh_disk_salt`]).
pub struct DiskSaltJob {
/// The footage it is taken for ([`Session::footage_fingerprint`]).
footage: u64,
slot: Arc<std::sync::Mutex<Option<u64>>>,
thread: std::thread::JoinHandle<()>,
}
@@ -137,10 +137,6 @@ impl Session {
// over the store of a roto test running beside it. Serialise them with the roto tests.
#[cfg(test)]
crate::tests::roto::hold_roto_cache_test_lock();
// Another setting changed (the disk cache stays as it is): its salt still holds, and
// stamping every footage file again would cost each settings change seconds with large
// sequences.
let kept = self.disk_cache.as_ref().is_some_and(|c| c.folder() == folder.as_path());
let dc = match &self.disk_cache {
Some(c) if c.folder() == folder.as_path() => {
c.set_max_bytes(max);
@@ -162,11 +158,17 @@ impl Session {
}
},
};
// Files reloaded while the cache was off: their entries go before anything is read from
// it (until the writer has deleted them they are misses, and no frame is linked).
if !self.pending_disk_drops.is_empty() {
dc.drop_files(std::mem::take(&mut self.pending_disk_drops));
}
self.disk_cache = Some(dc.clone());
// The reserve applies from the start (a new folder's watcher is idle).
self.poll_disk_space();
// Attaches the disk cache to the layer cache once the footage salt is known.
self.refresh_disk_salt(!kept);
// Used at once (no file is looked at first: each entry checks the source files it lists
// when it is read, #3184).
self.layer_cache.set_disk(Some(dc.clone()));
// Roto Brush / Object Matte segmentations persist there too (held weakly: switching the
// disk cache off drops it).
let store: std::sync::Arc<dyn effectcraft_effects::roto::SegStore> = dc.clone();
+1
View File
@@ -214,6 +214,7 @@ fn rendered_matte_of_pick(s: &mut Session, x: f64, y: f64, scale: f64) -> (Strin
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let buf = s.renderer(effectcraft_render::RenderOpts { scale, ..Default::default() }).content_buf(&ctx, layer).unwrap();
let (px, py) = buf.to_px([x, y]);
+263 -89
View File
@@ -76,11 +76,13 @@ fn disk_cache_settings_persistence_and_invalidation() {
assert!(s2.layer_cache.disk().is_none());
}
/// The comp key comes from the project in memory: a footage file that changes on disk doesn't
/// change it (the entries check their own files, #3184); edits that change pixels do.
#[test]
fn footage_changes_change_the_salt() {
let dir = folder("salt");
fn comp_content_keys_come_from_memory() {
let dir = folder("keys");
let mut s = session(&dir);
let foot = std::env::temp_dir().join(format!("effectcraft-salt-{}.png", std::process::id()));
let foot = std::env::temp_dir().join(format!("effectcraft-keys-{}.png", std::process::id()));
std::fs::write(&foot, b"one").unwrap();
s.edit("add", None, |p, _| {
let f = effectcraft_project::Footage {
@@ -108,12 +110,10 @@ fn footage_changes_change_the_salt() {
Ok(())
})
.unwrap();
let a = effectcraft_render::disk_cache::footage_salt(&s.project);
std::fs::write(&foot, b"two, longer").unwrap();
let b = effectcraft_render::disk_cache::footage_salt(&s.project);
assert_ne!(a, b);
let cid = project(&mut s);
let k1 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid);
std::fs::write(&foot, b"two, longer").unwrap();
assert_eq!(effectcraft_render::disk_cache::comp_content_key(&s.project, cid), k1, "no file is looked at");
s.execute("prop.set", json!({"layer": "#1", "path": "transform/opacity", "value": 50})).unwrap();
let k2 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid);
assert_ne!(k1, k2);
@@ -137,66 +137,51 @@ fn footage_changes_change_the_salt() {
assert_ne!(set_proxy(&mut s, false), k3);
}
/// The desktop app stamps footage files for the disk cache's salt on a worker thread: importing an
/// After Effects project with thousands of sequence frames on a network share stamped them all
/// inside the import's edit, freezing the UI for half a minute. Until the salt lands the disk
/// cache sits out (keys under the old salt may name other files); a newer footage change
/// supersedes a salt still being taken; a settings change keeps the salt.
/// No salt and no scan (#3184, replacing the background footage salt of #3106): the disk cache
/// is attached to the layer cache as soon as it is switched on, and stays attached through
/// edits that add footage (a large image sequence included), settings changes and opening a
/// project, none of which looks at a single file for it. The first render may use it at once.
#[test]
fn footage_salt_is_taken_off_the_ui_thread_and_the_disk_cache_waits_for_it() {
let dir = folder("bg-salt");
fn the_disk_cache_is_attached_at_once_and_edits_do_no_file_work() {
let dir = folder("attach");
let mut s = session(&dir);
s.disk_salt_background = true;
let at = |n: &str| std::env::temp_dir().join(format!("effectcraft-bg-salt-{}-{n}.png", std::process::id()));
let add = |s: &mut Session, path: &std::path::Path| {
let f = effectcraft_project::Footage {
path: path.to_string_lossy().to_string(),
kind: effectcraft_project::FootageKind::Still,
width: 1,
height: 1,
has_video: true,
..Default::default()
};
s.edit("add", None, |p, _| {
p.add_item("f.png", effectcraft_color::Label::None, None, effectcraft_project::ItemKind::Footage(f));
Ok(())
})
.unwrap();
let dc = s.disk_cache.clone().unwrap();
let attached = |s: &Session| s.layer_cache.disk().is_some_and(|d| std::sync::Arc::ptr_eq(&d, &dc));
assert!(attached(&s), "attached at once");
// A sequence of 5000 frames that don't even exist.
let seq_dir = std::env::temp_dir().join(format!("effectcraft-no-scan-{}", std::process::id()));
let f = effectcraft_project::Footage {
path: seq_dir.join("f_00000.exr").to_string_lossy().to_string(),
kind: effectcraft_project::FootageKind::Sequence,
sequence: (0..5000).map(|i| seq_dir.join(format!("f_{i:05}.exr")).to_string_lossy().to_string()).collect(),
width: 8,
height: 8,
has_video: true,
..Default::default()
};
let wait = |s: &mut Session| {
let t0 = std::time::Instant::now();
while s.poll_disk_salt() {
assert!(t0.elapsed().as_secs() < 20, "the salt never landed");
std::thread::sleep(std::time::Duration::from_millis(5));
}
};
assert!(s.layer_cache.disk().is_some(), "attached with its salt");
std::fs::write(at("a"), b"one").unwrap();
add(&mut s, &at("a"));
// The edit returned before the files were stamped, and the disk cache waits for the salt.
assert!(s.disk_salt.is_none() && s.layer_cache.disk().is_none());
// Edits that leave the footage alone don't start over (and may pick up the salt meanwhile).
s.execute("comp.new", json!({"name": "C", "width": 8, "height": 8, "frameRate": 30, "duration": 1})).unwrap();
let r = s.refresh_disk_salt(false);
assert!(r.is_none() || r == Some(effectcraft_render::disk_cache::footage_salt(&s.project)), "{r:?}");
// New footage while it is taken: the salt for the newest footage lands.
std::fs::write(at("b"), b"two").unwrap();
add(&mut s, &at("b"));
wait(&mut s);
let salt = effectcraft_render::disk_cache::footage_salt(&s.project);
assert_eq!(s.disk_salt.map(|(_, salt)| salt), Some(salt));
assert_eq!(s.refresh_disk_salt(false), Some(salt));
assert!(s.layer_cache.disk().is_some(), "attached again");
// Another setting keeps the salt and the disk cache attached.
s.edit("add", None, |p, _| {
p.add_item("seq", effectcraft_color::Label::None, None, effectcraft_project::ItemKind::Footage(f));
Ok(())
})
.unwrap();
assert!(attached(&s), "an import keeps it attached");
let cid = project(&mut s);
s.execute("prop.set", json!({"layer": "#1", "path": "transform/opacity", "value": 50})).unwrap();
s.execute_checked("prefs.set", json!({"key": "disk.diskCacheMaxGb", "value": 30})).unwrap();
assert!(s.layer_cache.disk().is_some() && s.disk_salt.map(|(_, salt)| salt) == Some(salt));
// Turned off and on again: stamped anew, in the background.
assert!(attached(&s), "edits and other settings keep it attached");
let same = (*s.project).clone();
s.replace_project(same, None);
assert!(attached(&s), "opening a project keeps it attached");
// The first render uses it: a solid reads no file, so nothing is checked either.
s.render(cid, s.time(), RenderOpts::default());
dc.flush();
assert!(dc.stats().entries > 0, "written by the first render");
assert_eq!(dc.file_checks(), 0, "no file looked at for the disk cache");
// Switched off and on: attached again at once.
s.execute_checked("prefs.set", json!({"key": "disk.diskCacheEnabled", "value": false})).unwrap();
assert!(s.layer_cache.disk().is_none());
s.execute_checked("prefs.set", json!({"key": "disk.diskCacheEnabled", "value": true})).unwrap();
wait(&mut s);
assert_eq!(s.refresh_disk_salt(false), Some(salt));
let _ = std::fs::remove_file(at("a"));
let _ = std::fs::remove_file(at("b"));
assert!(s.layer_cache.disk().is_some());
}
/// Marker edits keep the disk key (the sound of a playing preview must not wait for frames to
@@ -258,38 +243,227 @@ fn protected_markers_with_a_duration_change_the_content_key() {
assert_ne!(key(&s), k1, "its duration");
}
/// A project opened while a footage salt is still being taken must not keep that salt when it
/// names the same footage: its files may have changed since the worker read them.
/// With expressions the comp key covers what they can read, not the whole project (#3184,
/// #1492): an expression reading only its own comp keeps the key through edits of another comp
/// and of unused footage; `comp("Other")` makes "Other" count (and which comps have that name),
/// not a third comp; a computed name counts every item.
#[test]
fn opening_a_project_drops_the_footage_salt_being_taken() {
let dir = folder("bg-salt-open");
let mut s = session(&dir);
s.disk_salt_background = true;
let path = std::env::temp_dir().join(format!("effectcraft-bg-salt-open-{}.png", std::process::id()));
std::fs::write(&path, b"one").unwrap();
let f = effectcraft_project::Footage {
path: path.to_string_lossy().to_string(),
kind: effectcraft_project::FootageKind::Still,
width: 1,
height: 1,
has_video: true,
..Default::default()
fn expressions_reach_only_what_they_name() {
let mut s = Session::default();
let comp = |s: &mut Session, name: &str| {
let c = s.execute("comp.new", json!({"name": name, "width": 8, "height": 8, "frameRate": 30, "duration": 1})).unwrap()["comp"].as_u64().unwrap();
let l = s.execute("layer.newSolid", json!({"comp": c, "color": "#ff0000"})).unwrap()["layer"].as_u64().unwrap();
(effectcraft_project::ItemId(c), (c, l))
};
s.edit("add", None, |p, _| {
p.add_item("f.png", effectcraft_color::Label::None, None, effectcraft_project::ItemKind::Footage(f));
let (main, ml) = comp(&mut s, "Main");
let (_, ol) = comp(&mut s, "Other");
let (third, tl) = comp(&mut s, "Third");
let unused = s.edit("add", None, |p, _| {
let f = effectcraft_project::Footage { path: "/nowhere/a.png".into(), has_video: true, ..Default::default() };
Ok(p.add_item("a.png", effectcraft_color::Label::None, None, effectcraft_project::ItemKind::Footage(f)))
});
let unused = unused.unwrap();
let key = |s: &Session| effectcraft_render::disk_cache::comp_content_key(&s.project, main);
let expr = |s: &mut Session, (comp, layer): (u64, u64), text: &str| {
s.execute("prop.setExpression", json!({"comp": comp, "layer": layer, "path": "transform/opacity", "expression": text})).unwrap();
};
let opacity = |s: &mut Session, (comp, layer): (u64, u64), v: f64| {
s.execute("prop.set", json!({"comp": comp, "layer": layer, "path": "transform/opacity", "value": v})).unwrap();
};
let repath = |s: &mut Session, path: &str| {
let path = path.to_string();
s.edit("repath", None, move |p, _| {
if let Some(effectcraft_project::ItemKind::Footage(f)) = p.item_mut(unused).map(|i| &mut i.kind) {
f.path = path;
}
Ok(())
})
.unwrap();
};
// Its own comp only.
expr(&mut s, ml, "thisComp.layer(1).transform.rotation + wiggle(1, 1)[0]");
expr(&mut s, ol, "wiggle(2, 10)");
let k0 = key(&s);
opacity(&mut s, ol, 40.0);
expr(&mut s, tl, "time * 10");
repath(&mut s, "/nowhere/b.png");
s.execute("comp.addMarker", json!({"comp": third.0, "time": 0.5})).unwrap();
assert_eq!(key(&s), k0, "edits elsewhere keep the key");
// An edit in the comp itself changes it.
opacity(&mut s, ml, 30.0);
assert_ne!(key(&s), k0);
// `comp("Other")`: Other counts, Third doesn't.
expr(&mut s, ml, "comp(\"Other\").layer(1).transform.opacity");
let k1 = key(&s);
expr(&mut s, ol, "wiggle(3, 10)");
let k2 = key(&s);
assert_ne!(k2, k1, "the comp it names");
opacity(&mut s, tl, 20.0);
repath(&mut s, "/nowhere/c.png");
assert_eq!(key(&s), k2, "a comp it doesn't name");
// Another comp taking the name changes what it may read.
s.edit("rename", None, |p, _| {
p.item_mut(third).unwrap().name = "Other".into();
Ok(())
})
.unwrap();
// The file changes after the edit started the salt, before the same footage is opened again.
std::fs::write(&path, b"a longer content").unwrap();
let same = (*s.project).clone();
s.replace_project(same, None);
let t0 = std::time::Instant::now();
while s.poll_disk_salt() {
assert!(t0.elapsed().as_secs() < 20, "the salt never landed");
std::thread::sleep(std::time::Duration::from_millis(5));
let k3 = key(&s);
assert_ne!(k3, k2, "a second comp named Other");
// A computed name reaches every item.
expr(&mut s, ml, "comp(\"Oth\" + \"er\").layer(1).transform.opacity");
let k4 = key(&s);
repath(&mut s, "/nowhere/d.png");
assert_ne!(key(&s), k4, "unused footage counts with a computed name");
}
/// Markers count only for the comps whose expressions can read them (#1083): an expression in
/// another comp that reads markers leaves this comp's markers out of its key.
#[test]
fn markers_count_only_where_an_expression_can_read_them() {
let mut s = Session::default();
s.execute("comp.new", json!({"name": "M", "width": 8, "height": 8, "frameRate": 30, "duration": 2})).unwrap();
let cid = s.active_comp_id().unwrap();
s.execute("layer.newSolid", json!({"color": "#ff0000"})).unwrap();
let other = s.execute("comp.new", json!({"name": "N", "width": 8, "height": 8, "frameRate": 30, "duration": 2})).unwrap()["comp"].as_u64().unwrap();
let ol = s.execute("layer.newSolid", json!({"comp": other, "color": "#ff0000"})).unwrap()["layer"].as_u64().unwrap();
s.execute("prop.setExpression", json!({"comp": other, "layer": ol, "path": "transform/opacity", "expression": "thisComp.marker.numKeys"})).unwrap();
let key = |s: &Session| effectcraft_render::disk_cache::comp_content_key(&s.project, cid);
let k0 = key(&s);
s.execute("comp.addMarker", json!({"comp": cid.0, "time": 0.5, "comment": "beat"})).unwrap();
assert_eq!(key(&s), k0, "no expression of M reads its markers");
s.execute("prop.setExpression", json!({"comp": other, "layer": ol, "path": "transform/opacity", "expression": "comp(\"M\").marker.numKeys"})).unwrap();
let other_key = |s: &Session| effectcraft_render::disk_cache::comp_content_key(&s.project, effectcraft_project::ItemId(other));
let n0 = other_key(&s);
s.execute("comp.addMarker", json!({"comp": cid.0, "time": 1.0})).unwrap();
assert_ne!(other_key(&s), n0, "N reads M's markers");
assert_eq!(key(&s), k0, "M itself still doesn't");
}
/// File ▸ Reload Footage drops the disk entries made of the item's files (#3184): a file
/// replaced by one with the same size and modification time is read again (the check on hit
/// can't tell), and entries of other files stay.
#[test]
fn reload_footage_drops_the_entries_of_its_files() {
use crate::render::reads::{FileStamp, ReadFile, ReadFiles};
struct Probe;
impl crate::Importer for Probe {
fn probe(&self, path: &str) -> std::result::Result<effectcraft_project::Footage, String> {
Ok(effectcraft_project::Footage {
path: path.into(),
kind: effectcraft_project::FootageKind::Still,
width: 1,
height: 1,
has_video: true,
..Default::default()
})
}
}
let now = effectcraft_render::disk_cache::footage_salt(&s.project);
assert_eq!(s.disk_salt.map(|(_, salt)| salt), Some(now), "the salt reflects the file as it is now");
let _ = std::fs::remove_file(&path);
let dir = folder("reload");
let mut s = Session { importer: Some(std::sync::Arc::new(Probe)), ..Default::default() };
s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": &dir, "disk.diskCacheEnabled": true}})).unwrap();
let src = std::path::Path::new(&dir).join("src");
std::fs::create_dir_all(&src).unwrap();
let mut add = |name: &str| {
let p = src.join(name);
std::fs::write(&p, b"aa").unwrap();
let path = p.to_string_lossy().to_string();
let f = effectcraft_project::Footage {
path: path.clone(),
kind: effectcraft_project::FootageKind::Still,
width: 1,
height: 1,
has_video: true,
..Default::default()
};
let id = s.edit("add", None, |p, _| Ok(p.add_item(name, effectcraft_color::Label::None, None, effectcraft_project::ItemKind::Footage(f)))).unwrap();
(id, path)
};
let (a, a_path) = add("a.png");
let (_, b_path) = add("b.png");
let dc = s.disk_cache.clone().unwrap();
let read = |p: &str| -> ReadFiles { [ReadFile::new(p, FileStamp::native(p).unwrap())].into_iter().collect() };
dc.put_frame_now(1, 1, 1, &[1; 4], &read(&a_path));
dc.put_frame_now(2, 1, 1, &[2; 4], &read(&b_path));
dc.put_layer_now(3, &effectcraft_effects::Buf { img: effectcraft_raster::Image::new(1, 1), offset: [0.0; 2], scale: 1.0 }, &read(&a_path));
// Other bytes, same size, the old time back.
let t = std::fs::metadata(&a_path).unwrap().modified().unwrap();
std::fs::write(&a_path, b"AA").unwrap();
std::fs::File::options().append(true).open(&a_path).unwrap().set_modified(t).unwrap();
assert!(dc.contains(crate::render::disk_cache::Kind::Frame, 1));
s.execute_checked("file.reloadFootage", json!({"item": a.0})).unwrap();
dc.flush();
assert!(dc.get_frame(1, None).is_none(), "the reloaded file's frame renders again");
assert!(dc.get_layer(3, None).is_none(), "and its layer buffer");
assert!(dc.get_frame(2, None).is_some(), "another file's frame stays");
let _ = std::fs::remove_dir_all(&dir);
}
/// Reload Footage while the disk cache is off (#3184): when it is switched on later in the
/// session, the entries made of the reloaded files are dropped before anything is read from it
/// (a file replaced by one with the same size and time would pass the check on hit); entries of
/// other files stay.
#[test]
fn reload_footage_while_the_disk_cache_is_off_drops_its_entries_when_it_is_on() {
use crate::render::disk_cache::{DiskCache, Kind};
use crate::render::reads::{FileStamp, ReadFile, ReadFiles};
struct Probe;
impl crate::Importer for Probe {
fn probe(&self, path: &str) -> std::result::Result<effectcraft_project::Footage, String> {
Ok(effectcraft_project::Footage {
path: path.into(),
kind: effectcraft_project::FootageKind::Still,
width: 1,
height: 1,
has_video: true,
..Default::default()
})
}
}
let dir = folder("reload-off");
let src = std::path::Path::new(&dir).join("src");
std::fs::create_dir_all(&src).unwrap();
let mut s = Session { importer: Some(std::sync::Arc::new(Probe)), ..Default::default() };
let mut add = |name: &str| {
let p = src.join(name);
std::fs::write(&p, b"aa").unwrap();
let path = p.to_string_lossy().to_string();
let f = effectcraft_project::Footage {
path: path.clone(),
kind: effectcraft_project::FootageKind::Still,
width: 1,
height: 1,
has_video: true,
..Default::default()
};
let id = s.edit("add", None, |p, _| Ok(p.add_item(name, effectcraft_color::Label::None, None, effectcraft_project::ItemKind::Footage(f)))).unwrap();
(id, path)
};
let (a, a_path) = add("a.png");
let (_, b_path) = add("b.png");
// Entries from an earlier session in the cache folder.
let read = |p: &str| -> ReadFiles { [ReadFile::new(p, FileStamp::native(p).unwrap())].into_iter().collect() };
{
let dc = DiskCache::open(&dir, 1 << 30).unwrap();
dc.put_frame_now(1, 1, 1, &[1; 4], &read(&a_path));
dc.put_frame_now(2, 1, 1, &[2; 4], &read(&b_path));
dc.put_layer_now(3, &effectcraft_effects::Buf { img: effectcraft_raster::Image::new(1, 1), offset: [0.0; 2], scale: 1.0 }, &read(&a_path));
}
// Other bytes, same size, the old time back; reloaded while the cache is off.
let t = std::fs::metadata(&a_path).unwrap().modified().unwrap();
std::fs::write(&a_path, b"AA").unwrap();
std::fs::File::options().append(true).open(&a_path).unwrap().set_modified(t).unwrap();
assert!(s.disk_cache.is_none());
s.execute_checked("file.reloadFootage", json!({"item": a.0})).unwrap();
// Switched on: the reloaded file's entries are misses from the start, and then gone.
s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": &dir, "disk.diskCacheEnabled": true}})).unwrap();
let dc = s.disk_cache.clone().unwrap();
assert!(dc.get_frame(1, None).is_none(), "the reloaded file's frame renders again");
assert!(dc.get_layer(3, None).is_none(), "and its layer buffer");
dc.flush();
assert!(!dc.contains(Kind::Frame, 1) && !dc.contains(Kind::Layer, 3));
assert!(dc.get_frame(2, None).is_some(), "another file's frame stays");
assert!(s.pending_disk_drops.is_empty(), "applied once");
drop(dc);
drop(s);
let _ = std::fs::remove_dir_all(&dir);
}
+12 -2
View File
@@ -40,7 +40,17 @@ fn close3(a: [f64; 3], b: [f64; 3], tol: f64) -> bool {
fn frame(s: &Session, id: u64) -> (Vec3, Vec3) {
let c = cid(s);
let comp = s.project.comp(c).unwrap();
let ctx = EvalCtx { project: &s.project, comp_id: c, comp, time: Tick::ZERO, expr: None, footage: None, property_overrides: &[], media_overrides: &[] };
let ctx = EvalCtx {
project: &s.project,
comp_id: c,
comp,
time: Tick::ZERO,
expr: None,
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let (e, f, _) = layer_frame(&ctx, comp.layer(LayerId(id)).unwrap());
(e, f)
}
@@ -244,7 +254,7 @@ fn parented_camera_keeps_its_frame_under_an_offset_null() {
fn frame_at(s: &Session, id: u64, t: Tick) -> (Vec3, Vec3, Vec3) {
let c = cid(s);
let comp = s.project.comp(c).unwrap();
let ctx = EvalCtx { project: &s.project, comp_id: c, comp, time: t, expr: None, footage: None, property_overrides: &[], media_overrides: &[] };
let ctx = EvalCtx { project: &s.project, comp_id: c, comp, time: t, expr: None, footage: None, property_overrides: &[], media_overrides: &[], reads: None };
layer_frame(&ctx, comp.layer(LayerId(id)).unwrap())
}
+11 -10
View File
@@ -269,30 +269,31 @@ fn dpx_and_cineon_runs_import_as_sequences() {
assert_eq!(got, [("plate.[0001-0003].dpx".to_string(), 3), ("scan_[01-02].cin".to_string(), 2)], "{r}");
}
/// #1926: an edit that leaves the footage alone doesn't stamp every frame of its image sequences
/// again for the disk cache's salt (with large sequences each keystroke in a text field took over
/// half a second); Reload Footage does, and so does new footage.
/// #1926, #3184: an edit does no file work at all for the disk cache (it used to stamp every
/// frame of the project's image sequences for a salt; with large sequences each keystroke in a
/// text field took over half a second): the cache stays attached and no file is looked at, after
/// edits, after a frame of the sequence was rewritten, after Reload Footage and new footage.
/// Entries check the files they list when they are read instead.
#[test]
fn edits_keep_the_disk_salt_and_reload_stamps_the_files_again() {
fn edits_do_no_disk_cache_file_work() {
let dir = folder(&RUN);
let cache = std::env::temp_dir().join(format!("ec-seq-salt-{}", std::process::id()));
let cache = std::env::temp_dir().join(format!("ec-seq-nofiles-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&cache);
let mut s = session();
s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": cache.to_string_lossy(), "disk.diskCacheEnabled": true}})).unwrap();
let dc = s.disk_cache.clone().expect("a disk cache");
let attached = |s: &Session| s.layer_cache.disk().is_some_and(|d| std::sync::Arc::ptr_eq(&d, &dc));
s.execute("comp.new", json!({"name": "C", "width": 64, "height": 32, "frameRate": 30, "duration": 1})).unwrap();
let r = s.execute("file.import", json!({"paths": [at(&dir, "shot_0002.png")], "sequence": true, "addToComp": true})).unwrap();
let item = r["items"][0].clone();
let salt = s.refresh_disk_salt(false).expect("a disk cache");
// A frame rewritten in place: same footage settings, another size on disk.
std::fs::write(at(&dir, "shot_0004.png"), b"a new render of this frame").unwrap();
s.execute("layer.newText", json!({"text": "title"})).unwrap();
s.execute("layer.setText", json!({"text": "title 2"})).unwrap();
assert_eq!(s.refresh_disk_salt(false), Some(salt), "edits don't stamp the files again");
s.execute("file.reloadFootage", json!({"items": [item]})).unwrap();
let reloaded = s.refresh_disk_salt(false).unwrap();
assert_ne!(reloaded, salt, "Reload Footage stamps them again");
std::fs::write(at(&dir, "extra.png"), b"").unwrap();
s.execute("file.import", json!({"paths": [at(&dir, "extra.png")]})).unwrap();
assert_ne!(s.refresh_disk_salt(false), Some(reloaded), "new footage counts");
assert!(attached(&s), "attached all along");
assert_eq!(dc.file_checks(), 0, "no file looked at for the disk cache");
let _ = std::fs::remove_dir_all(&cache);
}