From 5b31b96cde1187eff124da080fb0755dfac2813d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 03:18:16 +0100 Subject: [PATCH] perf(media): tell the media pool which EXR footage effects read (#474) The pool learned that an effect reads a footage item's channels only at the first aux() call, so the first frame was decoded twice (picture, then in full), and the hint was never cleared after the effect was removed. The session now gives the footage source the whole set of files read by enabled 3D Channel effects (FootageSource::set_aux_footage) with every project change (Session::bump: edits, undo, redo, history jumps, rollbacks, project open), sending it only when it changed. The pool keeps that set and, as a safety net, what aux() learns; a new set drops what was learned. Refs #474 Co-Authored-By: Claude Opus 5.5 --- crates/automation/src/confine.rs | 3 + crates/effects/src/channel3d.rs | 19 +++ crates/effects/src/channel3d/tests.rs | 24 ++++ crates/effects/src/lib.rs | 2 +- crates/engine/src/lib.rs | 4 + crates/engine/src/session/edit.rs | 3 + crates/engine/src/session/render.rs | 39 ++++++ crates/engine/src/tests.rs | 1 + crates/engine/src/tests/aux_footage.rs | 175 +++++++++++++++++++++++++ crates/media/src/exr_channels.rs | 91 +++++++++++++ crates/media/src/pool.rs | 42 +++++- crates/render/src/footage.rs | 9 ++ docs/file-format-parity.md | 2 +- docs/footage.md | 10 +- 14 files changed, 412 insertions(+), 12 deletions(-) create mode 100644 crates/engine/src/tests/aux_footage.rs diff --git a/crates/automation/src/confine.rs b/crates/automation/src/confine.rs index 5f0a7a51a..7cfe11ac2 100644 --- a/crates/automation/src/confine.rs +++ b/crates/automation/src/confine.rs @@ -349,6 +349,9 @@ impl FootageSource for ConfinedFootage { fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { self.inside(footage).then(|| self.inner.aux(item, footage, t)).flatten() } + fn set_aux_footage(&self, paths: &[String]) { + self.inner.set_aux_footage(paths) + } fn model(&self, item: ItemId, footage: &Footage) -> Option> { self.inside(footage).then(|| self.inner.model(item, footage)).flatten() } diff --git a/crates/effects/src/channel3d.rs b/crates/effects/src/channel3d.rs index dc65c3255..59128eb7e 100644 --- a/crates/effects/src/channel3d.rs +++ b/crates/effects/src/channel3d.rs @@ -30,6 +30,25 @@ fn spec(id: &'static str, name: &'static str, params: Vec, ren EffectSpec { id, name, category: "3D Channel", params, render, gpu: false, float: true } } +/// The effects that read their layer's auxiliary channels ([`crate::EffectHost::aux`]): every +/// 3D Channel effect but 3D Layer Instance. +const READS_AUX: [&str; 8] = [ + "ec.3d.channelextract", + "ec.3d.cryptomatte", + "ec.3d.depthmatte", + "ec.3d.depthoffield", + "ec.3d.extractor", + "ec.3d.fog3d", + "ec.3d.idmatte", + "ec.3d.identifier", +]; + +/// Does effect `id` read its layer's auxiliary channels ([`crate::EffectHost::aux`])? The media +/// layer then decodes the footage's picture together with its channels (#474). +pub fn reads_aux(id: &str) -> bool { + READS_AUX.contains(&id) +} + fn aux(ctx: &EffectCtx) -> Option> { ctx.env.host?.aux() } diff --git a/crates/effects/src/channel3d/tests.rs b/crates/effects/src/channel3d/tests.rs index e69c2a6aa..9d9342a95 100644 --- a/crates/effects/src/channel3d/tests.rs +++ b/crates/effects/src/channel3d/tests.rs @@ -399,3 +399,27 @@ fn instance3d_replaces_the_pixels_with_the_referenced_layer() { let out = run_fx("ec.3d.instance3d", &[], solid(), 0.0, EffectEnv::default()); assert_eq!(out.img.data, solid().data); } + +/// #474: [`reads_aux`] names exactly the 3D Channel effects that ask the host for channels. +#[test] +fn reads_aux_names_the_effects_that_read_channels() { + struct Counting(AuxHost, std::sync::atomic::AtomicUsize); + impl EffectHost for Counting { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn aux(&self) -> Option> { + self.1.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.0.aux() + } + } + for s in specs() { + let host = Counting(scene(), Default::default()); + run_fx(s.id, &[], img(), 0.0, EffectEnv { host: Some(&host), ..Default::default() }); + assert_eq!(host.1.into_inner() > 0, reads_aux(s.id), "{}", s.id); + } + assert!(READS_AUX.iter().all(|id| specs().iter().any(|s| s.id == *id)), "every listed effect exists"); +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c865d4f96..c574509f3 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -115,7 +115,7 @@ pub use time_fx::{PixelMotion, TimewarpPlan, TwStep, pmb_plan, posterized_time, // effectcraft-gpu fx_tone. pub use ocio::color_stabilizer_maps; // GPU effects, part B (3D Channel, Immersive Video, colour management, simulation render passes). -pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, selection_hashes}; +pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, reads_aux, selection_hashes}; pub use ocio::{ColorOp, Straight, Tf, color_program}; pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, bubble_sample_bounds, drizzle_drops, mercury_blobs, sprite_plan}; pub use sim2::{ diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 0cc9a7189..54616fdbf 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -244,6 +244,9 @@ pub struct Session { pub footage_stamps: std::collections::HashMap, /// The disk cache's footage salt and the footage it was taken for ([`Session::refresh_disk_salt`]). pub disk_salt: Option<(u64, u64)>, + /// The footage files effects read the channels of, as last given to the footage source + /// ([`Session::refresh_aux_footage`]); `None` = not given yet. + pub aux_footage: Option>, /// How many script-issued commands are running ([`Session::execute_for_script`]): while one /// is, settings that guard what scripts may do can't change (see /// [`prefs::Scripting::same_permissions`]). @@ -350,6 +353,7 @@ impl Default for Session { stored_paths: Default::default(), footage_stamps: Default::default(), disk_salt: None, + aux_footage: None, script_calls: ScriptCalls::default(), } } diff --git a/crates/engine/src/session/edit.rs b/crates/engine/src/session/edit.rs index c2590abc3..d63ec5ead 100644 --- a/crates/engine/src/session/edit.rs +++ b/crates/engine/src/session/edit.rs @@ -98,6 +98,7 @@ impl Session { self.saved_revision = self.revision; } self.refresh_disk_salt(false); + self.refresh_aux_footage(); self.events.push(Event::ProjectChanged { revision: self.revision }); } @@ -252,6 +253,8 @@ impl Session { self.project = Arc::new(p); // Its files may have changed since they were last stamped. self.disk_salt = None; + // The set of footage effects read the channels of is sent again, even when it is empty. + self.aux_footage = None; self.history = History::default(); self.state = EditorState { snapping: true, view_layout: 1, ..Default::default() }; self.path = path; diff --git a/crates/engine/src/session/render.rs b/crates/engine/src/session/render.rs index 86127a510..24f6552dd 100644 --- a/crates/engine/src/session/render.rs +++ b/crates/engine/src/session/render.rs @@ -137,4 +137,43 @@ impl Session { self.layer_cache.clear(); self.drop_crypto_probe(); } + + /// Tell the footage source which files effects read the channels of ([`aux_footage`]), so + /// it can decode their pictures and channels at once from the first frame (#474). Called with + /// every project change ([`Session::bump`]); the set is only sent when it changed, or after + /// [`Session::replace_project`] cleared what was sent. + pub(crate) fn refresh_aux_footage(&mut self) { + let paths = aux_footage(&self.project); + if self.aux_footage.as_ref() != Some(&paths) { + self.footage.set_aux_footage(&paths); + self.aux_footage = Some(paths); + } + } +} + +/// The files (each footage item's [`effectcraft_project::Footage::path`], sorted, once each) of +/// the footage layers that have an enabled effect reading their channels +/// ([`effectcraft_effects::reads_aux`]), with the layer's Effects switch on. Several items can +/// use one file: it stays in while any of them is read. +pub(crate) fn aux_footage(project: &effectcraft_project::Project) -> Vec { + use effectcraft_project::{GroupKind, ItemKind, LayerSource}; + let mut paths: Vec = project + .comps() + .flat_map(|(_, c)| c.layers.iter()) + .filter(|l| l.switches.effects) + .filter(|l| { + l.effects() + .is_some_and(|fx| fx.groups().any(|g| g.enabled && matches!(&g.kind, GroupKind::Effect { effect } if effectcraft_effects::reads_aux(effect)))) + }) + .filter_map(|l| match &l.source { + LayerSource::Footage { item } => match &project.item(*item)?.kind { + ItemKind::Footage(f) if !f.path.is_empty() => Some(f.path.clone()), + _ => None, + }, + _ => None, + }) + .collect(); + paths.sort_unstable(); + paths.dedup(); + paths } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 013d2f24c..85d9353d3 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1,6 +1,7 @@ mod add_item_options; mod anim_presets; mod anim_tools; +mod aux_footage; mod blender; mod camera_track; mod codec_options; diff --git a/crates/engine/src/tests/aux_footage.rs b/crates/engine/src/tests/aux_footage.rs new file mode 100644 index 000000000..5a74d3b77 --- /dev/null +++ b/crates/engine/src/tests/aux_footage.rs @@ -0,0 +1,175 @@ +//! The footage files effects read the channels of (#474): the session gives the footage source +//! the whole set with every project change ([`Session::bump`]), so a multi-layer OpenEXR read by +//! EXtractoR or Cryptomatte decodes its picture and channels at once from the first frame, and +//! goes back to its picture alone once no effect reads it. + +use std::sync::{Arc, Mutex}; + +use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, Project}; +use effectcraft_raster::Image; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::json; + +use crate::Session; +use crate::render::{FootageSource, RenderOpts}; + +/// Records every set it is given, and the set it had when it was first asked for a frame. +#[derive(Default)] +struct Recorder { + sets: Mutex>>, + at_first_frame: Mutex>>>, +} + +impl Recorder { + fn last(&self) -> Option> { + self.sets.lock().unwrap().last().cloned() + } + fn calls(&self) -> usize { + self.sets.lock().unwrap().len() + } +} + +impl FootageSource for Recorder { + fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option> { + self.at_first_frame.lock().unwrap().get_or_insert_with(|| self.last()); + Some(Arc::new(Image::new(f.width, f.height))) + } + fn set_aux_footage(&self, paths: &[String]) { + self.sets.lock().unwrap().push(paths.to_vec()); + } +} + +struct Probe; +impl crate::Importer for Probe { + fn probe(&self, path: &str) -> Result { + Ok(Footage { + path: path.into(), + kind: FootageKind::Still, + width: 8, + height: 4, + has_video: true, + frame_rate: FrameRate::from_f64(24.0), + duration: Tick::from_seconds_f64(2.0), + ..Default::default() + }) + } +} + +fn session(rec: &Arc) -> Session { + let mut s = Session { footage: rec.clone(), importer: Some(Arc::new(Probe)), ..Default::default() }; + s.execute("comp.new", json!({"name": "C", "width": 8, "height": 4, "frameRate": 24, "duration": 2})).unwrap(); + s +} + +fn import(s: &mut Session, path: &str) -> serde_json::Value { + s.execute("file.import", json!({"paths": [path]})).unwrap()["items"][0].clone() +} + +/// Add a layer of `item` to the active comp; its layer id. +fn add_layer(s: &mut Session, item: &serde_json::Value) -> u64 { + s.execute("layer.addItem", json!({"item": item})).unwrap(); + s.state.selected_layers[0].0 +} + +fn paths(v: &[&str]) -> Option> { + Some(v.iter().map(|p| p.to_string()).collect()) +} + +/// Adding, disabling, removing an effect and undoing or redoing each change the set; edits that +/// don't change it don't send it again. +#[test] +fn effect_edits_undo_and_redo_update_the_set() { + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let item = import(&mut s, "/renders/a.exr"); + let l = add_layer(&mut s, &item); + assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "no effect reads it yet"); + s.execute_checked("effect.apply", json!({"effect": "Gaussian Blur", "layers": [l]})).unwrap(); + assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "a blur doesn't read channels"); + s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [l]})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"])); + assert!(s.undo()); + assert_eq!(rec.last(), paths(&[]), "undoing the add"); + assert!(s.redo()); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "redoing it"); + let calls = rec.calls(); + s.execute_checked("effect.toggle", json!({"layer": l, "effect": "Gaussian Blur", "value": false})).unwrap(); + assert_eq!(rec.calls(), calls, "the set didn't change: it isn't sent again"); + s.execute_checked("effect.toggle", json!({"layer": l, "effect": "EXtractoR", "value": false})).unwrap(); + assert_eq!(rec.last(), paths(&[]), "a disabled effect reads nothing"); + s.execute_checked("effect.toggle", json!({"layer": l, "effect": "EXtractoR", "value": true})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"])); + s.execute_checked("effect.remove", json!({"layer": l, "effect": "EXtractoR"})).unwrap(); + assert_eq!(rec.last(), paths(&[]), "removed"); + assert!(s.undo()); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "undoing the removal"); + assert!(s.redo()); + assert_eq!(rec.last(), paths(&[]), "redoing it"); + assert!(s.undo()); + s.execute_checked("effect.duplicate", json!({"layer": l, "effect": "EXtractoR"})).unwrap(); + s.execute_checked("effect.remove", json!({"layer": l, "effect": "EXtractoR"})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "the duplicate still reads it"); +} + +/// Opening a project gives the set before the first frame is rendered; opening another sends +/// it again even when it is empty. +#[test] +fn opening_a_project_gives_the_set_before_the_first_frame() { + let made = Arc::new(Recorder::default()); + let mut s = session(&made); + let item = import(&mut s, "/renders/crypto.exr"); + let l = add_layer(&mut s, &item); + s.execute_checked("effect.apply", json!({"effect": "Cryptomatte", "layers": [l]})).unwrap(); + let project: Project = (*s.project).clone(); + let comp = s.state.active_comp.unwrap(); + + let rec = Arc::new(Recorder::default()); + let mut s = Session { footage: rec.clone(), ..Default::default() }; + s.replace_project(project, None); + assert_eq!(rec.last(), paths(&["/renders/crypto.exr"])); + s.render(comp, Tick::ZERO, RenderOpts::default()); + assert_eq!(*rec.at_first_frame.lock().unwrap(), Some(paths(&["/renders/crypto.exr"])), "given before the first frame"); + s.replace_project(Project::default(), None); + assert_eq!(rec.last(), paths(&[])); + let calls = rec.calls(); + s.replace_project(Project::default(), None); + assert_eq!(rec.calls(), calls + 1, "each opened project gets its set"); +} + +/// Two footage items of one file: the file stays in while an effect reads either of them. +#[test] +fn two_items_of_one_file() { + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let a = import(&mut s, "/renders/shared.exr"); + let f = match &s.project.item(ItemId(a.as_u64().unwrap())).unwrap().kind { + ItemKind::Footage(f) => f.clone(), + k => panic!("{k:?}"), + }; + let b = s.edit("Add", None, |p, _| Ok(p.add_item("shared copy", crate::color::Label::None, None, ItemKind::Footage(f)))).unwrap(); + let la = add_layer(&mut s, &a); + let lb = add_layer(&mut s, &json!(b.0)); + s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [la]})).unwrap(); + s.execute_checked("effect.apply", json!({"effect": "ID Matte", "layers": [lb]})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/shared.exr"]), "once"); + s.execute_checked("effect.remove", json!({"layer": la, "effect": "EXtractoR"})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/shared.exr"]), "the other item still reads it"); + s.execute_checked("effect.remove", json!({"layer": lb, "effect": "ID Matte"})).unwrap(); + assert_eq!(rec.last(), paths(&[])); +} + +/// Replacing a layer's source or an item's file moves the set to the new file. +#[test] +fn source_and_footage_replacement_follow() { + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let a = import(&mut s, "/renders/a.exr"); + let b = import(&mut s, "/renders/b.exr"); + let l = add_layer(&mut s, &a); + s.execute_checked("effect.apply", json!({"effect": "Depth Matte", "layers": [l]})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"])); + s.execute_checked("layer.replaceSource", json!({"layers": [l], "item": b})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/b.exr"]), "the layer's new source"); + s.execute_checked("file.replaceFootage", json!({"item": b, "path": "/renders/c.exr"})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/c.exr"]), "the item's new file"); +} diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index 8a8f5edcc..3b5ee6084 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -652,6 +652,8 @@ impl ChannelCache { .get_or_init(|| { decoded_here = true; self.decodes.fetch_add(1, Ordering::Relaxed); + #[cfg(test)] + count_decode(); let _abandon = Abandon { cache: self, slot: &slot }; let bytes = read()?; // The planes it will allocate count against the budget while it decodes. @@ -726,11 +728,37 @@ pub(crate) fn forget(path: &str) { cache().forget(path); } +/// Stop keeping the decodes of the last pictures for their effects ([`HELD_FILES`]): which +/// footage effects read changed, so they may wait for effects that are gone. Their entries stay +/// cached as any other, least recently used out first. +pub(crate) fn release_held() { + cache().lock().held.clear(); +} + /// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { cache().get(path, Use::Channels, read).map(|d| d.aux.clone()) } +#[cfg(test)] +thread_local! { + /// Decodes of a still's pixels on this thread: the channel cache's and the `image` crate's + /// ([`count_decode`]). + static DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Count a decode of a file's pixels on this thread (tests). +#[cfg(test)] +pub(crate) fn count_decode() { + DECODES.with(|d| d.set(d.get() + 1)); +} + +/// The decodes [`count_decode`] counted on this thread since the last call. +#[cfg(test)] +pub(crate) fn take_decodes() -> u64 { + DECODES.with(|d| d.replace(0)) +} + #[cfg(test)] mod tests { use super::*; @@ -1035,6 +1063,69 @@ mod tests { assert_eq!(px, pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap().get(0, 0), "the same picture"); } + /// An EXR file with an unnamed RGB layer and a named `GlossDir.R` of `v` (2×2). + fn rgb_and_gloss(v: f32) -> Arc<[u8]> { + flat(&["R", "G", "B", "GlossDir.R"], (2, 2), v).into() + } + + /// #474: footage the project says effects read the channels of + /// ([`FootageSource::set_aux_footage`]) costs one decode from its first frame on: a single + /// still with EXtractoR decodes once. Without it the first frame decodes twice (the picture, + /// then the channels once `aux()` asks). + #[test] + fn given_aux_footage_decodes_once_from_the_first_frame() { + use effectcraft_render::FootageSource; + let (given, learned) = ("/474-given-still.exr", "/474-learned-still.exr"); + let pool = crate::MediaPool::new(); + let mut f = std::collections::HashMap::new(); + for p in [given, learned] { + pool.add_bytes(p, rgb_and_gloss(0.25)); + f.insert(p, crate::probe_bytes(p, rgb_and_gloss(0.25)).unwrap()); + } + pool.set_aux_footage(&[given.to_string()]); + take_decodes(); + let t = effectcraft_time::Tick::ZERO; + pool.frame_at(&f[given], t).unwrap(); + let aux = pool.aux(effectcraft_project::ItemId(1), &f[given], t).unwrap(); + assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); + assert_eq!(take_decodes(), 1, "the picture and the channels share one decode"); + // Learned only at the first `aux()`: the picture was decoded on its own before. + pool.frame_at(&f[learned], t).unwrap(); + pool.aux(effectcraft_project::ItemId(2), &f[learned], t).unwrap(); + assert_eq!(take_decodes(), 2); + } + + /// #474: once the project no longer names footage (its effect removed), later frames decode + /// only their picture again; what `aux()` learned is dropped with each new set too. + #[test] + fn a_new_aux_footage_set_returns_to_picture_only() { + use effectcraft_render::FootageSource; + let paths: Vec = (1..=4).map(|i| format!("/474-unset_{i:04}.exr")).collect(); + let pool = crate::MediaPool::new(); + for p in &paths { + pool.add_bytes(p, rgb_and_gloss(0.5)); + } + let mut f = crate::probe_bytes(&paths[0], rgb_and_gloss(0.5)).unwrap(); + f.kind = effectcraft_project::FootageKind::Sequence; + f.sequence = paths.clone(); + let at = |i: i64| f.frame_rate.tick_of(i); + let in_cache = |p: &str| cache().cached().iter().any(|c| c == p); + pool.set_aux_footage(std::slice::from_ref(&f.path)); + pool.frame_at(&f, at(0)).unwrap(); + assert!(in_cache(&paths[0]), "given: the picture comes from the channel decode"); + pool.set_aux_footage(&[]); + assert!(!cache().lock().held.iter().any(|(p, _)| *p == paths[0]), "no longer kept once no effect reads it"); + pool.frame_at(&f, at(1)).unwrap(); + assert!(!in_cache(&paths[1]), "no longer given: the picture alone"); + // Learned by `aux()`, then a new set without it: picture only again. + pool.aux(effectcraft_project::ItemId(1), &f, at(1)).unwrap(); + pool.frame_at(&f, at(2)).unwrap(); + assert!(in_cache(&paths[2]), "learned"); + pool.set_aux_footage(&[]); + pool.frame_at(&f, at(3)).unwrap(); + assert!(!in_cache(&paths[3]), "a new set drops what was learned"); + } + /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 /// around it), in every alpha interpretation and float curve. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 910f8238f..e9157a5ca 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -245,9 +245,25 @@ struct Inner { /// Movies that failed to open or decode (path to a readable reason; see /// [`FootageSource::problem`]). The first failure of a file is logged, later ones are not. problems: Mutex, String>>, - /// OpenEXR footage (by path) whose channels an effect has read (EXtractoR, Cryptomatte, the 3D + /// OpenEXR footage (by path) whose channels effects read (EXtractoR, Cryptomatte, the 3D /// Channel effects): its pictures come from the same cached decode (#474). - aux_footage: Mutex>, + aux_footage: Mutex, +} + +/// Footage whose channels effects read ([`Inner::aux_footage`]), by [`Footage::path`]. +#[derive(Default)] +struct AuxFootage { + /// What the project says ([`FootageSource::set_aux_footage`]). + given: HashSet, + /// Read through [`FootageSource::aux`] but not given (a reader the project doesn't name, or + /// a source that never gives a set). Cleared with each new set. + learned: HashSet, +} + +impl AuxFootage { + fn contains(&self, path: &str) -> bool { + self.given.contains(path) || self.learned.contains(path) + } } /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus @@ -868,6 +884,8 @@ impl Inner { { return Ok(img); } + #[cfg(test)] + crate::exr_channels::count_decode(); match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. @@ -964,6 +982,16 @@ impl FootageSource for MediaPool { crate::layered::rasterize_vector(&footage.path, &bytes, footage.layer.as_ref(), footage.page, scale).map(Arc::new) } + fn set_aux_footage(&self, paths: &[String]) { + let given: HashSet = paths.iter().cloned().collect(); + let mut wanted = lock(&self.inner.aux_footage); + if given != wanted.given || !wanted.learned.is_empty() { + crate::exr_channels::release_held(); + } + wanted.given = given; + wanted.learned.clear(); + } + fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { if footage.missing || !footage.has_video { return None; @@ -975,11 +1003,13 @@ impl FootageSource for MediaPool { let path = loc.key.path.clone(); { let mut wanted = lock(&self.inner.aux_footage); - // (a bound: names of footage items, never cleared otherwise) - if wanted.len() >= 4096 { - wanted.clear(); + if !wanted.contains(&footage.path) { + // (a bound: names of footage items, cleared otherwise only by a new set) + if wanted.learned.len() >= 4096 { + wanted.learned.clear(); + } + wanted.learned.insert(footage.path.clone()); } - wanted.insert(footage.path.clone()); } crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) } diff --git a/crates/render/src/footage.rs b/crates/render/src/footage.rs index db1561ad6..8db3530d3 100644 --- a/crates/render/src/footage.rs +++ b/crates/render/src/footage.rs @@ -48,6 +48,12 @@ pub trait FootageSource: Send + Sync { fn aux(&self, _item: ItemId, _footage: &Footage, _t: Tick) -> Option> { None } + /// The footage files (by [`Footage::path`]) whose channels effects of the project read + /// through [`FootageSource::aux`] (EXtractoR, Cryptomatte and the other 3D Channel effects), + /// replacing the set given before. A source that decodes a file's picture and its channels + /// separately can decode both at once for these from the first frame on. Sources that don't + /// ignore it. + fn set_aux_footage(&self, _paths: &[String]) {} /// The parsed 3D model of a [`effectcraft_project::FootageKind::Model`] item (Advanced 3D). fn model(&self, _item: ItemId, _footage: &Footage) -> Option> { None @@ -140,6 +146,9 @@ impl FootageSource for CancellableFootage<'_> { fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { self.inner.aux(item, footage, t) } + fn set_aux_footage(&self, paths: &[String]) { + self.inner.set_aux_footage(paths) + } fn model(&self, item: ItemId, footage: &Footage) -> Option> { self.inner.model(item, footage) } diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 648eadcee..f56a57319 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, except the first frame an effect reads and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte from the first frame on, except files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index 50bd4e7d1..2c88b15e8 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -122,10 +122,12 @@ vectors, positions and IDs are never the picture. EXtractoR reads every other la Footage on its own keeps only the channels of its picture (a single-part file still decompresses all of its pixels to read them). Once an effect reads an item's channels -(EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for the -picture and the effects together, and kept for a while: about 1 GB of decoded channels (512 MB -in the browser), a soft bound, as the newest file and files still in use stay whatever their -size. Several frames asking for the same file at once wait for one decode. +(EXtractoR, Cryptomatte, the 3D Channel effects), each frame, the first included, is decoded +once for the picture and the effects together, and kept for a while: about 1 GB of decoded +channels (512 MB in the browser), a soft bound, as the newest file and files still in use +stay whatever their size. Several frames asking for the same file at once wait for one +decode. Once no enabled effect reads the item any more, its frames go back to the picture's +channels alone. ## Replacing and reloading