Merge pull request #2929 from vangel76/perf/aux-footage-hint-474

perf(media): tell the media pool which EXR footage effects read (#474)

crates/automation/src/confine.rs

@@ -363,6 +363,9 @@ impl FootageSource for ConfinedFootage {
     fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<AuxChannels>> {
         self.inside(footage).then(|| self.inner.aux(item, footage, t)).flatten()
     }
+    fn set_aux_footage(&self, paths: &[String]) {
+        self.inner.set_aux_footage(paths)
+    }
     fn model(&self, item: ItemId, footage: &Footage) -> Option<Arc<effectcraft_model::Model>> {
         self.inside(footage).then(|| self.inner.model(item, footage)).flatten()
     }

crates/effects/src/channel3d.rs

@@ -30,6 +30,25 @@ fn spec(id: &'static str, name: &'static str, params: Vec<crate::ParamSpec>, ren
     EffectSpec { id, name, category: "3D Channel", params, render, gpu: false, float: true }
 }
 
+/// The effects that read their layer's auxiliary channels ([`crate::EffectHost::aux`]): every
+/// 3D Channel effect but 3D Layer Instance.
+const READS_AUX: [&str; 8] = [
+    "ec.3d.channelextract",
+    "ec.3d.cryptomatte",
+    "ec.3d.depthmatte",
+    "ec.3d.depthoffield",
+    "ec.3d.extractor",
+    "ec.3d.fog3d",
+    "ec.3d.idmatte",
+    "ec.3d.identifier",
+];
+
+/// Does effect `id` read its layer's auxiliary channels ([`crate::EffectHost::aux`])? The media
+/// layer then decodes the footage's picture together with its channels (#474).
+pub fn reads_aux(id: &str) -> bool {
+    READS_AUX.contains(&id)
+}
+
 fn aux(ctx: &EffectCtx) -> Option<Arc<AuxChannels>> {
     ctx.env.host?.aux()
 }

crates/effects/src/channel3d/tests.rs

@@ -399,3 +399,27 @@ fn instance3d_replaces_the_pixels_with_the_referenced_layer() {
     let out = run_fx("ec.3d.instance3d", &[], solid(), 0.0, EffectEnv::default());
     assert_eq!(out.img.data, solid().data);
 }
+
+/// #474: [`reads_aux`] names exactly the 3D Channel effects that ask the host for channels.
+#[test]
+fn reads_aux_names_the_effects_that_read_channels() {
+    struct Counting(AuxHost, std::sync::atomic::AtomicUsize);
+    impl EffectHost for Counting {
+        fn layer(&self, _: u64, _: bool) -> Option<LayerPixels> {
+            None
+        }
+        fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option<Vec<f32>> {
+            None
+        }
+        fn aux(&self) -> Option<Arc<AuxChannels>> {
+            self.1.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+            self.0.aux()
+        }
+    }
+    for s in specs() {
+        let host = Counting(scene(), Default::default());
+        run_fx(s.id, &[], img(), 0.0, EffectEnv { host: Some(&host), ..Default::default() });
+        assert_eq!(host.1.into_inner() > 0, reads_aux(s.id), "{}", s.id);
+    }
+    assert!(READS_AUX.iter().all(|id| specs().iter().any(|s| s.id == *id)), "every listed effect exists");
+}

crates/effects/src/lib.rs

@@ -115,7 +115,7 @@ pub use time_fx::{PixelMotion, TimewarpPlan, TwStep, pmb_plan, posterized_time,
 // effectcraft-gpu fx_tone.
 pub use ocio::color_stabilizer_maps;
 // GPU effects, part B (3D Channel, Immersive Video, colour management, simulation render passes).
-pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, selection_hashes};
+pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, reads_aux, selection_hashes};
 pub use ocio::{ColorOp, Straight, Tf, color_program};
 pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, bubble_sample_bounds, drizzle_drops, mercury_blobs, sprite_plan};
 pub use sim2::{

crates/engine/src/cmd/watch_folder.rs

@@ -88,6 +88,9 @@ fn render_project(host: &Session, path: &Path) -> Value {
         let failed = s.project.render_queue.iter().any(|i| matches!(i.status, RenderStatus::Failed(_)));
         Ok(json!({"state": if failed { "failed" } else { "done" }, "items": items}))
     })();
+    // The project shared `host`'s footage source and gave it its own set of footage effects read
+    // the channels of (#474): give back the host's.
+    host.footage.set_aux_footage(host.aux_footage.as_deref().unwrap_or(&[]));
     let mut v = match result {
         Ok(v) => v,
         Err(e) => json!({"state": "failed", "error": e, "items": []}),

crates/engine/src/lib.rs

@@ -255,6 +255,9 @@ pub struct Session {
     pub footage_stamps: std::collections::HashMap<String, footage_reload::Stamp>,
     /// The disk cache's footage salt and the footage it was taken for ([`Session::refresh_disk_salt`]).
     pub disk_salt: Option<(u64, u64)>,
+    /// The footage files effects read the channels of, as last given to the footage source
+    /// ([`Session::refresh_aux_footage`]); `None` = not given yet.
+    pub aux_footage: Option<Vec<String>>,
     /// How many script-issued commands are running ([`Session::execute_for_script`]): while one
     /// is, settings that guard what scripts may do can't change (see
     /// [`prefs::Scripting::same_permissions`]).
@@ -382,6 +385,7 @@ impl Default for Session {
             stored_paths: Default::default(),
             footage_stamps: Default::default(),
             disk_salt: None,
+            aux_footage: None,
             script_calls: ScriptCalls::default(),
         }
     }

crates/engine/src/session/edit.rs

@@ -98,6 +98,7 @@ impl Session {
             self.saved_revision = self.revision;
         }
         self.refresh_disk_salt(false);
+        self.refresh_aux_footage();
         self.events.push(Event::ProjectChanged { revision: self.revision });
     }
 
@@ -252,6 +253,8 @@ impl Session {
         self.project = Arc::new(p);
         // Its files may have changed since they were last stamped.
         self.disk_salt = None;
+        // The set of footage effects read the channels of is sent again, even when it is empty.
+        self.aux_footage = None;
         self.history = History::default();
         self.state = EditorState { snapping: true, view_layout: 1, ..Default::default() };
         self.path = path;

crates/engine/src/session/render.rs

@@ -144,4 +144,43 @@ impl Session {
         self.layer_cache.clear();
         self.drop_crypto_probe();
     }
+
+    /// Tell the footage source which files effects read the channels of ([`aux_footage`]), so
+    /// it can decode their pictures and channels at once from the first frame (#474). Called with
+    /// every project change ([`Session::bump`]); the set is only sent when it changed, or after
+    /// [`Session::replace_project`] cleared what was sent.
+    pub(crate) fn refresh_aux_footage(&mut self) {
+        let paths = aux_footage(&self.project);
+        if self.aux_footage.as_ref() != Some(&paths) {
+            self.footage.set_aux_footage(&paths);
+            self.aux_footage = Some(paths);
+        }
+    }
+}
+
+/// The files (each footage item's [`effectcraft_project::Footage::path`], sorted, once each) of
+/// the footage layers that have an enabled effect reading their channels
+/// ([`effectcraft_effects::reads_aux`]), with the layer's Effects switch on. Several items can
+/// use one file: it stays in while any of them is read.
+pub(crate) fn aux_footage(project: &effectcraft_project::Project) -> Vec<String> {
+    use effectcraft_project::{GroupKind, ItemKind, LayerSource};
+    let mut paths: Vec<String> = project
+        .comps()
+        .flat_map(|(_, c)| c.layers.iter())
+        .filter(|l| l.switches.effects)
+        .filter(|l| {
+            l.effects()
+                .is_some_and(|fx| fx.groups().any(|g| g.enabled && matches!(&g.kind, GroupKind::Effect { effect } if effectcraft_effects::reads_aux(effect))))
+        })
+        .filter_map(|l| match &l.source {
+            LayerSource::Footage { item } => match &project.item(*item)?.kind {
+                ItemKind::Footage(f) if !f.path.is_empty() => Some(f.path.clone()),
+                _ => None,
+            },
+            _ => None,
+        })
+        .collect();
+    paths.sort_unstable();
+    paths.dedup();
+    paths
 }

crates/engine/src/tests.rs

@@ -2,6 +2,7 @@ mod add_item_options;
 mod aep_import;
 mod anim_presets;
 mod anim_tools;
+mod aux_footage;
 mod blender;
 mod camera_track;
 mod codec_options;

crates/engine/src/tests/aux_footage.rs

@@ -0,0 +1,200 @@
+//! The footage files effects read the channels of (#474): the session gives the footage source
+//! the whole set with every project change ([`Session::bump`]), so a multi-layer OpenEXR read by
+//! EXtractoR or Cryptomatte decodes its picture and channels at once from the first frame, and
+//! goes back to its picture alone once no effect reads it.
+
+use std::sync::{Arc, Mutex};
+
+use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, Project};
+use effectcraft_raster::Image;
+use effectcraft_time::{FrameRate, Tick};
+use serde_json::json;
+
+use crate::Session;
+use crate::render::{FootageSource, RenderOpts};
+
+/// Records every set it is given, and the set it had when it was first asked for a frame.
+#[derive(Default)]
+struct Recorder {
+    sets: Mutex<Vec<Vec<String>>>,
+    at_first_frame: Mutex<Option<Option<Vec<String>>>>,
+}
+
+impl Recorder {
+    fn last(&self) -> Option<Vec<String>> {
+        self.sets.lock().unwrap().last().cloned()
+    }
+    fn calls(&self) -> usize {
+        self.sets.lock().unwrap().len()
+    }
+}
+
+impl FootageSource for Recorder {
+    fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option<Arc<Image>> {
+        self.at_first_frame.lock().unwrap().get_or_insert_with(|| self.last());
+        Some(Arc::new(Image::new(f.width, f.height)))
+    }
+    fn set_aux_footage(&self, paths: &[String]) {
+        self.sets.lock().unwrap().push(paths.to_vec());
+    }
+}
+
+struct Probe;
+impl crate::Importer for Probe {
+    fn probe(&self, path: &str) -> Result<Footage, String> {
+        Ok(Footage {
+            path: path.into(),
+            kind: FootageKind::Still,
+            width: 8,
+            height: 4,
+            has_video: true,
+            frame_rate: FrameRate::from_f64(24.0),
+            duration: Tick::from_seconds_f64(2.0),
+            ..Default::default()
+        })
+    }
+}
+
+fn session(rec: &Arc<Recorder>) -> Session {
+    let mut s = Session { footage: rec.clone(), importer: Some(Arc::new(Probe)), ..Default::default() };
+    s.execute("comp.new", json!({"name": "C", "width": 8, "height": 4, "frameRate": 24, "duration": 2})).unwrap();
+    s
+}
+
+fn import(s: &mut Session, path: &str) -> serde_json::Value {
+    s.execute("file.import", json!({"paths": [path]})).unwrap()["items"][0].clone()
+}
+
+/// Add a layer of `item` to the active comp; its layer id.
+fn add_layer(s: &mut Session, item: &serde_json::Value) -> u64 {
+    s.execute("layer.addItem", json!({"item": item})).unwrap();
+    s.state.selected_layers[0].0
+}
+
+fn paths(v: &[&str]) -> Option<Vec<String>> {
+    Some(v.iter().map(|p| p.to_string()).collect())
+}
+
+/// Adding, disabling, removing an effect and undoing or redoing each change the set; edits that
+/// don't change it don't send it again.
+#[test]
+fn effect_edits_undo_and_redo_update_the_set() {
+    let rec = Arc::new(Recorder::default());
+    let mut s = session(&rec);
+    let item = import(&mut s, "/renders/a.exr");
+    let l = add_layer(&mut s, &item);
+    assert_eq!(rec.last(), paths(&[]), "no effect reads it yet");
+    s.execute_checked("effect.apply", json!({"effect": "Gaussian Blur", "layers": [l]})).unwrap();
+    assert_eq!(rec.last(), paths(&[]), "a blur doesn't read channels");
+    s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [l]})).unwrap();
+    assert_eq!(rec.last(), paths(&["/renders/a.exr"]));
+    assert!(s.undo());
+    assert_eq!(rec.last(), paths(&[]), "undoing the add");
+    assert!(s.redo());
+    assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "redoing it");
+    let calls = rec.calls();
+    s.execute_checked("effect.toggle", json!({"layer": l, "effect": "Gaussian Blur", "value": false})).unwrap();
+    assert_eq!(rec.calls(), calls, "the set didn't change: it isn't sent again");
+    s.execute_checked("effect.toggle", json!({"layer": l, "effect": "EXtractoR", "value": false})).unwrap();
+    assert_eq!(rec.last(), paths(&[]), "a disabled effect reads nothing");
+    s.execute_checked("effect.toggle", json!({"layer": l, "effect": "EXtractoR", "value": true})).unwrap();
+    assert_eq!(rec.last(), paths(&["/renders/a.exr"]));
+    s.execute_checked("effect.remove", json!({"layer": l, "effect": "EXtractoR"})).unwrap();
+    assert_eq!(rec.last(), paths(&[]), "removed");
+    assert!(s.undo());
+    assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "undoing the removal");
+    assert!(s.redo());
+    assert_eq!(rec.last(), paths(&[]), "redoing it");
+    assert!(s.undo());
+    s.execute_checked("effect.duplicate", json!({"layer": l, "effect": "EXtractoR"})).unwrap();
+    s.execute_checked("effect.remove", json!({"layer": l, "effect": "EXtractoR"})).unwrap();
+    assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "the duplicate still reads it");
+}
+
+/// Opening a project gives the set before the first frame is rendered; opening another sends
+/// it again even when it is empty.
+#[test]
+fn opening_a_project_gives_the_set_before_the_first_frame() {
+    let made = Arc::new(Recorder::default());
+    let mut s = session(&made);
+    let item = import(&mut s, "/renders/crypto.exr");
+    let l = add_layer(&mut s, &item);
+    s.execute_checked("effect.apply", json!({"effect": "Cryptomatte", "layers": [l]})).unwrap();
+    let project: Project = (*s.project).clone();
+    let comp = s.state.active_comp.unwrap();
+
+    let rec = Arc::new(Recorder::default());
+    let mut s = Session { footage: rec.clone(), ..Default::default() };
+    s.replace_project(project, None);
+    assert_eq!(rec.last(), paths(&["/renders/crypto.exr"]));
+    s.render(comp, Tick::ZERO, RenderOpts::default());
+    assert_eq!(*rec.at_first_frame.lock().unwrap(), Some(paths(&["/renders/crypto.exr"])), "given before the first frame");
+    s.replace_project(Project::default(), None);
+    assert_eq!(rec.last(), paths(&[]));
+    let calls = rec.calls();
+    s.replace_project(Project::default(), None);
+    assert_eq!(rec.calls(), calls + 1, "each opened project gets its set");
+}
+
+/// Two footage items of one file: the file stays in while an effect reads either of them.
+#[test]
+fn two_items_of_one_file() {
+    let rec = Arc::new(Recorder::default());
+    let mut s = session(&rec);
+    let a = import(&mut s, "/renders/shared.exr");
+    let f = match &s.project.item(ItemId(a.as_u64().unwrap())).unwrap().kind {
+        ItemKind::Footage(f) => f.clone(),
+        k => panic!("{k:?}"),
+    };
+    let b = s.edit("Add", None, |p, _| Ok(p.add_item("shared copy", crate::color::Label::None, None, ItemKind::Footage(f)))).unwrap();
+    let la = add_layer(&mut s, &a);
+    let lb = add_layer(&mut s, &json!(b.0));
+    s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [la]})).unwrap();
+    s.execute_checked("effect.apply", json!({"effect": "ID Matte", "layers": [lb]})).unwrap();
+    assert_eq!(rec.last(), paths(&["/renders/shared.exr"]), "once");
+    s.execute_checked("effect.remove", json!({"layer": la, "effect": "EXtractoR"})).unwrap();
+    assert_eq!(rec.last(), paths(&["/renders/shared.exr"]), "the other item still reads it");
+    s.execute_checked("effect.remove", json!({"layer": lb, "effect": "ID Matte"})).unwrap();
+    assert_eq!(rec.last(), paths(&[]));
+}
+
+/// Replacing a layer's source or an item's file moves the set to the new file.
+#[test]
+fn source_and_footage_replacement_follow() {
+    let rec = Arc::new(Recorder::default());
+    let mut s = session(&rec);
+    let a = import(&mut s, "/renders/a.exr");
+    let b = import(&mut s, "/renders/b.exr");
+    let l = add_layer(&mut s, &a);
+    s.execute_checked("effect.apply", json!({"effect": "Depth Matte", "layers": [l]})).unwrap();
+    assert_eq!(rec.last(), paths(&["/renders/a.exr"]));
+    s.execute_checked("layer.replaceSource", json!({"layers": [l], "item": b})).unwrap();
+    assert_eq!(rec.last(), paths(&["/renders/b.exr"]), "the layer's new source");
+    s.execute_checked("file.replaceFootage", json!({"item": b, "path": "/renders/c.exr"})).unwrap();
+    assert_eq!(rec.last(), paths(&["/renders/c.exr"]), "the item's new file");
+}
+
+/// A watch-folder render opens its project in a session sharing the host's footage source: the
+/// host's set is given back afterwards.
+#[test]
+fn a_watch_folder_render_gives_the_host_set_back() {
+    let dir = std::env::temp_dir().join(format!("ec-aux-footage-watch-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&dir);
+    std::fs::create_dir_all(&dir).unwrap();
+    let mut other = Session::default();
+    other.execute("comp.new", json!({"name": "Idle", "width": 8, "height": 8, "duration": 1})).unwrap();
+    other.execute("file.saveAs", json!({"path": dir.join("idle.ecproj").to_string_lossy()})).unwrap();
+
+    let rec = Arc::new(Recorder::default());
+    let mut s = session(&rec);
+    let item = import(&mut s, "/renders/a.exr");
+    let l = add_layer(&mut s, &item);
+    s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [l]})).unwrap();
+    let calls = rec.calls();
+    let r = s.execute("file.watchFolder", json!({"folder": dir.to_string_lossy()})).unwrap();
+    assert_eq!(r["rendered"].as_array().map(Vec::len), Some(1), "{r}");
+    assert!(rec.calls() > calls + 1, "the watched project gave its own set");
+    assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "the host's set is back");
+    s.execute("file.watchFolder", json!({"stop": true})).unwrap();
+    let _ = std::fs::remove_dir_all(&dir);
+}

crates/gpu/src/tests/fx_depth.rs

@@ -208,6 +208,13 @@ fn on() -> Value {
     Value::Bool(true)
 }
 
+/// #474: every effect done here reads aux channels, so the session gives the media layer its
+/// footage ([`effectcraft_effects::reads_aux`]).
+#[test]
+fn every_effect_here_reads_aux() {
+    assert!(crate::fx_depth::IDS.iter().all(|i| effectcraft_effects::reads_aux(i)));
+}
+
 #[test]
 fn channel_extract() {
     case("ec.3d.channelextract", &[("blackPoint", n(0.0)), ("whitePoint", n(1200.0))]);

crates/media/src/exr_channels.rs

@@ -652,6 +652,8 @@ impl ChannelCache {
             .get_or_init(|| {
                 decoded_here = true;
                 self.decodes.fetch_add(1, Ordering::Relaxed);
+                #[cfg(test)]
+                count_decode(true);
                 let _abandon = Abandon { cache: self, slot: &slot };
                 let bytes = read()?;
                 // The planes it will allocate count against the budget while it decodes.
@@ -726,11 +728,43 @@ pub(crate) fn forget(path: &str) {
     cache().forget(path);
 }
 
+/// Stop keeping the decodes of the last pictures for their effects ([`HELD_FILES`]): which
+/// footage effects read changed, so they may wait for effects that are gone. Their entries stay
+/// cached as any other, least recently used out first.
+pub(crate) fn release_held() {
+    cache().lock().held.clear();
+}
+
 /// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`.
 pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option<Arc<[u8]>>) -> Option<Arc<AuxChannels>> {
     cache().get(path, Use::Channels, read).map(|d| d.aux.clone())
 }
 
+#[cfg(test)]
+thread_local! {
+    /// Decodes of a still's pixels on this thread ([`count_decode`]): the `image` crate's, then
+    /// the channel cache's.
+    static DECODES: std::cell::Cell<[u64; 2]> = const { std::cell::Cell::new([0; 2]) };
+}
+
+/// Count a decode of a file's pixels on this thread (tests): by the channel cache (`channels`)
+/// or by the `image` crate.
+#[cfg(test)]
+pub(crate) fn count_decode(channels: bool) {
+    DECODES.with(|d| {
+        let mut n = d.get();
+        n[usize::from(channels)] += 1;
+        d.set(n);
+    });
+}
+
+/// The decodes [`count_decode`] counted on this thread since the last call: by the `image` crate
+/// and by the channel cache.
+#[cfg(test)]
+pub(crate) fn take_decodes() -> [u64; 2] {
+    DECODES.with(|d| d.replace([0; 2]))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1035,6 +1069,70 @@ mod tests {
         assert_eq!(px, pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap().get(0, 0), "the same picture");
     }
 
+    /// An EXR file with an unnamed RGB layer and a named `GlossDir.R` of `v` (2×2).
+    fn rgb_and_gloss(v: f32) -> Arc<[u8]> {
+        flat(&["R", "G", "B", "GlossDir.R"], (2, 2), v).into()
+    }
+
+    /// #474: footage the project says effects read the channels of
+    /// ([`FootageSource::set_aux_footage`]) costs one decode from its first frame on: a single
+    /// still with EXtractoR decodes once. Without it the first frame decodes twice (the picture,
+    /// then the channels once `aux()` asks).
+    #[test]
+    fn given_aux_footage_decodes_once_from_the_first_frame() {
+        use effectcraft_render::FootageSource;
+        let (given, learned) = ("/474-given-still.exr", "/474-learned-still.exr");
+        let pool = crate::MediaPool::new();
+        let mut f = std::collections::HashMap::new();
+        for p in [given, learned] {
+            pool.add_bytes(p, rgb_and_gloss(0.25));
+            f.insert(p, crate::probe_bytes(p, rgb_and_gloss(0.25)).unwrap());
+        }
+        pool.set_aux_footage(&[given.to_string()]);
+        take_decodes();
+        let t = effectcraft_time::Tick::ZERO;
+        pool.frame_at(&f[given], t).unwrap();
+        let aux = pool.aux(effectcraft_project::ItemId(1), &f[given], t).unwrap();
+        assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25));
+        assert_eq!(take_decodes(), [0, 1], "the picture and the channels share one decode");
+        // Learned only at the first `aux()`: the picture was decoded on its own before.
+        pool.frame_at(&f[learned], t).unwrap();
+        pool.aux(effectcraft_project::ItemId(2), &f[learned], t).unwrap();
+        assert_eq!(take_decodes(), [1, 1]);
+    }
+
+    /// #474: once the project no longer names footage (its effect removed), later frames decode
+    /// only their picture again; what `aux()` learned is dropped with each new set too.
+    #[test]
+    fn a_new_aux_footage_set_returns_to_picture_only() {
+        use effectcraft_render::FootageSource;
+        let paths: Vec<String> = (1..=4).map(|i| format!("/474-unset_{i:04}.exr")).collect();
+        let pool = crate::MediaPool::new();
+        for p in &paths {
+            pool.add_bytes(p, rgb_and_gloss(0.5));
+        }
+        let mut f = crate::probe_bytes(&paths[0], rgb_and_gloss(0.5)).unwrap();
+        f.kind = effectcraft_project::FootageKind::Sequence;
+        f.sequence = paths.clone();
+        let at = |i: i64| f.frame_rate.tick_of(i);
+        // (counted on this thread, so other tests sharing the channel cache don't change them)
+        let frame = |i: i64| {
+            take_decodes();
+            pool.frame_at(&f, at(i)).unwrap();
+            take_decodes()
+        };
+        pool.set_aux_footage(std::slice::from_ref(&f.path));
+        assert_eq!(frame(0), [0, 1], "given: the picture comes from the channel decode");
+        pool.set_aux_footage(&[]);
+        assert!(!cache().lock().held.iter().any(|(p, _)| *p == paths[0]), "no longer kept once no effect reads it");
+        assert_eq!(frame(1), [1, 0], "no longer given: the picture alone");
+        // Learned by `aux()`, then a new set without it: picture only again.
+        pool.aux(effectcraft_project::ItemId(1), &f, at(1)).unwrap();
+        assert_eq!(frame(2), [0, 1], "learned");
+        pool.set_aux_footage(&[]);
+        assert_eq!(frame(3), [1, 0], "a new set drops what was learned");
+    }
+
     /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the
     /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0
     /// around it), in every alpha interpretation and float curve.

crates/media/src/pool.rs

@@ -245,9 +245,25 @@ struct Inner {
     /// Movies that failed to open or decode (path to a readable reason; see
     /// [`FootageSource::problem`]). The first failure of a file is logged, later ones are not.
     problems: Mutex<HashMap<Arc<str>, String>>,
-    /// OpenEXR footage (by path) whose channels an effect has read (EXtractoR, Cryptomatte, the 3D
+    /// OpenEXR footage (by path) whose channels effects read (EXtractoR, Cryptomatte, the 3D
     /// Channel effects): its pictures come from the same cached decode (#474).
-    aux_footage: Mutex<HashSet<String>>,
+    aux_footage: Mutex<AuxFootage>,
+}
+
+/// Footage whose channels effects read ([`Inner::aux_footage`]), by [`Footage::path`].
+#[derive(Default)]
+struct AuxFootage {
+    /// What the project says ([`FootageSource::set_aux_footage`]).
+    given: HashSet<String>,
+    /// Read through [`FootageSource::aux`] but not given (a reader the project doesn't name, or
+    /// a source that never gives a set). Cleared with each new set.
+    learned: HashSet<String>,
+}
+
+impl AuxFootage {
+    fn contains(&self, path: &str) -> bool {
+        self.given.contains(path) || self.learned.contains(path)
+    }
 }
 
 /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus
@@ -868,6 +884,8 @@ impl Inner {
         {
             return Ok(img);
         }
+        #[cfg(test)]
+        crate::exr_channels::count_decode(false);
         match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) {
             Ok(img) => Ok(dynamic_to_image(&img, op)),
             // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer.
@@ -964,6 +982,16 @@ impl FootageSource for MediaPool {
         crate::layered::rasterize_vector(&footage.path, &bytes, footage.layer.as_ref(), footage.page, scale).map(Arc::new)
     }
 
+    fn set_aux_footage(&self, paths: &[String]) {
+        let given: HashSet<String> = paths.iter().cloned().collect();
+        let mut wanted = lock(&self.inner.aux_footage);
+        if given != wanted.given || !wanted.learned.is_empty() {
+            crate::exr_channels::release_held();
+        }
+        wanted.given = given;
+        wanted.learned.clear();
+    }
+
     fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<effectcraft_raster::AuxChannels>> {
         if footage.missing || !footage.has_video {
             return None;
@@ -975,11 +1003,13 @@ impl FootageSource for MediaPool {
         let path = loc.key.path.clone();
         {
             let mut wanted = lock(&self.inner.aux_footage);
-            // (a bound: names of footage items, never cleared otherwise)
-            if wanted.len() >= 4096 {
-                wanted.clear();
+            if !wanted.contains(&footage.path) {
+                // (a bound: names of footage items, cleared otherwise only by a new set)
+                if wanted.learned.len() >= 4096 {
+                    wanted.learned.clear();
+                }
+                wanted.learned.insert(footage.path.clone());
             }
-            wanted.insert(footage.path.clone());
         }
         crate::exr_channels::cached(&path, || self.inner.read(&path).ok())
     }

crates/render/src/footage.rs

@@ -48,6 +48,12 @@ pub trait FootageSource: Send + Sync {
     fn aux(&self, _item: ItemId, _footage: &Footage, _t: Tick) -> Option<Arc<effectcraft_raster::AuxChannels>> {
         None
     }
+    /// The footage files (by [`Footage::path`]) whose channels effects of the project read
+    /// through [`FootageSource::aux`] (EXtractoR, Cryptomatte and the other 3D Channel effects),
+    /// replacing the set given before. A source that decodes a file's picture and its channels
+    /// separately can decode both at once for these from the first frame on. Sources that don't
+    /// ignore it.
+    fn set_aux_footage(&self, _paths: &[String]) {}
     /// The parsed 3D model of a [`effectcraft_project::FootageKind::Model`] item (Advanced 3D).
     fn model(&self, _item: ItemId, _footage: &Footage) -> Option<Arc<effectcraft_model::Model>> {
         None
@@ -140,6 +146,9 @@ impl FootageSource for CancellableFootage<'_> {
     fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<effectcraft_raster::AuxChannels>> {
         self.inner.aux(item, footage, t)
     }
+    fn set_aux_footage(&self, paths: &[String]) {
+        self.inner.set_aux_footage(paths)
+    }
     fn model(&self, item: ItemId, footage: &Footage) -> Option<Arc<effectcraft_model::Model>> {
         self.inner.model(item, footage)
     }

docs/file-format-parity.md

@@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na
 | PNG | R/W | yes | yes (sequence) | good |
 | JPEG | R/W | yes | yes (sequence) | good |
 | TIFF | R/W | yes | yes (sequence) | good |
-| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, except the first frame an effect reads and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 |
+| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte from the first frame on, except Media Replacement items, which are learned at their first frame, and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 |
 | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec |
 | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS |
 | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't |

docs/footage.md

@@ -122,10 +122,12 @@ vectors, positions and IDs are never the picture. EXtractoR reads every other la
 
 Footage on its own keeps only the channels of its picture (a single-part file still
 decompresses all of its pixels to read them). Once an effect reads an item's channels
-(EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for the
-picture and the effects together, and kept for a while: about 1 GB of decoded channels (512 MB
-in the browser), a soft bound, as the newest file and files still in use stay whatever their
-size. Several frames asking for the same file at once wait for one decode.
+(EXtractoR, Cryptomatte, the 3D Channel effects), each frame, the first included, is decoded
+once for the picture and the effects together, and kept for a while: about 1 GB of decoded
+channels (512 MB in the browser), a soft bound, as the newest file and files still in use
+stay whatever their size. Several frames asking for the same file at once wait for one
+decode. Once no enabled effect reads the item any more, its frames go back to the picture's
+channels alone.
 
 The Cryptomatte effect keys objects by their Cryptomatte ID passes, with the names from the
 file's manifest. Pick them with the eyedropper next to its Selection, then click in the