Merge pull request #3130 from vangel76/cryptomatte-matte-only

Cryptomatte: Matte Only puts the matte in RGB with alpha 1, as After Effects does

crates/effects/src/channel3d.rs

@@ -510,6 +510,18 @@ pub fn crypto_aux_image_with(a: &AuxChannels, layer: &str, sel: &[u32], what: Cr
     img
 }
 
+/// The hidden switch that keeps Matte Only as projects saved before #3127 render it: the matte in
+/// RGB and alpha. Fresh instances have it off (the matte in RGB over alpha 1, as After Effects'
+/// Cryptomatte plug-in shows Matte Only); `migrate::legacy_default` turns it on for older saves.
+pub const MATTE_ONLY_IN_ALPHA: &str = "matteOnlyInAlpha";
+
+/// A Matte Only pixel for matte `m`: grey `m` over alpha 1, or with `in_alpha` (older
+/// projects) `m` in every channel. The GPU shader (`fx_depth.wgsl`, op 7) does the same.
+#[inline]
+pub fn matte_only_px(m: f32, in_alpha: bool) -> Px {
+    if in_alpha { [m; 4] } else { [m, m, m, 1.0] }
+}
+
 fn cryptomatte(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let Some(a) = aux(ctx) else { return b };
     let layers = a.crypto_layers();
@@ -523,11 +535,12 @@ fn cryptomatte(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let idx = index_map(&b, &a);
     let display = ctx.params.e("display");
     let matte_only = ctx.params.b("matteOnly");
+    let in_alpha = ctx.params.b(MATTE_ONLY_IN_ALPHA);
     let planes = a.crypto_rank_planes(&layer);
     b.img.data.par_iter_mut().zip(idx.par_iter()).for_each_init(IdColorMemo::default, |memo, (px, &k)| {
         let m = k.map_or(0.0, |k| crypto_matte(&planes, &sel, k));
         if matte_only {
-            *px = [m, m, m, m];
+            *px = matte_only_px(m, in_alpha);
             return;
         }
         *px = match display {
@@ -643,6 +656,7 @@ pub fn specs() -> Vec<EffectSpec> {
                 text("selection", "Selection", ""),
                 p("display", "Display", Value::Enum(2), popup(&["Colors", "Matte", "Preview"])),
                 p("matteOnly", "Matte Only", Value::Bool(false), ParamUi::Checkbox),
+                p(MATTE_ONLY_IN_ALPHA, "Matte Only In Alpha", Value::Bool(false), ParamUi::Hidden),
             ],
             cryptomatte,
         ),

crates/effects/src/channel3d/tests.rs

@@ -258,15 +258,20 @@ fn cryptomatte_cpu_path_is_bit_exact_inside_and_outside_the_aux_area() {
     let names = "sphere, <7fc00000>, <7fc01234>, <80000000>";
     assert_eq!(selection_hashes(names, None), edge_selection());
     let src = img().data[0];
-    for (display, matte_only) in [(0, false), (1, false), (2, false), (0, true)] {
+    for (display, matte_only, in_alpha) in [(0, false, false), (1, false, false), (2, false, false), (0, true, false), (0, true, true)] {
         let out = run(
             "ec.3d.cryptomatte",
-            &[("selection", Value::Str(names.into())), ("display", Value::Enum(display)), ("matteOnly", Value::Bool(matte_only))],
+            &[
+                ("selection", Value::Str(names.into())),
+                ("display", Value::Enum(display)),
+                ("matteOnly", Value::Bool(matte_only)),
+                (MATTE_ONLY_IN_ALPHA, Value::Bool(in_alpha)),
+            ],
             &host,
         );
         let expect = |c: [f32; 3], m: f32| -> [f32; 4] {
             if matte_only {
-                return [m; 4];
+                return if in_alpha { [m; 4] } else { [m, m, m, 1.0] };
             }
             match display {
                 0 => {
@@ -279,24 +284,62 @@ fn cryptomatte_cpu_path_is_bit_exact_inside_and_outside_the_aux_area() {
         };
         for (x, y) in [(0u32, 0u32), (1, 0), (5, 0), (0, 1), (3, 1), (5, 1)] {
             let (c, m) = reduce_by_name(&a, "CryptoObject", &edge_selection(), (y * 6 + x) as usize);
-            assert_eq!(out.get(x as i64, y as i64).map(f32::to_bits), expect(c, m).map(f32::to_bits), "display {display} matteOnly {matte_only} at ({x}, {y})");
+            assert_eq!(
+                out.get(x as i64, y as i64).map(f32::to_bits),
+                expect(c, m).map(f32::to_bits),
+                "display {display} matteOnly {matte_only} inAlpha {in_alpha} at ({x}, {y})"
+            );
         }
         for (x, y) in [(6u32, 0u32), (19, 0), (0, 2), (12, 9)] {
             assert_eq!(
                 out.get(x as i64, y as i64).map(f32::to_bits),
                 expect([0.0; 3], 0.0).map(f32::to_bits),
-                "display {display} matteOnly {matte_only} outside the aux at ({x}, {y})"
+                "display {display} matteOnly {matte_only} inAlpha {in_alpha} outside the aux at ({x}, {y})"
             );
         }
     }
 }
 
+/// #3127: Matte Only shows the matte as After Effects' Cryptomatte plug-in does, in RGB over
+/// alpha 1 (its Matte Only render of 2026-11-05 is opaque black where nothing is selected), not
+/// in alpha. A partly covered pixel is grey, not translucent; the hidden Matte Only In Alpha
+/// switch (on for projects saved before) keeps the old matte in every channel.
+#[test]
+fn cryptomatte_matte_only_is_the_matte_in_rgb_over_alpha_one() {
+    let a = edge_aux(true);
+    let host = AuxHost(Arc::new(a.clone()));
+    let names = "sphere, <7fc00000>, <7fc01234>, <80000000>";
+    let only = |in_alpha: bool| {
+        run(
+            "ec.3d.cryptomatte",
+            &[("selection", Value::Str(names.into())), ("matteOnly", Value::Bool(true)), (MATTE_ONLY_IN_ALPHA, Value::Bool(in_alpha))],
+            &host,
+        )
+    };
+    let (new, old) = (only(false), only(true));
+    let partial = (0..6u32 * 2).map(|i| (i % 6, i / 6)).find(|&(x, y)| {
+        let m = reduce_by_name(&a, "CryptoObject", &edge_selection(), (y * 6 + x) as usize).1;
+        m > 0.0 && m < 1.0
+    });
+    let (x, y) = partial.expect("the edge aux has a partly covered pixel");
+    let m = reduce_by_name(&a, "CryptoObject", &edge_selection(), (y * 6 + x) as usize).1;
+    assert_eq!(new.get(x as i64, y as i64), [m, m, m, 1.0]);
+    assert_eq!(old.get(x as i64, y as i64), [m; 4]);
+    // Outside the aux area (and unselected): opaque black, as After Effects renders it.
+    assert_eq!(new.get(12, 9), [0.0, 0.0, 0.0, 1.0]);
+    assert_eq!(old.get(12, 9), [0.0; 4]);
+    // A fresh instance has the switch off.
+    let spec = crate::find("ec.3d.cryptomatte").unwrap();
+    let ps = spec.params.iter().find(|p| p.id == MATTE_ONLY_IN_ALPHA).expect("the hidden switch");
+    assert_eq!((&ps.default, &ps.ui), (&Value::Bool(false), &ParamUi::Hidden));
+}
+
 #[test]
 fn cryptomatte_selects_by_name_and_wildcard() {
     let h = scene();
     let m = run("ec.3d.cryptomatte", &[("selection", Value::Str("right".into())), ("matteOnly", Value::Bool(true))], &h);
-    assert_eq!(m.get(2, 5)[3], 0.0);
-    assert_eq!(m.get(12, 5)[3], 1.0);
+    assert_eq!(m.get(2, 5), [0.0, 0.0, 0.0, 1.0]);
+    assert_eq!(m.get(12, 5), [1.0; 4]);
     let w = run("ec.3d.cryptomatte", &[("selection", Value::Str("l*".into())), ("display", Value::Enum(1))], &h);
     assert_eq!(w.get(2, 5)[3], 1.0);
     assert_eq!(w.get(12, 5)[3], 0.0);

crates/effects/src/lib.rs

@@ -115,7 +115,9 @@ pub use time_fx::{PixelMotion, TimewarpPlan, TwStep, pmb_plan, posterized_time,
 // effectcraft-gpu fx_tone.
 pub use ocio::color_stabilizer_maps;
 // GPU effects, part B (3D Channel, Immersive Video, colour management, simulation render passes).
-pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, reads_aux, selection_hashes};
+pub use channel3d::{
+    CRYPTO_LAYERS, CryptoAux, MATTE_ONLY_IN_ALPHA, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, matte_only_px, reads_aux, selection_hashes,
+};
 pub use ocio::{ColorOp, Straight, Tf, color_program};
 pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, bubble_sample_bounds, drizzle_drops, mercury_blobs, sprite_plan};
 pub use sim2::{

crates/effects/src/migrate.rs

@@ -161,6 +161,8 @@ pub fn legacy_default(effect: &str, param: &str) -> Option<Value> {
         ("ec.key.advancedspill", "ultraSettings/tolerance") => Value::Scalar(100.0),
         // Transition: Block Dissolve's blocks had hard edges.
         ("ec.transition.blockdissolve", "softEdges") => Value::Bool(false),
+        // 3D Channel: Cryptomatte's Matte Only put the matte in alpha too (#3127).
+        ("ec.3d.cryptomatte", crate::channel3d::MATTE_ONLY_IN_ALPHA) => Value::Bool(true),
         _ => return None,
     })
 }
@@ -516,6 +518,21 @@ mod tests {
         assert_eq!(pr.ui, popup.ui);
     }
 
+    /// #3127: Cryptomatte's Matte Only moved the matte from RGBA to RGB over alpha 1; a project
+    /// saved before has no Matte Only In Alpha switch and gets it on, keeping its render. A fresh
+    /// instance has it off.
+    #[test]
+    fn cryptomatte_saved_before_matte_only_in_rgb_keeps_the_matte_in_alpha() {
+        let id = crate::channel3d::MATTE_ONLY_IN_ALPHA;
+        let mut next = 1;
+        let (spec, mut g) = instance("ec.3d.cryptomatte", &mut next);
+        assert_eq!(g.get(id).map(|p| p.value.clone()), Some(Value::Bool(false)));
+        g.children.retain(|c| c.match_id() != id);
+        assert!(upgrade_instance(spec, &mut g, &mut Ids(&mut next), SIZE));
+        let pr = g.get(id).expect("added on upgrade");
+        assert_eq!((&pr.value, &pr.ui), (&Value::Bool(true), &ParamUi::Hidden));
+    }
+
     /// #1797: Shift Channels' menu put Half On before Full Off; a project saved then keeps the
     /// option it chose, by label.
     #[test]

crates/gpu/src/fx_depth.rs

@@ -286,7 +286,7 @@ fn cryptomatte(e: &mut Enc, ctx: &EffectCtx, b: GBuf, a: &AuxChannels) -> Option
     let tex = e.g.upload_image(&img)?;
     let plane = lookup(e, &b, a, &tex, [0.5, 0.5], [0.0; 4]);
     let mut p = op(7);
-    p.u[0] = [7, pr.e("display"), pr.b("matteOnly") as u32, 0];
+    p.u[0] = [7, pr.e("display"), pr.b("matteOnly") as u32, pr.b(effectcraft_effects::MATTE_ONLY_IN_ALPHA) as u32];
     point(e, &p, b, &plane)
 }
 

crates/gpu/src/shaders/fx_depth.wgsl

@@ -198,11 +198,13 @@ fn fxd_point(@builtin(global_invocation_id) gid: vec3<u32>) {
                 }
             }
         }
-        // Cryptomatte (aux = (ID colours, selection coverage)). u[0] = (op, display, matte only).
+        // Cryptomatte (aux = (ID colours, selection coverage)). u[0] = (op, display, matte only,
+        // matte only in alpha). Matte Only is the matte in RGB over alpha 1, or in every channel
+        // for projects saved before #3127 (`effectcraft_effects::matte_only_px`).
         case 7u: {
             let m = q.w;
             if (P.u[0].z != 0u) {
-                o = vec4<f32>(m);
+                o = select(vec4<f32>(m, m, m, 1.0), vec4<f32>(m), P.u[0].w != 0u);
             } else if (P.u[0].y == 0u) {
                 o = vec4<f32>(q.xyz * (1.0 - 0.5 * m) + 0.5 * m, 1.0);
             } else if (P.u[0].y == 1u) {

crates/gpu/src/tests/fx_depth.rs

@@ -260,12 +260,21 @@ fn ids_and_cryptomatte() {
 
 /// The GPU skips the ID colours for displays that do not read them
 /// ([`effectcraft_effects::CryptoAux::for_display`]): every display × Matte Only must still match
-/// the CPU effect, so a shader that starts reading RGB in another mode fails here.
+/// the CPU effect, so a shader that starts reading RGB in another mode fails here. Matte Only is
+/// checked both ways: the matte in RGB over alpha 1 (#3127) and, for older projects, in alpha too.
 #[test]
 fn cryptomatte_every_display_matches_the_cpu() {
     for display in 0..3 {
-        for matte_only in [false, true] {
-            case("ec.3d.cryptomatte", &[("selection", Value::Str("beta, delta".into())), ("display", e(display)), ("matteOnly", Value::Bool(matte_only))]);
+        for (matte_only, in_alpha) in [(false, false), (true, false), (true, true)] {
+            case(
+                "ec.3d.cryptomatte",
+                &[
+                    ("selection", Value::Str("beta, delta".into())),
+                    ("display", e(display)),
+                    ("matteOnly", Value::Bool(matte_only)),
+                    (effectcraft_effects::MATTE_ONLY_IN_ALPHA, Value::Bool(in_alpha)),
+                ],
+            );
         }
     }
 }

crates/host/tests/it/cryptomatte_pick.rs

@@ -82,7 +82,8 @@ fn a_pick_at_preview_resolution_reads_the_pixel_the_preview_shows() {
     let r = s.execute_checked("effect.pickCryptomatte", json!({"effect": 1, "x": 2.5, "y": 1.5, "scale": 0.5})).unwrap();
     assert_eq!(r["name"], RIGHT, "{r}");
     let opts = effectcraft_engine::render::RenderOpts { scale: 0.5, ..Default::default() };
-    assert_eq!(s.render(s.active_comp_id().unwrap(), Tick::ZERO, opts).get(1, 0)[3], 1.0);
+    // Matte Only: the matte in RGB over alpha 1 (#3127).
+    assert_eq!(s.render(s.active_comp_id().unwrap(), Tick::ZERO, opts).get(1, 0), [1.0; 4]);
     // At full resolution the same point is Left's pixel.
     let r = s.execute_checked("effect.cryptomatteAt", json!({"effect": 1, "x": 2.5, "y": 1.5})).unwrap();
     assert_eq!(r["name"], "Left");
@@ -122,6 +123,6 @@ fn a_pick_on_a_precomp_follows_expressions() {
     let r = s.execute_checked("effect.pickCryptomatte", json!({"effect": 1, "x": 33.5, "y": 10.5})).unwrap();
     assert_eq!(r["selection"], "Ball", "{r}");
     let img = s.render(s.active_comp_id().unwrap(), Tick::ZERO, Default::default());
-    assert_eq!((img.get(33, 10)[3], img.get(20, 10)[3]), (1.0, 0.0));
+    assert_eq!((img.get(33, 10), img.get(20, 10)), ([1.0; 4], [0.0, 0.0, 0.0, 1.0]));
     assert_eq!(s.execute("effect.cryptomatteAt", json!({"effect": 1, "x": 20.5, "y": 10.5})).unwrap(), serde_json::Value::Null);
 }

docs/footage.md

@@ -131,7 +131,9 @@ channels alone.
 
 The Cryptomatte effect keys objects by their Cryptomatte ID passes, with the names from the
 file's manifest. Pick them with the eyedropper next to its Selection, then click in the
-Composition viewer (Shift adds, Alt or Ctrl removes).
+Composition viewer (Shift adds, Alt or Ctrl removes). Matte Only shows the matte as grey in
+RGB over full alpha, as After Effects' Cryptomatte plug-in does; projects saved before that
+change keep their matte in alpha as well.
 
 ## Replacing and reloading