Merge pull request #3210 from vangel76/grid-mask-3121

Effects: Grid, Checkerboard and Circle keep the layer's mask with Blending Mode None

crates/effects/src/generate.rs

@@ -53,7 +53,8 @@ fn gradient_ramp(ctx: &EffectCtx, mut b: Buf) -> Buf {
 }
 
 /// Blending Mode options of the shape generators (Checkerboard, Grid, Circle): "None" shows the
-/// generated pattern only; the others composite it onto the layer like layer blending modes.
+/// generated pattern only, inside the layer's alpha (see [`gen_alpha`]); the others composite it
+/// onto the layer like layer blending modes.
 pub(crate) const GEN_BLEND_MODES: [&str; 20] = [
     "None",
     "Normal",
@@ -103,7 +104,8 @@ pub fn gen_mode(i: u32) -> Option<BlendMode> {
 }
 
 /// Composite a generated premultiplied pixel `g` onto the original `o` with blending mode
-/// index `mode` (into [`GEN_BLEND_MODES`]).
+/// index `mode` (into [`GEN_BLEND_MODES`]). None returns `g` as is: the caller keeps the layer's
+/// alpha in `g` with [`gen_alpha`].
 #[inline]
 pub(crate) fn gen_blend(o: Px, g: Px, mode: u32) -> Px {
     match gen_mode(mode) {
@@ -112,6 +114,16 @@ pub(crate) fn gen_blend(o: Px, g: Px, mode: u32) -> Px {
     }
 }
 
+/// The generated alpha `a` with Blending Mode None (`keep_alpha`, `gen_mode(mode).is_none()`):
+/// scaled by the layer's alpha `layer_a`, so the pattern stays inside the layer's shape and its
+/// masks, which are applied before effects (#1381, #3121). The other modes composite onto the
+/// layer as After Effects' blending modes do and keep `a`; their alpha outside the layer is not
+/// verified against After Effects. Mirrored by `gen_alpha` in `fx_generate.wgsl`.
+#[inline]
+pub(crate) fn gen_alpha(a: f32, layer_a: f32, keep_alpha: bool) -> f32 {
+    if keep_alpha { a * layer_a } else { a }
+}
+
 /// 4-Color Gradient's weights are inverse squared distances softened by Blend²: each point
 /// weighs 1 / (d² + Blend²) (pixels). Fitted to After Effects 2026 renders at Blend 10, 50, 100
 /// and 300 (power 2.00 at every Blend, softening 0 / 2367 / 9817 / 90127 ≈ Blend²; mean error
@@ -134,6 +146,7 @@ fn four_color(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let jitter = four_color_jitter(ctx.params.f("jitter"));
     let op = ctx.params.f("opacity") as f32 / 100.0;
     let mode = ctx.params.e("blendingMode");
+    let keep_alpha = gen_mode(mode).is_none();
     let seed = ctx.seed;
     b.img.rows_mut().for_each(|(y, row)| {
         for (x, px) in row.iter_mut().enumerate() {
@@ -155,7 +168,7 @@ fn four_color(ctx: &EffectCtx, mut b: Buf) -> Buf {
             }
             // None keeps the layer's alpha (#1381), as the Gradient Ramp does: the gradient fills
             // a shape or text, not its bounding box.
-            let a = if gen_mode(mode).is_none() { op * px[3] } else { op };
+            let a = gen_alpha(op, px[3], keep_alpha);
             *px = gen_blend(*px, [c[0] * a, c[1] * a, c[2] * a, a], mode);
         }
     });
@@ -185,6 +198,7 @@ fn checkerboard(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let c = ctx.params.color("color");
     let op = ctx.params.f("opacity") as f32 / 100.0;
     let mode = ctx.params.e("blendingMode");
+    let keep_alpha = gen_mode(mode).is_none();
     // Signed distance (in feather units) to the nearest cell edge, positive in even cells.
     let axis = |p: f64, size: f64, f: f64| -> f64 {
         let u = p / size;
@@ -197,8 +211,8 @@ fn checkerboard(ctx: &EffectCtx, mut b: Buf) -> Buf {
         for (x, px) in row.iter_mut().enumerate() {
             let sx = axis(x as f64 + 0.5 - anchor.0, w, fw);
             let sy = axis(y as f64 + 0.5 - anchor.1, h, fh);
-            let cov = ((0.5 + 0.5 * sx * sy) as f32).clamp(0.0, 1.0) * op;
-            *px = gen_blend(*px, [c[0] * cov, c[1] * cov, c[2] * cov, cov], mode);
+            let a = gen_alpha(((0.5 + 0.5 * sx * sy) as f32).clamp(0.0, 1.0) * op, px[3], keep_alpha);
+            *px = gen_blend(*px, [c[0] * a, c[1] * a, c[2] * a, a], mode);
         }
     });
     b
@@ -214,6 +228,7 @@ fn grid(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let c = ctx.params.color("color");
     let op = ctx.params.f("opacity") as f32 / 100.0;
     let mode = ctx.params.e("blendingMode");
+    let keep_alpha = gen_mode(mode).is_none();
     b.img.rows_mut().for_each(|(y, row)| {
         for (x, px) in row.iter_mut().enumerate() {
             let gx = (x as f64 + 0.5 - anchor.0).rem_euclid(w);
@@ -225,7 +240,7 @@ fn grid(ctx: &EffectCtx, mut b: Buf) -> Buf {
             if invert {
                 cov = 1.0 - cov;
             }
-            let a = cov * op;
+            let a = gen_alpha(cov * op, px[3], keep_alpha);
             *px = gen_blend(*px, [c[0] * a, c[1] * a, c[2] * a, a], mode);
         }
     });
@@ -262,6 +277,7 @@ fn circle(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let op = ctx.params.f("opacity") as f32 / 100.0;
     let invert = ctx.params.b("invert");
     let mode = ctx.params.e("blendingMode");
+    let keep_alpha = gen_mode(mode).is_none();
     b.img.rows_mut().for_each(|(y, row)| {
         for (x, px) in row.iter_mut().enumerate() {
             let d = ((x as f64 + 0.5 - c.0).powi(2) + (y as f64 + 0.5 - c.1).powi(2)).sqrt();
@@ -272,7 +288,7 @@ fn circle(ctx: &EffectCtx, mut b: Buf) -> Buf {
             if invert {
                 cov = 1.0 - cov;
             }
-            let a = cov * op;
+            let a = gen_alpha(cov * op, px[3], keep_alpha);
             *px = gen_blend(*px, [color[0] * a, color[1] * a, color[2] * a, a], mode);
         }
     });
@@ -594,6 +610,47 @@ mod generator_tests {
         assert!(o.img.get(0, 5)[0] > 0.9 && o.img.get(5, 5)[3] < 0.01);
     }
 
+    /// With Blending Mode None, a shape generator's pattern shows only where the layer has alpha
+    /// (#1381, #3121: masks come before effects). `set` makes the pattern cover (5, 5) and (35, 5).
+    fn assert_none_stays_inside_the_layer_alpha(id: &str, set: &[(&'static str, Value)]) {
+        // Three bands: transparent, half and fully opaque, premultiplied.
+        let mut img = Image::filled(40, 24, [1.0, 0.0, 0.0, 1.0]);
+        for (i, p) in img.data.iter_mut().enumerate() {
+            let a = [0.0, 0.5, 1.0][i % 40 / 14];
+            *p = p.map(|c| c * a);
+        }
+        for op in [100.0, 50.0] {
+            let mut set = set.to_vec();
+            set.push(("opacity", num(op)));
+            let o = run_fx(id, &set, img.clone(), 0.0, EffectEnv::default());
+            let k = op as f32 / 100.0;
+            for (src, out) in img.data.iter().zip(&o.img.data) {
+                assert!(out[3] <= src[3] * k + 1e-6, "{id} opacity {op}: alpha {} became {}", src[3], out[3]);
+                assert!(out[..3].iter().all(|&c| c <= out[3] + 1e-6), "{id} opacity {op}: premultiplied colour above alpha");
+            }
+            assert!(o.img.get(5, 5)[3] < 1e-6, "{id} opacity {op}: the pattern shows where the layer is transparent");
+            assert!(o.img.get(35, 5)[3] > 0.9 * k, "{id} opacity {op}: the pattern shows on the opaque layer: {:?}", o.img.get(35, 5));
+        }
+    }
+
+    #[test]
+    fn grid_none_stays_inside_the_layer_alpha() {
+        // Inverted, the grid covers nearly every pixel, wherever the lines fall.
+        assert_none_stays_inside_the_layer_alpha("ec.generate.grid", &[("width", num(10.0)), ("border", num(2.0)), ("invertGrid", Value::Bool(true))]);
+    }
+
+    #[test]
+    fn checkerboard_none_stays_inside_the_layer_alpha() {
+        // 8-pixel squares from the anchor at 0: (5, 5) and (35, 5) lie in on squares.
+        assert_none_stays_inside_the_layer_alpha("ec.generate.checkerboard", &[("width", num(8.0))]);
+    }
+
+    #[test]
+    fn circle_none_stays_inside_the_layer_alpha() {
+        // A disk larger than the image covers every pixel.
+        assert_none_stays_inside_the_layer_alpha("ec.generate.circle", &[("center", Value::Vec2([20.0, 12.0])), ("radius", num(100.0))]);
+    }
+
     #[test]
     fn circle_edge_makes_a_ring() {
         let c = ("center", Value::Vec2([16.0, 16.0]));

crates/gpu/src/shaders/fx_extra.wgsl

@@ -29,8 +29,9 @@ fn fxe_circle(@builtin(global_invocation_id) gid: vec3<u32>) {
     if (P.u[0].z != 0u) {
         cov = 1.0 - cov;
     }
-    let a = cov * P.f[1].z;
-    textureStore(out, p, gen_blend(textureLoad(src, p, 0), vec4<f32>(P.f[2].xyz * a, a), P.u[0].x));
+    let o = textureLoad(src, p, 0);
+    let a = gen_alpha(cov * P.f[1].z, o.w, P.u[0].x);
+    textureStore(out, p, gen_blend(o, vec4<f32>(P.f[2].xyz * a, a), P.u[0].x));
 }
 
 // ---------------------------------------------------------------- Ellipse (generate2::ellipse)

crates/gpu/src/shaders/fx_generate.wgsl

@@ -32,6 +32,12 @@ fn gen_blend(o: vec4<f32>, g: vec4<f32>, mode: u32) -> vec4<f32> {
     return blend_pixel(mode, o, g, 0.5);
 }
 
+// generate::gen_alpha: with None (mode 255) the generated alpha `a` is scaled by the layer's
+// alpha, so the pattern stays inside the layer and its masks; other modes keep `a`.
+fn gen_alpha(a: f32, layer_a: f32, mode: u32) -> f32 {
+    return select(a, a * layer_a, mode == 255u);
+}
+
 // A (hi, lo) pair from `data`.
 fn gen_hl(i: u32) -> vec2<f32> {
     return vec2<f32>(data[i], data[i + 1u]);
@@ -466,9 +472,10 @@ fn gen_checker(@builtin(global_invocation_id) gid: vec3<u32>) {
     }
     let sx = gen_axis(f32(p.x) + P.f[0].x, P.f[0].z, P.f[1].x);
     let sy = gen_axis(f32(p.y) + P.f[0].y, P.f[0].w, P.f[1].y);
-    let cov = clamp(0.5 + 0.5 * sx * sy, 0.0, 1.0) * P.f[1].z;
+    let o = textureLoad(src, p, 0);
+    let a = gen_alpha(clamp(0.5 + 0.5 * sx * sy, 0.0, 1.0) * P.f[1].z, o.w, P.u[0].x);
     let c = P.f[2].xyz;
-    textureStore(out, p, gen_blend(textureLoad(src, p, 0), vec4<f32>(c * cov, cov), P.u[0].x));
+    textureStore(out, p, gen_blend(o, vec4<f32>(c * a, a), P.u[0].x));
 }
 
 // f32::rem_euclid.
@@ -502,8 +509,9 @@ fn gen_grid(@builtin(global_invocation_id) gid: vec3<u32>) {
     if (P.u[0].y != 0u) {
         cov = 1.0 - cov;
     }
-    let a = cov * P.f[1].z;
-    textureStore(out, p, gen_blend(textureLoad(src, p, 0), vec4<f32>(P.f[2].xyz * a, a), P.u[0].x));
+    let o = textureLoad(src, p, 0);
+    let a = gen_alpha(cov * P.f[1].z, o.w, P.u[0].x);
+    textureStore(out, p, gen_blend(o, vec4<f32>(P.f[2].xyz * a, a), P.u[0].x));
 }
 
 // u[0] = (mode, seed); f[0..4] = points (x, y); f[4..8] = colours;
@@ -535,7 +543,7 @@ fn gen_fourcolor(@builtin(global_invocation_id) gid: vec3<u32>) {
     }
     let o = textureLoad(src, p, 0);
     // None keeps the layer's alpha, as the CPU path does (#1381).
-    let op = select(P.f[8].z, P.f[8].z * o.w, P.u[0].x == 255u);
+    let op = gen_alpha(P.f[8].z, o.w, P.u[0].x);
     textureStore(out, p, gen_blend(o, vec4<f32>(c * op, op), P.u[0].x));
 }
 

crates/render/src/tests.rs

@@ -927,6 +927,30 @@ fn effects_see_layer_masks() {
     assert!(img.get(150, 50)[3] > 0.99, "mode None mask does not cut the layer");
 }
 
+/// Masks come before effects: Grid (Blending Mode None) draws only inside a masked solid (#3121).
+#[test]
+fn grid_stays_inside_the_layer_mask() {
+    let (mut p, cid, comp) = setup();
+    let mut l = solid(&mut p, &comp, [1.0, 1.0, 1.0], 200, 100);
+    let mut next = p.next_id;
+    let m = build::mask(&mut Ids(&mut next), "Mask 1", ShapePath::rect([50.0, 50.0], 40.0, 40.0), MaskMode::Add, [255, 255, 0]);
+    p.next_id = next;
+    l.props.sub_mut("masks").unwrap().children.push(m.into());
+    // Inverted, the grid covers nearly every pixel, so the check does not depend on where the
+    // lines fall.
+    let set = [("width", Value::Scalar(10.0)), ("border", Value::Scalar(2.0)), ("invertGrid", Value::Bool(true))];
+    add_effect(&mut p, &mut l, "ec.generate.grid", [200.0, 100.0], &set);
+    p.comp_mut(cid).unwrap().layers.push(l);
+    let img = render_frame(&p, cid, Tick::ZERO, 1.0);
+    for y in 0..100 {
+        for x in 100..200 {
+            assert!(img.get(x, y)[3] < 0.01, "grid outside the mask at ({x}, {y}): {:?}", img.get(x, y));
+        }
+    }
+    assert!(img.get(15, 15)[3] < 0.01, "grid left of and above the mask: {:?}", img.get(15, 15));
+    assert!(img.get(55, 55)[3] > 0.5, "grid inside the mask: {:?}", img.get(55, 55));
+}
+
 /// An open mask (the Pen still drawing it) leaves the layer whole; closed, it masks (#290).
 #[test]
 fn open_masks_make_no_transparency() {