Effects: Grid, Checkerboard and Circle keep the layer's mask with Blending Mode None

Masks are applied to a layer before its effects, but with the default
Blending Mode None these generators replaced each pixel with the
generated pattern, so the pattern drew over the whole layer and the
mask was lost. With None the generated alpha is now scaled by the
layer's alpha, as 4-Color Gradient already did (#1381). A shared
gen_alpha helper (CPU) and its WGSL twin hold the rule for Grid,
Checkerboard, Circle and 4-Color Gradient.

Only None keeps the layer's alpha: the other modes composite the
pattern onto the layer as After Effects' blending modes do, and their
alpha outside the layer is not verified against After Effects, so they
are unchanged.

Fixes #3121

crates/effects/src/generate.rs

@@ -53,7 +53,8 @@ fn gradient_ramp(ctx: &EffectCtx, mut b: Buf) -> Buf {
 }
 
 /// Blending Mode options of the shape generators (Checkerboard, Grid, Circle): "None" shows the
-/// generated pattern only; the others composite it onto the layer like layer blending modes.
+/// generated pattern only, inside the layer's alpha (see [`gen_alpha`]); the others composite it
+/// onto the layer like layer blending modes.
 pub(crate) const GEN_BLEND_MODES: [&str; 20] = [
     "None",
     "Normal",
@@ -103,7 +104,8 @@ pub fn gen_mode(i: u32) -> Option<BlendMode> {
 }
 
 /// Composite a generated premultiplied pixel `g` onto the original `o` with blending mode
-/// index `mode` (into [`GEN_BLEND_MODES`]).
+/// index `mode` (into [`GEN_BLEND_MODES`]). None returns `g` as is: the caller keeps the layer's
+/// alpha in `g` with [`gen_alpha`].
 #[inline]
 pub(crate) fn gen_blend(o: Px, g: Px, mode: u32) -> Px {
     match gen_mode(mode) {
@@ -112,6 +114,16 @@ pub(crate) fn gen_blend(o: Px, g: Px, mode: u32) -> Px {
     }
 }
 
+/// The generated alpha `a` with Blending Mode None (`keep_alpha`, `gen_mode(mode).is_none()`):
+/// scaled by the layer's alpha `layer_a`, so the pattern stays inside the layer's shape and its
+/// masks, which are applied before effects (#1381, #3121). The other modes composite onto the
+/// layer as After Effects' blending modes do and keep `a`; their alpha outside the layer is not
+/// verified against After Effects. Mirrored by `gen_alpha` in `fx_generate.wgsl`.
+#[inline]
+pub(crate) fn gen_alpha(a: f32, layer_a: f32, keep_alpha: bool) -> f32 {
+    if keep_alpha { a * layer_a } else { a }
+}
+
 /// 4-Color Gradient's weights are inverse squared distances softened by Blend²: each point
 /// weighs 1 / (d² + Blend²) (pixels). Fitted to After Effects 2026 renders at Blend 10, 50, 100
 /// and 300 (power 2.00 at every Blend, softening 0 / 2367 / 9817 / 90127 ≈ Blend²; mean error
@@ -128,6 +140,7 @@ fn four_color(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let jitter = (ctx.params.f("jitter") / 100.0).clamp(0.0, 1.0) as f32;
     let op = ctx.params.f("opacity") as f32 / 100.0;
     let mode = ctx.params.e("blendingMode");
+    let keep_alpha = gen_mode(mode).is_none();
     let seed = ctx.seed;
     b.img.rows_mut().for_each(|(y, row)| {
         for (x, px) in row.iter_mut().enumerate() {
@@ -149,7 +162,7 @@ fn four_color(ctx: &EffectCtx, mut b: Buf) -> Buf {
             }
             // None keeps the layer's alpha (#1381), as the Gradient Ramp does: the gradient fills
             // a shape or text, not its bounding box.
-            let a = if gen_mode(mode).is_none() { op * px[3] } else { op };
+            let a = gen_alpha(op, px[3], keep_alpha);
             *px = gen_blend(*px, [c[0] * a, c[1] * a, c[2] * a, a], mode);
         }
     });
@@ -179,6 +192,7 @@ fn checkerboard(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let c = ctx.params.color("color");
     let op = ctx.params.f("opacity") as f32 / 100.0;
     let mode = ctx.params.e("blendingMode");
+    let keep_alpha = gen_mode(mode).is_none();
     // Signed distance (in feather units) to the nearest cell edge, positive in even cells.
     let axis = |p: f64, size: f64, f: f64| -> f64 {
         let u = p / size;
@@ -191,8 +205,8 @@ fn checkerboard(ctx: &EffectCtx, mut b: Buf) -> Buf {
         for (x, px) in row.iter_mut().enumerate() {
             let sx = axis(x as f64 + 0.5 - anchor.0, w, fw);
             let sy = axis(y as f64 + 0.5 - anchor.1, h, fh);
-            let cov = ((0.5 + 0.5 * sx * sy) as f32).clamp(0.0, 1.0) * op;
-            *px = gen_blend(*px, [c[0] * cov, c[1] * cov, c[2] * cov, cov], mode);
+            let a = gen_alpha(((0.5 + 0.5 * sx * sy) as f32).clamp(0.0, 1.0) * op, px[3], keep_alpha);
+            *px = gen_blend(*px, [c[0] * a, c[1] * a, c[2] * a, a], mode);
         }
     });
     b
@@ -208,6 +222,7 @@ fn grid(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let c = ctx.params.color("color");
     let op = ctx.params.f("opacity") as f32 / 100.0;
     let mode = ctx.params.e("blendingMode");
+    let keep_alpha = gen_mode(mode).is_none();
     b.img.rows_mut().for_each(|(y, row)| {
         for (x, px) in row.iter_mut().enumerate() {
             let gx = (x as f64 + 0.5 - anchor.0).rem_euclid(w);
@@ -219,7 +234,7 @@ fn grid(ctx: &EffectCtx, mut b: Buf) -> Buf {
             if invert {
                 cov = 1.0 - cov;
             }
-            let a = cov * op;
+            let a = gen_alpha(cov * op, px[3], keep_alpha);
             *px = gen_blend(*px, [c[0] * a, c[1] * a, c[2] * a, a], mode);
         }
     });
@@ -256,6 +271,7 @@ fn circle(ctx: &EffectCtx, mut b: Buf) -> Buf {
     let op = ctx.params.f("opacity") as f32 / 100.0;
     let invert = ctx.params.b("invert");
     let mode = ctx.params.e("blendingMode");
+    let keep_alpha = gen_mode(mode).is_none();
     b.img.rows_mut().for_each(|(y, row)| {
         for (x, px) in row.iter_mut().enumerate() {
             let d = ((x as f64 + 0.5 - c.0).powi(2) + (y as f64 + 0.5 - c.1).powi(2)).sqrt();
@@ -266,7 +282,7 @@ fn circle(ctx: &EffectCtx, mut b: Buf) -> Buf {
             if invert {
                 cov = 1.0 - cov;
             }
-            let a = cov * op;
+            let a = gen_alpha(cov * op, px[3], keep_alpha);
             *px = gen_blend(*px, [color[0] * a, color[1] * a, color[2] * a, a], mode);
         }
     });
@@ -572,6 +588,47 @@ mod generator_tests {
         assert!(o.img.get(0, 5)[0] > 0.9 && o.img.get(5, 5)[3] < 0.01);
     }
 
+    /// With Blending Mode None, a shape generator's pattern shows only where the layer has alpha
+    /// (#1381, #3121: masks come before effects). `set` makes the pattern cover (5, 5) and (35, 5).
+    fn assert_none_stays_inside_the_layer_alpha(id: &str, set: &[(&'static str, Value)]) {
+        // Three bands: transparent, half and fully opaque, premultiplied.
+        let mut img = Image::filled(40, 24, [1.0, 0.0, 0.0, 1.0]);
+        for (i, p) in img.data.iter_mut().enumerate() {
+            let a = [0.0, 0.5, 1.0][i % 40 / 14];
+            *p = p.map(|c| c * a);
+        }
+        for op in [100.0, 50.0] {
+            let mut set = set.to_vec();
+            set.push(("opacity", num(op)));
+            let o = run_fx(id, &set, img.clone(), 0.0, EffectEnv::default());
+            let k = op as f32 / 100.0;
+            for (src, out) in img.data.iter().zip(&o.img.data) {
+                assert!(out[3] <= src[3] * k + 1e-6, "{id} opacity {op}: alpha {} became {}", src[3], out[3]);
+                assert!(out[..3].iter().all(|&c| c <= out[3] + 1e-6), "{id} opacity {op}: premultiplied colour above alpha");
+            }
+            assert!(o.img.get(5, 5)[3] < 1e-6, "{id} opacity {op}: the pattern shows where the layer is transparent");
+            assert!(o.img.get(35, 5)[3] > 0.9 * k, "{id} opacity {op}: the pattern shows on the opaque layer: {:?}", o.img.get(35, 5));
+        }
+    }
+
+    #[test]
+    fn grid_none_stays_inside_the_layer_alpha() {
+        // Inverted, the grid covers nearly every pixel, wherever the lines fall.
+        assert_none_stays_inside_the_layer_alpha("ec.generate.grid", &[("width", num(10.0)), ("border", num(2.0)), ("invertGrid", Value::Bool(true))]);
+    }
+
+    #[test]
+    fn checkerboard_none_stays_inside_the_layer_alpha() {
+        // 8-pixel squares from the anchor at 0: (5, 5) and (35, 5) lie in on squares.
+        assert_none_stays_inside_the_layer_alpha("ec.generate.checkerboard", &[("width", num(8.0))]);
+    }
+
+    #[test]
+    fn circle_none_stays_inside_the_layer_alpha() {
+        // A disk larger than the image covers every pixel.
+        assert_none_stays_inside_the_layer_alpha("ec.generate.circle", &[("center", Value::Vec2([20.0, 12.0])), ("radius", num(100.0))]);
+    }
+
     #[test]
     fn circle_edge_makes_a_ring() {
         let c = ("center", Value::Vec2([16.0, 16.0]));

crates/gpu/src/shaders/fx_extra.wgsl

@@ -29,8 +29,9 @@ fn fxe_circle(@builtin(global_invocation_id) gid: vec3<u32>) {
     if (P.u[0].z != 0u) {
         cov = 1.0 - cov;
     }
-    let a = cov * P.f[1].z;
-    textureStore(out, p, gen_blend(textureLoad(src, p, 0), vec4<f32>(P.f[2].xyz * a, a), P.u[0].x));
+    let o = textureLoad(src, p, 0);
+    let a = gen_alpha(cov * P.f[1].z, o.w, P.u[0].x);
+    textureStore(out, p, gen_blend(o, vec4<f32>(P.f[2].xyz * a, a), P.u[0].x));
 }
 
 // ---------------------------------------------------------------- Ellipse (generate2::ellipse)

crates/gpu/src/shaders/fx_generate.wgsl

@@ -32,6 +32,12 @@ fn gen_blend(o: vec4<f32>, g: vec4<f32>, mode: u32) -> vec4<f32> {
     return blend_pixel(mode, o, g, 0.5);
 }
 
+// generate::gen_alpha: with None (mode 255) the generated alpha `a` is scaled by the layer's
+// alpha, so the pattern stays inside the layer and its masks; other modes keep `a`.
+fn gen_alpha(a: f32, layer_a: f32, mode: u32) -> f32 {
+    return select(a, a * layer_a, mode == 255u);
+}
+
 // A (hi, lo) pair from `data`.
 fn gen_hl(i: u32) -> vec2<f32> {
     return vec2<f32>(data[i], data[i + 1u]);
@@ -466,9 +472,10 @@ fn gen_checker(@builtin(global_invocation_id) gid: vec3<u32>) {
     }
     let sx = gen_axis(f32(p.x) + P.f[0].x, P.f[0].z, P.f[1].x);
     let sy = gen_axis(f32(p.y) + P.f[0].y, P.f[0].w, P.f[1].y);
-    let cov = clamp(0.5 + 0.5 * sx * sy, 0.0, 1.0) * P.f[1].z;
+    let o = textureLoad(src, p, 0);
+    let a = gen_alpha(clamp(0.5 + 0.5 * sx * sy, 0.0, 1.0) * P.f[1].z, o.w, P.u[0].x);
     let c = P.f[2].xyz;
-    textureStore(out, p, gen_blend(textureLoad(src, p, 0), vec4<f32>(c * cov, cov), P.u[0].x));
+    textureStore(out, p, gen_blend(o, vec4<f32>(c * a, a), P.u[0].x));
 }
 
 // f32::rem_euclid.
@@ -502,8 +509,9 @@ fn gen_grid(@builtin(global_invocation_id) gid: vec3<u32>) {
     if (P.u[0].y != 0u) {
         cov = 1.0 - cov;
     }
-    let a = cov * P.f[1].z;
-    textureStore(out, p, gen_blend(textureLoad(src, p, 0), vec4<f32>(P.f[2].xyz * a, a), P.u[0].x));
+    let o = textureLoad(src, p, 0);
+    let a = gen_alpha(cov * P.f[1].z, o.w, P.u[0].x);
+    textureStore(out, p, gen_blend(o, vec4<f32>(P.f[2].xyz * a, a), P.u[0].x));
 }
 
 // u[0] = (mode, seed); f[0..4] = points (x, y); f[4..8] = colours;
@@ -535,7 +543,7 @@ fn gen_fourcolor(@builtin(global_invocation_id) gid: vec3<u32>) {
     }
     let o = textureLoad(src, p, 0);
     // None keeps the layer's alpha, as the CPU path does (#1381).
-    let op = select(P.f[8].z, P.f[8].z * o.w, P.u[0].x == 255u);
+    let op = gen_alpha(P.f[8].z, o.w, P.u[0].x);
     textureStore(out, p, gen_blend(o, vec4<f32>(c * op, op), P.u[0].x));
 }
 

crates/render/src/tests.rs

@@ -784,6 +784,30 @@ fn effects_see_layer_masks() {
     assert!(img.get(150, 50)[3] > 0.99, "mode None mask does not cut the layer");
 }
 
+/// Masks come before effects: Grid (Blending Mode None) draws only inside a masked solid (#3121).
+#[test]
+fn grid_stays_inside_the_layer_mask() {
+    let (mut p, cid, comp) = setup();
+    let mut l = solid(&mut p, &comp, [1.0, 1.0, 1.0], 200, 100);
+    let mut next = p.next_id;
+    let m = build::mask(&mut Ids(&mut next), "Mask 1", ShapePath::rect([50.0, 50.0], 40.0, 40.0), MaskMode::Add, [255, 255, 0]);
+    p.next_id = next;
+    l.props.sub_mut("masks").unwrap().children.push(m.into());
+    // Inverted, the grid covers nearly every pixel, so the check does not depend on where the
+    // lines fall.
+    let set = [("width", Value::Scalar(10.0)), ("border", Value::Scalar(2.0)), ("invertGrid", Value::Bool(true))];
+    add_effect(&mut p, &mut l, "ec.generate.grid", [200.0, 100.0], &set);
+    p.comp_mut(cid).unwrap().layers.push(l);
+    let img = render_frame(&p, cid, Tick::ZERO, 1.0);
+    for y in 0..100 {
+        for x in 100..200 {
+            assert!(img.get(x, y)[3] < 0.01, "grid outside the mask at ({x}, {y}): {:?}", img.get(x, y));
+        }
+    }
+    assert!(img.get(15, 15)[3] < 0.01, "grid left of and above the mask: {:?}", img.get(15, 15));
+    assert!(img.get(55, 55)[3] > 0.5, "grid inside the mask: {:?}", img.get(55, 55));
+}
+
 /// An open mask (the Pen still drawing it) leaves the layer whole; closed, it masks (#290).
 #[test]
 fn open_masks_make_no_transparency() {