Masks: set the corner roundness of a Rounded Rectangle mask (#3077)

* Masks: set the corner roundness of a Rounded Rectangle mask (#3067)

layer.addMask takes a roundness (corner radius in px) for shape rounded, and the new
mask.setRoundness changes it afterwards on a rectangular mask.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* Mask roundness: refuse ellipses, keep feather points and key correspondence

mask.setRoundness accepted any closed path whose vertices lie on its bounding
box, so an ellipse was replaced by a rectangle. It now accepts only a plain
rectangle (four corner vertices, straight sides) or a rounded one (eight
vertices, straight sides alternating with circular corners).

The rebuilt path keeps the old start vertex and direction, and its feather
points, so an animated path's other keys and the feather points still line
up. A change between square and rounded corners changes the vertex count; it
is refused when the path is keyed or has feather points instead of leaving
keys with different counts or dropping the points.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/engine/src/cmd/layer.rs

@@ -182,7 +182,7 @@ pub fn specs() -> Vec<CommandSpec> {
             "New Mask",
             ["Layer", "Mask"],
             Some("Cmd+Shift+N"),
-            "{layer?, shape?: rect|ellipse|rounded|polygon|star, rect? [x,y,w,h] (layer space; a polygon or star fits its width), mode?}",
+            "{layer?, shape?: rect|ellipse|rounded|polygon|star, rect? [x,y,w,h] (layer space; a polygon or star fits its width), roundness? (a rounded rectangle's corner radius in px; default 20), mode?}",
             has_layers,
             add_mask
         ),

crates/engine/src/cmd/layer/contents.rs

@@ -17,8 +17,17 @@ pub(super) fn add_mask(s: &mut Session, p: &Value) -> Result<Value> {
         None => (if w > 0.0 { w / 2.0 } else { 0.0 }, h / 2.0, w * 0.6, h * 0.6),
     };
     // The shape tools' masks: rectangle, ellipse, rounded rectangle, polygon or star.
-    let path = crate::cmd::shape_tool::mask_path(shape, [cx, cy], [rw, rh])
+    let radius = match p.get("roundness").filter(|v| !v.is_null()) {
+        None => None,
+        Some(v) => {
+            Some(v.as_f64().filter(|r| r.is_finite() && *r >= 0.0).ok_or_else(|| bad("layer.addMask", "`roundness` is a corner radius in pixels, 0 or more"))?)
+        }
+    };
+    let path = crate::cmd::shape_tool::mask_path_rounded(shape, [cx, cy], [rw, rh], radius)
         .ok_or_else(|| bad("layer.addMask", format!("unknown shape `{shape}`; one of rect, ellipse, rounded, polygon, star")))?;
+    if radius.is_some() && !matches!(shape, "rounded" | "roundedRect") {
+        return Err(bad("layer.addMask", "`roundness` only applies to shapes `rounded` and `roundedRect`"));
+    }
     let mode = str_p(p, "mode").and_then(MaskMode::from_name).unwrap_or(MaskMode::Add);
     let cycle = s.prefs.appearance.cycle_mask_colors;
     let uid = s.edit("New Mask", None, |proj, _| {

crates/engine/src/cmd/mask.rs

@@ -229,6 +229,120 @@ fn set_closed(s: &mut Session, p: &Value) -> Result<Value> {
     Ok(json!(c))
 }
 
+/// Rebuild a rectangular mask or shape path with corner radius `radius` (layer pixels; 0 gives
+/// square corners). The rectangle is the path's bounding box. Only a plain rectangle (four
+/// corner vertices, straight sides) or a rounded one (eight vertices, straight sides alternating
+/// with circular corners) is reshaped; anything else (an ellipse, a star) is refused. The new path
+/// keeps the old one's start vertex and direction, so the feather points and the other keys of an
+/// animated path still line up; a change between square and rounded corners changes the vertex
+/// count, which is refused when the path is keyed or has feather points.
+fn set_roundness(s: &mut Session, p: &Value) -> Result<Value> {
+    let c = "mask.setRoundness";
+    let radius = p.get("roundness").and_then(Value::as_f64).filter(|r| r.is_finite() && *r >= 0.0);
+    let radius = radius.ok_or_else(|| bad(c, "need `roundness`: a corner radius in pixels, 0 or more"))?;
+    let (cid, lid) = layer_p(s, p, c)?;
+    let key = p.get("mask").cloned().ok_or_else(|| bad(c, "missing `mask` (uid, index or name)"))?;
+    let t = s.time();
+    s.edit(c, merge_p(p), |proj, _| {
+        let l = layer_mut(proj, cid, lid)?;
+        let lt = l.layer_time(t);
+        let g = path_group(&mut l.props, &key).ok_or_else(|| bad(c, "no such mask or shape path"))?;
+        let roto = super::paths::is_roto(g);
+        let pr = g.get_mut("path").ok_or_else(|| bad(c, "mask has no path"))?;
+        let KV::Path(mut sp) = pr.value_at(lt) else { return Err(bad(c, "not a path")) };
+        sp.fit_tangents();
+        let (lo, hi) = rect_box(&sp).ok_or_else(|| bad(c, "the path is not a closed rectangle or rounded rectangle"))?;
+        let size = [hi[0] - lo[0], hi[1] - lo[1]];
+        let bez = effectcraft_path::rect(size, [(lo[0] + hi[0]) / 2.0, (lo[1] + hi[1]) / 2.0], radius);
+        let mut new = effectcraft_path::from_kurbo(&bez).into_iter().next().ok_or_else(|| bad(c, "could not rebuild the path"))?;
+        if new.vertices.len() != sp.vertices.len() {
+            if !pr.keys.is_empty() {
+                return Err(bad(c, "the path is keyed: changing between square and rounded corners would change its vertex count"));
+            }
+            if !sp.feather.is_empty() {
+                return Err(bad(c, "the path has feather points: changing between square and rounded corners would change its vertex count"));
+            }
+        }
+        align_like(&mut new, &sp);
+        new.feather = std::mem::take(&mut sp.feather);
+        if roto {
+            super::paths::roto_smooth(&mut new);
+        }
+        pr.set_value_at(lt, KV::Path(new));
+        Ok(())
+    })?;
+    Ok(Value::Null)
+}
+
+/// The bounding box of `sp` when it is a closed plain rectangle (four corner vertices, straight
+/// sides) or rounded rectangle (eight vertices on the box's edges, straight sides along the edges
+/// alternating with circular corners whose two ends are as far apart across as down).
+fn rect_box(sp: &ShapePath) -> Option<([f64; 2], [f64; 2])> {
+    let first = *sp.vertices.first()?;
+    let (mut lo, mut hi) = (first, first);
+    for v in &sp.vertices {
+        lo = [lo[0].min(v[0]), lo[1].min(v[1])];
+        hi = [hi[0].max(v[0]), hi[1].max(v[1])];
+    }
+    let n = sp.vertices.len();
+    if !sp.closed || !(n == 4 || n == 8) || hi[0] <= lo[0] || hi[1] <= lo[1] {
+        return None;
+    }
+    let eps = 1e-9 * (hi[0] - lo[0]).max(hi[1] - lo[1]).max(1.0);
+    let x_edge = |v: [f64; 2]| v[0] == lo[0] || v[0] == hi[0];
+    let y_edge = |v: [f64; 2]| v[1] == lo[1] || v[1] == hi[1];
+    let zero = |t: Option<&[f64; 2]>| t.is_none_or(|t| *t == [0.0, 0.0]);
+    let seg = |i: usize| (sp.vertices[i], sp.vertices[(i + 1) % n]);
+    let straight = |i: usize| {
+        let (a, b) = seg(i);
+        zero(sp.out_tangents.get(i)) && zero(sp.in_tangents.get((i + 1) % n)) && ((a[0] == b[0] && x_edge(a)) || (a[1] == b[1] && y_edge(a)))
+    };
+    let corner = |i: usize| {
+        let (a, b) = seg(i);
+        let (dx, dy) = ((b[0] - a[0]).abs(), (b[1] - a[1]).abs());
+        dx > eps && (dx - dy).abs() <= eps.max(dx * 1e-9) && ((x_edge(a) && y_edge(b)) || (y_edge(a) && x_edge(b)))
+    };
+    let ok = if n == 4 {
+        sp.vertices.iter().all(|v| x_edge(*v) && y_edge(*v)) && (0..n).all(straight)
+    } else {
+        sp.vertices.iter().all(|v| x_edge(*v) || y_edge(*v)) && (0..2).any(|k| (0..n).all(|i| if i % 2 == k { straight(i) } else { corner(i) }))
+    };
+    ok.then_some((lo, hi))
+}
+
+/// Turn the rebuilt closed path `new` so it runs the same way round as `old` and starts at the
+/// vertex nearest `old`'s first one, keeping vertex correspondence for keys and feather points.
+fn align_like(new: &mut ShapePath, old: &ShapePath) {
+    let area = |sp: &ShapePath| {
+        let n = sp.vertices.len();
+        (0..n)
+            .map(|i| {
+                let (a, b) = (sp.vertices[i], sp.vertices[(i + 1) % n]);
+                a[0] * b[1] - b[0] * a[1]
+            })
+            .sum::<f64>()
+    };
+    if (area(new) > 0.0) != (area(old) > 0.0) {
+        // Reverse: vertex order flips and each vertex's in and out tangents swap; keep vertex 0.
+        new.vertices.reverse();
+        std::mem::swap(&mut new.in_tangents, &mut new.out_tangents);
+        new.in_tangents.reverse();
+        new.out_tangents.reverse();
+        let n = new.vertices.len();
+        if n > 0 {
+            new.vertices.rotate_right(1);
+            new.in_tangents.rotate_right(1);
+            new.out_tangents.rotate_right(1);
+        }
+    }
+    let Some(o) = old.vertices.first() else { return };
+    let d = |v: &[f64; 2]| (v[0] - o[0]).powi(2) + (v[1] - o[1]).powi(2);
+    let start = new.vertices.iter().enumerate().min_by(|a, b| d(a.1).total_cmp(&d(b.1))).map_or(0, |(i, _)| i);
+    new.vertices.rotate_left(start);
+    new.in_tangents.rotate_left(start);
+    new.out_tangents.rotate_left(start);
+}
+
 /// Selected (or given) vertices grouped per (layer, mask).
 fn vertex_groups(s: &Session, p: &Value) -> Result<std::collections::BTreeMap<(u64, Uid), Vec<usize>>> {
     let comp = s.active_comp().ok_or(EngineError::NoComp)?;
@@ -649,6 +763,7 @@ pub fn specs() -> Vec<CommandSpec> {
         cmd!("mask.addVertex", "Add Mask Vertex", [], None, "{layer?, mask, point: [x,y], in?, out?, index?}", has_layers, add_vertex),
         cmd!("mask.setVertex", "Set Mask Vertex", [], None, "{layer?, mask, index, point?, in?, out?, merge?}", has_layers, set_vertex),
         cmd!("mask.setClosed", "Closed", ["Layer", "Mask and Shape Path"], None, "{layer?, mask, closed?}", has_layers, set_closed),
+        cmd!("mask.setRoundness", "Set Mask Roundness", [], None, "{layer?, mask, roundness (corner radius in px; 0 = square)}", has_layers, set_roundness),
         cmd!("mask.selectVertices", "Select Mask Vertices", [], None, "{vertices: [{layer, mask, index}], add?, toggle?}", has_comp, select_vertices),
         cmd!("mask.moveVertices", "Move Mask Vertices", [], None, "{vertices?: [{layer, mask, index}], delta: [dx,dy], merge?}", has_vertices, move_vertices),
         cmd!("mask.deleteVertices", "Delete Mask Vertices", [], None, "{vertices?}", has_vertices, delete_vertices),

crates/engine/src/cmd/shape_tool.rs

@@ -216,10 +216,16 @@ pub(crate) fn shape_group(ids: &mut Ids, kind: &str, size: [f64; 2], paint: &Sha
 /// of every one of the five kinds, so a caller that only wants to draw it (the viewer's
 /// drag ghost) cannot drift from what `shape.newShape` / `layer.addMask` commit.
 pub fn mask_path(kind: &str, centre: [f64; 2], size: [f64; 2]) -> Option<ShapePath> {
+    mask_path_rounded(kind, centre, size, None)
+}
+
+/// [`mask_path`] with the rounded rectangle's corner radius (layer pixels) given; `None` keeps
+/// the tool's default. Other kinds ignore it.
+pub fn mask_path_rounded(kind: &str, centre: [f64; 2], size: [f64; 2], radius: Option<f64>) -> Option<ShapePath> {
     let bez = match kind {
         "rect" | "rectangle" => return Some(ShapePath::rect(centre, size[0], size[1])),
         "ellipse" => return Some(ShapePath::ellipse(centre, size[0], size[1])),
-        "rounded" | "roundedRect" => effectcraft_path::rect(size, centre, roundness()),
+        "rounded" | "roundedRect" => effectcraft_path::rect(size, centre, radius.unwrap_or_else(roundness)),
         k => {
             let (star, points, outer, inner) = polystar_of(k, size)?;
             effectcraft_path::polystar(effectcraft_path::Polystar {

crates/engine/src/tests/stubs.rs

@@ -667,3 +667,109 @@ fn free_transform_drag_undoes_in_one_step() {
     s.execute("edit.redo", json!({})).unwrap();
     assert_eq!(mask_path(&s, l, m), after, "redo restores the whole drag");
 }
+
+fn bounds(p: &effectcraft_keyframe::ShapePath) -> ([f64; 2], [f64; 2]) {
+    let lo = p.vertices.iter().fold([f64::MAX; 2], |a, v| [a[0].min(v[0]), a[1].min(v[1])]);
+    let hi = p.vertices.iter().fold([f64::MIN; 2], |a, v| [a[0].max(v[0]), a[1].max(v[1])]);
+    (lo, hi)
+}
+
+#[test]
+fn rounded_mask_roundness_is_set_on_creation_and_editable() {
+    let mut s = comp();
+    let l = s.execute("layer.newSolid", json!({"color": "#ffffff", "width": 400, "height": 300})).unwrap()["layer"].as_u64().unwrap();
+    // A 20 px radius by default; an explicit radius moves the first vertex.
+    let m = s.execute("layer.addMask", json!({"layer": l, "shape": "rounded", "rect": [100, 100, 200, 100]})).unwrap()["mask"].as_u64().unwrap();
+    let default = mask_path(&s, l, m);
+    assert_eq!(default.vertices[0], [300.0, 120.0]);
+    let m2 =
+        s.execute("layer.addMask", json!({"layer": l, "shape": "rounded", "rect": [100, 100, 200, 100], "roundness": 30})).unwrap()["mask"].as_u64().unwrap();
+    assert_eq!(mask_path(&s, l, m2).vertices[0], [300.0, 130.0]);
+    assert!(s.execute("layer.addMask", json!({"layer": l, "shape": "ellipse", "roundness": 5})).is_err());
+    assert!(s.execute("layer.addMask", json!({"layer": l, "shape": "rounded", "roundness": -1})).is_err());
+    // Edit it afterwards: same box, new radius; 0 is a plain rectangle; undo restores.
+    s.execute("mask.setRoundness", json!({"layer": l, "mask": m, "roundness": 50})).unwrap();
+    let p = mask_path(&s, l, m);
+    assert_eq!(p.vertices[0], [300.0, 150.0]);
+    assert_eq!(bounds(&p), ([100.0, 100.0], [300.0, 200.0]));
+    s.execute("mask.setRoundness", json!({"layer": l, "mask": m, "roundness": 0})).unwrap();
+    let p = mask_path(&s, l, m);
+    assert_eq!(p.vertices.len(), 4);
+    assert_eq!(bounds(&p), ([100.0, 100.0], [300.0, 200.0]));
+    s.execute("edit.undo", json!({})).unwrap();
+    assert_eq!(mask_path(&s, l, m).vertices[0], [300.0, 150.0]);
+}
+
+#[test]
+fn set_roundness_refuses_what_is_not_a_rectangle() {
+    let mut s = comp();
+    let l = s.execute("layer.newSolid", json!({"color": "#ffffff", "width": 400, "height": 300})).unwrap()["layer"].as_u64().unwrap();
+    let star = s.execute("layer.addMask", json!({"layer": l, "shape": "star"})).unwrap()["mask"].as_u64().unwrap();
+    let before = mask_path(&s, l, star);
+    assert!(s.execute("mask.setRoundness", json!({"layer": l, "mask": star, "roundness": 10})).is_err());
+    assert_eq!(mask_path(&s, l, star), before, "a refused edit leaves the mask alone");
+    assert!(s.execute("mask.setRoundness", json!({"layer": l, "mask": star})).is_err(), "needs a roundness");
+    // A diamond inscribed in its box touches every edge but is not a rectangle.
+    s.execute("mask.new", json!({"layer": l, "vertices": [[150, 100], [200, 150], [150, 200], [100, 150]], "closed": true})).unwrap();
+    let diamond = layer(&s, l).masks().unwrap().groups().last().unwrap().uid;
+    assert!(s.execute("mask.setRoundness", json!({"layer": l, "mask": diamond, "roundness": 10})).is_err());
+    assert_eq!(mask_path(&s, l, diamond).vertices.len(), 4);
+    // "roundness": null is unset, and an unknown shape is reported as such.
+    s.execute("layer.addMask", json!({"layer": l, "shape": "rounded", "roundness": null})).unwrap();
+    let e = s.execute("layer.addMask", json!({"layer": l, "shape": "foo", "roundness": 5})).unwrap_err().to_string();
+    assert!(e.contains("unknown shape"), "{e}");
+}
+
+#[test]
+fn set_roundness_refuses_an_ellipse_and_keeps_feather_points_and_keys_in_step() {
+    let mut s = comp();
+    let l = s.execute("layer.newSolid", json!({"color": "#ffffff", "width": 400, "height": 300})).unwrap()["layer"].as_u64().unwrap();
+    let add = |s: &mut Session, shape: &str, rect: [u32; 4], extra: serde_json::Value| {
+        let mut p = json!({"layer": l, "shape": shape, "rect": rect});
+        if let (Some(p), Some(e)) = (p.as_object_mut(), extra.as_object()) {
+            p.extend(e.clone());
+        }
+        s.execute("layer.addMask", p).unwrap()["mask"].as_u64().unwrap()
+    };
+    // An ellipse (or a circle) has its four vertices on its box's edges and curved sides, but it
+    // is not a rectangle: refused, unchanged.
+    for rect in [[100, 100, 200, 100], [100, 100, 100, 100]] {
+        let e = add(&mut s, "ellipse", rect, json!({}));
+        let before = mask_path(&s, l, e);
+        assert!(s.execute("mask.setRoundness", json!({"layer": l, "mask": e, "roundness": 10})).is_err(), "{rect:?}");
+        assert_eq!(mask_path(&s, l, e), before);
+    }
+    // A feather point on the top side survives a rounded-to-rounded edit, still on the top side.
+    let m = add(&mut s, "rounded", [100, 100, 200, 100], json!({"roundness": 30}));
+    s.execute("mask.featherPoint.add", json!({"layer": l, "mask": m, "point": [200, 90]})).unwrap();
+    let on_top = |p: &effectcraft_keyframe::ShapePath| {
+        let f = &p.feather[0];
+        let n = p.vertices.len();
+        p.vertices[f.segment % n][1] == 100.0 && p.vertices[(f.segment + 1) % n][1] == 100.0
+    };
+    assert!(on_top(&mask_path(&s, l, m)));
+    s.execute("mask.setRoundness", json!({"layer": l, "mask": m, "roundness": 50})).unwrap();
+    let p = mask_path(&s, l, m);
+    assert_eq!((p.vertices.len(), p.feather.len()), (8, 1));
+    assert!(on_top(&p), "{p:?}");
+    // Square corners would drop four vertices under the feather point: refused.
+    assert!(s.execute("mask.setRoundness", json!({"layer": l, "mask": m, "roundness": 0})).is_err());
+    assert_eq!(mask_path(&s, l, m), p);
+    // A keyed path: a rounded one is re-rounded with its vertex count kept, so its keys still
+    // interpolate; a square one can't gain the corner vertices on one key only.
+    let keys = |s: &Session, m: u64| -> Vec<usize> {
+        let g = layer(s, l).masks().unwrap().find_group(m).unwrap().clone();
+        g.get("path").unwrap().keys.iter().map(|k| k.value.as_path().unwrap().vertices.len()).collect()
+    };
+    let r = add(&mut s, "rounded", [100, 100, 200, 100], json!({"roundness": 20}));
+    let q = add(&mut s, "rect", [100, 100, 200, 100], json!({}));
+    for (i, mask) in [(4, r), (5, q)] {
+        s.execute("time.set", json!({"time": 0})).unwrap();
+        s.execute("prop.addKey", json!({"layer": l, "path": format!("masks/#{i}/path"), "time": 0.0})).unwrap();
+        s.execute("time.set", json!({"time": 1})).unwrap();
+        let ok = s.execute("mask.setRoundness", json!({"layer": l, "mask": mask, "roundness": 40})).is_ok();
+        assert_eq!(ok, mask == r);
+    }
+    assert_eq!(keys(&s, r), [8, 8]);
+    assert_eq!(keys(&s, q), [4]);
+}

docs/agents.md

@@ -318,8 +318,10 @@ build real projects through these interfaces; they are worked examples of everyt
   channels (`{"layers":[{"name":"diffuse","rgba":["diffuse.R","diffuse.G","diffuse.B",""]}]}`);
   EXtractoR shows them when its `red` / `green` / `blue` / `alpha` are set to those names
   (`prop.set {"path":"effects/#1/red","value":"depth.Z"}`). Effect Controls offers them as popups.
-* **Masks**: `layer.addMask {layer, shape: rect|ellipse, rect}` first, then `layer.mask.set
-  {field: feather|expansion|opacity, value}`.
+* **Masks**: `layer.addMask {layer, shape: rect|ellipse|rounded, rect, roundness}` first, then
+  `layer.mask.set {field: feather|expansion|opacity, value}`. A Rounded Rectangle mask's corner
+  radius (`roundness`, layer pixels) is set when it's added and changed later with
+  `mask.setRoundness {layer, mask, roundness}` (0 = square corners); it only reshapes a rectangular mask.
 * **Renders**: `renderQueue.add {comp, format: h264|prores|webm|png…, channels: rgba,
   proresProfile: 4444, output}` then `renderQueue.render {"wait": true}`. Check a render by
   importing it (`file.import`, `file.newCompFromSelection`) and `render_frame {transparent: true}`