S2.71: the boundary view outlines each object in its own colour

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/engine/src/analysis/roto.rs

@@ -293,7 +293,7 @@ pub(crate) fn params_at(p: &Project, cid: ItemId, layer: &Layer, g: &PropGroup,
 }
 
 /// The layer's source size (layer pixels).
-pub(crate) fn layer_size(p: &Project, comp: &effectcraft_project::Comp, layer: &Layer) -> [f64; 2] {
+pub fn layer_size(p: &Project, comp: &effectcraft_project::Comp, layer: &Layer) -> [f64; 2] {
     effectcraft_render::effect_bounds(p, comp, layer).0
 }
 

crates/ui-egui/src/panels/roto_tool.rs

@@ -21,6 +21,17 @@ fn roto_group(layer: &Layer) -> Option<&effectcraft_engine::project::PropGroup>
     layer.effects()?.groups().nth(roto_index(layer)?)
 }
 
+/// The colour of object `o`'s outline: object 0 keeps the chosen boundary colour, the others cycle through a palette.
+pub fn object_color(o: u32, base: [f64; 3]) -> Color32 {
+    const PALETTE: [[u8; 3]; 8] =
+        [[60, 200, 255], [255, 170, 40], [150, 230, 60], [180, 110, 255], [255, 225, 60], [40, 210, 170], [255, 90, 80], [90, 130, 255]];
+    if o == 0 {
+        return c32(base, 255);
+    }
+    let [r, g, b] = PALETTE[(o as usize - 1) % PALETTE.len()];
+    Color32::from_rgb(r, g, b)
+}
+
 /// A stroke being drawn.
 #[derive(Clone, Debug)]
 struct Drag {
@@ -94,6 +105,22 @@ pub fn draw_overlay(
             let alpha: Vec<f32> = (0..w * h).map(|i| a_at((i % w) as i64, (i / w) as i64)).collect();
             let inside = |x: i64, y: i64| x >= 0 && y >= 0 && x < w as i64 && y < h as i64 && alpha[y as usize * w + x as usize] >= 0.5;
             let mut px = vec![Color32::TRANSPARENT; w * h];
+            // With several objects the boundary view outlines each in its own colour.
+            let per_object: Vec<(Color32, Vec<bool>)> = match (o.view.as_str(), roto_group(layer)) {
+                ("alphaBoundary", Some(g)) => {
+                    let params = effectcraft_engine::effects::flatten_params(g, &mut |p| p.value.clone());
+                    let size = effectcraft_engine::roto::layer_size(&app.session.project, comp, layer);
+                    let f = fx::frame_of(layer.layer_time(time).seconds(), comp.frame_rate.as_f64());
+                    fx::object_mattes(&params, size, comp.frame_rate.as_f64(), f)
+                        .into_iter()
+                        .map(|(obj, seg)| {
+                            let seg = fx::resample(&seg, w, h);
+                            (object_color(obj, o.boundary_color), seg.matte.iter().map(|m| *m != 0).collect())
+                        })
+                        .collect()
+                }
+                _ => vec![],
+            };
             let bc = c32(o.boundary_color, 255);
             let oc = c32(o.overlay_color, (o.overlay_opacity / 100.0 * 255.0).clamp(0.0, 255.0) as u8);
             // View Search Region: the band around the boundary (yellow).
@@ -117,7 +144,20 @@ pub fn draw_overlay(
                         _ => {
                             let me = a >= 0.5;
                             let edge = me && [(-1, 0), (1, 0), (0, -1), (0, 1)].iter().any(|(dx, dy)| !inside(x + dx, y + dy));
-                            if edge { bc } else { Color32::TRANSPARENT }
+                            if per_object.is_empty() {
+                                if edge { bc } else { Color32::TRANSPARENT }
+                            } else {
+                                // the outline of every object whose own matte has an edge here
+                                let inside_of =
+                                    |m: &Vec<bool>, x: i64, y: i64| x >= 0 && y >= 0 && x < w as i64 && y < h as i64 && m[y as usize * w + x as usize];
+                                per_object
+                                    .iter()
+                                    .rev()
+                                    .find(|(_, m)| {
+                                        inside_of(m, x, y) && [(-1, 0), (1, 0), (0, -1), (0, 1)].iter().any(|(dx, dy)| !inside_of(m, x + dx, y + dy))
+                                    })
+                                    .map_or(Color32::TRANSPARENT, |(c, _)| *c)
+                            }
                         }
                     };
                     if let Some((sd, r)) = &search {
@@ -389,6 +429,7 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
         let next = all.iter().copied().find(|o| *o > current).or_else(|| all.first().copied()).unwrap_or(0);
         let _ = app.session.execute("roto.options", json!({"object": next}));
     }
+    p.circle_filled(pos2(r.min.x + 9.0, r.center().y), 4.0, object_color(current, app.session.state.roto.boundary_color));
     app.auto.add("layerPanel.roto.object", r, &label);
     x += 90.0;
     let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));

crates/ui-egui/tests/ui_roto_objects.rs

@@ -0,0 +1,97 @@
+//! A Roto Brush instance with two objects: the Layer panel's boundary view outlines each in its own colour, and
+//! Show Object limits the matte to one (egui_kittest, a real render). A stand-in tracker follows two regions, so no
+//! weights are needed. `ROTO_OBJECTS_SNAPSHOT=/abs/out.png` also saves the rendered panel.
+
+use std::sync::Arc;
+
+use effectcraft_engine::Session;
+use effectcraft_segment::{ClipFrame, MaskModel, ModelInfo, ObjectTracker, Prompt, TrackRequest, TrackedFrame};
+use effectcraft_ui_egui::EffectcraftApp;
+use effectcraft_ui_egui::dock::PanelKind;
+use egui_kittest::Harness;
+use serde_json::json;
+
+/// Follows two fixed regions of the layer: object 0 on the left, object 1 on the right.
+struct TwoRegions;
+
+impl MaskModel for TwoRegions {
+    fn info(&self) -> &'static ModelInfo {
+        &effectcraft_segment::SAM2_TINY
+    }
+    fn segment(&self, _: &[[f32; 3]], _: usize, _: usize, _: &Prompt) -> effectcraft_segment::Result<Vec<f32>> {
+        Err("no weights".into())
+    }
+    fn tracker(&self) -> Option<&dyn ObjectTracker> {
+        Some(self)
+    }
+}
+
+impl ObjectTracker for TwoRegions {
+    fn track(
+        &self,
+        req: &TrackRequest,
+        frame: &mut dyn FnMut(i64) -> Option<Arc<ClipFrame>>,
+        _: &dyn Fn() -> bool,
+        out: &mut dyn FnMut(usize, i64, TrackedFrame),
+    ) -> effectcraft_segment::Result<()> {
+        for g in req.first..req.first + req.count as i64 {
+            let img = frame(g).ok_or("no frame")?;
+            for o in 0..req.objects.len() {
+                let (x0, x1) = if o == 0 { (img.w / 8, img.w * 3 / 8) } else { (img.w * 5 / 8, img.w * 7 / 8) };
+                let prob = (0..img.w * img.h)
+                    .map(|i| {
+                        let (x, y) = (i % img.w, i / img.w);
+                        f32::from(x >= x0 && x < x1 && y > img.h / 4 && y < img.h * 3 / 4)
+                    })
+                    .collect();
+                out(o, g, TrackedFrame { prob, w: img.w, h: img.h, confidence: 0.99, occluded: false });
+            }
+        }
+        Ok(())
+    }
+}
+
+/// Pixels of the viewer (above the panel's bar, whose object swatch has the same colour) within a few levels of `want`.
+fn count(image: &image::RgbaImage, want: [u8; 3]) -> usize {
+    let limit = image.height() * 54 / 100;
+    image.enumerate_pixels().filter(|(_, y, p)| *y < limit && p.0[..3].iter().zip(want).all(|(a, b)| a.abs_diff(b) <= 12)).count()
+}
+
+#[test]
+fn the_boundary_view_outlines_each_object_and_show_object_picks_one() {
+    let mut s = Session::default();
+    s.execute("comp.new", json!({"name": "View", "width": 640, "height": 360, "duration": 4})).unwrap();
+    s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 80, "height": 80})).unwrap();
+    let id = s.active_comp().unwrap().layers.iter().find(|l| l.name == "Box").unwrap().id.0;
+    for (object, x) in [(0, 22.0), (1, 58.0)] {
+        s.execute("roto.stroke", json!({"layer": id, "object": object, "points": [[x, 38.0], [x, 42.0]], "radius": 4})).unwrap();
+    }
+    effectcraft_engine::effects::roto::set_model(Some(Arc::new(TwoRegions)));
+    s.execute("roto.propagate", json!({"layer": id, "direction": "both", "wait": true})).unwrap();
+    let st = s.execute("roto.status", json!({"layer": id})).unwrap();
+    assert_eq!(st["objects"], json!([{"id": 0, "strokes": 1}, {"id": 1, "strokes": 1}]), "{st}");
+
+    // object 0 keeps the boundary colour (pink by default), object 1 is the first palette colour (blue-green)
+    let (pink, cyan) = ([255, 64, 191], [60, 200, 255]);
+    let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(move |_| EffectcraftApp::new(s));
+    h.run_steps(3);
+    h.state_mut().session.execute("layer.select", json!({"layers": [id]})).unwrap();
+    let ctx = h.ctx.clone();
+    effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "layer.openLayer", json!({"layer": id})).unwrap();
+    h.run_steps(4);
+    assert_eq!(h.state().ui.focused, PanelKind::Layer);
+    let both = h.render().expect("the panel renders");
+    if let Some(path) = std::env::var_os("ROTO_OBJECTS_SNAPSHOT") {
+        both.save(path).unwrap();
+    }
+    let (a, b) = (count(&both, pink), count(&both, cyan));
+    assert!(a > 50 && b > 50, "both objects are outlined: {a} pink, {b} cyan pixels");
+
+    // Show Object 2 (object id 1): only its outline remains
+    h.state_mut().session.execute("prop.set", json!({"layer": id, "path": "effects/#1/rotoBrushMatte/object", "value": 2})).unwrap();
+    h.run_steps(4);
+    let one = h.render().expect("the panel renders");
+    let (a, b) = (count(&one, pink), count(&one, cyan));
+    assert!(a < 10 && b > 50, "only object 1 is shown: {a} pink, {b} cyan pixels");
+    effectcraft_engine::effects::roto::set_model(None);
+}

docs/roto-brush-sam2.md

@@ -43,6 +43,14 @@ lies under it without cutting a hole in the others. Refine Edge strokes belong t
 no objects: all foreground strokes make one object, and background strokes apply to it as before. Saved projects are
 unchanged (strokes without an object are object 0, and nothing is written for object 0).
 
+**Seeing and choosing objects.** The Layer panel's boundary view outlines each object in its own colour (object 1 in the
+chosen boundary colour, the others from a palette), and the picker button carries the current object's colour. **Show
+Object** (Roto Brush Matte, a slider: 0 shows all objects together, N shows object N alone) limits the instance's matte to
+one object, so an effect after it can treat the pieces separately; an object the instance does not have shows nothing, and
+with one object, or on frames the classic engine made, the setting changes nothing. Each object's matte is kept next to
+the united one, in the session and in the saved tracking (`tracked` stores them only when there are several objects), so
+Show Object renders the same after the project is reopened. Freeze records the Show Object that was set.
+
 This is not several Roto Brush instances sharing a pass: each instance reads the layer after the effects above it, so
 instances can see different pixels. Objects inside one instance always see the same ones.