S2.50: effectcraft-cli bench --roto on synthetic clips

Synthetic Roto Brush clips with known mattes (movement, lighting change,
occlusion, leaving and re-entering) and a runner that compares the classical
engine with MobileSAM when installed. Needs no weights; SAM 2 plugs in later.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Cargo.lock

@@ -1394,7 +1394,9 @@ dependencies = [
  "effectcraft-gpu",
  "effectcraft-host",
  "effectcraft-render",
+ "effectcraft-segment",
  "effectcraft-time",
+ "effectcraft-track",
  "effectcraft-ui-egui",
  "image",
  "serde_json",

apps/effectcraft-cli/Cargo.toml

@@ -20,6 +20,8 @@ effectcraft-ui-egui = { workspace = true }
 effectcraft-render = { workspace = true }
 effectcraft-gpu = { workspace = true }
 effectcraft-time = { workspace = true }
+effectcraft-track = { workspace = true }
+effectcraft-segment = { workspace = true }
 serde_json = { workspace = true }
 image = { workspace = true }
 cpu-time = "1"

apps/effectcraft-cli/src/main.rs

@@ -27,6 +27,8 @@
 //!     (--gpu: CPU vs GPU ms/frame for every comp at Full and Half)
 //! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N]   everyday-operation timings
 //!     on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)
+//! effectcraft-cli bench --roto [--frames N] [--json]   Roto Brush trackers on synthetic clips with known mattes
+//!     (classical engine, MobileSAM when installed; IoU, flicker, re-appearance, seconds per frame)
 //! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE    run an After Effects-style script
 //! effectcraft-cli mcp [--autosave | --bridge PORT]            MCP server on stdio
 //!
@@ -124,6 +126,8 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|se
   bench --ops [--small] [--layers N] [--comps N] [--footage N]
                                            everyday operations on a large generated project: startup, open,
                                            save, auto-save, edits + undo/redo, timeline, Project panel
+  bench --roto [--frames N] [--json]       Roto Brush trackers on synthetic clips with known mattes: movement,
+                                           lighting change, occlusion, leaving and re-entering the frame
   script FILE.jsx [F.ecproj] | --eval CODE run JavaScript with the After Effects-style object model
                                            (app.project, comps, layers, properties…); prints writeLn
                                            output and the result; errors exit 1 with file:line:col
@@ -137,6 +141,7 @@ options: --project F.ecproj | --demo | --empty   --save | --save-as F   --bridge
 /// Options that take a value.
 const VALUED: &[&str] = &[
     "--n",
+    "--frames",
     "--play",
     "--layers",
     "--comps",
@@ -199,6 +204,7 @@ const FLAGS: &[&str] = &[
     "--work-area",
     "--queue",
     "--ops",
+    "--roto",
     "--small",
     "--adv3d",
     "--autosave",
@@ -976,6 +982,9 @@ fn bench_cmd(args: &Args) -> Result<(), Failure> {
     if args.flag("--ops") {
         return bench_ops(args);
     }
+    if args.flag("--roto") {
+        return bench_roto(args);
+    }
     let mut s = effectcraft_host::session();
     match &args.project {
         Some(p) => s.execute("file.open", json!({"path": p})),
@@ -998,6 +1007,37 @@ fn bench_cmd(args: &Args) -> Result<(), Failure> {
     Ok(())
 }
 
+/// `bench --roto`: the Roto Brush's trackers on synthetic clips whose true mattes are known.
+fn bench_roto(args: &Args) -> Result<(), Failure> {
+    use effectcraft_segment::{Loaded, MOBILE_SAM, MaskModel};
+    use effectcraft_track::roto::bench;
+    let frames = args.num("--frames")?.unwrap_or(24.0).clamp(12.0, 240.0) as usize;
+    let session = effectcraft_host::session();
+    let mobilesam: Option<std::sync::Arc<dyn MaskModel>> =
+        session.model_path(&MOBILE_SAM).and_then(|p| std::fs::read(p).ok()).and_then(|bytes| effectcraft_segment::load(MOBILE_SAM.id, &bytes).ok()).and_then(
+            |m| match m {
+                Loaded::Mask(m) => Some(m),
+                _ => None,
+            },
+        );
+    if mobilesam.is_none() {
+        note!("bench --roto: MobileSAM is not installed; comparing the classical engine only");
+    }
+    let mut runs = vec![];
+    for sc in bench::scenarios(frames) {
+        runs.push(bench::run_flow(&sc, "classical", None));
+        if let Some(m) = &mobilesam {
+            runs.push(bench::run_flow(&sc, "mobilesam", Some(m.as_ref())));
+        }
+    }
+    if args.flag("--json") {
+        say!("{}", serde_json::to_string_pretty(&runs).map_err(|e| Failure::Error(e.to_string()))?);
+    } else {
+        say!("{}", bench::table(&runs));
+    }
+    Ok(())
+}
+
 /// `bench --ops`: everyday operations on a large generated project (200 comps, 5,000 layers in
 /// the main comp, 300 footage items, 20-deep precomp nesting, expressions): startup to first
 /// frame, open, save, auto-save, small and large edits with undo/redo, first frame, timeline

crates/track/src/roto.rs

@@ -31,6 +31,7 @@
 //! deterministic function of its *chain*: the base frame's strokes, then every frame's strokes
 //! on the way out to it ([`RotoData::chain_keys`]).
 
+pub mod bench;
 pub mod gmm;
 pub mod matting;
 pub mod maxflow;

crates/track/src/roto/bench.rs

@@ -0,0 +1,264 @@
+//! Roto Brush benchmark: synthetic clips with known mattes (movement, colour change, occlusion,
+//! leaving and re-entering the frame) and the metrics that compare trackers on them.
+//!
+//! Needs no weights: the classical engine always runs, and a trained [`MaskModel`] (MobileSAM,
+//! later SAM 2) is added when the caller has one. `effectcraft-cli bench --roto` prints the table.
+
+use effectcraft_raster::Image;
+use effectcraft_segment::MaskModel;
+use serde::Serialize;
+use web_time::Instant;
+
+use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, segment_with};
+
+pub const W: u32 = 200;
+pub const H: u32 = 150;
+const R: f64 = 32.0;
+
+/// A clip with the true matte of the visible object on every frame.
+pub struct Scenario {
+    pub name: &'static str,
+    pub what: &'static str,
+    pub frames: Vec<Image>,
+    /// Visible-object matte per frame (`W×H`, 0/1); empty when the object is out of view.
+    pub truth: Vec<Vec<u8>>,
+    /// Frame the strokes are painted on.
+    pub base: usize,
+    pub strokes: Vec<Stroke>,
+}
+
+/// One tracker's result on one scenario.
+#[derive(Clone, Debug, Serialize)]
+pub struct Run {
+    pub scenario: &'static str,
+    pub tracker: String,
+    pub mean_iou: f64,
+    pub min_iou: f64,
+    /// Mean IoU between consecutive mattes minus the same for the truth, as a loss (0 is steady).
+    pub flicker: f64,
+    pub secs_per_frame: f64,
+    /// Mean IoU over the frames after the object's last absence (re-appearance), if it had one.
+    pub after_return: Option<f64>,
+    pub per_frame: Vec<f64>,
+}
+
+fn noise(x: f64, y: f64, seed: u32) -> f64 {
+    fn h(i: i64, j: i64, s: u32) -> f64 {
+        let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ i64::from(s).wrapping_mul(83492791)) as u64;
+        v ^= v >> 13;
+        v = v.wrapping_mul(0x5bd1e995);
+        v ^= v >> 15;
+        (v % 10_000) as f64 / 10_000.0
+    }
+    let mut out = 0.0;
+    for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] {
+        let (fx, fy) = (x / cell, y / cell);
+        let (i, j) = (fx.floor() as i64, fy.floor() as i64);
+        let (tx, ty) = (fx - i as f64, fy - j as f64);
+        let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty));
+        let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx;
+        let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx;
+        out += amp * (a + (b - a) * sy);
+    }
+    out
+}
+
+/// What a frame shows: the disk centre, an optional occluding bar `[x0, y0, x1, y1]` and a
+/// brightness gain applied to the whole frame.
+#[derive(Clone, Copy)]
+struct Look {
+    c: [f64; 2],
+    bar: Option<[f64; 4]>,
+    gain: f32,
+}
+
+fn render(l: Look) -> (Image, Vec<u8>) {
+    let mut img = Image::new(W, H);
+    let mut gt = vec![0u8; (W * H) as usize];
+    for y in 0..H {
+        for x in 0..W {
+            let p = [f64::from(x) + 0.5, f64::from(y) + 0.5];
+            let d = (p[0] - l.c[0]).hypot(p[1] - l.c[1]);
+            let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32;
+            let n = noise(p[0] - l.c[0], p[1] - l.c[1], 11);
+            let f = [(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32];
+            let (bn, bm) = (noise(p[0], p[1], 5), noise(p[0] * 1.7, p[1] * 1.7, 9));
+            let b = [(0.05 + 0.25 * bm) as f32, (0.3 + 0.4 * bn) as f32, (0.45 + 0.4 * bm) as f32];
+            let mut px = [0.0f32; 3];
+            for k in 0..3 {
+                px[k] = f[k] * a + b[k] * (1.0 - a);
+            }
+            let under_bar = l.bar.is_some_and(|r| p[0] >= r[0] && p[0] < r[2] && p[1] >= r[1] && p[1] < r[3]);
+            if under_bar {
+                let t = noise(p[0] * 0.8, p[1] * 0.8, 23) as f32;
+                px = [0.15 + 0.1 * t, 0.15 + 0.1 * t, 0.2 + 0.1 * t];
+            }
+            let g = l.gain;
+            img.set(x, y, [(px[0] * g).min(1.0), (px[1] * g).min(1.0), (px[2] * g).min(1.0), 1.0]);
+            if let Some(v) = gt.get_mut((y * W + x) as usize) {
+                *v = (d <= R && !under_bar) as u8;
+            }
+        }
+    }
+    (img, gt)
+}
+
+fn line(kind: StrokeKind, frame: i64, a: [f64; 2], b: [f64; 2], radius: f64) -> Stroke {
+    let pt = |k: u32| {
+        let t = f64::from(k) / 10.0;
+        [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]
+    };
+    Stroke { kind, frame, radius, points: (0..=10).map(pt).collect() }
+}
+
+/// The strokes an artist would paint on the base frame: one through the disk, two around it.
+fn strokes_for(c: [f64; 2]) -> Vec<Stroke> {
+    vec![
+        line(StrokeKind::Fg, 0, [c[0] - 18.0, c[1] - 6.0], [c[0] + 18.0, c[1] + 6.0], 5.0),
+        line(StrokeKind::Bg, 0, [c[0] - 50.0, c[1] - 45.0], [c[0] + 50.0, c[1] - 45.0], 5.0),
+        line(StrokeKind::Bg, 0, [c[0] + 50.0, c[1] + 45.0], [c[0] - 45.0, c[1] + 45.0], 5.0),
+    ]
+}
+
+fn scenario(name: &'static str, what: &'static str, n: usize, look: impl Fn(usize) -> Look) -> Scenario {
+    let (mut frames, mut truth) = (Vec::with_capacity(n), Vec::with_capacity(n));
+    for f in 0..n {
+        let (img, gt) = render(look(f));
+        frames.push(img);
+        truth.push(gt);
+    }
+    Scenario { name, what, frames, truth, base: 0, strokes: strokes_for(look(0).c) }
+}
+
+/// The benchmark clips, `n` frames each (at least 12, so every event happens).
+pub fn scenarios(n: usize) -> Vec<Scenario> {
+    let n = n.clamp(12, 240);
+    let k = n as f64 / 24.0;
+    let drift = move |f: usize| {
+        let t = f as f64 / k;
+        [60.0 + 3.1 * t, 70.0 + 1.3 * t + 4.0 * (t * 0.3).sin()]
+    };
+    vec![
+        scenario("move", "disk drifts across a textured background", n, move |f| Look { c: drift(f), bar: None, gain: 1.0 }),
+        scenario("light", "same motion while the whole frame brightens by 40 %", n, move |f| Look {
+            c: drift(f),
+            bar: None,
+            gain: 1.0 + 0.4 * f as f32 / n as f32,
+        }),
+        scenario("occlusion", "a bar crosses and hides most of the disk, then clears it", n, move |f| {
+            let t = f as f64 / n as f64;
+            let c = drift(f);
+            let x = -40.0 + 320.0 * ((t - 0.3) / 0.4).clamp(0.0, 1.0);
+            Look { c, bar: (0.3..0.7).contains(&t).then_some([x - 40.0, 0.0, x + 40.0, f64::from(H)]), gain: 1.0 }
+        }),
+        scenario("reenter", "disk leaves through the right edge and comes back", n, move |f| {
+            let t = f as f64 / n as f64;
+            let x = 60.0 + 190.0 * (1.0 - (2.0 * t - 1.0).powi(2));
+            Look { c: [x, 70.0 + 4.0 * (t * 6.0).sin()], bar: None, gain: 1.0 }
+        }),
+    ]
+}
+
+fn metrics(sc: &Scenario, tracker: &str, mattes: &[Vec<u8>], secs: f64) -> Run {
+    let per_frame: Vec<f64> = mattes.iter().zip(&sc.truth).map(|(m, t)| iou(m, t)).collect();
+    let n = per_frame.len().max(1) as f64;
+    let steady = |v: &[Vec<u8>]| {
+        let pairs: Vec<f64> = v.windows(2).filter_map(|w| Some(iou(w.first()?, w.get(1)?))).collect();
+        pairs.iter().sum::<f64>() / pairs.len().max(1) as f64
+    };
+    let last_gap = sc.truth.iter().rposition(|t| t.iter().all(|v| *v == 0));
+    let after_return = last_gap.and_then(|g| {
+        let tail = per_frame.get(g + 1..)?;
+        (!tail.is_empty()).then(|| tail.iter().sum::<f64>() / tail.len() as f64)
+    });
+    Run {
+        scenario: sc.name,
+        tracker: tracker.to_string(),
+        mean_iou: per_frame.iter().sum::<f64>() / n,
+        min_iou: per_frame.iter().copied().fold(1.0, f64::min),
+        flicker: (steady(&sc.truth) - steady(mattes)).max(0.0),
+        secs_per_frame: secs / n,
+        after_return,
+        per_frame,
+    }
+}
+
+/// Segment the base frame from the strokes and propagate to the end of the clip, as the Roto
+/// Brush does, with `model` (or none: the classical engine).
+pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) -> Run {
+    let opts = SegOpts::default();
+    let start = Instant::now();
+    let refs: Vec<&Stroke> = sc.strokes.iter().collect();
+    let mut segs: Vec<Option<FrameSeg>> = vec![None; sc.frames.len()];
+    if let Some(first) = sc.frames.get(sc.base) {
+        let base = segment_with(first, &refs, None, &opts, 0.0, model);
+        if let Some(slot) = segs.get_mut(sc.base) {
+            *slot = Some(base);
+        }
+    }
+    for f in sc.base + 1..sc.frames.len() {
+        let (Some(prev_img), Some(img)) = (sc.frames.get(f - 1), sc.frames.get(f)) else { break };
+        let Some(Some(prev)) = segs.get(f - 1) else { break };
+        let next = propagate_with(prev_img, prev, img, &[], &opts, 0.0, model);
+        if let Some(slot) = segs.get_mut(f) {
+            *slot = Some(next);
+        }
+    }
+    let secs = start.elapsed().as_secs_f64();
+    let empty = vec![0u8; (W * H) as usize];
+    let mattes: Vec<Vec<u8>> = segs.into_iter().map(|s| s.map_or_else(|| empty.clone(), |s| s.matte)).collect();
+    metrics(sc, tracker, &mattes, secs)
+}
+
+/// A fixed-width table of the runs, grouped by scenario.
+pub fn table(runs: &[Run]) -> String {
+    let mut out = format!("{:<10} {:<12} {:>8} {:>8} {:>8} {:>10} {:>9}\n", "clip", "tracker", "meanIoU", "minIoU", "flicker", "after ret.", "s/frame");
+    for r in runs {
+        let back = r.after_return.map_or_else(|| "-".to_string(), |v| format!("{v:.3}"));
+        out.push_str(&format!(
+            "{:<10} {:<12} {:>8.3} {:>8.3} {:>8.3} {:>10} {:>9.3}\n",
+            r.scenario, r.tracker, r.mean_iou, r.min_iou, r.flicker, back, r.secs_per_frame
+        ));
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn area(v: &[u8]) -> usize {
+        v.iter().filter(|x| **x != 0).count()
+    }
+
+    #[test]
+    fn scenarios_have_the_events_they_claim() {
+        let all = scenarios(24);
+        assert_eq!(all.len(), 4);
+        for s in &all {
+            assert_eq!((s.frames.len(), s.truth.len()), (24, 24), "{}", s.name);
+        }
+        let full = all.first().map(|s| area(&s.truth[0])).unwrap_or(0);
+        assert!(full > 3000, "{full}");
+        let occ = all.iter().find(|s| s.name == "occlusion").map(|s| s.truth.iter().map(|t| area(t)).min().unwrap_or(0)).unwrap_or(full);
+        assert!(occ < full / 2, "occluded minimum {occ} of {full}");
+        let gone = all.iter().find(|s| s.name == "reenter").map(|s| s.truth.iter().filter(|t| area(t) == 0).count()).unwrap_or(0);
+        assert!(gone > 0, "the disk never leaves");
+        assert!(all.iter().find(|s| s.name == "reenter").is_some_and(|s| area(s.truth.last().map_or(&[][..], |v| v)) > 0), "it must come back");
+    }
+
+    #[test]
+    fn classical_tracks_plain_motion_and_the_table_lists_it() {
+        let all = scenarios(24);
+        let Some(sc) = all.first() else { return };
+        let run = run_flow(sc, "classical", None);
+        assert!(run.mean_iou > 0.9, "mean IoU {}", run.mean_iou);
+        assert_eq!(run.per_frame.len(), 24);
+        assert!(table(&[run]).contains("classical"));
+    }
+
+    #[test]
+    fn tiny_clip_requests_are_clamped() {
+        assert!(scenarios(0).iter().all(|s| s.frames.len() == 12));
+    }
+}