From b72637bb55e38d05d2f996a127df39cd56f6276a Mon Sep 17 00:00:00 2001
From: Kristian Myram <kristian.myram@gmail.com>
Date: Thu, 15 Oct 2026 07:16:50 +0200
Subject: [PATCH] S2.50: bench --roto compares classical, MobileSAM and SAM 2

--mobilesam FILE; results recorded in docs/roto-brush-sam2.md (re-entry clip:
0.632 classical, 0.499 MobileSAM, 0.975 SAM 2).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 apps/effectcraft-cli/src/main.rs | 17 +++++++++--------
 docs/roto-brush-sam2.md          | 22 +++++++++++++---------
 2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs
index b60918304..27710a4a2 100644
--- a/apps/effectcraft-cli/src/main.rs
+++ b/apps/effectcraft-cli/src/main.rs
@@ -27,7 +27,7 @@
 //!     (--gpu: CPU vs GPU ms/frame for every comp at Full and Half)
 //! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N]   everyday-operation timings
 //!     on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)
-//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--json]   Roto Brush trackers on synthetic clips with known mattes
+//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--json]   Roto Brush trackers on synthetic clips with known mattes
 //!     (classical engine, MobileSAM and SAM 2.1 Tiny when installed; IoU, flicker, re-appearance, seconds per frame)
 //! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE    run an After Effects-style script
 //! effectcraft-cli mcp [--autosave | --bridge PORT]            MCP server on stdio
@@ -126,7 +126,7 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|se
   bench --ops [--small] [--layers N] [--comps N] [--footage N]
                                            everyday operations on a large generated project: startup, open,
                                            save, auto-save, edits + undo/redo, timeline, Project panel
-  bench --roto [--frames N] [--sam2 FILE] [--json]
+  bench --roto [--frames N] [--sam2 FILE] [--mobilesam FILE] [--json]
                                            Roto Brush trackers on synthetic clips with known mattes: movement,
                                            lighting change, occlusion, leaving and re-entering the frame
   script FILE.jsx [F.ecproj] | --eval CODE run JavaScript with the After Effects-style object model
@@ -144,6 +144,7 @@ const VALUED: &[&str] = &[
     "--n",
     "--frames",
     "--sam2",
+    "--mobilesam",
     "--play",
     "--layers",
     "--comps",
@@ -1015,13 +1016,13 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {
     use effectcraft_track::roto::bench;
     let frames = args.num("--frames")?.unwrap_or(24.0).clamp(12.0, 240.0) as usize;
     let session = effectcraft_host::session();
+    // `--mobilesam FILE` uses the weights at FILE; otherwise the installed ones.
+    let mobilesam_file = args.opt("--mobilesam").map(std::path::PathBuf::from).or_else(|| session.model_path(&MOBILE_SAM));
     let mobilesam: Option<std::sync::Arc<dyn MaskModel>> =
-        session.model_path(&MOBILE_SAM).and_then(|p| std::fs::read(p).ok()).and_then(|bytes| effectcraft_segment::load(MOBILE_SAM.id, &bytes).ok()).and_then(
-            |m| match m {
-                Loaded::Mask(m) => Some(m),
-                _ => None,
-            },
-        );
+        mobilesam_file.and_then(|p| std::fs::read(p).ok()).and_then(|bytes| effectcraft_segment::load(MOBILE_SAM.id, &bytes).ok()).and_then(|m| match m {
+            Loaded::Mask(m) => Some(m),
+            _ => None,
+        });
     if mobilesam.is_none() {
         note!("bench --roto: MobileSAM is not installed; comparing the classical engine only");
     }
diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md
index b2c356e74..efa7a5eb8 100644
--- a/docs/roto-brush-sam2.md
+++ b/docs/roto-brush-sam2.md
@@ -44,14 +44,18 @@ clip costs one encode per frame, so a 300-frame clip takes on the order of 17 mi
 backend; GPU work is Phase 6 of the plan (the early Hiera blocks are memory-bound and have headroom on the CPU too).
 
 ## Benchmark
-`effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors]` runs synthetic clips with known mattes. 12 frames,
-debug build, mean IoU (after the object returns):
+`effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt]` runs synthetic clips
+with known mattes through all three. 12 frames, debug build, this PC; mean IoU (mean IoU after the object returns), seconds
+per frame:
 
-| Clip | Classical | SAM 2.1 Tiny |
-|---|---|---|
-| move | 0.982 | 0.993 |
-| light | 0.981 | 0.992 |
-| occlusion | 0.978 | 0.988 |
-| reenter | 0.632 (0.000) | 0.975 (0.967) |
+| Clip | Classical | MobileSAM | SAM 2.1 Tiny |
+|---|---|---|---|
+| move | 0.982 | 0.992 | 0.993 |
+| light | 0.981 | 0.992 | 0.992 |
+| occlusion | 0.978 | 0.881 | 0.988 |
+| reenter | 0.632 (0.000) | 0.499 (0.725) | 0.975 (0.967) |
+| s/frame | 0.01 | 1.1 | 8.2 |
 
-MobileSAM's column appears when it is installed.
+MobileSAM prompts every frame independently from the flow-warped matte, so it follows the object while the flow does and
+loses it when it is hidden; SAM 2's streaming memory is what lets it re-acquire the object. The synthetic clips are easy
+(one textured disc), so the "move" and "light" rows mostly show all three working; the re-entry row is the difference.