From ac8765bf67b42dd6b4bcbc75b3fe09cf02432848 Mon Sep 17 00:00:00 2001 From: Kristian Myram Date: Sat, 17 Oct 2026 14:31:25 +0200 Subject: [PATCH] S2.65: help and bench follow all four SAM 2 sizes; tests for the size configs and detection Co-Authored-By: Claude Sonnet 5.5 --- apps/effectcraft-cli/src/main.rs | 9 +++++---- crates/engine/src/analysis/models.rs | 2 +- crates/engine/src/tests/models.rs | 6 +++++- crates/sam2/src/lib.rs | 30 ++++++++++++++++++++++++++++ docs/roto-brush-sam2.md | 2 +- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index e6a3352dc..1c528c8fb 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -28,7 +28,7 @@ //! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N] everyday-operation timings //! on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel) //! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--fixes] [--clip FRAMES --truth MASKS] [--json] Roto Brush trackers on synthetic clips with known mattes -//! (classical engine, MobileSAM and SAM 2.1 Tiny when installed; IoU, flicker, re-appearance, seconds per frame) +//! (classical engine, MobileSAM and SAM 2.1 (any installed size) when installed; IoU, flicker, re-appearance, seconds per frame) //! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script //! effectcraft-cli mcp [--autosave | --bridge PORT] MCP server on stdio //! @@ -1030,11 +1030,11 @@ fn bench_roto(args: &Args) -> Result<(), Failure> { note!("bench --roto: MobileSAM is not installed; comparing the classical engine only"); } // `--sam2 FILE` uses the weights at FILE; otherwise the installed ones. - let sam2_file = args.opt("--sam2").map(std::path::PathBuf::from).or_else(|| session.model_path(&effectcraft_segment::SAM2_TINY)); + let sam2_file = args.opt("--sam2").map(std::path::PathBuf::from).or_else(|| effectcraft_sam2::mask::SIZES.iter().find_map(|m| session.model_path(m))); let sam2: Option> = sam2_file.and_then(|p| std::fs::read(p).ok()).and_then(|bytes| effectcraft_sam2::mask::load(&bytes).ok()).map(|m| m as std::sync::Arc); if sam2.is_none() { - note!("bench --roto: SAM 2.1 Tiny is not installed; no SAM 2 column"); + note!("bench --roto: no SAM 2.1 size is installed; no SAM 2 column"); } // `--fixes`: also count the correction strokes each needs before every frame reaches IoU 0.9 (up to 6). let fixes = args.flag("--fixes"); @@ -1053,7 +1053,8 @@ fn bench_roto(args: &Args) -> Result<(), Failure> { runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "mobilesam", Some(m.as_ref()))))); } if let Some(m) = &sam2 { - runs.extend(measure(&sc, &|s| bench::run_tracker(s, "sam2-tiny", m.as_ref()))); + let label = m.info().id.replacen("sam2.1-hiera-", "sam2-", 1); + runs.extend(measure(&sc, &|s| bench::run_tracker(s, &label, m.as_ref()))); } } if args.flag("--json") { diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rs index d6d9ef63b..a00dd660e 100644 --- a/crates/engine/src/analysis/models.rs +++ b/crates/engine/src/analysis/models.rs @@ -544,7 +544,7 @@ pub fn specs() -> Vec { "Use Roto Brush Model", [], None, - "{id: classical|mobilesam|sam2.1-hiera-tiny, wait?} (Roto Brush 2.0 / 3.0 use it; wait: loaded before returning)", + "{id: classical|mobilesam|sam2.1-hiera-tiny|sam2.1-hiera-small|sam2.1-hiera-base-plus|sam2.1-hiera-large, wait?} (Roto Brush 2.0 / 3.0 use it; wait: loaded before returning)", always, |s, p| select(s, p, Task::Mask) ), diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rs index ee05ea6b8..3dee87e74 100644 --- a/crates/engine/src/tests/models.rs +++ b/crates/engine/src/tests/models.rs @@ -37,7 +37,11 @@ fn models_are_listed_chosen_and_verified() { let list = s.execute_checked(&format!("{prefix}.models"), json!({})).unwrap(); let ids: Vec<&str> = list["models"].as_array().unwrap().iter().map(|m| m["id"].as_str().unwrap()).collect(); // Roto Brush also offers SAM 2.1 in four sizes (video trackers; see crates/sam2). - let expect: Vec<&str> = if task == Task::Mask { vec!["classical", model, "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] } else { vec!["classical", model] }; + let expect: Vec<&str> = if task == Task::Mask { + vec!["classical", model, "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] + } else { + vec!["classical", model] + }; assert_eq!(ids, expect); let m = &list["models"][1]; assert_eq!((m["licence"].as_str(), m["installed"].as_bool()), (Some("Apache-2.0"), Some(false))); diff --git a/crates/sam2/src/lib.rs b/crates/sam2/src/lib.rs index 09cbb1466..f2aba0390 100644 --- a/crates/sam2/src/lib.rs +++ b/crates/sam2/src/lib.rs @@ -107,6 +107,36 @@ mod tests { assert!(Size::Large.repo().ends_with("large")); } + #[test] + fn configs_match_the_released_sizes() { + let blocks = |s| hiera::HieraConfig::for_size(s).blocks.iter().sum::(); + assert_eq!([Size::Tiny, Size::Small, Size::BasePlus, Size::Large].map(blocks), [12, 16, 24, 48]); + for s in [Size::Tiny, Size::Small, Size::BasePlus, Size::Large] { + let c = hiera::HieraConfig::for_size(s); + assert_eq!((c.embed_dims.len(), c.heads.len(), c.windows.len()), (4, 4, 4)); + assert!(c.global_blocks.iter().all(|g| *g < blocks(s))); + } + } + + #[test] + fn size_comes_from_the_patch_embedding_and_depth() { + use candle_core::{DType, Device, Tensor}; + let make = |width: usize, deep: bool| { + let mut m = std::collections::HashMap::new(); + m.insert("vision_encoder.backbone.patch_embed.projection.weight".to_string(), Tensor::zeros((width, 3, 7, 7), DType::F32, &Device::Cpu).unwrap()); + if deep { + m.insert("vision_encoder.backbone.blocks.15.layer_norm1.weight".to_string(), Tensor::zeros(4, DType::F32, &Device::Cpu).unwrap()); + } + weights::Weights::from_map(m).unwrap() + }; + assert_eq!(Size::detect(&make(96, false)).unwrap(), Size::Tiny); + assert_eq!(Size::detect(&make(96, true)).unwrap(), Size::Small); + assert_eq!(Size::detect(&make(112, false)).unwrap(), Size::BasePlus); + assert_eq!(Size::detect(&make(144, false)).unwrap(), Size::Large); + assert!(Size::detect(&make(64, false)).is_err()); + assert!(Size::detect(&weights::Weights::from_map(Default::default()).unwrap()).is_err()); + } + #[test] fn errors_say_what_failed() { assert!(Error::Input("empty frame".into()).to_string().contains("empty frame")); diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md index 56044f3fb..b25506b2b 100644 --- a/docs/roto-brush-sam2.md +++ b/docs/roto-brush-sam2.md @@ -106,7 +106,7 @@ encoder's 64 × 64 level matches PyTorch to 7e-6, the memory attention to 1.8e-5 times the pieces on both devices (`EFFECTCRAFT_SAM2_PROFILE=1` prints a frame's breakdown). ## Benchmark -`effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt]` runs synthetic clips with +`effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors (any size; default: the first installed)] [--mobilesam mobile_sam.pt]` runs synthetic clips with known mattes through all three. 24 frames, debug build, SAM 2 on the GPU; mean IoU (mean IoU after the object returns), seconds per frame: