From a0ec3e2da4d0e15f079d90fb26b98f5f4ef22ab8 Mon Sep 17 00:00:00 2001 From: Kristian Myram Date: Sat, 17 Oct 2026 17:13:43 +0200 Subject: [PATCH] S2.73: Tiny is the recommended size (benchmarked per size); stroke count and remove button per object; download test bench --roto per size: no larger size tracked better on the synthetic clips and Large lost the object after it re-entered; the hints say what was measured. The bar shows the current object's stroke count and can remove its strokes. An env-gated test runs the in-app download of a size. Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/tests/models.rs | 23 ++++++++++++++++++ crates/sam2/src/mask.rs | 32 ++++++++++++++----------- crates/ui-egui/src/panels/roto_tool.rs | 21 ++++++++++++---- crates/ui-egui/tests/ui_roto_objects.rs | 3 +++ docs/roto-brush-sam2.md | 24 ++++++++++++++++--- 5 files changed, 82 insertions(+), 21 deletions(-) diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rs index 1dbc507aa..ed0eecf9b 100644 --- a/crates/engine/src/tests/models.rs +++ b/crates/engine/src/tests/models.rs @@ -160,3 +160,26 @@ fn an_idle_roto_model_is_unloaded() { crate::effects::roto::set_model(None); let _ = std::fs::remove_dir_all(d); } + +/// With `EFFECTCRAFT_DOWNLOAD_SIZE=sam2.1-hiera-small` (or another registry id): the in-app download (Settings ▸ Roto +/// Brush ▸ Download) fetches the pinned file over the network, verifies it, installs it with its notice and loads it. +/// Skipped without the variable: it needs the network and hundreds of megabytes. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn downloading_a_size_installs_and_loads_it() { + let Ok(id) = std::env::var("EFFECTCRAFT_DOWNLOAD_SIZE") else { return }; + crate::tests_roto::hold_roto_cache_test_lock(); + let (mut s, d) = session("download"); + let r = s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap(); + assert_eq!(r["id"], id.as_str(), "{r}"); + let dir = s.models_dir().unwrap(); + let info = effectcraft_segment::info(&id).unwrap(); + let file = dir.join(info.file_name); + assert_eq!(std::fs::metadata(&file).unwrap().len(), info.size, "the whole file arrived"); + assert!(dir.join(format!("{}.NOTICE.txt", info.file_name)).exists()); + assert!(!dir.join(format!("{}.part", info.file_name)).exists(), "no partial file is left"); + s.execute_checked("roto.model.select", json!({"id": id})).unwrap(); + wait_loaded(&mut s, Task::Mask); + assert_eq!(s.models.status(Task::Mask).active.as_deref(), Some(id.as_str())); + let _ = std::fs::remove_dir_all(d); +} diff --git a/crates/sam2/src/mask.rs b/crates/sam2/src/mask.rs index d840d4052..8470bd4f3 100644 --- a/crates/sam2/src/mask.rs +++ b/crates/sam2/src/mask.rs @@ -148,25 +148,29 @@ pub fn load(bytes: &[u8]) -> std::result::Result, String> { Ok(Arc::new(Sam2Mask::with_info(Arc::new(model), info))) } -/// The size to suggest on this computer: Small when a GPU is usable (no slower to run than Tiny there, and more accurate), -/// Tiny otherwise (the bigger backbones take several seconds a frame on a CPU). +/// The size to suggest: Tiny. On the benchmark clips (`bench --roto`, 24 frames, a GPU) no larger size tracked better, they +/// cost 1.0x (Small), 1.15x (Base+) and 1.7x (Large) the time, and Large lost the object after it re-entered the frame. pub fn recommended() -> &'static ModelInfo { - if crate::gpu::Gpu::shared().is_ok() { &effectcraft_segment::SAM2_SMALL } else { &SAM2_TINY } + &SAM2_TINY } -/// A one-line note for the Settings list: what suits this computer, and what the bigger sizes cost without a GPU. +/// A one-line note for the Settings list: what was measured for this size, and what the bigger ones cost without a GPU. pub fn hint(id: &str) -> Option { - if !SIZES.iter().any(|m| m.id == id) { - return None; - } let gpu = crate::gpu::Gpu::shared().is_ok(); - Some(match (id == recommended().id, gpu, id) { - (true, true, _) => "Recommended: a GPU was found, and this size is no slower than Tiny on it.".to_string(), - (true, false, _) => "Recommended: no GPU was found, and this is the quickest on a CPU.".to_string(), - (false, false, "sam2.1-hiera-base-plus" | "sam2.1-hiera-large") => "Slow without a GPU (several seconds a frame); none was found.".to_string(), - (false, true, "sam2.1-hiera-large") => "For difficult footage; about 0.7 s of GPU time to look at each frame.".to_string(), - _ => return None, - }) + Some( + match id { + "sam2.1-hiera-tiny" => "Recommended: the smallest download, and no larger size tracked better on the benchmark clips.", + "sam2.1-hiera-small" => "As fast as Tiny on a GPU and no better on the benchmark clips; worth a try on difficult footage.", + "sam2.1-hiera-base-plus" if !gpu => "Slow without a GPU (several seconds a frame); none was found.", + "sam2.1-hiera-base-plus" => "About 15% slower than Tiny on a GPU and no better on the benchmark clips.", + "sam2.1-hiera-large" if !gpu => "Slow without a GPU (many seconds a frame); none was found.", + "sam2.1-hiera-large" => { + "About 1.7 times Tiny's time on a GPU; it lost the object after it re-entered in one benchmark clip. Try it on difficult footage." + } + _ => return None, + } + .to_string(), + ) } /// The registry entries this crate loads. diff --git a/crates/ui-egui/src/panels/roto_tool.rs b/crates/ui-egui/src/panels/roto_tool.rs index 56fa39b0a..0b42befbe 100644 --- a/crates/ui-egui/src/panels/roto_tool.rs +++ b/crates/ui-egui/src/panels/roto_tool.rs @@ -417,8 +417,9 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte let objects = fx::data(¶ms).objects(); let current = app.session.state.roto.object; let known = objects.contains(¤t); - let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(86.0, 20.0)); - let label = tr_args("Object {}", &[&(current + 1)]); + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(104.0, 20.0)); + let strokes = fx::data(¶ms).strokes.iter().filter(|st| st.object == current && st.kind.is_segmentation()).count(); + let label = if strokes == 0 { tr_args("Object {}", &[&(current + 1)]) } else { tr_args("Object {} ({})", &[&(current + 1), &strokes]) }; if widgets::text_button(ui, r, &label, false, &t, egui::Id::new(("roto-object", uid))).clicked() { // cycle through the objects that have strokes (and the empty one being drawn) let mut all = objects.clone(); @@ -431,14 +432,26 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte } p.circle_filled(pos2(r.min.x + 9.0, r.center().y), 4.0, object_color(current, app.session.state.roto.boundary_color)); app.auto.add("layerPanel.roto.object", r, &label); - x += 90.0; + x += 108.0; let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0)); if widgets::text_button(ui, r, "+", false, &t, egui::Id::new(("roto-object-new", uid))).clicked() { let next = objects.iter().copied().chain([current]).max().map_or(0, |m| m + 1); let _ = app.session.execute("roto.options", json!({"object": next})); } app.auto.add("layerPanel.roto.newObject", r, "New object"); - x += 34.0; + x += 30.0; + // remove the current object's strokes (the object stays selected; with none left it is simply empty) + if strokes > 0 { + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0)); + let c = widgets::text_button(ui, r, "×", false, &t, egui::Id::new(("roto-object-clear", uid))).on_hover_text(tr("Remove this object's strokes")); + if c.clicked() + && let Err(e) = app.session.execute("roto.clearStrokes", json!({"layer": layer.id.0, "effect": uid, "object": current})) + { + app.ui.status = e.to_string(); + } + app.auto.add("layerPanel.roto.clearObject", r, "Remove object strokes"); + x += 30.0; + } if running { let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(60.0, 20.0)); if widgets::text_button(ui, r, tr("Stop"), false, &t, egui::Id::unique("roto-stop")).clicked() { diff --git a/crates/ui-egui/tests/ui_roto_objects.rs b/crates/ui-egui/tests/ui_roto_objects.rs index ee5efc020..5d16b0ada 100644 --- a/crates/ui-egui/tests/ui_roto_objects.rs +++ b/crates/ui-egui/tests/ui_roto_objects.rs @@ -84,6 +84,9 @@ fn the_boundary_view_outlines_each_object_and_show_object_picks_one() { if let Some(path) = std::env::var_os("ROTO_OBJECTS_SNAPSHOT") { both.save(path).unwrap(); } + // the bar offers to remove the current object's strokes, and says how many it has + assert!(h.state().auto.find("layerPanel.roto.clearObject").is_some(), "the remove button is on screen"); + assert!(h.state().auto.find("layerPanel.roto.object").is_some_and(|e| e.label.contains("(1)")), "the object's stroke count is on the button"); let (a, b) = (count(&both, pink), count(&both, cyan)); assert!(a > 50 && b > 50, "both objects are outlined: {a} pink, {b} cyan pixels"); diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md index 4e1792404..72c48e4e1 100644 --- a/docs/roto-brush-sam2.md +++ b/docs/roto-brush-sam2.md @@ -65,9 +65,27 @@ pinned to a Hugging Face commit with its SHA-256. All run the same tracker; a bi | SAM 2.1 Base+ (`sam2.1-hiera-base-plus`) | 323.5 MB | 0.37 s | encoder 2.3e-5 | | SAM 2.1 Large (`sam2.1-hiera-large`) | 897.9 MB | 0.72 s | encoder 5.2e-4 | -Settings marks the size that suits the computer: **Small** when a usable GPU is found (it encodes no slower than -Tiny there and is more accurate), **Tiny** otherwise; Base+ and Large say so when they would be slow on a CPU. The list -carries `recommended` and `hint` per model (`roto.models`). +### How the sizes compare on the benchmark clips +`effectcraft-cli bench --roto --frames 24 --fixes --sam2 FILE`, release build, SAM 2 on the GPU (RX 7900 GRE), one run each. +Mean IoU (after the object returns, in brackets) and seconds per tracked frame: + +| Clip | Tiny | Small | Base+ | Large | +|---|---|---|---|---| +| move | 0.992 | 0.988 | 0.986 | 0.994 | +| light | 0.990 | 0.985 | 0.987 | 0.993 | +| occlusion | 0.989 (0.987) | 0.988 (0.984) | 0.987 (0.984) | 0.989 (0.987) | +| reenter | 0.982 (0.975) | 0.981 (0.970) | 0.978 (0.966) | **0.694 (0.110)** | +| s / tracked frame | 0.69 | 0.70 | 0.79 | 1.17 | + +The classic engine scores 0.98 / 0.98 / 0.49 / 0.62 on the same clips. Peak tracker state is 88 MB for every size (the +weights are not counted). The clips are synthetic and easy, so they cannot tell the sizes apart on accuracy: **no larger +size tracked better here, and Large lost the object after it left and came back** (IoU 0.11 after the return, where +Tiny, Small and Base+ found it again). That is the model's behaviour on this clip, not the port: +Large matches PyTorch stage by stage. Whether a larger size helps on real footage is not measured. + +Settings therefore marks **Tiny** as recommended (the smallest download, and nothing larger did better), and says what was +measured for the others; Base+ and Large also warn when no GPU was found (they take several seconds a frame on a CPU). +The list carries `recommended` and `hint` per model (`roto.models`). ### Half-precision weights (optional) `EFFECTCRAFT_SAM2_F16=1` keeps the weight matrices on the GPU as half precision, halving their memory (Large: about