mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-31 07:38:24 +00:00
S2.73: Tiny is the recommended size (benchmarked per size); stroke count and remove button per object; download test
bench --roto per size: no larger size tracked better on the synthetic clips and Large lost the object after it re-entered; the hints say what was measured. The bar shows the current object's stroke count and can remove its strokes. An env-gated test runs the in-app download of a size. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ba6fee9863
commit
a0ec3e2da4
5 files changed
+82
-21
No files matched your search
@@ -160,3 +160,26 @@ fn an_idle_roto_model_is_unloaded() {
|
||||
crate::effects::roto::set_model(None);
|
||||
let _ = std::fs::remove_dir_all(d);
|
||||
}
|
||||
|
||||
/// With `EFFECTCRAFT_DOWNLOAD_SIZE=sam2.1-hiera-small` (or another registry id): the in-app download (Settings ▸ Roto
|
||||
/// Brush ▸ Download) fetches the pinned file over the network, verifies it, installs it with its notice and loads it.
|
||||
/// Skipped without the variable: it needs the network and hundreds of megabytes.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[test]
|
||||
fn downloading_a_size_installs_and_loads_it() {
|
||||
let Ok(id) = std::env::var("EFFECTCRAFT_DOWNLOAD_SIZE") else { return };
|
||||
crate::tests_roto::hold_roto_cache_test_lock();
|
||||
let (mut s, d) = session("download");
|
||||
let r = s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap();
|
||||
assert_eq!(r["id"], id.as_str(), "{r}");
|
||||
let dir = s.models_dir().unwrap();
|
||||
let info = effectcraft_segment::info(&id).unwrap();
|
||||
let file = dir.join(info.file_name);
|
||||
assert_eq!(std::fs::metadata(&file).unwrap().len(), info.size, "the whole file arrived");
|
||||
assert!(dir.join(format!("{}.NOTICE.txt", info.file_name)).exists());
|
||||
assert!(!dir.join(format!("{}.part", info.file_name)).exists(), "no partial file is left");
|
||||
s.execute_checked("roto.model.select", json!({"id": id})).unwrap();
|
||||
wait_loaded(&mut s, Task::Mask);
|
||||
assert_eq!(s.models.status(Task::Mask).active.as_deref(), Some(id.as_str()));
|
||||
let _ = std::fs::remove_dir_all(d);
|
||||
}
|
||||
+18
-14
@@ -148,25 +148,29 @@ pub fn load(bytes: &[u8]) -> std::result::Result<Arc<Sam2Mask>, String> {
|
||||
Ok(Arc::new(Sam2Mask::with_info(Arc::new(model), info)))
|
||||
}
|
||||
|
||||
/// The size to suggest on this computer: Small when a GPU is usable (no slower to run than Tiny there, and more accurate),
|
||||
/// Tiny otherwise (the bigger backbones take several seconds a frame on a CPU).
|
||||
/// The size to suggest: Tiny. On the benchmark clips (`bench --roto`, 24 frames, a GPU) no larger size tracked better, they
|
||||
/// cost 1.0x (Small), 1.15x (Base+) and 1.7x (Large) the time, and Large lost the object after it re-entered the frame.
|
||||
pub fn recommended() -> &'static ModelInfo {
|
||||
if crate::gpu::Gpu::shared().is_ok() { &effectcraft_segment::SAM2_SMALL } else { &SAM2_TINY }
|
||||
&SAM2_TINY
|
||||
}
|
||||
|
||||
/// A one-line note for the Settings list: what suits this computer, and what the bigger sizes cost without a GPU.
|
||||
/// A one-line note for the Settings list: what was measured for this size, and what the bigger ones cost without a GPU.
|
||||
pub fn hint(id: &str) -> Option<String> {
|
||||
if !SIZES.iter().any(|m| m.id == id) {
|
||||
return None;
|
||||
}
|
||||
let gpu = crate::gpu::Gpu::shared().is_ok();
|
||||
Some(match (id == recommended().id, gpu, id) {
|
||||
(true, true, _) => "Recommended: a GPU was found, and this size is no slower than Tiny on it.".to_string(),
|
||||
(true, false, _) => "Recommended: no GPU was found, and this is the quickest on a CPU.".to_string(),
|
||||
(false, false, "sam2.1-hiera-base-plus" | "sam2.1-hiera-large") => "Slow without a GPU (several seconds a frame); none was found.".to_string(),
|
||||
(false, true, "sam2.1-hiera-large") => "For difficult footage; about 0.7 s of GPU time to look at each frame.".to_string(),
|
||||
_ => return None,
|
||||
})
|
||||
Some(
|
||||
match id {
|
||||
"sam2.1-hiera-tiny" => "Recommended: the smallest download, and no larger size tracked better on the benchmark clips.",
|
||||
"sam2.1-hiera-small" => "As fast as Tiny on a GPU and no better on the benchmark clips; worth a try on difficult footage.",
|
||||
"sam2.1-hiera-base-plus" if !gpu => "Slow without a GPU (several seconds a frame); none was found.",
|
||||
"sam2.1-hiera-base-plus" => "About 15% slower than Tiny on a GPU and no better on the benchmark clips.",
|
||||
"sam2.1-hiera-large" if !gpu => "Slow without a GPU (many seconds a frame); none was found.",
|
||||
"sam2.1-hiera-large" => {
|
||||
"About 1.7 times Tiny's time on a GPU; it lost the object after it re-entered in one benchmark clip. Try it on difficult footage."
|
||||
}
|
||||
_ => return None,
|
||||
}
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
/// The registry entries this crate loads.
|
||||
|
||||
@@ -417,8 +417,9 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
|
||||
let objects = fx::data(¶ms).objects();
|
||||
let current = app.session.state.roto.object;
|
||||
let known = objects.contains(¤t);
|
||||
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(86.0, 20.0));
|
||||
let label = tr_args("Object {}", &[&(current + 1)]);
|
||||
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(104.0, 20.0));
|
||||
let strokes = fx::data(¶ms).strokes.iter().filter(|st| st.object == current && st.kind.is_segmentation()).count();
|
||||
let label = if strokes == 0 { tr_args("Object {}", &[&(current + 1)]) } else { tr_args("Object {} ({})", &[&(current + 1), &strokes]) };
|
||||
if widgets::text_button(ui, r, &label, false, &t, egui::Id::new(("roto-object", uid))).clicked() {
|
||||
// cycle through the objects that have strokes (and the empty one being drawn)
|
||||
let mut all = objects.clone();
|
||||
@@ -431,14 +432,26 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
|
||||
}
|
||||
p.circle_filled(pos2(r.min.x + 9.0, r.center().y), 4.0, object_color(current, app.session.state.roto.boundary_color));
|
||||
app.auto.add("layerPanel.roto.object", r, &label);
|
||||
x += 90.0;
|
||||
x += 108.0;
|
||||
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));
|
||||
if widgets::text_button(ui, r, "+", false, &t, egui::Id::new(("roto-object-new", uid))).clicked() {
|
||||
let next = objects.iter().copied().chain([current]).max().map_or(0, |m| m + 1);
|
||||
let _ = app.session.execute("roto.options", json!({"object": next}));
|
||||
}
|
||||
app.auto.add("layerPanel.roto.newObject", r, "New object");
|
||||
x += 34.0;
|
||||
x += 30.0;
|
||||
// remove the current object's strokes (the object stays selected; with none left it is simply empty)
|
||||
if strokes > 0 {
|
||||
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));
|
||||
let c = widgets::text_button(ui, r, "×", false, &t, egui::Id::new(("roto-object-clear", uid))).on_hover_text(tr("Remove this object's strokes"));
|
||||
if c.clicked()
|
||||
&& let Err(e) = app.session.execute("roto.clearStrokes", json!({"layer": layer.id.0, "effect": uid, "object": current}))
|
||||
{
|
||||
app.ui.status = e.to_string();
|
||||
}
|
||||
app.auto.add("layerPanel.roto.clearObject", r, "Remove object strokes");
|
||||
x += 30.0;
|
||||
}
|
||||
if running {
|
||||
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(60.0, 20.0));
|
||||
if widgets::text_button(ui, r, tr("Stop"), false, &t, egui::Id::unique("roto-stop")).clicked() {
|
||||
|
||||
@@ -84,6 +84,9 @@ fn the_boundary_view_outlines_each_object_and_show_object_picks_one() {
|
||||
if let Some(path) = std::env::var_os("ROTO_OBJECTS_SNAPSHOT") {
|
||||
both.save(path).unwrap();
|
||||
}
|
||||
// the bar offers to remove the current object's strokes, and says how many it has
|
||||
assert!(h.state().auto.find("layerPanel.roto.clearObject").is_some(), "the remove button is on screen");
|
||||
assert!(h.state().auto.find("layerPanel.roto.object").is_some_and(|e| e.label.contains("(1)")), "the object's stroke count is on the button");
|
||||
let (a, b) = (count(&both, pink), count(&both, cyan));
|
||||
assert!(a > 50 && b > 50, "both objects are outlined: {a} pink, {b} cyan pixels");
|
||||
|
||||
|
||||
+21
-3
@@ -65,9 +65,27 @@ pinned to a Hugging Face commit with its SHA-256. All run the same tracker; a bi
|
||||
| SAM 2.1 Base+ (`sam2.1-hiera-base-plus`) | 323.5 MB | 0.37 s | encoder 2.3e-5 |
|
||||
| SAM 2.1 Large (`sam2.1-hiera-large`) | 897.9 MB | 0.72 s | encoder 5.2e-4 |
|
||||
|
||||
Settings marks the size that suits the computer: **Small** when a usable GPU is found (it encodes no slower than
|
||||
Tiny there and is more accurate), **Tiny** otherwise; Base+ and Large say so when they would be slow on a CPU. The list
|
||||
carries `recommended` and `hint` per model (`roto.models`).
|
||||
### How the sizes compare on the benchmark clips
|
||||
`effectcraft-cli bench --roto --frames 24 --fixes --sam2 FILE`, release build, SAM 2 on the GPU (RX 7900 GRE), one run each.
|
||||
Mean IoU (after the object returns, in brackets) and seconds per tracked frame:
|
||||
|
||||
| Clip | Tiny | Small | Base+ | Large |
|
||||
|---|---|---|---|---|
|
||||
| move | 0.992 | 0.988 | 0.986 | 0.994 |
|
||||
| light | 0.990 | 0.985 | 0.987 | 0.993 |
|
||||
| occlusion | 0.989 (0.987) | 0.988 (0.984) | 0.987 (0.984) | 0.989 (0.987) |
|
||||
| reenter | 0.982 (0.975) | 0.981 (0.970) | 0.978 (0.966) | **0.694 (0.110)** |
|
||||
| s / tracked frame | 0.69 | 0.70 | 0.79 | 1.17 |
|
||||
|
||||
The classic engine scores 0.98 / 0.98 / 0.49 / 0.62 on the same clips. Peak tracker state is 88 MB for every size (the
|
||||
weights are not counted). The clips are synthetic and easy, so they cannot tell the sizes apart on accuracy: **no larger
|
||||
size tracked better here, and Large lost the object after it left and came back** (IoU 0.11 after the return, where
|
||||
Tiny, Small and Base+ found it again). That is the model's behaviour on this clip, not the port:
|
||||
Large matches PyTorch stage by stage. Whether a larger size helps on real footage is not measured.
|
||||
|
||||
Settings therefore marks **Tiny** as recommended (the smallest download, and nothing larger did better), and says what was
|
||||
measured for the others; Base+ and Large also warn when no GPU was found (they take several seconds a frame on a CPU).
|
||||
The list carries `recommended` and `hint` per model (`roto.models`).
|
||||
|
||||
### Half-precision weights (optional)
|
||||
`EFFECTCRAFT_SAM2_F16=1` keeps the weight matrices on the GPU as half precision, halving their memory (Large: about
|
||||
|
||||
Reference in new issue
Block a user