#1539: preserve stroke corrections and current main APIs

This commit is contained in:
Kristian Myram committed 2026-10-30 08:49:47 +01:00
1 parent fa46bbcb88
commit baa1ca31b3
14 files changed
+94 -93

No files matched your search

Generated
+1 -1
View File
@@ -1838,7 +1838,7 @@ dependencies = [
"effectcraft-segment",
"half",
"log",
"pollster 0.4.0",
"pollster 1.0.1",
"rayon",
"wgpu",
]
-41
View File
@@ -870,44 +870,3 @@ fn render_without_progress_writes_no_progress_lines() {
let stderr = render_demo_stderr("no-progress.gif", &[]);
assert!(!stderr.lines().any(|l| l.starts_with("progress")), "{stderr}");
}
/// `bench --roto --clip FRAMES --truth MASKS` runs the trackers on the user's own image sequences: here a bright disc
/// drifting over a dark textured frame, with its true masks. Mismatched sequences are refused with a message.
#[test]
fn bench_roto_runs_on_a_users_clip() {
let (frames, masks) = (tmp("clip-frames"), tmp("clip-masks"));
for d in [&frames, &masks] {
let _ = std::fs::remove_dir_all(d);
std::fs::create_dir_all(d).unwrap();
}
let (w, h) = (96u32, 72u32);
for f in 0..6u32 {
let (cx, cy) = (30.0 + 4.0 * f as f32, 36.0f32);
let (mut img, mut mask) = (image::RgbImage::new(w, h), image::GrayImage::new(w, h));
for y in 0..h {
for x in 0..w {
let inside = ((x as f32 - cx).powi(2) + (y as f32 - cy).powi(2)).sqrt() < 14.0;
let tex = 40 + ((x * 7 + y * 13) % 30) as u8;
img.put_pixel(x, y, if inside { image::Rgb([230, 120, 40 + (x % 20) as u8]) } else { image::Rgb([tex / 2, tex, tex + 20]) });
mask.put_pixel(x, y, image::Luma([if inside { 255 } else { 0 }]));
}
}
img.save(frames.join(format!("f{f:03}.png"))).unwrap();
mask.save(masks.join(format!("f{f:03}.png"))).unwrap();
}
let (f, m) = (frames.to_str().unwrap(), masks.to_str().unwrap());
let out = bin().args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6"]).output().unwrap();
let text = String::from_utf8_lossy(&out.stdout).to_string();
assert!(out.status.success(), "{text}\n{}", String::from_utf8_lossy(&out.stderr));
let row = text.lines().find(|l| l.contains("classical")).unwrap_or_else(|| panic!("no classical row: {text}"));
let mean: f64 = row.split_whitespace().nth(2).unwrap().parse().unwrap();
assert!(mean > 0.8, "classical mean IoU on the user's clip: {mean}\n{text}");
// one mask short: refused
std::fs::remove_file(masks.join("f005.png")).unwrap();
let out = bin().args(["bench", "--roto", "--clip", f, "--truth", m]).output().unwrap();
assert!(!out.status.success());
assert!(String::from_utf8_lossy(&out.stderr).contains("must match"), "{}", String::from_utf8_lossy(&out.stderr));
// --clip without --truth
let out = bin().args(["bench", "--roto", "--clip", f]).output().unwrap();
assert!(!out.status.success());
}
+1 -1
View File
@@ -696,7 +696,7 @@ impl Chain {
let strokes = self.data.strokes_of(f, obj);
let p = match frame(f) {
Some(img) => rb::tracker_prompt_boxed(&strokes, self.opts.scale, &rb::segment(&img, &strokes, None, &self.opts, self.refine_radius)),
None => rb::tracker_prompt(&strokes, self.opts.scale),
None => return Err(format!("cannot read Roto Brush prompt frame {f}")),
};
if p.points.iter().any(|(_, fg)| *fg) {
prompts.push((f, p));
+37 -32
View File
@@ -101,37 +101,8 @@ fn models_are_listed_chosen_and_verified() {
s.execute_checked("roto.model.select", json!({"id": "classical"})).unwrap();
assert!(crate::effects::roto::model_for(&params).is_none());
}
// The SAM 2.1 sizes, each from its own file (EFFECTCRAFT_SAM2, _SAM2_SMALL, _SAM2_BASE_PLUS, _SAM2_LARGE =
// path/to/model.safetensors of facebook/sam2.1-hiera-*): installed with its notice, loaded (verified, native
// only) and used by Roto Brush 3.0, then dropped and removed.
#[cfg(not(target_arch = "wasm32"))]
for (var, id) in [
("EFFECTCRAFT_SAM2", "sam2.1-hiera-tiny"),
("EFFECTCRAFT_SAM2_SMALL", "sam2.1-hiera-small"),
("EFFECTCRAFT_SAM2_BASE_PLUS", "sam2.1-hiera-base-plus"),
("EFFECTCRAFT_SAM2_LARGE", "sam2.1-hiera-large"),
] {
let Ok(path) = std::env::var(var) else { continue };
let r = s.execute_checked("roto.model.install", json!({"path": path})).unwrap();
assert_eq!(r["id"], id, "{var}");
let dir = s.models_dir().unwrap();
let notice = std::fs::read_to_string(dir.join(format!("{id}.safetensors.NOTICE.txt"))).unwrap();
assert!(notice.contains("Meta AI") && notice.contains("Apache-2.0"), "{notice}");
s.execute_checked("roto.model.select", json!({"id": id})).unwrap();
wait_loaded(&mut s, Task::Mask);
assert_eq!(s.models.status(Task::Mask).active.as_deref(), Some(id), "{var}");
let params = crate::effects::Params { values: [("version".to_string(), effectcraft_keyframe::Value::Enum(2))].into_iter().collect() };
assert_eq!(crate::effects::roto::model_for(&params).map(|m| m.info().id), Some(id));
// the list says it is installed, and exactly one size is recommended
let list = s.execute_checked("roto.models", json!({})).unwrap();
let entry = list["models"].as_array().unwrap().iter().find(|m| m["id"] == id).unwrap();
assert_eq!(entry["installed"], true);
assert_eq!(list["models"].as_array().unwrap().iter().filter(|m| m["recommended"] == true).count(), 1);
s.execute_checked("roto.model.select", json!({"id": "classical"})).unwrap();
assert!(crate::effects::roto::model_for(&params).is_none());
assert_eq!(s.execute_checked("roto.model.remove", json!({"id": id})).unwrap()["removed"], true);
assert!(!dir.join(format!("{id}.safetensors")).exists(), "{id} removed");
}
verify_installed_sam2_sizes(&mut s);
// EFFECTCRAFT_FACE_LANDMARKER=path/to/face_landmarker.task: installed with its notice,
// loaded for face tracking, dropped when the classic tracker is chosen, removed.
if let Ok(path) = std::env::var("EFFECTCRAFT_FACE_LANDMARKER") {
@@ -150,6 +121,40 @@ fn models_are_listed_chosen_and_verified() {
let _ = std::fs::remove_dir_all(d);
}
#[cfg(not(target_arch = "wasm32"))]
fn verify_installed_sam2_sizes(s: &mut Session) {
// The SAM 2.1 sizes, each from its own file (EFFECTCRAFT_SAM2, _SAM2_SMALL, _SAM2_BASE_PLUS, _SAM2_LARGE =
// path/to/model.safetensors of facebook/sam2.1-hiera-*): installed with its notice, loaded (verified, native
// only) and used by Roto Brush 3.0, then dropped and removed.
for (var, id) in [
("EFFECTCRAFT_SAM2", "sam2.1-hiera-tiny"),
("EFFECTCRAFT_SAM2_SMALL", "sam2.1-hiera-small"),
("EFFECTCRAFT_SAM2_BASE_PLUS", "sam2.1-hiera-base-plus"),
("EFFECTCRAFT_SAM2_LARGE", "sam2.1-hiera-large"),
] {
let Ok(path) = std::env::var(var) else { continue };
let r = s.execute_checked("roto.model.install", json!({"path": path})).unwrap();
assert_eq!(r["id"], id, "{var}");
let dir = s.models_dir().unwrap();
let notice = std::fs::read_to_string(dir.join(format!("{id}.safetensors.NOTICE.txt"))).unwrap();
assert!(notice.contains("Meta AI") && notice.contains("Apache-2.0"), "{notice}");
s.execute_checked("roto.model.select", json!({"id": id})).unwrap();
wait_loaded(s, Task::Mask);
assert_eq!(s.models.status(Task::Mask).active.as_deref(), Some(id), "{var}");
let params = crate::effects::Params { values: [("version".to_string(), effectcraft_keyframe::Value::Enum(2))].into_iter().collect() };
assert_eq!(crate::effects::roto::model_for(&params).map(|m| m.info().id), Some(id));
// the list says it is installed, and exactly one size is recommended
let list = s.execute_checked("roto.models", json!({})).unwrap();
let entry = list["models"].as_array().unwrap().iter().find(|m| m["id"] == id).unwrap();
assert_eq!(entry["installed"], true);
assert_eq!(list["models"].as_array().unwrap().iter().filter(|m| m["recommended"] == true).count(), 1);
s.execute_checked("roto.model.select", json!({"id": "classical"})).unwrap();
assert!(crate::effects::roto::model_for(&params).is_none());
assert_eq!(s.execute_checked("roto.model.remove", json!({"id": id})).unwrap()["removed"], true);
assert!(!dir.join(format!("{id}.safetensors")).exists(), "{id} removed");
}
}
/// A Roto Brush model that has been idle for ten minutes is unloaded (the choice stays), one in use is not,
/// and a model that does not track use is never unloaded.
#[test]
@@ -167,7 +172,7 @@ fn an_idle_roto_model_is_unloaded() {
self.0
}
}
crate::tests_roto::hold_roto_cache_test_lock();
crate::tests::roto::hold_roto_cache_test_lock();
let (mut s, d) = session("idle");
for (idle, unloaded) in [(Some(5.0), false), (None, false), (Some(601.0), true)] {
let m: Arc<dyn MaskModel> = Arc::new(Fake(idle));
@@ -256,7 +261,7 @@ fn reloading_for_a_stroke_does_not_block_or_retry_a_failed_load() {
#[test]
fn downloading_a_size_installs_and_loads_it() {
let Ok(id) = std::env::var("EFFECTCRAFT_DOWNLOAD_SIZE") else { return };
crate::tests_roto::hold_roto_cache_test_lock();
crate::tests::roto::hold_roto_cache_test_lock();
let (mut s, d) = session("download");
let info = effectcraft_segment::info(&id).unwrap();
if info.url.is_empty() {
+1 -1
View File
@@ -237,7 +237,7 @@ impl GpuEncoder {
let px = g.upload(&pixels.contiguous()?.flatten_all()?.to_vec1::<f32>()?);
let (mut h, mut w) = (ph / 4, pw / 4);
let tokens = g.zeros(h * w * hidden)?;
g.patch_embed(&px, &self.patch_w, &self.patch_b, &tokens, ph, pw, hidden)?;
g.patch_embed(&px, &self.patch_w, &self.patch_b, &tokens, (ph, pw, hidden))?;
let mut x = g.zeros(h * w * hidden)?;
g.add(&tokens, &self.pos_buffer(h, w)?, &x, h * w * hidden)?;
let mut stages: Vec<(Buf, usize, usize, usize)> = vec![];
+1 -1
View File
@@ -169,7 +169,7 @@ impl Gpu {
}
/// The patch embedding: `[3, h, w]` pixels, weight `[co, 3, 7, 7]`, bias `[co]`; output `[h/4, w/4, co]`.
pub fn patch_embed(&self, pixels: &Buf, weight: &Buf, bias: &Buf, out: &Buf, h: usize, w: usize, co: usize) -> Result<()> {
pub fn patch_embed(&self, pixels: &Buf, weight: &Buf, bias: &Buf, out: &Buf, (h, w, co): (usize, usize, usize)) -> Result<()> {
let (ho, wo) = ((h + 6 - 7) / 4 + 1, (w + 6 - 7) / 4 + 1);
need(pixels, 3 * h * w, "patch_embed pixels")?;
need(weight, co * 3 * 49, "patch_embed weight")?;
+1 -1
View File
@@ -340,7 +340,7 @@ fn patch_embed_rope_and_upsample_match_the_cpu_definitions() {
let (gp, gw, gb) = (g.upload(&px), g.upload(&wt), g.upload(&bias));
let (ho, wo) = (h / 4, w / 4);
let out = g.zeros(ho * wo * co).unwrap();
g.patch_embed(&gp, &gw, &gb, &out, h, w, co).unwrap();
g.patch_embed(&gp, &gw, &gb, &out, (h, w, co)).unwrap();
let t = Tensor::from_vec(px, (1, 3, h, w), &Device::Cpu).unwrap();
let k = Tensor::from_vec(wt, (co, 3, 7, 7), &Device::Cpu).unwrap();
let want =
+8 -8
View File
@@ -950,18 +950,18 @@ fn points_along(s: &Stroke, scale: f64, max: usize) -> Vec<[f32; 2]> {
/// SAM 2 was trained on a handful of clicks: with sixteen or more points it doubts its own mask
/// (measured: confidence 0.25 with 16 points, 0.00 with 32, 0.99 with 1 to 7), so this picks at
/// most [`TRACKER_FG_POINTS`] foreground and [`TRACKER_BG_POINTS`] background points, spread
/// along the strokes.
pub fn tracker_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
/// along the strokes. Like [`stroke_prompt`], later corrections win over earlier paint.
pub fn tracker_prompt(strokes: &[&Stroke], scale: f64, w: usize, h: usize) -> Prompt {
let hard = resolve(strokes, StrokeKind::Fg, StrokeKind::Bg, scale, w, h);
let prompt = stroke_prompt(strokes, &hard, scale, w, h);
let mut points = vec![];
for (kind, fg, max) in [(StrokeKind::Fg, true, TRACKER_FG_POINTS), (StrokeKind::Bg, false, TRACKER_BG_POINTS)] {
let all: Vec<[f32; 2]> = strokes.iter().filter(|s| s.kind == kind).flat_map(|s| points_along(s, scale, 3)).collect();
for (fg, max) in [(true, TRACKER_FG_POINTS), (false, TRACKER_BG_POINTS)] {
let all: Vec<[f32; 2]> = prompt.points.iter().filter_map(|(p, label)| (*label == fg).then_some(*p)).collect();
let n = all.len().min(max);
// evenly spaced picks, keeping the first and the last
points.extend((0..n).filter_map(|i| all.get(if n > 1 { i * (all.len() - 1) / (n - 1) } else { all.len() / 2 }).map(|p| (*p, fg))));
}
// a box the user drew on the frame prompts the tracker too
let bbox = strokes.iter().filter(|s| s.kind == StrokeKind::Box).find_map(|s| box_of(s, scale));
Prompt { points, bbox, ..Default::default() }
Prompt { points, ..prompt }
}
/// [`tracker_prompt`] plus a box around the classic engine's matte of the same strokes. Clicks alone
@@ -969,7 +969,7 @@ pub fn tracker_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
/// multi-part figure, against 1.00 with the box); the box says how far the object reaches. A box
/// stroke the user drew wins over the matte's.
pub fn tracker_prompt_boxed(strokes: &[&Stroke], scale: f64, classic: &FrameSeg) -> Prompt {
let mut p = tracker_prompt(strokes, scale);
let mut p = tracker_prompt(strokes, scale, classic.w, classic.h);
if p.bbox.is_none() {
let (w, h) = (classic.w, classic.h);
let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0usize, 0usize);
+36 -3
View File
@@ -591,13 +591,46 @@ fn a_correction_keeps_the_model_prompted_with_what_is_left() {
fn tracker_prompts_carry_a_handful_of_points() {
let st = base_strokes(0);
let refs: Vec<&Stroke> = st.iter().collect();
let p = tracker_prompt(&refs, 1.0);
let p = tracker_prompt(&refs, 1.0, W as usize, H as usize);
let (fg, bg) = (p.points.iter().filter(|(_, f)| *f).count(), p.points.iter().filter(|(_, f)| !*f).count());
assert!((1..=TRACKER_FG_POINTS).contains(&fg) && (1..=TRACKER_BG_POINTS).contains(&bg), "{fg} fg, {bg} bg");
// a long, finely sampled stroke still gives at most the limit, and an empty one gives none
let long = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: (0..400).map(|i| [i as f64 * 0.4, 50.0]).collect(), object: 0 };
assert!(tracker_prompt(&[&long], 1.0).points.len() <= TRACKER_FG_POINTS);
assert!(tracker_prompt(&[], 1.0).points.is_empty());
assert!(tracker_prompt(&[&long], 1.0, W as usize, H as usize).points.len() <= TRACKER_FG_POINTS);
assert!(tracker_prompt(&[], 1.0, W as usize, H as usize).points.is_empty());
}
/// Tracker clicks follow the same correction order as the classic matte and model prompts.
#[test]
fn tracker_prompts_follow_latest_corrections_and_keep_remaining_foreground() {
let (w, h) = (100, 80);
let green = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 15.0, points: vec![[50.0, 40.0]], object: 0 };
let red = Stroke { kind: StrokeKind::Bg, radius: 16.0, ..green.clone() };
for strokes in [&[&green, &red][..], &[&red, &green][..]] {
let classic = FrameSeg::empty(w, h);
for prompt in [tracker_prompt(strokes, 1.0, w, h), tracker_prompt_boxed(strokes, 1.0, &classic)] {
let expected = std::ptr::eq(strokes[1], &green);
assert_eq!(prompt.points.iter().any(|(_, fg)| *fg), expected, "{:?}", prompt.points);
let hard = resolve(strokes, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);
for (point, fg) in &prompt.points {
assert_eq!(owner(&hard, w, h, *point), Some(if *fg { HARD_FG } else { HARD_BG }));
}
assert!(!prompt.points.is_empty());
}
}
// A small correction removes the sampled centre, while most of the foreground dot remains.
let tap = Stroke { radius: 2.0, ..red };
let strokes = [&green, &tap];
let hard = resolve(&strokes, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);
for prompt in [tracker_prompt(&strokes, 1.0, w, h), tracker_prompt_boxed(&strokes, 1.0, &FrameSeg::empty(w, h))] {
assert!(prompt.points.iter().any(|(_, fg)| *fg));
assert!(prompt.points.iter().any(|(_, fg)| !*fg));
for (point, fg) in &prompt.points {
assert_eq!(owner(&hard, w, h, *point), Some(if *fg { HARD_FG } else { HARD_BG }));
}
assert!(prompt.points.iter().filter(|(_, fg)| *fg).count() <= TRACKER_FG_POINTS);
assert!(prompt.points.iter().filter(|(_, fg)| !*fg).count() <= TRACKER_BG_POINTS);
}
}
#[test]
+3 -3
View File
@@ -452,7 +452,7 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(104.0, 20.0));
let strokes = fx::data(&params).strokes.iter().filter(|st| st.object == current && st.kind.is_segmentation()).count();
let label = if strokes == 0 { tr_args("Object {}", &[&(current + 1)]) } else { tr_args("Object {} ({})", &[&(current + 1), &strokes]) };
if widgets::text_button(ui, r, &label, false, &t, egui::Id::new(("roto-object", uid))).clicked() {
if widgets::text_button(ui, r, &label, false, &t, egui::Id::unique(("roto-object", uid))).clicked() {
// cycle through the objects that have strokes (and the empty one being drawn)
let mut all = objects.clone();
if !known {
@@ -466,7 +466,7 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
app.auto.add("layerPanel.roto.object", r, &label);
x += 108.0;
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));
if widgets::text_button(ui, r, "+", false, &t, egui::Id::new(("roto-object-new", uid))).clicked() {
if widgets::text_button(ui, r, "+", false, &t, egui::Id::unique(("roto-object-new", uid))).clicked() {
let next = objects.iter().copied().chain([current]).max().map_or(0, |m| m + 1);
let _ = app.session.execute("roto.options", json!({"object": next}));
}
@@ -475,7 +475,7 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
// remove the current object's strokes (the object stays selected; with none left it is simply empty)
if strokes > 0 {
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));
let c = widgets::text_button(ui, r, "×", false, &t, egui::Id::new(("roto-object-clear", uid))).on_hover_text(tr("Remove this object's strokes"));
let c = widgets::text_button(ui, r, "×", false, &t, egui::Id::unique(("roto-object-clear", uid))).on_hover_text(tr("Remove this object's strokes"));
if c.clicked()
&& let Err(e) = app.session.execute("roto.clearStrokes", json!({"layer": layer.id.0, "effect": uid, "object": current}))
{
+2
View File
@@ -72,6 +72,8 @@ mod ui_quick_offset;
mod ui_ram_preview;
mod ui_render_queue;
mod ui_render_queue_keyboard;
mod ui_roto_objects;
mod ui_roto_span_confidence;
mod ui_script_console;
mod ui_scriptui;
mod ui_scrolling;
@@ -59,6 +59,7 @@ fn count(image: &image::RgbaImage, want: [u8; 3]) -> usize {
#[test]
fn the_boundary_view_outlines_each_object_and_show_object_picks_one() {
let _gpu = crate::gpu_test::lock();
let mut s = Session::default();
s.execute("comp.new", json!({"name": "View", "width": 640, "height": 360, "duration": 4})).unwrap();
s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 80, "height": 80})).unwrap();
@@ -64,6 +64,7 @@ fn close(got: [u8; 4], want: [u8; 3]) -> bool {
#[test]
fn the_span_bar_shows_the_trackers_confidence() {
let _gpu = crate::gpu_test::lock();
let mut s = Session::default();
s.execute("comp.new", json!({"name": "View", "width": 640, "height": 360, "duration": 4})).unwrap();
s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 80, "height": 80})).unwrap();
+1 -1
View File
@@ -3,7 +3,7 @@
<!-- Generated from the effect registry (crates/effects/src/catalog.rs); do not edit by hand.
Regenerate: UPDATE_DOCS=1 cargo test -p effectcraft-effects --lib effects_doc_is_current -->
EffectCraft ships 308 effects with 3034 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
EffectCraft ships 308 effects with 2987 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
- **GPU**: 280 effects also run on the GPU compositor with identical results.
- **32**: 308 effects process 32-bit float (HDR, overbright) pixels without clamping.