From 0605406c50da50f6d40a22c836221bba98adc63d Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 20 Oct 2026 03:26:53 +0200 Subject: [PATCH 1/6] fix(linux): enable font-free Wayland window controls --- Cargo.lock | 41 +++++++++++++++++- NOTICE | 42 +++++++++++++++++++ apps/effectcraft/Cargo.toml | 3 ++ packaging/env.sh | 2 +- .../flatpak/ai.storyteller.effectcraft.yml | 1 + packaging/linux/package.sh | 2 + 6 files changed, 88 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 412fb6672..0c26f0a07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1495,7 +1495,7 @@ dependencies = [ "effectcraft-raster", "kurbo", "linesweeper", - "tiny-skia", + "tiny-skia 0.12.0", ] [[package]] @@ -4581,6 +4581,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sctk-adwaita" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6277f0217056f77f1d8f49f2950ac6c278c0d607c45f5ee99328d792ede24ec" +dependencies = [ + "log", + "smithay-client-toolkit 0.19.2", + "tiny-skia 0.11.4", +] + [[package]] name = "self_cell" version = "1.3.0" @@ -5217,6 +5228,20 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-skia" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d13394d44dae3207b52a326c0c85a8bf87f1541f23b0d143811088497b09ab" +dependencies = [ + "arrayref", + "arrayvec", + "bytemuck", + "cfg-if", + "log", + "tiny-skia-path 0.11.4", +] + [[package]] name = "tiny-skia" version = "0.12.0" @@ -5228,7 +5253,18 @@ dependencies = [ "bytemuck", "cfg-if", "log", - "tiny-skia-path", + "tiny-skia-path 0.12.0", +] + +[[package]] +name = "tiny-skia-path" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e7fc0c2e86a30b117d0462aa261b72b7a99b7ebd7deb3a14ceda95c5bdc93" +dependencies = [ + "arrayref", + "bytemuck", + "strict-num", ] [[package]] @@ -6505,6 +6541,7 @@ dependencies = [ "raw-window-handle", "redox_syscall 0.4.1", "rustix 0.38.44", + "sctk-adwaita", "smithay-client-toolkit 0.19.2", "smol_str", "tracing", diff --git a/NOTICE b/NOTICE index 43bd2e2a9..718d12cef 100644 --- a/NOTICE +++ b/NOTICE @@ -20,3 +20,45 @@ The optional PDF document reader uses craft-pdf and Hayro-syntax 0.7.2, MIT OR A The shared patched syntax source is centralized in storytold/craft-libs, transferred from PdfCraft revision 9ed3abe4907a127d36b12fbfeee71d7e26248da1. This parser-only use embeds no Hayro fonts, CMaps or image codecs; EffectCraft keeps its existing fonts and drawing filters. + +Linux Wayland window decorations +------------------------------- +sctk-adwaita 0.10.1 (https://github.com/PolyMeilex/sctk-adwaita, +commit d5f343c45c04d4a026db57f76760819b7325db00) is used by winit on Linux +for vector window controls. Its title-font features are disabled: no Cantarell +font is embedded by this dependency and no font file is added to this repo. + +sctk-adwaita's MIT licence: + +MIT License + +Copyright (c) 2022 Bartłomiej Maryńczak + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Bundled application fonts +------------------------- +Inter (assets/fonts/Inter-*.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-Inter.txt. +Noto Serif (assets/fonts/NotoSerif-Regular.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-NotoSerif.txt. +JetBrains Mono (assets/fonts/JetBrainsMono-Regular.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-JetBrainsMono.txt. + +The optional PDF document reader uses craft-pdf and Hayro-syntax 0.7.2, MIT OR Apache-2.0. +The shared patched syntax source is centralized in storytold/craft-libs, transferred from +PdfCraft revision 9ed3abe4907a127d36b12fbfeee71d7e26248da1. This parser-only use embeds no +Hayro fonts, CMaps or image codecs; EffectCraft keeps its existing fonts and drawing filters. diff --git a/apps/effectcraft/Cargo.toml b/apps/effectcraft/Cargo.toml index 5ad6c80d1..52ed374ae 100644 --- a/apps/effectcraft/Cargo.toml +++ b/apps/effectcraft/Cargo.toml @@ -27,6 +27,9 @@ cpal = { workspace = true } log = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] +# Native GNOME/Wayland controls without a bundled title font. +winit = { version = "0.30", default-features = false, features = ["wayland-csd-adwaita-notitle"] } + # Settings ▸ Appearance ▸ Sync with System: the XDG desktop portal's colour scheme. Already in # Cargo.lock (accesskit, through eframe), so it adds no new crates. zbus = "5.19" diff --git a/packaging/env.sh b/packaging/env.sh index 6a1bfa286..f1943f777 100644 --- a/packaging/env.sh +++ b/packaging/env.sh @@ -47,7 +47,7 @@ warn() { # Copy licence and readme files that exist into a package directory. copy_docs() { local dest="$1" f - for f in README.md LICENSE LICENSE-MIT LICENSE-APACHE COPYRIGHT; do + for f in README.md LICENSE LICENSE-MIT LICENSE-APACHE COPYRIGHT NOTICE; do if [ -f "$ROOT/$f" ]; then cp "$ROOT/$f" "$dest/"; fi done } diff --git a/packaging/linux/flatpak/ai.storyteller.effectcraft.yml b/packaging/linux/flatpak/ai.storyteller.effectcraft.yml index be9f898ed..7dd7a8d68 100644 --- a/packaging/linux/flatpak/ai.storyteller.effectcraft.yml +++ b/packaging/linux/flatpak/ai.storyteller.effectcraft.yml @@ -55,6 +55,7 @@ modules: build-commands: - cargo --offline build --release --locked -p effectcraft -p effectcraft-cli - install -Dm755 target/release/effectcraft /app/bin/effectcraft + - install -Dm644 NOTICE /app/share/doc/effectcraft/NOTICE - install -Dm755 target/release/effectcraft-cli /app/bin/effectcraft-cli - install -Dm644 packaging/linux/ai.storyteller.effectcraft.desktop /app/share/applications/ai.storyteller.effectcraft.desktop - install -Dm644 packaging/linux/ai.storyteller.effectcraft.mime.xml /app/share/mime/packages/ai.storyteller.effectcraft.xml diff --git a/packaging/linux/package.sh b/packaging/linux/package.sh index f149e663d..504d2c5b8 100755 --- a/packaging/linux/package.sh +++ b/packaging/linux/package.sh @@ -106,6 +106,8 @@ if has appimage; then APPDIR="$WORK/EffectCraft.AppDir" cp -R "$STAGE" "$APPDIR" mv "$APPDIR/usr/share/doc" "$WORK/doc-unused" + # Keep sctk-adwaita's MIT notice in the AppImage. + install -Dm644 "$ROOT/NOTICE" "$APPDIR/usr/share/doc/effectcraft/NOTICE" ln -s usr/bin/effectcraft "$APPDIR/AppRun" cp "$HERE/$APP_ID.desktop" "$APPDIR/$APP_ID.desktop" cp "$ROOT/assets/app-icon/hicolor/256x256/apps/$APP_ID.png" "$APPDIR/$APP_ID.png" From 812d463dfde541276b1452b64decd93d10dc9661 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 31 Oct 2026 11:12:24 +0100 Subject: [PATCH 2/6] fix(packaging): remove duplicate font and Hayro notices --- NOTICE | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/NOTICE b/NOTICE index ce3ed2e03..4a79e69fd 100644 --- a/NOTICE +++ b/NOTICE @@ -58,17 +58,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -Bundled application fonts -------------------------- -Inter (assets/fonts/Inter-*.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-Inter.txt. -Noto Serif (assets/fonts/NotoSerif-Regular.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-NotoSerif.txt. -JetBrains Mono (assets/fonts/JetBrainsMono-Regular.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-JetBrainsMono.txt. - -The optional PDF document reader uses craft-pdf and Hayro-syntax 0.7.2, MIT OR Apache-2.0. -The shared patched syntax source is centralized in storytold/craft-libs, transferred from -PdfCraft revision 9ed3abe4907a127d36b12fbfeee71d7e26248da1. This parser-only use embeds no -Hayro fonts, CMaps or image codecs; EffectCraft keeps its existing fonts and drawing filters. - SIL Open Font License 1.1 Applies to Inter (assets/fonts/Inter-*.ttf), Noto Serif (assets/fonts/NotoSerif-Regular.ttf), From 425b69249fa6d30d80d4cea72ccaf6b7fb87f0ed Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 31 Oct 2026 11:12:26 +0100 Subject: [PATCH 3/6] fix(flatpak): install NOTICE only once --- packaging/linux/flatpak/ai.storyteller.effectcraft.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/packaging/linux/flatpak/ai.storyteller.effectcraft.yml b/packaging/linux/flatpak/ai.storyteller.effectcraft.yml index ee7a31b74..7dd7a8d68 100644 --- a/packaging/linux/flatpak/ai.storyteller.effectcraft.yml +++ b/packaging/linux/flatpak/ai.storyteller.effectcraft.yml @@ -56,7 +56,6 @@ modules: - cargo --offline build --release --locked -p effectcraft -p effectcraft-cli - install -Dm755 target/release/effectcraft /app/bin/effectcraft - install -Dm644 NOTICE /app/share/doc/effectcraft/NOTICE - - install -Dm644 NOTICE /app/share/doc/effectcraft/NOTICE - install -Dm755 target/release/effectcraft-cli /app/bin/effectcraft-cli - install -Dm644 packaging/linux/ai.storyteller.effectcraft.desktop /app/share/applications/ai.storyteller.effectcraft.desktop - install -Dm644 packaging/linux/ai.storyteller.effectcraft.mime.xml /app/share/mime/packages/ai.storyteller.effectcraft.xml From 9d7baea5cce3ddb4a1a33a0f1f6c3de439eddae3 Mon Sep 17 00:00:00 2001 From: ash Date: Mon, 2 Nov 2026 18:48:41 +0530 Subject: [PATCH 4/6] Fix #2823: Propagate boxes SAM 2 around the matte shown on the stroked frame (#2826) * Fix #2823: Propagate boxes SAM 2 around the matte shown on the stroked frame The tracker's box prompt came from the classic graph cut of the strokes, which can leak into the background; SAM 2 then filled the whole box. Build the box from the model's own matte (segment_with), the one shown before Propagate. segment_with falls back to the classic cut when the model can't segment, so non-model paths are unchanged. Co-Authored-By: Claude Opus 5.5 * Fix #2823: box a lone object around the matte shown on correction frames too A correction frame shows a matte propagated from the frame before, not a fresh model cut, so the tracker's box came from a different matte than the one on screen. With one object the box now comes from Chain::ensure, the matte shown on that frame. Several objects share one shown matte, so each keeps the model's cut of its own strokes. Cancelling while the shown matte is computed stops instead of falling back, and the prompt frame is read only when needed. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- crates/effects/src/roto.rs | 67 +++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs index d6f27b9bf..b53fa5d65 100644 --- a/crates/effects/src/roto.rs +++ b/crates/effects/src/roto.rs @@ -686,8 +686,9 @@ impl Chain { let (Some(&need_lo), Some(&need_hi)) = (needed.first(), needed.last()) else { return Ok(Some(TrackSummary::default())); }; - // Each object has its own prompts: a few clicks from its strokes and a box around what the classic engine - // makes of them on that frame (clicks alone pick a part of the object, not all of it). + // Each object has its own prompts: a few clicks from its strokes and a box saying how far it reaches (clicks + // alone pick a part of the object). SAM 2 fills the box it is given, so a lone object's box surrounds the matte + // shown on that frame (base or correction); with several objects, each gets the model's cut of its own strokes. let objects = self.data.objects(); let several = objects.len() > 1; let mut by_object: Vec> = vec![]; @@ -698,10 +699,16 @@ impl Chain { if !strokes.iter().any(|s| s.kind == StrokeKind::Fg) { continue; } - let p = match frame(f) { - Some(img) => rb::tracker_prompt_boxed(&strokes, self.opts.scale, &rb::segment(&img, &strokes, None, &self.opts, self.refine_radius)), - None => return Err(format!("cannot read Roto Brush prompt frame {f}")), + let shown = if several { None } else { self.ensure(f, &mut *frame, cancel) }; + let matte = match shown { + Some(shown) => shown, + None if cancel() => return Err("tracking cancelled".into()), + None => match frame(f) { + Some(img) => Arc::new(rb::segment_with(&img, &strokes, None, &self.opts, self.refine_radius, self.model.as_deref())), + None => return Err(format!("cannot read Roto Brush prompt frame {f}")), + }, }; + let p = rb::tracker_prompt_boxed(&strokes, self.opts.scale, &matte); if p.points.iter().any(|(_, fg)| *fg) { prompts.push((f, p)); } @@ -1470,6 +1477,56 @@ mod tests { } } + /// A model that sees only the left part of the disk, and records the tracker's box prompts. + #[derive(Default)] + struct LeftPart(std::sync::Mutex)>>); + impl effectcraft_segment::MaskModel for LeftPart { + fn info(&self) -> &'static effectcraft_segment::ModelInfo { + &effectcraft_segment::SAM2_TINY + } + fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, _: &effectcraft_segment::Prompt) -> effectcraft_segment::Result> { + Ok((0..w * h).map(|i| f32::from((26..36).contains(&(i % w)) && (30..50).contains(&(i / w)))).collect()) + } + fn tracker(&self) -> Option<&dyn effectcraft_segment::ObjectTracker> { + Some(self) + } + } + impl effectcraft_segment::ObjectTracker for LeftPart { + fn track( + &self, + req: &effectcraft_segment::TrackRequest, + _: &mut dyn FnMut(i64) -> Option>, + _: &dyn Fn() -> bool, + _: &mut dyn FnMut(usize, i64, effectcraft_segment::TrackedFrame), + ) -> effectcraft_segment::Result<()> { + *self.0.lock().unwrap() = req.objects.iter().flatten().map(|(f, p)| (*f, p.bbox)).collect(); + Ok(()) + } + } + + #[test] + fn the_tracker_box_surrounds_the_matte_shown_on_each_stroked_frame() { + let mut data = RotoData::default(); + data.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[30.0, 40.0]], object: 0 }, 3, [0, 3]); + data.add(Stroke { kind: StrokeKind::Fg, frame: 2, radius: 3.0, points: vec![[50.0, 40.0]], object: 0 }, 3, [0, 3]); + data.span = [0, 3]; + let mut params = params_with(&data); + params.values.insert(INPUT_KEY.into(), Value::Str("tracker-box-from-shown-matte".into())); + let mut chain = Chain::new(¶ms, [100.0, 80.0], 25.0, 1.0); + let fake = Arc::new(LeftPart::default()); + chain.model = Some(fake.clone()); + chain.keys = data.chain_keys(seed_with(¶ms, [100.0, 80.0], 25.0, effectcraft_segment::SAM2_TINY.id)); + let mut get = |f: i64| Some(Arc::new(frame(f as f64 / 25.0))); + let expected: Vec<(i64, Option<[f32; 4]>)> = [0, 2] + .into_iter() + .map(|f| (f, rb::tracker_prompt_boxed(&data.strokes_of(f, 0), 1.0, &chain.ensure(f, &mut get, &|| false).expect("a shown matte")).bbox)) + .collect(); + assert!(expected.iter().all(|(_, b)| b.is_some())); + assert!(expected[0].1.unwrap()[2] < 44.0, "the base frame shows the model's part of the disk: {expected:?}"); + chain.track(&mut get, &|| false, &mut |_| {}).unwrap(); + assert_eq!(*fake.0.lock().unwrap(), expected, "the base frame and the correction frame are boxed around what they show"); + } + #[test] fn prompt_segmentation_reads_only_frames_with_that_objects_foreground() { let mut data = RotoData::default(); From f15b619a92805ba898a55ae0c9a49691e4cea825 Mon Sep 17 00:00:00 2001 From: ash Date: Mon, 2 Nov 2026 22:31:39 +0530 Subject: [PATCH 5/6] Follow-up to #2823: box each of several objects around its whole matte on correction frames (#2830) * Follow-up to #2823: box each of several objects around its whole matte on correction frames With several objects, a correction frame's box came from the model's cut of that object's strokes on the frame alone. A stroke on part of the object boxed only that part, and SAM 2 followed only that part from there on. Each object's matte is now carried to the frame with its own strokes, from its first prompted frame or its nearest tracked frame, by the walk Chain::ensure uses (now Chain::carry). These mattes are not cached, since object keys hold tracked results. Co-Authored-By: Claude Opus 5.5 * Follow-up to #2823: carry corrections before the base from the base outwards Prompt frames were visited in ascending order, but before the base the carry walks towards it, so each correction there re-walked from the root. Visit each side from the base outwards and sort the prompts back into frame order for the tracker. Co-Authored-By: Claude Opus 5.5 * Follow-up to #2823: find an object's carry root in one pass over its strokes The root search scanned every frame's strokes from the prompt frame to the base, for every prompt. Take the object's foreground stroke on the way that is nearest the base in one pass instead; the root is the same. Co-Authored-By: Claude Opus 5.5 * M13.35: reproduce an earlier object correction losing tracked matte * M13.35: carry earlier object corrections from valid tracked mattes --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: FredApps --- crates/effects/src/roto.rs | 251 ++++++++++++++++++++++++++++++++++--- 1 file changed, 235 insertions(+), 16 deletions(-) diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs index b53fa5d65..fe6191714 100644 --- a/crates/effects/src/roto.rs +++ b/crates/effects/src/roto.rs @@ -543,25 +543,74 @@ impl Chain { } None }; + self.carry(f, None, frame, cancel, &lookup, &mut |g, seg| { + let k = *self.keys.get(&g)?; + store(k, sc, seg.clone()); + if let Some(st) = seg_store() { + st.put_seg(k, sc, seg); + } + Some(()) + }) + } + + /// [`Chain::ensure`]'s walk: the matte carried to `f` from the nearest frame towards the base that `lookup` has, + /// else from the root cut afresh from its strokes. `object: None` carries every stroke from the base frame (the shown + /// matte); `Some(o)` carries only `o`'s, from its foreground-stroked frame nearest the base on the way. `keep` + /// gets every frame made on the way. + fn carry( + &self, + f: i64, + object: Option, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, + lookup: &dyn Fn(i64) -> Option>, + keep: &mut dyn FnMut(i64, &Arc) -> Option<()>, + ) -> Option> { + let strokes = |g: i64| match object { + Some(o) => self.data.strokes_of(g, o), + None => self.data.strokes_at(g), + }; if let Some(s) = lookup(f) { return Some(s); } let base = self.data.base?; + let root = match object { + Some(o) => { + // one pass over the strokes, not one per frame on the way + let on_the_way = |g: i64| (g == base || g.cmp(&base) == f.cmp(&base)) && g.abs_diff(base) <= f.abs_diff(base); + let fg = self.data.strokes.iter().filter(|s| s.object == o && s.kind == StrokeKind::Fg && on_the_way(s.frame)); + fg.map(|s| s.frame).min_by_key(|g| g.abs_diff(base)).unwrap_or(f) + } + None => base, + }; if !self.data.in_span(f) { return None; } - // Walk towards the base to the nearest available frame. + // A new first stroke can still have a valid tracked matte nearer the base. Look for it + // without reading those frames; without one, start afresh at the object's first stroke. let mut path = vec![f]; + let mut untracked_len = 1; let mut g = f; let mut start: Option> = None; - while let Some(src) = self.data.source_of(g) { + while g != base + && let Some(src) = self.data.source_of(g) + { + if cancel() { + return None; + } if let Some(s) = lookup(src) { start = Some(s); break; } path.push(src); + if src == root { + untracked_len = path.len(); + } g = src; } + if start.is_none() { + path.truncate(untracked_len); + } path.reverse(); let mut prev_img: Option> = None; let mut cur = start; @@ -570,8 +619,8 @@ impl Chain { return None; } let img = frame(g)?; - let strokes = self.data.strokes_at(g); - let seg = match (&cur, g == base) { + let strokes = strokes(g); + let seg = match (&cur, object.is_none() && g == root) { (_, true) | (None, _) => rb::segment_with(&img, &strokes, None, &self.opts, self.refine_radius, self.model.as_deref()), (Some(p), false) => { let src = self.data.source_of(g)?; @@ -587,11 +636,7 @@ impl Chain { } }; let seg = Arc::new(seg); - let k = *self.keys.get(&g)?; - store(k, sc, seg.clone()); - if let Some(st) = seg_store() { - st.put_seg(k, sc, &seg); - } + keep(g, &seg)?; cur = Some(seg); prev_img = Some(img); } @@ -674,7 +719,7 @@ impl Chain { max_pending_bytes: usize, ) -> Result, String> { let Some(tracker) = self.model.as_ref().and_then(|m| m.tracker()) else { return Ok(None) }; - let (Some(_), true) = (self.data.base, self.data.span[1] >= self.data.span[0]) else { return Err("no base frame".into()) }; + let (Some(base), true) = (self.data.base, self.data.span[1] >= self.data.span[0]) else { return Err("no base frame".into()) }; // Frames of the span that already have a tracked result (this session's, or the saved // tracking) are kept: only the rest is tracked. A correction stroke changes the keys of the // frames from it on (both directions), so only those come back here. @@ -687,21 +732,35 @@ impl Chain { return Ok(Some(TrackSummary::default())); }; // Each object has its own prompts: a few clicks from its strokes and a box saying how far it reaches (clicks - // alone pick a part of the object). SAM 2 fills the box it is given, so a lone object's box surrounds the matte - // shown on that frame (base or correction); with several objects, each gets the model's cut of its own strokes. + // alone pick a part of the object). SAM 2 fills the box it is given, so the box surrounds the object's matte + // on that frame: the one shown for a lone object; with several, each object's own, carried with its own strokes + // as the shown one is (from its first prompted frame, or the nearest tracked or prompted frame), so a correction stroke on + // part of an object still boxes all of it. Those per-object mattes are not cached: their keys mean tracked results. let objects = self.data.objects(); let several = objects.len() > 1; let mut by_object: Vec> = vec![]; for &obj in &objects { let mut prompts = vec![]; - for f in self.data.stroke_frames().into_iter().filter(|f| self.data.in_span(*f)) { + let mut carried: Option<(i64, Arc)> = None; + // from the base outwards on each side, so each carry goes on from the one before + let mut frames: Vec = self.data.stroke_frames().into_iter().filter(|f| self.data.in_span(*f)).collect(); + frames.sort_by_key(|f| (*f < base, f.abs_diff(base))); + for f in frames { let strokes = self.data.strokes_of(f, obj); if !strokes.iter().any(|s| s.kind == StrokeKind::Fg) { continue; } - let shown = if several { None } else { self.ensure(f, &mut *frame, cancel) }; - let matte = match shown { - Some(shown) => shown, + let own = if several { + // a later correction frame carries on from the one before it (only the last can be on its way) + let lookup = |g| self.object_seg(g, obj).or_else(|| carried.as_ref().filter(|(k, _)| *k == g).map(|(_, s)| s.clone())); + let own = self.carry(f, Some(obj), &mut *frame, cancel, &lookup, &mut |_, _| Some(())); + carried = own.clone().map(|s| (f, s)); + own + } else { + self.ensure(f, &mut *frame, cancel) + }; + let matte = match own { + Some(own) => own, None if cancel() => return Err("tracking cancelled".into()), None => match frame(f) { Some(img) => Arc::new(rb::segment_with(&img, &strokes, None, &self.opts, self.refine_radius, self.model.as_deref())), @@ -713,6 +772,7 @@ impl Chain { prompts.push((f, p)); } } + prompts.sort_by_key(|q| q.0); by_object.push(prompts); } // objects without a foreground prompt cannot be followed @@ -1527,6 +1587,165 @@ mod tests { assert_eq!(*fake.0.lock().unwrap(), expected, "the base frame and the correction frame are boxed around what they show"); } + /// A model that cuts only a small square around each foreground click, and records every object's box prompts. + #[derive(Default)] + struct ClickSquares(std::sync::Mutex)>>>); + impl effectcraft_segment::MaskModel for ClickSquares { + fn info(&self) -> &'static effectcraft_segment::ModelInfo { + &effectcraft_segment::SAM2_TINY + } + fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, p: &effectcraft_segment::Prompt) -> effectcraft_segment::Result> { + let near = |x: f32, y: f32| p.points.iter().any(|(c, fg)| *fg && (c[0] - x).abs() <= 4.0 && (c[1] - y).abs() <= 4.0); + Ok((0..w * h).map(|i| f32::from(near((i % w) as f32 + 0.5, (i / w) as f32 + 0.5))).collect()) + } + fn tracker(&self) -> Option<&dyn effectcraft_segment::ObjectTracker> { + Some(self) + } + } + impl effectcraft_segment::ObjectTracker for ClickSquares { + fn track( + &self, + req: &effectcraft_segment::TrackRequest, + _: &mut dyn FnMut(i64) -> Option>, + _: &dyn Fn() -> bool, + _: &mut dyn FnMut(usize, i64, effectcraft_segment::TrackedFrame), + ) -> effectcraft_segment::Result<()> { + *self.0.lock().unwrap() = req.objects.iter().map(|o| o.iter().map(|(f, p)| (*f, p.bbox)).collect()).collect(); + Ok(()) + } + } + + #[test] + fn a_correction_stroke_on_part_of_one_of_several_objects_boxes_the_whole_object() { + let mut data = RotoData::default(); + data.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[30.0, 40.0], [50.0, 40.0]], object: 0 }, 3, [0, 3]); + data.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[88.0, 72.0]], object: 1 }, 3, [0, 3]); + data.add(Stroke { kind: StrokeKind::Fg, frame: 2, radius: 2.0, points: vec![[54.0, 40.0]], object: 0 }, 3, [0, 3]); + data.span = [0, 3]; + let mut params = params_with(&data); + params.values.insert(INPUT_KEY.into(), Value::Str("object-correction-box".into())); + let mut chain = Chain::new(¶ms, [100.0, 80.0], 25.0, 1.0); + let fake = Arc::new(ClickSquares::default()); + chain.model = Some(fake.clone()); + chain.keys = data.chain_keys(seed_with(¶ms, [100.0, 80.0], 25.0, effectcraft_segment::SAM2_TINY.id)); + chain.track(&mut |f| Some(Arc::new(frame(f as f64 / 25.0))), &|| false, &mut |_| {}).unwrap(); + let boxes = fake.0.lock().unwrap().clone(); + let box_of = |o: usize, f: i64| boxes[o].iter().find(|(g, _)| *g == f).and_then(|(_, b)| *b).expect("a box prompt"); + let (base, correction, other) = (box_of(0, 0), box_of(0, 2), box_of(1, 0)); + assert!(correction[0] <= base[0] + 6.0, "the correction frame boxes the object carried from frame 0 {base:?}, not only the stroke: {correction:?}"); + assert!(correction[2] >= 54.0 && correction[2] - correction[0] >= base[2] - base[0], "the box holds the whole object and the stroke: {correction:?}"); + assert!(correction[2] < other[0], "object 1 {other:?} stays out of object 0's box {correction:?}"); + } + + #[test] + fn a_correction_after_propagate_carries_each_object_on_from_its_tracked_matte() { + let mut data = RotoData::default(); + data.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[30.0, 40.0]], object: 0 }, 3, [0, 3]); + data.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[15.0, 15.0]], object: 1 }, 3, [0, 3]); + data.span = [0, 3]; + let mut params = params_with(&data); + params.values.insert(INPUT_KEY.into(), Value::Str("object-correction-after-propagate".into())); + let mut chain = Chain::new(¶ms, [100.0, 80.0], 25.0, 1.0); + chain.model = Some(Arc::new(TwoObjects(Default::default()))); + chain.keys = data.chain_keys(seed_with(¶ms, [100.0, 80.0], 25.0, effectcraft_segment::SAM2_TINY.id)); + assert_eq!(chain.track(&mut |f| Some(Arc::new(frame(f as f64 / 25.0))), &|| false, &mut |_| {}).unwrap().unwrap().frames, 4); + data.add(Stroke { kind: StrokeKind::Fg, frame: 2, radius: 2.0, points: vec![[56.0, 40.0]], object: 0 }, 3, [0, 3]); + chain.data = Arc::new(data.clone()); + chain.keys = data.chain_keys(seed_with(¶ms, [100.0, 80.0], 25.0, effectcraft_segment::SAM2_TINY.id)); + let fake = Arc::new(ClickSquares::default()); + chain.model = Some(fake.clone()); + let mut prompt_reads = vec![]; + let mut get = |f: i64| { + if fake.0.lock().unwrap().is_empty() { + prompt_reads.push(f); + } + Some(Arc::new(frame(f as f64 / 25.0))) + }; + chain.track(&mut get, &|| false, &mut |_| {}).unwrap(); + assert_eq!(prompt_reads, [2, 1], "frame 2 carries on from the tracked frame 1; tracked frame 0 boxes both objects as it is"); + let boxes = fake.0.lock().unwrap().clone(); + let correction = boxes[0].iter().find(|(g, _)| *g == 2).and_then(|(_, b)| *b).expect("a box prompt"); + // the tracked disk carried to frame 2 spans x 29..59 + assert!(correction[0] <= 29.0 && correction[2] >= 59.0, "the box holds the whole disk: {correction:?}"); + } + + #[test] + fn a_correction_before_an_objects_first_stroke_keeps_its_tracked_matte() { + for base in [0, 4] { + let first = if base == 0 { 3 } else { 1 }; + let prior = if base == 0 { 1 } else { 3 }; + let mut data = RotoData::default(); + data.add(Stroke { kind: StrokeKind::Fg, frame: base, radius: 3.0, points: vec![[40.0, 40.0]], object: 0 }, 4, [0, 4]); + data.add(Stroke { kind: StrokeKind::Fg, frame: first, radius: 3.0, points: vec![[15.0, 15.0]], object: 1 }, 4, [0, 4]); + data.span = [0, 4]; + let mut params = params_with(&data); + params.values.insert(INPUT_KEY.into(), Value::Str(format!("earlier-object-correction-{base}"))); + let mut chain = Chain::new(¶ms, [100.0, 80.0], 25.0, 1.0); + chain.model = Some(Arc::new(TwoObjects(Default::default()))); + let seed = seed_with(¶ms, [100.0, 80.0], 25.0, effectcraft_segment::SAM2_TINY.id); + chain.keys = data.chain_keys(seed); + let image = |f: i64| { + let mut img = frame(f as f64 / 25.0); + // Match the second object's tracked square, so the carry has a real image edge. + for y in 8..22 { + for x in 8..22 { + img.set(x, y, [0.95, 0.9, 0.1, 1.0]); + } + } + Arc::new(img) + }; + assert_eq!(chain.track(&mut |f| Some(image(f)), &|| false, &mut |_| {}).unwrap().unwrap().frames, 5); + let prior_key = chain.keys[&prior]; + assert!(chain.object_seg(prior, 1).is_some()); + data.add(Stroke { kind: StrokeKind::Fg, frame: 2, radius: 1.0, points: vec![[19.0, 15.0]], object: 1 }, 4, [0, 4]); + chain.data = Arc::new(data.clone()); + chain.keys = data.chain_keys(seed); + assert_eq!(chain.keys[&prior], prior_key, "the nearer tracked frame is still valid"); + let fake = Arc::new(ClickSquares::default()); + chain.model = Some(fake.clone()); + let mut prompt_reads = vec![]; + let mut get = |f: i64| { + if fake.0.lock().unwrap().is_empty() { + prompt_reads.push(f); + } + Some(image(f)) + }; + chain.track(&mut get, &|| false, &mut |_| {}).unwrap(); + assert_eq!(prompt_reads, [2, prior, first, 2], "corrections carry from the valid tracked matte in either direction"); + let boxes = fake.0.lock().unwrap().clone(); + let correction = boxes[1].iter().find(|(g, _)| *g == 2).and_then(|(_, b)| *b).expect("a correction box"); + assert!(correction[0] <= 10.0 && correction[2] >= 22.0, "the box keeps the whole tracked square, not only the correction: {correction:?}"); + assert!(correction[2] < 30.0, "the other object's disk stays outside this box: {correction:?}"); + } + } + + #[test] + fn corrections_before_the_base_carry_on_from_each_other() { + let mut data = RotoData::default(); + for (frame, object) in [(3, 0), (3, 1), (2, 0), (1, 0)] { + data.add(Stroke { kind: StrokeKind::Fg, frame, radius: 3.0, points: vec![[35.0, 40.0]], object }, 3, [0, 3]); + } + data.span = [0, 3]; + assert_eq!(data.base, Some(3)); + let mut params = params_with(&data); + params.values.insert(INPUT_KEY.into(), Value::Str("corrections-before-the-base".into())); + let mut chain = Chain::new(¶ms, [100.0, 80.0], 25.0, 1.0); + let fake = Arc::new(ClickSquares::default()); + chain.model = Some(fake.clone()); + chain.keys = data.chain_keys(seed_with(¶ms, [100.0, 80.0], 25.0, effectcraft_segment::SAM2_TINY.id)); + let mut prompt_reads = vec![]; + let mut get = |f: i64| { + if fake.0.lock().unwrap().is_empty() { + prompt_reads.push(f); + } + Some(Arc::new(frame(f as f64 / 25.0))) + }; + chain.track(&mut get, &|| false, &mut |_| {}).unwrap(); + assert_eq!(prompt_reads, [3, 2, 3, 1, 2, 3], "object 0: the base, then 2 from 3, then 1 from 2; object 1: the base"); + let frames: Vec> = fake.0.lock().unwrap().iter().map(|o| o.iter().map(|q| q.0).collect()).collect(); + assert_eq!(frames, [vec![1, 2, 3], vec![3]], "prompts reach the tracker in frame order"); + } + #[test] fn prompt_segmentation_reads_only_frames_with_that_objects_foreground() { let mut data = RotoData::default(); From 7870b5916ae0749d497d880989683bf2946d6c8b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 2 Nov 2026 19:26:55 -0500 Subject: [PATCH 6/6] Release 0.21.0 Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 74 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d27b959f..c43ba9d9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1412,7 +1412,7 @@ dependencies = [ [[package]] name = "effectcraft" -version = "0.20.0" +version = "0.21.0" dependencies = [ "cpal", "craft-update", @@ -1438,7 +1438,7 @@ dependencies = [ [[package]] name = "effectcraft-automation" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-engine", "effectcraft-host", @@ -1451,14 +1451,14 @@ dependencies = [ [[package]] name = "effectcraft-av1enc" -version = "0.20.0" +version = "0.21.0" dependencies = [ "filmcraft-av1", ] [[package]] name = "effectcraft-cli" -version = "0.20.0" +version = "0.21.0" dependencies = [ "cpu-time", "effectcraft-automation", @@ -1477,7 +1477,7 @@ dependencies = [ [[package]] name = "effectcraft-color" -version = "0.20.0" +version = "0.21.0" dependencies = [ "proptest", "serde", @@ -1485,7 +1485,7 @@ dependencies = [ [[package]] name = "effectcraft-effects" -version = "0.20.0" +version = "0.21.0" dependencies = [ "chrono", "effectcraft-color", @@ -1503,7 +1503,7 @@ dependencies = [ [[package]] name = "effectcraft-engine" -version = "0.20.0" +version = "0.21.0" dependencies = [ "chrono", "craft-docpath", @@ -1541,7 +1541,7 @@ dependencies = [ [[package]] name = "effectcraft-export" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-av1enc", "effectcraft-color", @@ -1571,7 +1571,7 @@ dependencies = [ [[package]] name = "effectcraft-expr" -version = "0.20.0" +version = "0.21.0" dependencies = [ "boa_engine", "effectcraft-color", @@ -1588,7 +1588,7 @@ dependencies = [ [[package]] name = "effectcraft-geom" -version = "0.20.0" +version = "0.21.0" dependencies = [ "proptest", "serde", @@ -1596,7 +1596,7 @@ dependencies = [ [[package]] name = "effectcraft-gpu" -version = "0.20.0" +version = "0.21.0" dependencies = [ "bytemuck", "effectcraft-color", @@ -1618,14 +1618,14 @@ dependencies = [ [[package]] name = "effectcraft-hevcenc" -version = "0.20.0" +version = "0.21.0" dependencies = [ "filmcraft-hevc", ] [[package]] name = "effectcraft-host" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-engine", "effectcraft-export", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "effectcraft-interchange" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-color", "effectcraft-keyframe", @@ -1669,7 +1669,7 @@ dependencies = [ [[package]] name = "effectcraft-keyframe" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-color", "effectcraft-geom", @@ -1681,7 +1681,7 @@ dependencies = [ [[package]] name = "effectcraft-lottie" -version = "0.20.0" +version = "0.21.0" dependencies = [ "craft-docpath", "effectcraft-color", @@ -1699,7 +1699,7 @@ dependencies = [ [[package]] name = "effectcraft-media" -version = "0.20.0" +version = "0.21.0" dependencies = [ "craft-docpath", "effectcraft-color", @@ -1731,7 +1731,7 @@ dependencies = [ [[package]] name = "effectcraft-model" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-geom", "image", @@ -1744,14 +1744,14 @@ dependencies = [ [[package]] name = "effectcraft-opusenc" -version = "0.20.0" +version = "0.21.0" dependencies = [ "filmcraft-opus", ] [[package]] name = "effectcraft-path" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-geom", "effectcraft-keyframe", @@ -1763,7 +1763,7 @@ dependencies = [ [[package]] name = "effectcraft-pdf" -version = "0.20.0" +version = "0.21.0" dependencies = [ "craft-pdf", "effectcraft-raster", @@ -1779,7 +1779,7 @@ dependencies = [ [[package]] name = "effectcraft-plugin" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-effects", "effectcraft-keyframe", @@ -1791,7 +1791,7 @@ dependencies = [ [[package]] name = "effectcraft-project" -version = "0.20.0" +version = "0.21.0" dependencies = [ "craft-ecproj-format", "effectcraft-color", @@ -1805,14 +1805,14 @@ dependencies = [ [[package]] name = "effectcraft-psd" -version = "0.20.0" +version = "0.21.0" dependencies = [ "miniz_oxide 0.9.1", ] [[package]] name = "effectcraft-raster" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-color", "effectcraft-geom", @@ -1821,7 +1821,7 @@ dependencies = [ [[package]] name = "effectcraft-render" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-color", "effectcraft-effects", @@ -1843,7 +1843,7 @@ dependencies = [ [[package]] name = "effectcraft-sam2" -version = "0.20.0" +version = "0.21.0" dependencies = [ "bytemuck", "candle-core", @@ -1858,7 +1858,7 @@ dependencies = [ [[package]] name = "effectcraft-script" -version = "0.20.0" +version = "0.21.0" dependencies = [ "boa_engine", "effectcraft-engine", @@ -1876,7 +1876,7 @@ dependencies = [ [[package]] name = "effectcraft-segment" -version = "0.20.0" +version = "0.21.0" dependencies = [ "miniz_oxide 0.9.1", "ndarray", @@ -1885,7 +1885,7 @@ dependencies = [ [[package]] name = "effectcraft-svg" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-geom", "effectcraft-path", @@ -1897,7 +1897,7 @@ dependencies = [ [[package]] name = "effectcraft-text" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-keyframe", "harfrust", @@ -1909,7 +1909,7 @@ dependencies = [ [[package]] name = "effectcraft-time" -version = "0.20.0" +version = "0.21.0" dependencies = [ "proptest", "serde", @@ -1917,7 +1917,7 @@ dependencies = [ [[package]] name = "effectcraft-track" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-raster", "effectcraft-segment", @@ -1929,7 +1929,7 @@ dependencies = [ [[package]] name = "effectcraft-ui-egui" -version = "0.20.0" +version = "0.21.0" dependencies = [ "chrono", "craft-quit", @@ -1965,11 +1965,11 @@ dependencies = [ [[package]] name = "effectcraft-vp9enc" -version = "0.20.0" +version = "0.21.0" [[package]] name = "effectcraft-web" -version = "0.20.0" +version = "0.21.0" dependencies = [ "effectcraft-engine", "effectcraft-export", @@ -7623,7 +7623,7 @@ checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" [[package]] name = "xtask" -version = "0.20.0" +version = "0.21.0" dependencies = [ "chrono", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 9e63ffc26..f5f84775f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*", "apps/*", "xtask"] default-members = ["crates/*", "apps/*"] [workspace.package] -version = "0.20.0" +version = "0.21.0" edition = "2024" license = "MIT OR Apache-2.0" rust-version = "1.95"