diff --git a/apps/effectcraft-cli/tests/it/cli.rs b/apps/effectcraft-cli/tests/it/cli.rs index ed6d98086..8bdfb3c00 100644 --- a/apps/effectcraft-cli/tests/it/cli.rs +++ b/apps/effectcraft-cli/tests/it/cli.rs @@ -683,6 +683,7 @@ fn render_bitrate_applies_to_vp9_webm() { #[test] fn bench_roto_runs_on_a_users_clip() { let (frames, masks) = (tmp("clip-frames"), tmp("clip-masks")); + let config = frames.join("config"); for d in [&frames, &masks] { let _ = std::fs::remove_dir_all(d); std::fs::create_dir_all(d).unwrap(); @@ -703,7 +704,7 @@ fn bench_roto_runs_on_a_users_clip() { mask.save(masks.join(format!("f{f:03}.png"))).unwrap(); } let (f, m) = (frames.to_str().unwrap(), masks.to_str().unwrap()); - let out = bin().args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6"]).output().unwrap(); + let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6"]).output().unwrap(); let text = String::from_utf8_lossy(&out.stdout).to_string(); assert!(out.status.success(), "{text}\n{}", String::from_utf8_lossy(&out.stderr)); let row = text.lines().find(|l| l.contains("classical")).unwrap_or_else(|| panic!("no classical row: {text}")); @@ -711,11 +712,11 @@ fn bench_roto_runs_on_a_users_clip() { assert!(mean > 0.8, "classical mean IoU on the user's clip: {mean}\n{text}"); // one mask short: refused std::fs::remove_file(masks.join("f005.png")).unwrap(); - let out = bin().args(["bench", "--roto", "--clip", f, "--truth", m]).output().unwrap(); + let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f, "--truth", m]).output().unwrap(); assert!(!out.status.success()); assert!(String::from_utf8_lossy(&out.stderr).contains("must match"), "{}", String::from_utf8_lossy(&out.stderr)); // --clip without --truth - let out = bin().args(["bench", "--roto", "--clip", f]).output().unwrap(); + let out = bin().env("EFFECTCRAFT_CONFIG_DIR", &config).args(["bench", "--roto", "--clip", f]).output().unwrap(); assert!(!out.status.success()); } @@ -730,11 +731,11 @@ fn bench_roto_reports_why_a_named_model_file_failed() { std::fs::write(&bogus, b"not the weights").unwrap(); for flag in ["--sam2", "--mobilesam"] { let missing = dir.join("missing.bin"); - let out = bin().args(["bench", "--roto", flag, missing.to_str().unwrap()]).output().unwrap(); + let out = bin().env("EFFECTCRAFT_CONFIG_DIR", dir.join("config")).args(["bench", "--roto", flag, missing.to_str().unwrap()]).output().unwrap(); let err = String::from_utf8_lossy(&out.stderr).to_string(); assert!(!out.status.success(), "{flag}: {err}"); assert!(err.contains(flag) && err.contains("missing.bin"), "{flag} names the file that could not be read: {err}"); - let out = bin().args(["bench", "--roto", flag, bogus.to_str().unwrap()]).output().unwrap(); + let out = bin().env("EFFECTCRAFT_CONFIG_DIR", dir.join("config")).args(["bench", "--roto", flag, bogus.to_str().unwrap()]).output().unwrap(); let err = String::from_utf8_lossy(&out.stderr).to_string(); assert!(!out.status.success(), "{flag}: {err}"); assert!(err.contains("does not match"), "{flag} says the file is not the published weights: {err}"); diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs index 8046c4773..e66400416 100644 --- a/crates/effects/src/roto.rs +++ b/crates/effects/src/roto.rs @@ -624,6 +624,20 @@ impl Chain { } } + /// Discard a failed track completely before falling back to the classic engine. + pub fn forget_tracking(&mut self) { + let mut objects = self.data.objects(); + if let Some(saved) = &self.saved { + objects.extend(saved.cache.objects.keys().copied()); + objects.sort_unstable(); + objects.dedup(); + } + let keys = self.keys.values().chain(self.saved.iter().flat_map(|saved| saved.keys.values())).copied(); + forget(keys.flat_map(|key| std::iter::once(key).chain(objects.iter().copied().map(move |object| object_key(key, object))))); + self.saved = None; + self.model = None; + } + /// Follow the object through the span with the model's tracker, from the strokes on their /// frames (only the frames without a tracked result yet: after a correction stroke, the frames it /// changes, tracked on from the nearest prompted frame), and cache every frame's segmentation (under the tracker's keys, with its @@ -1471,6 +1485,24 @@ mod tests { let seg = chain.object_seg(1, 1).expect("the saved object matte"); assert_eq!(seg.matte.len(), 100 * 80); assert!(cached(object_key(other[&1], 1), 1.0).is_none(), "the other model's cache holds none of it"); + for key in [keys[&1], other[&1]] { + for scale in [1.0, 0.5] { + store(tracked_key(key), scale, seg.clone()); + store(object_key(key, 1), scale, seg.clone()); + } + trust_cache().lock().unwrap().insert(tracked_key(key), FrameTrust { confidence: 0.9, occluded: false }); + } + chain.forget_tracking(); + assert!(chain.model.is_none()); + assert!(!chain.saved_has(1)); + for key in [keys[&1], other[&1]] { + for scale in [1.0, 0.5] { + assert!(cached(tracked_key(key), scale).is_none()); + assert!(cached(object_key(key, 1), scale).is_none()); + } + assert!(trust(key).is_none()); + } + assert!(chain.ensure(1, &mut |_| None, &|| false).is_none(), "neither saved nor in-memory tracking survives fallback"); } #[test] diff --git a/crates/engine/src/analysis/roto.rs b/crates/engine/src/analysis/roto.rs index 03b287d6e..67b0afc68 100644 --- a/crates/engine/src/analysis/roto.rs +++ b/crates/engine/src/analysis/roto.rs @@ -477,6 +477,9 @@ async fn run_work(w: Work, shared: &RotoShared) { Ok(_) => fx::tracks_of(&chain).map(|t| t.to_json()).unwrap_or_default(), Err(_) => String::new(), }); + if result.is_err() { + chain.forget_tracking(); + } let mut s = lock(&shared.state); match result { Ok(summary) => s.tracked = summary, diff --git a/crates/engine/src/tests/roto.rs b/crates/engine/src/tests/roto.rs index afc3adad9..e686962e1 100644 --- a/crates/engine/src/tests/roto.rs +++ b/crates/engine/src/tests/roto.rs @@ -937,9 +937,9 @@ fn a_stroke_change_makes_the_saved_tracking_stale() { let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap()); } -/// A model whose tracker always fails: Roto Brush must carry on with the classic engine and say so. +/// A tracker that emits a completed frame before failing must leave only classic results behind. #[test] -fn a_failing_tracker_falls_back_to_the_classic_engine() { +fn a_partially_failing_tracker_falls_back_to_the_classic_engine() { use effectcraft_segment::{ClipFrame, MaskModel, ModelInfo, ObjectTracker, Prompt, TrackRequest, TrackedFrame}; use std::sync::Arc; struct Broken; @@ -957,29 +957,51 @@ fn a_failing_tracker_falls_back_to_the_classic_engine() { impl ObjectTracker for Broken { fn track( &self, - _: &TrackRequest, - _: &mut dyn FnMut(i64) -> Option>, + req: &TrackRequest, + frame: &mut dyn FnMut(i64) -> Option>, _: &dyn Fn() -> bool, - _: &mut dyn FnMut(usize, i64, TrackedFrame), + out: &mut dyn FnMut(usize, i64, TrackedFrame), ) -> effectcraft_segment::Result<()> { - Err("the tracker broke".into()) + let img = frame(req.first).ok_or("missing test frame")?; + for object in 0..req.objects.len() { + out(object, req.first, TrackedFrame { prob: vec![1.0; img.w * img.h], w: img.w, h: img.h, confidence: 0.9, occluded: false }); + } + Err("the tracker broke after emitting a frame".into()) } } hold_roto_cache_test_lock(); - let (mut s, clip, _) = painted(center, "disk-broken-tracker.mov"); + let (mut s, clip, uid) = painted(center, "disk-partially-broken-tracker.mov"); + s.execute_checked("roto.stroke", json!({"layer": clip.0, "object": 1, "points": [center(0)], "radius": 4})).unwrap(); effectcraft_effects::roto::set_model(Some(Arc::new(Broken))); let r = s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})); + let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 0})); + let comp = s.active_comp().unwrap(); + let layer = comp.layer(clip).unwrap(); + let params = crate::roto::params_static(layer.props.find_group(uid).unwrap()); + let size = crate::roto::layer_size(&s.project, comp, layer); + let mut chain = effectcraft_effects::roto::Chain::new(¶ms, size, comp.frame_rate.as_f64(), 1.0); + let index = layer.effects().unwrap().groups().position(|g| g.uid == uid).unwrap(); + let mut source = + crate::roto::FrameSource::new(&s.project, s.footage.as_ref(), s.expr.as_deref(), &s.layer_cache, s.active_comp_id().unwrap(), comp, layer, index, size); + let mut get = |f: i64| source.get(f); + let actual: Option> = (0..=20).map(|f| chain.ensure(f, &mut get, &|| false)).collect(); + let retained_object = chain.object_seg(0, 0).is_some() || chain.object_seg(0, 1).is_some(); + let retained_trust = chain.trust_at(0); + // Build the reference only after capturing the fallback, with no model or cached union. + effectcraft_effects::roto::forget(chain.keys.values().copied()); + chain.model = None; + let expected: Option> = (0..=20).map(|f| chain.ensure(f, &mut get, &|| false)).collect(); effectcraft_effects::roto::set_model(None); let r = r.unwrap(); assert_eq!(r["frames"], 21, "{r}"); - let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 10})).unwrap(); - assert_eq!(st["source"], "classical", "{st}"); + let st = st.unwrap(); + assert_ne!(st["source"], "tracker", "{st}"); assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 0); - let mut worst: f64 = 1.0; - for f in 0..=20 { - worst = worst.min(iou_at(&mut s, clip, f, center(f))); + assert!(!retained_object, "partial per-object mattes must be discarded"); + assert_eq!(retained_trust, None, "partial tracker confidence must be discarded"); + for (frame, (actual, expected)) in actual.unwrap().into_iter().zip(expected.unwrap()).enumerate() { + assert!(actual == expected, "fallback frame {frame} differs from the classic engine"); } - assert!(worst > 0.9, "the classic engine should have carried the clip: {worst}"); } /// Without a tracking model the Roto Brush chain is exactly the classic engine: every frame diff --git a/crates/sam2/src/mask.rs b/crates/sam2/src/mask.rs index 911ff2e00..382f9b514 100644 --- a/crates/sam2/src/mask.rs +++ b/crates/sam2/src/mask.rs @@ -19,10 +19,7 @@ const MAX_PIXELS: usize = 100_000_000; /// Resize `w x h` RGB (0..1) to the model input and normalise it: `[1, 3, 1024, 1024]`. pub fn preprocess(rgb: &[[f32; 3]], w: usize, h: usize) -> Result { - let pixels = w.checked_mul(h).filter(|n| *n > 0 && *n == rgb.len() && w <= MAX_SIDE && h <= MAX_SIDE && *n <= MAX_PIXELS); - let Some(pixels) = pixels else { - return Err(Error::Input(format!("a {w} x {h} frame needs {} pixels (at most {MAX_PIXELS}), got {}", w.saturating_mul(h), rgb.len()))); - }; + let pixels = validate_frame_size(w, h, rgb.len())?; const MEAN: [f32; 3] = [0.485, 0.456, 0.406]; const STD: [f32; 3] = [0.229, 0.224, 0.225]; let mut planar = vec![0f32; 3 * pixels]; @@ -40,6 +37,12 @@ pub fn preprocess(rgb: &[[f32; 3]], w: usize, h: usize) -> Result { Ok(x.broadcast_sub(&mean)?.broadcast_div(&std)?) } +pub(crate) fn validate_frame_size(w: usize, h: usize, len: usize) -> Result { + w.checked_mul(h) + .filter(|n| *n > 0 && *n == len && w <= MAX_SIDE && h <= MAX_SIDE && *n <= MAX_PIXELS) + .ok_or_else(|| Error::Input(format!("a {w} x {h} frame needs {} pixels (at most {MAX_PIXELS}), got {len}", w.saturating_mul(h)))) +} + fn fingerprint(rgb: &[[f32; 3]], w: usize, h: usize) -> u64 { let mut hash = 0xcbf2_9ce4_8422_2325u64 ^ ((w as u64) << 32 | h as u64); for px in rgb { diff --git a/crates/sam2/src/neck.rs b/crates/sam2/src/neck.rs index c897e9818..3a80ede9e 100644 --- a/crates/sam2/src/neck.rs +++ b/crates/sam2/src/neck.rs @@ -13,23 +13,47 @@ use crate::weights::Weights; use crate::{Error, Result}; /// Sine position encoding `[1, 2 * num_pos_features, h, w]`, normalised (`Sam2SinePositionEmbedding`). -/// It depends only on the sizes, so each is built once (it was 150 ms of a 2.7 s frame). +/// Only small tables are retained globally; the neck owns its larger feature-pyramid tables. pub fn sine_position(num_pos_features: usize, h: usize, w: usize) -> Result { - static CACHE: OnceLock>> = OnceLock::new(); - let cache = CACHE.get_or_init(Default::default); - if let Some(t) = cache.lock().unwrap_or_else(PoisonError::into_inner).get(&(num_pos_features, h, w)) { - return Ok(t.clone()); + static CACHE: OnceLock = OnceLock::new(); + CACHE.get_or_init(|| PositionCache::new(1024 * 1024 / size_of::())).get(num_pos_features, h, w) +} + +const MAX_POSITION_ELEMENTS: usize = 128 * 1024 * 1024 / size_of::(); + +struct PositionCache { + entries: Mutex>, + max_elements: usize, +} + +impl PositionCache { + fn new(max_elements: usize) -> Self { + Self { entries: Mutex::new(HashMap::new()), max_elements } } - let t = build_sine_position(num_pos_features, h, w)?; - let mut map = cache.lock().unwrap_or_else(PoisonError::into_inner); - if map.len() >= 16 { - map.clear(); + + fn get(&self, features: usize, h: usize, w: usize) -> Result { + let key = (features, h, w); + if let Some(t) = self.entries.lock().unwrap_or_else(PoisonError::into_inner).get(&key) { + return Ok(t.clone()); + } + let t = build_sine_position(features, h, w)?; + let count = t.elem_count(); + if count <= self.max_elements { + let mut map = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let retained = map.values().fold(0usize, |n, t| n.saturating_add(t.elem_count())); + if map.len() >= 16 || retained.saturating_add(count) > self.max_elements { + map.clear(); + } + map.insert(key, t.clone()); + } + Ok(t) } - map.insert((num_pos_features, h, w), t.clone()); - Ok(t) } fn build_sine_position(num_pos_features: usize, h: usize, w: usize) -> Result { + let count = num_pos_features.checked_mul(2).and_then(|n| n.checked_mul(h)).and_then(|n| n.checked_mul(w)); + let count = + count.filter(|n| *n > 0 && *n <= MAX_POSITION_ELEMENTS).ok_or_else(|| Error::Input("sine position dimensions are empty or exceed 128 MiB".into()))?; let eps = 1e-6f32; let scale = 2.0 * std::f32::consts::PI; let dim_t: Vec = (0..num_pos_features).map(|i| 10000f32.powf(2.0 * (i / 2) as f32 / num_pos_features as f32)).collect(); @@ -39,7 +63,7 @@ fn build_sine_position(num_pos_features: usize, h: usize, w: usize) -> Result, Vec<_>) = ((0..h).map(|k| axis(h, k)).collect(), (0..w).map(|k| axis(w, k)).collect()); - let mut out = vec![0f32; 2 * num_pos_features * h * w]; + let mut out = vec![0f32; count]; for y in 0..h { for x in 0..w { for f in 0..num_pos_features { @@ -75,6 +99,7 @@ pub struct Neck { convs: Vec, top_down: Vec, pos_features: usize, + positions: PositionCache, } /// What the neck produces, both in the neck's own order (lowest resolution first, all four @@ -97,7 +122,7 @@ impl Neck { }) }) .collect::>>()?; - Ok(Neck { convs, top_down: top_down.to_vec(), pos_features: fpn / 2 }) + Ok(Neck { convs, top_down: top_down.to_vec(), pos_features: fpn / 2, positions: PositionCache::new(MAX_POSITION_ELEMENTS) }) } /// `stages`: Hiera stage outputs, NHWC, highest resolution first. @@ -119,7 +144,7 @@ impl Neck { } _ => lateral, }; - positions.push(sine_position(self.pos_features, cur.dim(2)?, cur.dim(3)?)?); + positions.push(self.positions.get(self.pos_features, cur.dim(2)?, cur.dim(3)?)?); features.push(cur.clone()); prev = Some(cur); } @@ -160,3 +185,31 @@ impl VisionEncoder { Ok(ImageFeatures { features: keep(out.features), positions: keep(out.positions) }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn position_tables_reject_empty_overflowing_and_oversized_dimensions() { + for (features, h, w) in [(0, 1, 1), (1, 0, 1), (1, 1, 0), (usize::MAX, 2, 2), (128, usize::MAX, 2), (128, 1024, 1024)] { + assert!(matches!(sine_position(features, h, w), Err(Error::Input(_))), "{features}, {h}, {w}"); + } + } + + #[test] + fn position_cache_bounds_retained_elements_and_preserves_the_encoding() { + let cache = PositionCache::new(64); + let first = cache.get(2, 2, 4).unwrap(); + assert_eq!(first.dims(), &[1, 4, 2, 4]); + assert_eq!(first.flatten_all().unwrap().to_vec1::().unwrap(), cache.get(2, 2, 4).unwrap().flatten_all().unwrap().to_vec1::().unwrap()); + cache.get(2, 3, 4).unwrap(); + let entries = cache.entries.lock().unwrap(); + assert_eq!(entries.len(), 1, "older tables are evicted to fit the budget"); + assert!(entries.values().map(Tensor::elem_count).sum::() <= 64); + drop(entries); + let large = cache.get(2, 8, 8).unwrap(); + assert_eq!(large.dims(), &[1, 4, 8, 8]); + assert_eq!(cache.entries.lock().unwrap().len(), 1, "a table over budget is returned without retention"); + } +} diff --git a/crates/sam2/src/resize.rs b/crates/sam2/src/resize.rs index 67e96e9af..fdbcbab48 100644 --- a/crates/sam2/src/resize.rs +++ b/crates/sam2/src/resize.rs @@ -5,8 +5,9 @@ use candle_core::{Device, Tensor}; +use crate::Result; +use crate::mask::validate_frame_size; use crate::prompt::INPUT_SIZE; -use crate::{Error, Result}; /// For each output index: the first input index and the normalised triangle-filter weights. fn axis_weights(n_in: usize, n_out: usize) -> Vec<(usize, Vec)> { @@ -35,13 +36,13 @@ fn axis_weights(n_in: usize, n_out: usize) -> Vec<(usize, Vec)> { /// Antialiased bilinear resize of a row-major `h x w` image to `oh x ow`. pub fn antialias_bilinear(src: &[f32], (h, w): (usize, usize), (oh, ow): (usize, usize)) -> Result> { - if h == 0 || w == 0 || oh == 0 || ow == 0 || src.len() != h * w { - return Err(Error::Input(format!("cannot resize {} values as {w}x{h} to {ow}x{oh}", src.len()))); - } + validate_frame_size(w, h, src.len())?; + let out_len = validate_frame_size(ow, oh, oh.saturating_mul(ow))?; + let mid_len = validate_frame_size(ow, h, h.saturating_mul(ow))?; let xs = axis_weights(w, ow); let ys = axis_weights(h, oh); // horizontal pass: h x ow - let mut mid = vec![0f32; h * ow]; + let mut mid = vec![0f32; mid_len]; for y in 0..h { let row = src.get(y * w..(y + 1) * w).unwrap_or(&[]); for (x, (x0, wt)) in xs.iter().enumerate() { @@ -52,7 +53,7 @@ pub fn antialias_bilinear(src: &[f32], (h, w): (usize, usize), (oh, ow): (usize, } } // vertical pass: oh x ow - let mut out = vec![0f32; oh * ow]; + let mut out = vec![0f32; out_len]; for (y, (y0, wt)) in ys.iter().enumerate() { for x in 0..ow { let acc: f32 = wt.iter().enumerate().map(|(k, a)| a * mid.get((y0 + k) * ow + x).copied().unwrap_or(0.0)).sum(); @@ -67,6 +68,7 @@ pub fn antialias_bilinear(src: &[f32], (h, w): (usize, usize), (oh, ow): (usize, /// A binary mask of any size as the model's mask prompt: `[1, 1, 1024, 1024]` of 0 and 1 (resized /// like the reference: antialiased bilinear, then thresholded at one half). pub fn mask_to_model(mask: &[u8], w: usize, h: usize) -> Result { + validate_frame_size(w, h, mask.len())?; let side = INPUT_SIZE; let f: Vec = mask.iter().map(|v| f32::from(*v != 0)).collect(); let data = if (w, h) == (side, side) { f } else { antialias_bilinear(&f, (h, w), (side, side))?.into_iter().map(|v| f32::from(v >= 0.5)).collect() }; @@ -77,6 +79,15 @@ pub fn mask_to_model(mask: &[u8], w: usize, h: usize) -> Result { mod tests { use super::*; + #[test] + fn resize_rejects_overflowing_oversized_and_mismatched_grids() { + for (input, output) in [((usize::MAX, 2), (1, 1)), ((1, 1), (usize::MAX, 2)), ((1, 1), (16_385, 1)), ((0, 1), (1, 1))] { + assert!(antialias_bilinear(&[1.0], input, output).is_err(), "{input:?} to {output:?}"); + } + assert!(mask_to_model(&[1], usize::MAX, 2).is_err()); + assert!(mask_to_model(&[1], INPUT_SIZE, INPUT_SIZE).is_err()); + } + #[test] fn a_constant_stays_constant_and_weights_sum_to_one() { let out = antialias_bilinear(&[3.0; 64 * 64], (64, 64), (16, 16)).unwrap(); diff --git a/crates/sam2/src/track.rs b/crates/sam2/src/track.rs index ad920efe7..7e73a1c74 100644 --- a/crates/sam2/src/track.rs +++ b/crates/sam2/src/track.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use candle_core::Tensor; use effectcraft_segment::{ClipFrame, ObjectTracker, TrackRequest, TrackedFrame}; -use crate::mask::{Sam2Mask, preprocess}; +use crate::mask::{Sam2Mask, preprocess, validate_frame_size}; use crate::prompt::{INPUT_SIZE, Prompt}; use crate::resize::mask_to_model; use crate::session::{Cancel, Direction, FrameMask, FrameSource, ObjectId, VideoTracker}; @@ -36,8 +36,12 @@ impl FrameSource for Source<'_> { } fn frame(&self, index: usize) -> Result { - let f = (self.fetch.borrow_mut())(self.first + index as i64) - .ok_or_else(|| Error::Input(format!("frame {} is not available", self.first + index as i64)))?; + let number = i64::try_from(index) + .ok() + .filter(|_| index < self.count) + .and_then(|offset| self.first.checked_add(offset)) + .ok_or_else(|| Error::Input(format!("frame index {index} is outside the clip")))?; + let f = (self.fetch.borrow_mut())(number).ok_or_else(|| Error::Input(format!("frame {number} is not available")))?; self.sizes.borrow_mut().insert(index, (f.w, f.h)); preprocess(&f.rgb, f.w, f.h) } @@ -59,6 +63,7 @@ impl Sam2Mask { if req.count == 0 || req.count > MAX_CLIP { return Err(Error::Input(format!("a tracking request needs 1 to {MAX_CLIP} frames, got {}", req.count))); } + req.first.checked_add((req.count - 1) as i64).ok_or_else(|| Error::Input("the clip's last frame overflows its frame number".into()))?; if req.objects.is_empty() || req.objects.len() > MAX_OBJECTS || req.objects.iter().any(|o| o.is_empty()) { return Err(Error::Input(format!("tracking needs 1 to {MAX_OBJECTS} objects, each with a prompt on at least one frame"))); } @@ -69,12 +74,14 @@ impl Sam2Mask { let mut prompt_frames = vec![]; for (oi, prompts) in req.objects.iter().enumerate() { for (frame, p) in prompts { - let idx = usize::try_from(frame - req.first) - .ok() + let idx = frame + .checked_sub(req.first) + .and_then(|offset| usize::try_from(offset).ok()) .filter(|i| *i < req.count) .ok_or_else(|| Error::Input(format!("prompt frame {frame} is outside the clip")))?; if !sizes.borrow().contains_key(&idx) { let f = (source.fetch.borrow_mut())(*frame).ok_or_else(|| Error::Input(format!("frame {frame} is not available")))?; + validate_frame_size(f.w, f.h, f.rgb.len())?; sizes.borrow_mut().insert(idx, (f.w, f.h)); } prompt_frames.push((oi, idx, p)); @@ -159,3 +166,16 @@ impl ObjectTracker for Sam2Mask { r } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn source_rejects_overflowing_and_out_of_range_indices_before_fetching() { + let mut fetch = |_| panic!("invalid indices must not fetch a frame"); + let source = Source { first: i64::MAX, count: 2, fetch: RefCell::new(&mut fetch), sizes: Default::default() }; + assert!(matches!(source.frame(1), Err(Error::Input(_)))); + assert!(matches!(source.frame(usize::MAX), Err(Error::Input(_)))); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 0bfe4023e..c4bebd9ed 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** minor (crate table checked against `crates/`: effect count 306, VP9 inter frames) · **Target:** Adobe After Effects 2026 +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-29 · **Change:** minor (SAM 2 crate added to the layer table) · **Target:** Adobe After Effects 2026 EffectCraft is a stack of small Rust crates. The engine knows nothing about the user interface; the egui frontend, the command-line tool and the MCP server all sit on top of the same `Session` @@ -39,6 +39,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | | L3 | `export` | Render queue encoding: H.264, HEVC and AV1 MP4, ProRes, WebM (VP9 + alpha or AV1, Opus), PNG/JPEG/TIFF/EXR/Targa sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | +| L3 | `sam2` | SAM 2.1 image segmentation and video object tracking for Roto Brush, with candle CPU inference and optional wgpu acceleration; native only, empty on wasm32, with user-installed weights | | L3 | `interchange` | Premiere Pro interop through timeline interchange (FCP7 XML, FCPXML, OTIO, EDL, AAF, OMF via FilmCraft's `filmcraft-interchange`): sequences ↔ compositions (§6a) | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L3 | `plugin` | Effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1; the wasmi interpreter behind the `wasm` feature, on for native hosts) into the effect registry ([plugins.md](plugins.md)) |