Roto Brush, Object Matte: decode frozen mattes only at the layer's full-resolution grid (#2941)

* Roto Brush, Object Matte: decode frozen mattes only at the layer's full-resolution grid

* address review: frozen matte size mismatch falls back to computed matte

* address review: frozen-size-mismatch test compares against a render with no cache

crates/effects/src/object_matte.rs

@@ -301,7 +301,8 @@ pub fn frozen_alpha(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, w:
     let fr = fc.frames.get(&f)?;
     let [fw, fh] = fc.size;
     let expected = fw.checked_mul(fh)?;
-    if fw == 0 || fh == 0 || expected > rle::MAX_PLANE_BYTES {
+    // Only the layer's own full-resolution grid (see Roto Brush's frozen mattes).
+    if fc.size != roto::frozen_grid(layer_size) || fw == 0 || fh == 0 || expected > rle::MAX_PLANE_BYTES {
         return None;
     }
     let a = rle::decode_bounded(&fr.a, expected)?;
@@ -719,6 +720,12 @@ mod tests {
         p.values.insert("objectPoint".into(), Value::Vec2([40.0, 40.0]));
         p.values.insert(FROZEN.into(), Value::Str(fc2.to_json()));
         assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
+        // A compact plane claiming a size other than the layer's grid is not decoded.
+        let mut fc3 = fc.clone();
+        fc3.size = [4000, 4000];
+        fc3.frames.insert(2, roto::freeze_frame(&vec![1.0; 4000 * 4000], &[]));
+        p.values.insert(FROZEN.into(), Value::Str(fc3.to_json()));
+        assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
     }
 
     /// Mattes frozen with a model stay usable in a session where that model is not loaded (a

crates/effects/src/roto.rs

@@ -378,10 +378,22 @@ pub fn purge() {
     c.bytes = 0;
 }
 
+/// The pixel size of the layer-space grid at `scale` (`round(size · scale)`, at least 1).
+fn grid_size(layer_size: [f64; 2], scale: f64) -> (u32, u32) {
+    (((layer_size[0] * scale).round() as u32).max(1), ((layer_size[1] * scale).round() as u32).max(1))
+}
+
+/// The full-resolution layer grid frozen mattes are stored at. A frozen cache claiming any
+/// other size did not come from freezing this layer and is not decoded.
+pub fn frozen_grid(layer_size: [f64; 2]) -> [usize; 2] {
+    let (w, h) = grid_size(layer_size, 1.0);
+    [w as usize, h as usize]
+}
+
 /// The layer-space grid (`round(size · scale)` pixels) of a buffer.
 pub fn layer_grid(buf: &Buf, layer_size: [f64; 2]) -> Image {
     let s = if buf.scale > 0.0 { buf.scale } else { 1.0 };
-    let (w, h) = (((layer_size[0] * s).round() as u32).max(1), ((layer_size[1] * s).round() as u32).max(1));
+    let (w, h) = grid_size(layer_size, s);
     let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64);
     let mut img = Image::new(w, h);
     img.rows_mut().for_each(|(y, row)| {
@@ -1212,8 +1224,11 @@ pub fn freeze_frame(alpha: &[f32], band: &[u8]) -> FrozenFrame {
 fn matte_for(ctx: &EffectCtx, f: i64, img: &Arc<Image>, scale: f64) -> Option<(Vec<f32>, Vec<u8>, f64)> {
     let (w, h) = (img.width as usize, img.height as usize);
     // Frozen mattes.
+    // Only the layer's own full-resolution grid: a project cannot make a tiny layer decode (and
+    // convert to floats) planes as large as the RLE allows. Any other size falls through to a computed matte.
     if let Some(fc) = frozen(ctx.params)
         && frozen_applies(&fc, ctx.params, ctx.layer_size, ctx.fps())
+        && fc.size == frozen_grid(ctx.layer_size)
         && let Some(fr) = fc.frames.get(&f)
     {
         let [fw, fh] = fc.size;
@@ -1409,6 +1424,33 @@ mod tests {
         params
     }
 
+    #[test]
+    fn mismatched_frozen_size_falls_back_to_a_computed_matte() {
+        let mut d = RotoData::default();
+        d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[35.0, 40.0], [45.0, 40.0]], object: 0 }, 20, [0, 24]);
+        let mut params = params_with(&d);
+        let fc =
+            FrozenCache { key: frozen_key(&params, [100.0, 80.0], 25.0), size: [10, 8], frames: BTreeMap::from([(5, freeze_frame(&[1.0; 80], &[0; 80]))]) };
+        let host = Seq;
+        let render = |params: &Params| {
+            let ctx = EffectCtx {
+                params,
+                time: 5.0 / 25.0 + 0.001,
+                layer_size: [100.0, 80.0],
+                seed: 1,
+                adjustment: false,
+                env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() },
+            };
+            crate::apply(&specs()[0], &ctx, Buf { img: frame(5.0 / 25.0 + 0.001), offset: [0.0; 2], scale: 1.0 }).img
+        };
+        let computed = render(&params);
+        params.values.insert(FROZEN.into(), Value::Str(fc.to_json()));
+        let out = render(&params);
+        assert!(out.data[(40 * 100 + 50) as usize][3] > 0.5, "the disk is missing from the output");
+        // The cache's all-opaque matte would leave the background opaque too; the fallback must match a render with no cache.
+        assert!(out.data == computed.data, "the mismatched frozen cache was used");
+    }
+
     #[test]
     fn renders_the_segmented_disk_on_demand() {
         let mut d = RotoData::default();
@@ -1980,6 +2022,18 @@ mod tests {
         }
     }
 
+    /// Frozen mattes are only decoded at the size the freeze pass stores: the layer grid at
+    /// full resolution.
+    #[test]
+    fn frozen_grid_is_the_full_resolution_layer_grid() {
+        for size in [[100.0, 80.0], [99.6, 0.2], [1.0, 1.0]] {
+            let buf = Buf { img: Image::new(3, 3), offset: [0.0; 2], scale: 1.0 };
+            let g = layer_grid(&buf, size);
+            assert_eq!(frozen_grid(size), [g.width as usize, g.height as usize]);
+        }
+        assert_eq!(frozen_grid([100.0, 80.0]), [100, 80]);
+    }
+
     #[test]
     fn saved_tracking_decoders_validate_dimensions_and_plane_lengths() {
         let valid = rle::encode(&[255, 0, 255, 0]);