From e7c208434b6a966f67731aa98abe625651f52778 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 00:27:45 +0100 Subject: [PATCH 01/25] #474: pin OpenEXR pictures and aux channels to exact hashes Synthetic single-layer, multi-layer, multi-part and windowed files in half, float and uint, with RLE, ZIP, PIZ and DWAA, decoded through the media pool picture first and channels first. The hashes are taken on main before the decode is reworked. Co-Authored-By: Claude Opus 5.5 --- crates/media/tests/it/exr_golden.rs | 338 ++++++++++++++++++++++++++++ crates/media/tests/it/main.rs | 1 + 2 files changed, 339 insertions(+) create mode 100644 crates/media/tests/it/exr_golden.rs diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs new file mode 100644 index 000000000..498944882 --- /dev/null +++ b/crates/media/tests/it/exr_golden.rs @@ -0,0 +1,338 @@ +//! OpenEXR pictures and aux channels pinned to exact hashes (#474). +//! +//! Synthetic files (single-layer, multi-layer with and without an unnamed RGB layer, multi-part, +//! data windows smaller and larger than the display window, half / float / uint channels, RLE, +//! ZIP, PIZ and DWAA) are decoded through the media pool as the renderer asks for them: the +//! picture in each alpha interpretation, and the channels EXtractoR and Cryptomatte read. The +//! hashes were taken before the picture and the channels started sharing one decode, so a change +//! to how either is decoded must leave every bit as it was. Only bit-exact float steps are +//! pinned (linear light, no `powf`), so the hashes hold on every platform; DWAA (whose decoder +//! uses `powf`) and the sRGB / gamma 2.4 curves are checked for agreement between the two ways a +//! frame can be decoded instead. Re-pin with `EXR_GOLDEN_PRINT=1` after an intended change. + +use std::sync::Arc; + +use effectcraft_media::{MediaPool, probe_bytes}; +use effectcraft_project::{AlphaMode, Footage, ItemId}; +use effectcraft_render::{FloatCurve, FootageSource}; +use effectcraft_time::Tick; +use exr::prelude::*; + +const W: usize = 37; +const H: usize = 23; + +/// A deterministic value for pixel (x, y) of channel `c`: negative, in range and above 1. +fn value(x: usize, y: usize, c: usize) -> f32 { + ((x * 7 + y * 13 + c * 29) % 97) as f32 / 40.0 - 0.3 +} + +/// A deterministic alpha in 0..=1 (0 included, for the premultiplied divide). +fn alpha(x: usize, y: usize) -> f32 { + ((x * 3 + y * 5) % 11) as f32 / 10.0 +} + +#[derive(Clone, Copy)] +enum Kind { + Half, + Float, + Uint, +} + +/// A channel `name` of `size` filled from [`value`] (or [`alpha`] for an `A` / `a` channel). +fn channel(name: &str, kind: Kind, size: (usize, usize), c: usize) -> AnyChannel { + let is_alpha = name.rsplit('.').next().is_some_and(|n| n.eq_ignore_ascii_case("a")); + let px = |i: usize| { + let (x, y) = (i % size.0, i / size.0); + if is_alpha { alpha(x, y) } else { value(x, y, c) } + }; + let n = size.0 * size.1; + let samples = match kind { + Kind::Half => FlatSamples::F16((0..n).map(|i| f16::from_f32(px(i))).collect()), + Kind::Float => FlatSamples::F32((0..n).map(px).collect()), + Kind::Uint => FlatSamples::U32((0..n).map(|i| (i as u32).wrapping_mul(2_654_435_761) >> 8).collect()), + }; + AnyChannel::new(name, samples) +} + +fn channels(list: &[(&str, Kind)], size: (usize, usize)) -> AnyChannels { + AnyChannels::sort(list.iter().enumerate().map(|(c, (n, k))| channel(n, *k, size, c)).collect::>().into()) +} + +fn encoding(compression: Compression) -> Encoding { + Encoding { compression, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing } +} + +fn with_manifest(mut attrs: LayerAttributes) -> LayerAttributes { + attrs.other.insert(Text::from("cryptomatte/abc1234/name"), AttributeValue::Text(Text::from("CryptoObject"))); + attrs.other.insert(Text::from("cryptomatte/abc1234/manifest"), AttributeValue::Text(Text::from(r#"{"bunny":"13851a76","floor":"0a1b2c3d"}"#))); + attrs +} + +/// One part: `list` channels over a data window of `size` at `pos`. +fn part(name: Option<&str>, list: &[(&str, Kind)], size: (usize, usize), pos: (i32, i32), compression: Compression) -> Layer> { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, with_manifest(attrs), encoding(compression), channels(list, size)) +} + +/// Encode `layers` with the display window `display` (position, size). +fn write(layers: Vec>>, display: ((i32, i32), (usize, usize))) -> Vec { + let mut attrs = ImageAttributes::new(IntegerBounds::new(display.0, display.1)); + attrs.pixel_aspect = 1.0; + let image = Image::from_layers(attrs, layers); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().non_parallel().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() +} + +const RGBA_HALF: [(&str, Kind); 4] = [("R", Kind::Half), ("G", Kind::Half), ("B", Kind::Half), ("A", Kind::Half)]; +const CRYPTO: [(&str, Kind); 4] = + [("CryptoObject00.r", Kind::Float), ("CryptoObject00.g", Kind::Float), ("CryptoObject00.b", Kind::Float), ("CryptoObject00.a", Kind::Float)]; + +/// The synthetic files: (name, bytes, whether the hashes are pinned). +fn files() -> Vec<(&'static str, Vec, bool)> { + let full = (W, H); + let origin = ((0, 0), full); + let cat = |a: &[(&'static str, Kind)], b: &[(&'static str, Kind)]| -> Vec<(&'static str, Kind)> { a.iter().chain(b).copied().collect() }; + let layered: Vec<(&str, Kind)> = cat( + &[("Image.R", Kind::Half), ("Image.G", Kind::Half), ("Image.B", Kind::Half), ("Image.A", Kind::Half), ("depth.Z", Kind::Float), ("ObjectID", Kind::Uint)], + &CRYPTO, + ); + let unnamed_plus: Vec<(&str, Kind)> = + cat(&cat(&RGBA_HALF, &[("Diffuse.R", Kind::Half), ("Diffuse.G", Kind::Half), ("Diffuse.B", Kind::Half), ("Z", Kind::Float)]), &CRYPTO); + let rgb_float = [("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("depth.Z", Kind::Float)]; + vec![ + ("rgba_half_zip16", write(vec![part(None, &RGBA_HALF, full, (0, 0), Compression::ZIP16)], origin), true), + ("rgb_float_piz", write(vec![part(None, &[("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("Z", Kind::Float)], full, (0, 0), Compression::PIZ)], origin), true), + ("layered_rle", write(vec![part(None, &layered, full, (0, 0), Compression::RLE)], origin), true), + ("unnamed_rgb_plus_layers_zip1", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::ZIP1)], origin), true), + ( + "multipart_zip16", + write( + vec![ + part(Some("Image"), &RGBA_HALF, full, (0, 0), Compression::ZIP16), + part(Some("depth"), &[("Z", Kind::Float)], full, (0, 0), Compression::ZIP16), + part(Some("CryptoObject00"), &[("r", Kind::Float), ("g", Kind::Float), ("b", Kind::Float), ("a", Kind::Float)], full, (0, 0), Compression::ZIP16), + ], + origin, + ), + true, + ), + ( + // The first part with R, G and B is not the beauty pass. + "multipart_diffuse_then_beauty_zip1", + write( + vec![ + part(Some("diffuse"), &[("R", Kind::Half), ("G", Kind::Half), ("B", Kind::Half)], full, (0, 0), Compression::ZIP1), + part(Some("beauty"), &RGBA_HALF, (20, 10), (5, 7), Compression::ZIP1), + part(Some("depth"), &[("Z", Kind::Float)], full, (0, 0), Compression::ZIP1), + ], + origin, + ), + true, + ), + ( + "multipart_blender5_rle", + write( + vec![ + part(Some("ViewLayer.Depth"), &[("ViewLayer.Depth.Z", Kind::Float)], full, (0, 0), Compression::RLE), + part( + Some("ViewLayer.Combined"), + &[("ViewLayer.Combined.R", Kind::Half), ("ViewLayer.Combined.G", Kind::Half), ("ViewLayer.Combined.B", Kind::Half), ("ViewLayer.Combined.A", Kind::Half)], + full, + (0, 0), + Compression::RLE, + ), + ], + origin, + ), + true, + ), + ("unnamed_rgb_window_smaller_zip16", write(vec![part(None, &unnamed_plus, (20, 10), (5, 7), Compression::ZIP16)], origin), true), + ("rgb_float_window_larger_piz", write(vec![part(None, &rgb_float, (45, 30), (-1, -5), Compression::PIZ)], ((3, -2), full)), true), + ("layered_window_smaller_zip16", write(vec![part(None, &layered, (20, 10), (5, 7), Compression::ZIP16)], origin), true), + ("layered_window_larger_piz", write(vec![part(None, &layered, (45, 30), (-4, -3), Compression::PIZ)], origin), true), + ("unnamed_rgb_plus_layers_dwaa", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::DWAA(None))], origin), false), + ("layered_dwaa", write(vec![part(None, &layered, full, (0, 0), Compression::DWAA(None))], origin), false), + ] +} + +/// FNV-1a over 32-bit words. +fn fnv(words: impl IntoIterator) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for w in words { + for b in w.to_le_bytes() { + h ^= u64::from(b); + h = h.wrapping_mul(0x0100_0000_01b3); + } + } + h +} + +fn picture_hash(pool: &MediaPool, f: &Footage, float: FloatCurve, size: Option<(u32, u32)>) -> u64 { + let img = pool.frame_with(ItemId(1), f, Tick::ZERO, size, float).expect("picture"); + fnv([img.width, img.height].into_iter().chain(img.data.iter().flat_map(|p| p.map(f32::to_bits)))) +} + +fn aux_hash(pool: &MediaPool, f: &Footage) -> u64 { + let Some(aux) = pool.aux(ItemId(1), f, Tick::ZERO) else { return 0 }; + let mut words = vec![aux.width, aux.height, aux.scale.to_bits() as u32]; + for (name, plane) in &aux.channels { + words.extend(name.bytes().map(u32::from)); + words.extend(plane.iter().map(|v| v.to_bits())); + } + for (name, objects) in &aux.manifests { + words.extend(name.bytes().map(u32::from)); + for (o, h) in objects { + words.extend(o.bytes().map(u32::from)); + words.push(*h); + } + } + fnv(words) +} + +/// The interpretations a picture is decoded in: (label, footage, curve, size, pinned). +fn variants(f: &Footage) -> Vec<(String, Footage, FloatCurve, Option<(u32, u32)>, bool)> { + let mut out = vec![]; + for (mode, label) in [(None, "probed"), (Some(AlphaMode::Straight), "straight"), (Some(AlphaMode::Premultiplied), "premul"), (Some(AlphaMode::Ignore), "ignore")] { + let mut g = f.clone(); + if let Some(m) = mode { + g.alpha = m; + } + g.premul_color = [0.2, 0.4, 0.6]; + out.push((format!("{label}/linear"), g.clone(), FloatCurve::Linear, None, true)); + out.push((format!("{label}/srgb"), g.clone(), FloatCurve::Srgb, None, false)); + out.push((format!("{label}/gamma24"), g.clone(), FloatCurve::Gamma24, None, false)); + } + let preserve = Footage { preserve_rgb: true, ..f.clone() }; + out.push(("preserve_rgb".into(), preserve, FloatCurve::Srgb, None, true)); + out.push(("half_size/srgb".into(), f.clone(), FloatCurve::Srgb, Some((f.width.div_ceil(2), f.height.div_ceil(2))), false)); + out +} + +/// Every hash of `file`: the size and alpha import finds, each picture and the channels, decoded +/// picture first (the footage alone) or channels first (an effect reads them). +fn hashes(name: &str, bytes: &[u8], aux_first: bool) -> Vec<(String, u64, bool)> { + let order = if aux_first { "aux-first" } else { "picture-first" }; + let path = format!("/exr-golden-474/{order}/{name}.exr"); + let b: Arc<[u8]> = bytes.into(); + let pool = MediaPool::new(); + pool.add_bytes(&path, b.clone()); + let f = probe_bytes(&path, b).expect("probe"); + let mut out = vec![("probe".to_string(), fnv([f.width, f.height, f.alpha as u32]), true)]; + let aux = if aux_first { Some(aux_hash(&pool, &f)) } else { None }; + for (label, g, float, size, pinned) in variants(&f) { + out.push((label, picture_hash(&pool, &g, float, size), pinned)); + } + out.push(("aux".into(), aux.unwrap_or_else(|| aux_hash(&pool, &f)), true)); + out +} + +/// Hashes taken on main before #474's shared decode (`EXR_GOLDEN_PRINT=1`). +const PINNED: &[(&str, &str, u64)] = &[ + ("rgba_half_zip16", "probe", 0x0a32e074a28c1f66), + ("rgba_half_zip16", "probed/linear", 0xa84600283dce98f6), + ("rgba_half_zip16", "straight/linear", 0xf028e1367a2fbb30), + ("rgba_half_zip16", "premul/linear", 0xa84600283dce98f6), + ("rgba_half_zip16", "ignore/linear", 0xa90a57f9a67f7468), + ("rgba_half_zip16", "preserve_rgb", 0xc8e0050f5efe75b7), + ("rgba_half_zip16", "aux", 0xdd555e6cf67d75f3), + ("rgb_float_piz", "probe", 0x6a38346c4c5662d5), + ("rgb_float_piz", "probed/linear", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "straight/linear", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "premul/linear", 0x515f476c176229c9), + ("rgb_float_piz", "ignore/linear", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "preserve_rgb", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "aux", 0x554623d237c2e5ef), + ("layered_rle", "probe", 0x0a32e074a28c1f66), + ("layered_rle", "probed/linear", 0xa84600283dce98f6), + ("layered_rle", "straight/linear", 0xf028e1367a2fbb30), + ("layered_rle", "premul/linear", 0xa84600283dce98f6), + ("layered_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_rle", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_zip1", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_zip1", "probed/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_zip1", "straight/linear", 0xf028e1367a2fbb30), + ("unnamed_rgb_plus_layers_zip1", "premul/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_zip1", "ignore/linear", 0xa90a57f9a67f7468), + ("unnamed_rgb_plus_layers_zip1", "preserve_rgb", 0xc8e0050f5efe75b7), + ("unnamed_rgb_plus_layers_zip1", "aux", 0x27cc9b53d0862db1), + ("multipart_zip16", "probe", 0x0a32e074a28c1f66), + ("multipart_zip16", "probed/linear", 0xa84600283dce98f6), + ("multipart_zip16", "straight/linear", 0xf028e1367a2fbb30), + ("multipart_zip16", "premul/linear", 0xa84600283dce98f6), + ("multipart_zip16", "ignore/linear", 0xa90a57f9a67f7468), + ("multipart_zip16", "preserve_rgb", 0xc8e0050f5efe75b7), + ("multipart_zip16", "aux", 0x567ad99a6690d03d), + ("multipart_diffuse_then_beauty_zip1", "probe", 0x6a38346c4c5662d5), + ("multipart_diffuse_then_beauty_zip1", "probed/linear", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "straight/linear", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "premul/linear", 0x7ab18932e25c5cc8), + ("multipart_diffuse_then_beauty_zip1", "ignore/linear", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "preserve_rgb", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "aux", 0x6724d6dfd56a04cb), + ("multipart_blender5_rle", "probe", 0x0a32e074a28c1f66), + ("multipart_blender5_rle", "probed/linear", 0xa84600283dce98f6), + ("multipart_blender5_rle", "straight/linear", 0xf028e1367a2fbb30), + ("multipart_blender5_rle", "premul/linear", 0xa84600283dce98f6), + ("multipart_blender5_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("multipart_blender5_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("multipart_blender5_rle", "aux", 0x522ae4819dd92e89), + ("unnamed_rgb_window_smaller_zip16", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_window_smaller_zip16", "probed/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_window_smaller_zip16", "straight/linear", 0x2c73f9acaa936420), + ("unnamed_rgb_window_smaller_zip16", "premul/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_window_smaller_zip16", "ignore/linear", 0x3b86057d18a16bf0), + ("unnamed_rgb_window_smaller_zip16", "preserve_rgb", 0x5b999014e4a64bb2), + ("unnamed_rgb_window_smaller_zip16", "aux", 0x8f78d3233450acf3), + ("rgb_float_window_larger_piz", "probe", 0x6a38346c4c5662d5), + ("rgb_float_window_larger_piz", "probed/linear", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "straight/linear", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "premul/linear", 0x0998eefc456dcdc9), + ("rgb_float_window_larger_piz", "ignore/linear", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "preserve_rgb", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "aux", 0x7ccc9f86e6ff6140), + ("layered_window_smaller_zip16", "probe", 0x0a32e074a28c1f66), + ("layered_window_smaller_zip16", "probed/linear", 0x19eb3b1a88d108a7), + ("layered_window_smaller_zip16", "straight/linear", 0x2c73f9acaa936420), + ("layered_window_smaller_zip16", "premul/linear", 0x19eb3b1a88d108a7), + ("layered_window_smaller_zip16", "ignore/linear", 0x3b86057d18a16bf0), + ("layered_window_smaller_zip16", "preserve_rgb", 0x5b999014e4a64bb2), + ("layered_window_smaller_zip16", "aux", 0x443480a1d789e63c), + ("layered_window_larger_piz", "probe", 0x0a32e074a28c1f66), + ("layered_window_larger_piz", "probed/linear", 0x443afe4029748836), + ("layered_window_larger_piz", "straight/linear", 0xb32b44880f746826), + ("layered_window_larger_piz", "premul/linear", 0x443afe4029748836), + ("layered_window_larger_piz", "ignore/linear", 0xaa01706accce2cc4), + ("layered_window_larger_piz", "preserve_rgb", 0xb2cbb40b10bc56b4), + ("layered_window_larger_piz", "aux", 0xb7d91c8eea56a772), +]; + +#[test] +fn exr_pictures_and_channels_are_bit_identical() { + let print = std::env::var_os("EXR_GOLDEN_PRINT").is_some(); + let mut failures = vec![]; + for (name, bytes, pinned_file) in files() { + let first = hashes(name, &bytes, false); + let second = hashes(name, &bytes, true); + for ((label, h, pinned), (_, h2, _)) in first.iter().zip(&second) { + if h != h2 { + failures.push(format!("{name} {label}: picture first {h:#018x}, channels first {h2:#018x}")); + } + if print { + println!(" (\"{name}\", \"{label}\", {h:#018x}),{}", if *pinned && pinned_file { "" } else { " // not pinned" }); + continue; + } + if !(*pinned && pinned_file) { + continue; + } + match PINNED.iter().find(|(n, l, _)| *n == name && l == label) { + Some((_, _, want)) if want == h => {} + Some((_, _, want)) => failures.push(format!("{name} {label}: {h:#018x}, pinned {want:#018x}")), + None => failures.push(format!("{name} {label}: {h:#018x} is not pinned")), + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rs index 69bed4a9f..08c5ec389 100644 --- a/crates/media/tests/it/main.rs +++ b/crates/media/tests/it/main.rs @@ -3,6 +3,7 @@ mod aac_layout; mod downmix; mod exif_orientation; +mod exr_golden; mod footage; mod open_in_place; mod tga; From 684e05859fb4e1ad3b5ba45cc92720ae010ea12e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 01:47:24 +0100 Subject: [PATCH 02/25] EXR export: no compand / tone mapping, and the output space's own curve OpenEXR renders (Render Queue sequences and Save Frame As ProEXR) received the frame after Project Settings > HDR, so Compand and Tone Map (the default of every OCIO project) compressed highlights and changed values below 1.0 before the writer decoded them, always with the sRGB curve. - RenderOpts::scene_output: the top-level comp converts to the output space without the HDR step (CPU and GPU walks, via Renderer::output_pipe). - Pipe::output_linear: the writers decode with the output space's curve (the Working Gamma's when unmanaged) instead of a fixed sRGB curve. - Render Queue EXR jobs and ProEXR set the flag; PSD layers, 8-bit stills, movies and the viewer keep the HDR step. The flag is part of the viewer frame and disk cache keys (opts_hash). Golden hashes, taken before the change, pin PNG/TIFF/Targa bytes for 8/16/32 bpc x Clip/Compand/ToneMap, Clip-mode EXR samples with an sRGB output, and the layered PSD bytes. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/frame_export.rs | 28 +- crates/engine/src/tests/frame_export.rs | 108 ++++++++ crates/export/src/lib.rs | 14 +- crates/export/src/pipeline.rs | 2 + crates/export/tests/it/exr_scene_linear.rs | 293 +++++++++++++++++++++ crates/export/tests/it/main.rs | 1 + crates/gpu/src/tests.rs | 28 ++ crates/gpu/src/walk.rs | 5 +- crates/render/src/accel.rs | 6 + crates/render/src/color.rs | 14 + crates/render/src/opts.rs | 6 + crates/render/src/pipeline.rs | 5 +- crates/render/src/tests/color.rs | 46 ++++ crates/ui-egui/src/app/viewer_frames.rs | 1 + crates/ui-egui/src/frames/tests.rs | 8 + docs/architecture.md | 6 +- 16 files changed, 548 insertions(+), 23 deletions(-) create mode 100644 crates/export/tests/it/exr_scene_linear.rs diff --git a/crates/engine/src/cmd/frame_export.rs b/crates/engine/src/cmd/frame_export.rs index d8b49159c..24b6f0c3e 100644 --- a/crates/engine/src/cmd/frame_export.rs +++ b/crates/engine/src/cmd/frame_export.rs @@ -27,12 +27,15 @@ struct Isolated { image: Image, } -fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64) -> Image { +/// `scene_output`: a float output (ProEXR), rendered without the HDR step +/// ([`RenderOpts::scene_output`]). +fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64, scene_output: bool) -> Image { let opts = RenderOpts { scale, backend: Backend::Auto, nested_switches: s.prefs.general.switches_affect_nested_comps, draft_shadows: s.prefs.three_d.realtime_shadows, + scene_output, ..Default::default() }; let mut r = Renderer::new(project, s.footage.as_ref(), opts); @@ -41,10 +44,11 @@ fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64 r.comp_frame(comp, t) } -/// The comp frame and each visible layer rendered alone (top layer first). -fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Image, Vec)> { +/// The comp frame and each visible layer rendered alone (top layer first). `scene_output`: as +/// [`render_with`]. +fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64, scene_output: bool) -> Result<(Image, Vec)> { let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; - let full = render_with(s, &s.project, cid, t, scale); + let full = render_with(s, &s.project, cid, t, scale, scene_output); let any_solo = comp.layers.iter().any(|l| l.switches.solo && l.has_video()); let ids: Vec = comp .layers @@ -72,7 +76,7 @@ fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Image, Vec< } } let Some((name, blend, opacity)) = info else { continue }; - let image = render_with(s, &p, cid, t, scale); + let image = render_with(s, &p, cid, t, scale, scene_output); out.push(Isolated { name, blend, opacity, image }); } Ok((full, out)) @@ -111,7 +115,7 @@ fn alpha_bounds(img: &Image) -> Option<(u32, u32, u32, u32)> { pub fn layered_psd(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Vec, usize, u32, u32)> { use effectcraft_psd::Rect; use effectcraft_psd::write::{WDoc, WLayer, write}; - let (full, layers) = isolate(s, cid, t, scale)?; + let (full, layers) = isolate(s, cid, t, scale, false)?; let mut doc = WDoc::new(full.width, full.height); let n = layers.len(); // File order is bottom first. @@ -136,10 +140,6 @@ pub fn layered_psd(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Vec Ok((write(&doc), n, full.width, full.height)) } -fn srgb_to_linear(c: f32) -> f32 { - if c <= 0.04045 { c / 12.92 } else { ((c + 0.055) / 1.055).powf(2.4) } -} - /// A unique EXR layer name (no dots: they separate the layer from the channel). Printable ASCII only: /// the `exr` crate panics on characters it cannot store in a channel name (#1702). fn exr_name(name: &str, used: &mut Vec) -> String { @@ -158,15 +158,19 @@ fn exr_name(name: &str, used: &mut Vec) -> String { /// The multi-layer EXR bytes of a frame and the channel names. pub fn layered_exr(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Vec, Vec, u32, u32)> { use exr::prelude::*; - let (full, layers) = isolate(s, cid, t, scale)?; + // The composite and every layer without the HDR step, decoded with the output space's + // curve (linear light in the output space's primaries). + let (full, layers) = isolate(s, cid, t, scale, true)?; + let dec = effectcraft_render::color::Pipe::of(&s.project.settings).output_linear(); let (w, h) = (full.width as usize, full.height as usize); let planes = |img: &effectcraft_raster::Image| -> [Vec; 4] { let mut out: [Vec; 4] = Default::default(); for p in &img.data { let sp = straight(*p); let a = sp[3]; + let lin = dec.apply([sp[0].max(0.0), sp[1].max(0.0), sp[2].max(0.0)]); for c in 0..3 { - out[c].push(srgb_to_linear(sp[c].max(0.0)) * a); + out[c].push(lin[c] * a); } out[3].push(a); } diff --git a/crates/engine/src/tests/frame_export.rs b/crates/engine/src/tests/frame_export.rs index a8dc938fb..2686e810e 100644 --- a/crates/engine/src/tests/frame_export.rs +++ b/crates/engine/src/tests/frame_export.rs @@ -230,3 +230,111 @@ fn save_frame_as_queued_respects_frame_default_template() { s.execute_checked("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "still", "name": "JPEG Sequence"})).unwrap(); assert!(s.execute_checked("comp.saveFrameAs", json!({"queue": true})).is_ok()); } + +fn fnv(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ *b as u64).wrapping_mul(0x0100_0000_01b3)) +} + +/// A 32×32 comp in a managed project (sRGB working space, linearized): a full-frame over-range +/// solid under a 16×16 in-range Multiply solid at 50%. +fn hdr_frame(bit_depth: u64, hdr: &str) -> Session { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Frame", "width": 32, "height": 32, "frameRate": 24, "duration": 1})).unwrap(); + s.execute("file.projectSettings", json!({"bitDepth": bit_depth, "workingSpace": "srgb", "linearize": true, "hdr": hdr})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Bright", "color": [4.0, 2.0, 0.5]})).unwrap(); + let box_ = s.execute("layer.newSolid", json!({"name": "Box", "color": [0.85, 0.9, 0.2], "width": 16, "height": 16})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.setBlendMode", json!({"layers": [box_], "mode": "Multiply"})).unwrap(); + s.execute("prop.set", json!({"layer": box_, "path": "transform/opacity", "value": 50})).unwrap(); + s +} + +/// Photoshop Layers keeps the HDR step (it is a display-referred output): its bytes for each bit +/// depth × HDR mode. +#[test] +fn photoshop_layers_bytes_are_unchanged() { + let dir = out_dir("frame-psd-golden"); + let mut got = vec![]; + for depth in [8, 16, 32] { + for hdr in ["clip", "compand", "toneMap"] { + let mut s = hdr_frame(depth, hdr); + let path = dir.join(format!("{depth}-{hdr}.psd")).to_string_lossy().to_string(); + s.execute("comp.saveFrameAsPsd", json!({"path": path})).unwrap(); + got.push(fnv(&std::fs::read(&path).unwrap())); + } + } + // The hashes before ProEXR skipped the HDR step. + let want: [u64; 9] = [ + 0x3b921d4d9fc3119f, + 0x1886d026edf36a7f, + 0x6b564aa7fb50139f, + 0xabd585992079fc9f, + 0x88ca38726eaa557f, + 0xa3636606d7a4927f, + 0x7385ed211e9815ff, + 0x7385ed211e9815ff, + 0xfb59fa3f3bdfac3f, + ]; + assert_eq!(got, want); +} + +/// Every sample of a ProEXR file, channels in name order. +fn proexr_samples(path: &str) -> Vec<(String, Vec)> { + use exr::prelude::*; + let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_file(path).unwrap(); + let mut out: Vec<(String, Vec)> = img.layer_data[0] + .channel_data + .list + .iter() + .map(|c| match &c.sample_data { + FlatSamples::F32(v) => (c.name.to_string(), v.clone()), + o => panic!("{o:?}"), + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out +} + +/// ProEXR samples in Clip mode (sRGB output) for each bit depth. +#[test] +fn proexr_clip_samples_are_unchanged() { + let dir = out_dir("frame-exr-clip-golden"); + let mut got = vec![]; + for depth in [8, 16, 32] { + let mut s = hdr_frame(depth, "clip"); + let path = dir.join(format!("{depth}.exr")).to_string_lossy().to_string(); + s.execute("comp.saveFrameAsExr", json!({"path": path})).unwrap(); + let bytes: Vec = + proexr_samples(&path).iter().flat_map(|(n, v)| n.bytes().chain(v.iter().flat_map(|x| x.to_bits().to_le_bytes())).collect::>()).collect(); + got.push(fnv(&bytes)); + } + // The hashes before ProEXR skipped the HDR step. + let want: [u64; 3] = [0x4b9ecc87e6db7d03, 0xe06a35d302e71503, 0x694ca552f3efc903]; + assert_eq!(got, want); +} + +/// ProEXR ignores compand and tone mapping: the composite and every isolated layer hold the Clip +/// mode's samples, and over-range values stay. +#[test] +fn proexr_layers_are_not_tone_mapped() { + let dir = out_dir("frame-exr-hdr"); + let samples = |depth: u64, hdr: &str| { + let mut s = hdr_frame(depth, hdr); + let path = dir.join(format!("{depth}-{hdr}.exr")).to_string_lossy().to_string(); + s.execute("comp.saveFrameAsExr", json!({"path": path})).unwrap(); + proexr_samples(&path) + }; + for depth in [8, 16, 32] { + let clip = samples(depth, "clip"); + assert!(clip.iter().any(|(n, _)| n == "Bright.R") && clip.iter().any(|(n, _)| n == "Box.R"), "{:?}", clip.iter().map(|c| &c.0).collect::>()); + for hdr in ["compand", "toneMap"] { + assert!(samples(depth, hdr) == clip, "{depth} bpc {hdr}"); + } + } + let tone = samples(32, "toneMap"); + let at = |n: &str| tone.iter().find(|c| c.0 == n).unwrap().1[0]; + // The solid's colour is sRGB-encoded in the working space (4.0 is about 25.3 in linear light). + let lin = effectcraft_color::srgb_to_linear; + for (n, want) in [("Bright.R", lin(4.0)), ("Bright.G", lin(2.0)), ("Bright.B", lin(0.5)), ("R", lin(4.0))] { + assert!((at(n) - want).abs() <= 1e-4 * want, "{n}: {} vs {want}", at(n)); + } +} diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 1b69a87bd..2d0d38071 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -302,7 +302,10 @@ fn write_still(cx: &Cx, file: &mut W, img: &I OutputFormat::ExrSequence => { // Linear light, premultiplied (the EXR convention); RGB keeps alpha = 1 over the background. // Colour is floored at 0 but not capped at 1: a 32 bpc frame's over-range values survive (#339). - let lin = |v: f32| if v <= 0.04045 { v / 12.92 } else { ((v + 0.055) / 1.055).powf(2.4) }; + // The frame was rendered without the HDR step (`RenderOpts::scene_output`) and is + // decoded with the output space's curve, in the output space's primaries. + let dec = effectcraft_render::color::Pipe::of(&cx.project.settings).output_linear(); + let lin3 = |c: [f32; 3]| dec.apply(c.map(|v| v.max(0.0))); let mut data = Vec::with_capacity((w * h * 4) as usize); for y in 0..h { for x in 0..w { @@ -310,17 +313,16 @@ fn write_still(cx: &Cx, file: &mut W, img: &I let (c, a) = match channels { Channels::Alpha => { let a = p[3].clamp(0.0, 1.0); - ([lin(a); 3], 1.0) + (lin3([a; 3]), 1.0) } Channels::Rgba => { let a = p[3].clamp(0.0, 1.0); - let s = |c: f32| if a > 0.0 { lin((c / a).max(0.0)) * a } else { 0.0 }; - ([s(p[0]), s(p[1]), s(p[2])], a) + let c = if a > 0.0 { lin3([p[0] / a, p[1] / a, p[2] / a]).map(|v| v * a) } else { [0.0; 3] }; + (c, a) } Channels::Rgb => { let k = 1.0 - p[3].clamp(0.0, 1.0); - let c = |i: usize| lin((p[i] + comp.background[i] * k).max(0.0)); - ([c(0), c(1), c(2)], 1.0) + (lin3([0, 1, 2].map(|i| p[i] + comp.background[i] * k)), 1.0) } }; data.extend_from_slice(&[c[0], c[1], c[2], a]); diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index 8e8c90561..3742ae0a2 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -251,6 +251,8 @@ impl<'a> Cx<'a> { roi: None, nested_switches: self.job.nested_switches, draft_shadows: true, + // OpenEXR keeps over-range values: no HDR compand / tone mapping on its frames. + scene_output: self.job.output.format == effectcraft_project::render_queue::OutputFormat::ExrSequence, proxy: s.proxy_use, }; let mut r = Renderer::new(&self.project, self.job.footage, opts); diff --git a/crates/export/tests/it/exr_scene_linear.rs b/crates/export/tests/it/exr_scene_linear.rs new file mode 100644 index 000000000..99a419894 --- /dev/null +++ b/crates/export/tests/it/exr_scene_linear.rs @@ -0,0 +1,293 @@ +//! OpenEXR output holds the comp's values in linear light without Project Settings ▸ HDR +//! (compand / tone map), and decodes with the output space's own curve; 8-bit writers keep the +//! HDR step and their exact bytes. + +use std::path::{Path, PathBuf}; + +use effectcraft_color::{ColorSpace, Conversion, Label}; +use effectcraft_export::render_queue::{Channels, OutputFormat, OutputModule, RenderSettings, TimeSpan}; +use effectcraft_export::{Job, export}; +use effectcraft_project::{BitDepth, Comp, HdrMode, ItemId, ItemKind, LayerSource, Project, Solid, build}; +use effectcraft_render::NoFootage; +use effectcraft_time::{FrameRate, Tick}; + +const W: u32 = 48; +const H: u32 = 16; +/// Three nested bands, outermost first: in range, inside the compand knee, over range. +const BANDS: [[f32; 3]; 3] = [[0.2, 0.5, 0.9], [0.85, 0.9, 0.95], [4.0, 2.0, 0.5]]; + +fn out_dir(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("effectcraft-exr-scene-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).expect("mkdir"); + d +} + +/// A one-frame comp of [`BANDS`] (full width, 2/3, 1/3, centred) with the given colour settings. +fn project(depth: BitDepth, hdr: HdrMode, ws: Option, linearize: bool, out: Option) -> (Project, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = depth; + p.settings.hdr = hdr; + p.settings.working_space = ws; + p.settings.linearize = linearize; + p.settings.output_space = out; + let mut comp = Comp::new(W, H, FrameRate::new(10, 1), Tick::from_seconds_f64(0.1)); + comp.background = [0.0, 0.0, 0.0]; + for (i, color) in BANDS.into_iter().enumerate() { + let w = W - W / 3 * i as u32; + let sid = p.add_item(&format!("Band {i}"), Label::Red, None, ItemKind::Solid(Solid { color, width: w, height: H, pixel_aspect: 1.0 })); + let l = build::layer(&mut p, &comp, &format!("Band {i}"), LayerSource::Solid { item: sid }, (w, H), None); + comp.layers.insert(0, l); + } + let cid = p.add_item("Comp 1", Label::Sandstone, None, ItemKind::Comp(comp.into())); + (p, cid) +} + +/// Export the comp's one frame as a `fmt` sequence; the frame file's path. +fn render(p: &Project, cid: ItemId, fmt: OutputFormat, channels: Channels, dir: &Path, ext: &str) -> PathBuf { + let mut om = OutputModule::for_format(fmt); + om.channels = channels; + let s = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() }; + let path = dir.join(format!("f_[#####].{ext}")).to_string_lossy().to_string(); + let job = Job { + project: p, + footage: &NoFootage, + expr: None, + accel: None, + comp: cid, + settings: &s, + output: &om, + path: &path, + sink: None, + options: Default::default(), + nested_switches: true, + }; + export(&job, &mut |_| true).expect("export"); + dir.join(format!("f_00000.{ext}")) +} + +fn fnv(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ *b as u64).wrapping_mul(0x0100_0000_01b3)) +} + +const DEPTHS: [BitDepth; 3] = [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32]; +const MODES: [HdrMode; 3] = [HdrMode::Clip, HdrMode::Compand, HdrMode::ToneMap]; +const STILLS: [(OutputFormat, Channels, &str); 3] = [ + (OutputFormat::PngSequence, Channels::Rgba, "png"), + (OutputFormat::TiffSequence, Channels::Rgb, "tif"), + (OutputFormat::TargaSequence, Channels::Rgba, "tga"), +]; + +/// The bytes of every 8-bit still for each bit depth × HDR mode, in a managed linear sRGB +/// project with a Display P3 output. +fn still_hashes() -> Vec { + let mut out = vec![]; + for depth in DEPTHS { + for mode in MODES { + let (p, cid) = project(depth, mode, Some(ColorSpace::Srgb), true, Some(ColorSpace::DisplayP3)); + for (fmt, ch, ext) in STILLS { + let d = out_dir(&format!("golden-{depth:?}-{mode:?}-{ext}")); + let bytes = std::fs::read(render(&p, cid, fmt, ch, &d, ext)).expect("read"); + out.push(fnv(&bytes)); + let _ = std::fs::remove_dir_all(&d); + } + } + } + out +} + +/// [`still_hashes`] before float outputs skipped the HDR step: PNG, TIFF and Targa (8-bit +/// writers, which keep compand / tone mapping) are byte for byte the same. +const STILL_GOLDEN: [u64; 27] = [ + 0xd0cf67f45d6b6b68, + 0x712d42f3e204cbdb, + 0x021dc457f259eb87, + 0x88a16b1ca2db8805, + 0xc6fb221b89179bdb, + 0x965636561aaabb87, + 0x85c831b7694662db, + 0x8875a5c5232a33db, + 0x42d8f3c110bd0987, + 0xd555a6850fe66f04, + 0x920b49ee0f9900e4, + 0x94e4f43742f2c8ce, + 0xab0792254da43348, + 0x647a60c0a33237ec, + 0x65cba80bc3ecff08, + 0x0726be1b60eb7276, + 0xffd0315633cb88db, + 0x4f6a13131fb7a175, + 0x07a706a7e100b51d, + 0x718600eaa65a6f80, + 0x53489230b732f638, + 0x13024d498aa2552c, + 0x3a3dae30c9e25f83, + 0xc3191a3e860aa0c5, + 0xe5d7d08ecc3ddece, + 0x29a748f729fe5c8a, + 0x78d3b9aa9e107774, +]; + +#[test] +fn eight_bit_stills_are_unchanged() { + assert_eq!(still_hashes(), STILL_GOLDEN); +} + +/// The decoded samples of a still EXR frame, row-major RGBA. +fn exr_samples(path: &Path) -> Vec { + image::open(path).expect("exr").to_rgba32f().into_raw() +} + +fn bits_hash(v: &[f32]) -> u64 { + fnv(&v.iter().flat_map(|x| x.to_bits().to_le_bytes()).collect::>()) +} + +/// Clip-mode EXR samples with an sRGB output, for each colour setup × bit depth × channels. +fn exr_clip_hashes() -> Vec { + let mut out = vec![]; + for (ws, linearize) in [(None, false), (Some(ColorSpace::Srgb), false), (Some(ColorSpace::Srgb), true), (Some(ColorSpace::AcesCg), true)] { + for depth in DEPTHS { + let (p, cid) = project(depth, HdrMode::Clip, ws, linearize, None); + for ch in [Channels::Rgba, Channels::Rgb, Channels::Alpha] { + let d = out_dir(&format!("clip-{ws:?}-{linearize}-{depth:?}-{ch:?}")); + out.push(bits_hash(&exr_samples(&render(&p, cid, OutputFormat::ExrSequence, ch, &d, "exr")))); + let _ = std::fs::remove_dir_all(&d); + } + } + } + out +} + +/// [`exr_clip_hashes`] before: Clip-mode EXRs with an sRGB output keep every sample. +const EXR_CLIP_GOLDEN: [u64; 36] = [ + 0x9f54fea3d3abcb25, + 0x9f54fea3d3abcb25, + 0x7d40ad9df8600325, + 0x379f5ef9e35ce325, + 0x379f5ef9e35ce325, + 0x7d40ad9df8600325, + 0xb60de6a90263b525, + 0xb60de6a90263b525, + 0x7d40ad9df8600325, + 0x9f54fea3d3abcb25, + 0x9f54fea3d3abcb25, + 0x7d40ad9df8600325, + 0x379f5ef9e35ce325, + 0x379f5ef9e35ce325, + 0x7d40ad9df8600325, + 0xb60de6a90263b525, + 0xb60de6a90263b525, + 0x7d40ad9df8600325, + 0x27858400549ac325, + 0x27858400549ac325, + 0x7d40ad9df8600325, + 0x379f5ef9e35ce325, + 0x379f5ef9e35ce325, + 0x7d40ad9df8600325, + 0xb60de6a90263b525, + 0xb60de6a90263b525, + 0x7d40ad9df8600325, + 0x31d03697119d9325, + 0x31d03697119d9325, + 0x7d40ad9df8600325, + 0xff61dc513c834b25, + 0xff61dc513c834b25, + 0x7d40ad9df8600325, + 0x0c57d24a44f45925, + 0x0c57d24a44f45925, + 0x7d40ad9df8600325, +]; + +#[test] +fn exr_clip_with_srgb_output_is_unchanged() { + assert_eq!(exr_clip_hashes(), EXR_CLIP_GOLDEN); +} + +/// Compand and tone mapping leave EXR output alone: the file holds the Clip mode's samples, for +/// every bit depth, channel set and colour setup (wide working spaces, a Display P3 output). +#[test] +fn exr_ignores_compand_and_tone_mapping() { + let setups = [ + (Some(ColorSpace::Srgb), true, None), + (Some(ColorSpace::Srgb), false, Some(ColorSpace::DisplayP3)), + (Some(ColorSpace::AcesCg), true, None), + (Some(ColorSpace::Rec2020), true, Some(ColorSpace::Rec709)), + ]; + for (ws, linearize, out) in setups { + for depth in DEPTHS { + for ch in [Channels::Rgba, Channels::Rgb] { + let samples = |mode: HdrMode| { + let (p, cid) = project(depth, mode, ws, linearize, out); + let d = out_dir(&format!("modes-{ws:?}-{linearize}-{out:?}-{depth:?}-{ch:?}-{mode:?}")); + let v = exr_samples(&render(&p, cid, OutputFormat::ExrSequence, ch, &d, "exr")); + let _ = std::fs::remove_dir_all(&d); + v + }; + let clip = samples(HdrMode::Clip); + for mode in [HdrMode::Compand, HdrMode::ToneMap] { + assert!(samples(mode) == clip, "{ws:?} {linearize} {out:?} {depth:?} {ch:?} {mode:?}"); + } + } + } + } +} + +/// The centre pixel (the innermost band) of a one-frame EXR render. +fn exr_centre(p: &Project, cid: ItemId, name: &str) -> [f32; 4] { + let d = out_dir(name); + let v = exr_samples(&render(p, cid, OutputFormat::ExrSequence, Channels::Rgba, &d, "exr")); + let _ = std::fs::remove_dir_all(&d); + let o = ((H / 2 * W + W / 2) * 4) as usize; + [v[o], v[o + 1], v[o + 2], v[o + 3]] +} + +fn assert_near(got: [f32; 4], want: [f32; 3], what: &str) { + for i in 0..3 { + assert!((got[i] - want[i]).abs() <= 1e-4 * want[i].abs().max(1.0), "{what}: channel {i}: got {got:?}, want {want:?}"); + } + assert_eq!(got[3], 1.0, "{what}: {got:?}"); +} + +/// In a 32 bpc linear project with compand or tone mapping, the EXR holds the comp's scene-linear +/// values in the output space's primaries: 4.0 stays 4.0. (Solid colours are authored in the +/// working space's encoding; ACES spaces are linear.) +#[test] +fn exr_holds_scene_linear_values_in_output_primaries() { + let [_, _, band] = BANDS; + for mode in [HdrMode::Compand, HdrMode::ToneMap] { + let cases = [ + // ACEScg in and out: nothing converts. + (ColorSpace::AcesCg, ColorSpace::AcesCg, band), + // ACEScg working space, Display P3 output: the primaries convert, the values stay linear. + (ColorSpace::AcesCg, ColorSpace::DisplayP3, Conversion::new(ColorSpace::AcesCg, true, ColorSpace::DisplayP3, true).expect("matrix").apply(band)), + // Linearized Rec. 2020 working space, sRGB output. + (ColorSpace::Rec2020, ColorSpace::Srgb, Conversion::new(ColorSpace::Rec2020, false, ColorSpace::Srgb, true).expect("matrix").apply(band)), + // Linearized sRGB in and out. + (ColorSpace::Srgb, ColorSpace::Srgb, band.map(effectcraft_color::srgb_to_linear)), + ]; + for (ws, out, want) in cases { + let (p, cid) = project(BitDepth::Bpc32, mode, Some(ws), true, Some(out)); + let what = format!("{mode:?} {ws:?} to {out:?}"); + assert_near(exr_centre(&p, cid, &format!("lin-{ws:?}-{out:?}-{mode:?}")), want.map(|v| v.max(0.0)), &what); + } + } + let (p, cid) = project(BitDepth::Bpc32, HdrMode::ToneMap, Some(ColorSpace::AcesCg), true, Some(ColorSpace::AcesCg)); + assert_eq!(exr_centre(&p, cid, "lin-exact")[..3], band, "4.0 is exactly 4.0"); +} + +/// The EXR is decoded with the output space's curve, not always sRGB's: a Rec. 709 output (pure +/// 2.4 power curve) and an unmanaged project at Working Gamma 2.4 give back the linear values. +#[test] +fn exr_decodes_with_the_output_curve() { + let [_, _, band] = BANDS; + // A linearized sRGB working space (same primaries as Rec. 709): the authored colour, decoded. + let (p, cid) = project(BitDepth::Bpc32, HdrMode::Clip, Some(ColorSpace::Srgb), true, Some(ColorSpace::Rec709)); + assert_near(exr_centre(&p, cid, "curve-709"), band.map(effectcraft_color::srgb_to_linear), "Rec. 709 output"); + // Unmanaged at 2.4: the solid's colour is encoded with the 2.4 power curve. + let (mut p, cid) = project(BitDepth::Bpc32, HdrMode::Clip, None, false, None); + p.settings.working_gamma = 2.4; + assert_near(exr_centre(&p, cid, "curve-gamma24"), band.map(|v| v.powf(2.4)), "Working Gamma 2.4"); + // Working Gamma 2.2 keeps the sRGB curve. + let (p, cid) = project(BitDepth::Bpc32, HdrMode::Clip, None, false, None); + assert_near(exr_centre(&p, cid, "curve-gamma22"), band.map(effectcraft_color::srgb_to_linear), "Working Gamma 2.2"); +} diff --git a/crates/export/tests/it/main.rs b/crates/export/tests/it/main.rs index a5b9fa99a..87548f9e5 100644 --- a/crates/export/tests/it/main.rs +++ b/crates/export/tests/it/main.rs @@ -2,6 +2,7 @@ mod audio_sample_positions; mod banding; +mod exr_scene_linear; mod h264_gpu_yuv; mod hevc_av1_roundtrip; #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index 44e8922a6..4c09faa29 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -1315,3 +1315,31 @@ fn owned_devices_favour_memory_usage() { assert_eq!(d.label, Some("effectcraft gpu")); assert_eq!(d.required_limits.max_texture_dimension_2d, wgpu::Limits::default().max_texture_dimension_2d); } + +/// A float output (`RenderOpts::scene_output`) skips Project Settings ▸ HDR on the GPU as on the +/// CPU: both compositors agree, and the frame is the Clip frame with its over-range values. +#[test] +fn scene_output_skips_the_hdr_step_on_the_gpu() { + use effectcraft_project::HdrMode; + for mode in [HdrMode::Compand, HdrMode::ToneMap] { + let mut s = blend_scene(BitDepth::Bpc32, BlendMode::Normal); + let mut bright = s.solid([3.0, 1.5, 0.5], 30, 20); + bright.blend_mode = BlendMode::Add; + s.push(bright); + s.p.settings.working_space = Some(ColorSpace::Srgb); + s.p.settings.linearize = true; + s.p.settings.hdr = mode; + let scene = RenderOpts { scene_output: true, ..opts() }; + check(&format!("scene output {mode:?}"), compare_at(&s, scene, Tick::ZERO), 0.0); + let Some(g) = gpu() else { continue }; + let mut r = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Gpu, ..scene }); + r.accel = Some(g); + let img = g.render(&r, s.cid, Tick::ZERO).expect("the GPU renders the frame"); + let mut clip = s.p.clone(); + clip.settings.hdr = HdrMode::Clip; + let want = Renderer::new(&clip, &Pattern, RenderOpts { backend: Backend::Cpu, ..opts() }).comp_frame_cpu(s.cid, Tick::ZERO); + let d = diff(&want, &img, tolerance(BitDepth::Bpc32)); + assert_eq!(d.over, 0, "{mode:?}: max {} at {:?}", d.max, d.worst); + assert!(img.data.iter().any(|p| p[3] > 0.0 && p[0] / p[3] > 1.0), "{mode:?}: over-range values survive"); + } +} diff --git a/crates/gpu/src/walk.rs b/crates/gpu/src/walk.rs index fdcb58dd4..1b6687395 100644 --- a/crates/gpu/src/walk.rs +++ b/crates/gpu/src/walk.rs @@ -45,8 +45,9 @@ pub(crate) fn render<'g>(e: &mut Enc<'g>, r: &Renderer, comp_id: ItemId, t: Tick if let Some(l) = pipe.levels { canvas = ops::quantize(e, &canvas, l); } + let out = r.output_pipe(); if r.depth() == 0 - && let Some(c) = pipe.output() + && let Some(c) = out.output() { canvas = ops::convert(e, &canvas, &c); if let Some(l) = pipe.levels { @@ -54,7 +55,7 @@ pub(crate) fn render<'g>(e: &mut Enc<'g>, r: &Renderer, comp_id: ItemId, t: Tick } } if r.depth() == 0 - && let Some((lin, mode, enc)) = pipe.output_hdr() + && let Some((lin, mode, enc)) = out.output_hdr() { on_cpu(e, &mut canvas, |img| { effectcraft_render::color::output_hdr(img, lin, mode, enc); diff --git a/crates/render/src/accel.rs b/crates/render/src/accel.rs index e6bbd2113..613853e69 100644 --- a/crates/render/src/accel.rs +++ b/crates/render/src/accel.rs @@ -174,6 +174,12 @@ impl<'a> Renderer<'a> { self.pipe } + /// The colour pipeline of the top-level output steps: the project's, without the HDR step + /// for a float output ([`crate::RenderOpts::scene_output`]). + pub fn output_pipe(&self) -> color::Pipe { + if self.opts.scene_output { self.pipe.scene_output() } else { self.pipe } + } + /// Nesting depth (0 = the top-level comp). pub fn depth(&self) -> usize { self.depth diff --git a/crates/render/src/color.rs b/crates/render/src/color.rs index d27526553..54d6d82e0 100644 --- a/crates/render/src/color.rs +++ b/crates/render/src/color.rs @@ -117,6 +117,20 @@ impl Pipe { Conversion::new(ws, self.linear, self.out, false) } + /// The pipe of a float output ([`crate::RenderOpts::scene_output`]): the same, without + /// the HDR step (Clip), so [`Pipe::output`] converts to the output space as it is. + pub fn scene_output(self) -> Pipe { + Pipe { hdr: HdrMode::Clip, ..self } + } + + /// The top-level frame of a [`Pipe::scene_output`] render (output-encoded, straight colour) + /// → linear light in the output space's primaries: the output space's curve, or the Working + /// Gamma's when unmanaged (sRGB at 2.2, a pure 2.4 power curve at 2.4). The identity for a + /// linear output (ACES). + pub fn output_linear(&self) -> Conversion { + Conversion::linearize(if self.space.is_some() { self.out } else { self.curve() }) + } + /// Output with HDR compand / tone mapping (standard-dynamic-range outputs only): working /// space → linear output primaries, the HDR curve, then the output's encoding. pub fn output_hdr(&self) -> Option<(Option, HdrMode, Conversion)> { diff --git a/crates/render/src/opts.rs b/crates/render/src/opts.rs index c6175bf39..ae8d3435f 100644 --- a/crates/render/src/opts.rs +++ b/crates/render/src/opts.rs @@ -28,6 +28,11 @@ pub struct RenderOpts { pub nested_switches: bool, /// Realtime Shadows in Draft (Settings ▸ 3D): draft renders (`draft`) still cast shadows. pub draft_shadows: bool, + /// A float output (OpenEXR): the top-level comp leaves the working space for the output + /// space without Project Settings ▸ HDR (compand / tone mapping are for standard-dynamic-range + /// displays and outputs), so over-range values reach the file as they are. The writer + /// decodes the frame with [`crate::color::Pipe::output_linear`]. + pub scene_output: bool, } impl RenderOpts { @@ -49,6 +54,7 @@ impl Default for RenderOpts { backend: Backend::Cpu, nested_switches: true, draft_shadows: true, + scene_output: false, proxy: effectcraft_project::render_queue::ProxyUse::CurrentSettings, } } diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index a4927144f..7ecf2f536 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -473,14 +473,15 @@ impl<'a> Renderer<'a> { self.pipe.quantize(&mut canvas); } // The top-level comp leaves the working space for the (sRGB) display / output. + let out = self.output_pipe(); if self.depth == 0 - && let Some(c) = self.pipe.output() + && let Some(c) = out.output() { color::convert(&mut canvas, &c); self.pipe.quantize(&mut canvas); } if self.depth == 0 - && let Some((lin, mode, enc)) = self.pipe.output_hdr() + && let Some((lin, mode, enc)) = out.output_hdr() { color::output_hdr(&mut canvas, lin, mode, enc); self.pipe.quantize(&mut canvas); diff --git a/crates/render/src/tests/color.rs b/crates/render/src/tests/color.rs index 29e4d62c2..de3ad19c6 100644 --- a/crates/render/src/tests/color.rs +++ b/crates/render/src/tests/color.rs @@ -339,3 +339,49 @@ fn float_footage_is_encoded_once_with_the_working_gamma() { assert_eq!(render(2.4, Some(ColorSpace::Srgb), linear_light), srgb); } } + +/// A float output (`RenderOpts::scene_output`) skips Project Settings ▸ HDR: its top-level frame +/// is the Clip frame in every mode, while other renders keep compand / tone mapping. +#[test] +fn scene_output_skips_the_hdr_step() { + use effectcraft_project::HdrMode; + for depth in [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32] { + let (mut p, cid, _) = setup(depth); + p.settings.working_space = Some(ColorSpace::Srgb); + p.settings.linearize = true; + add_solid(&mut p, cid, [4.0, 2.0, 0.5], BlendMode::Normal, 100.0); + let frame = |p: &Project, scene_output: bool| { + Renderer::new(p, &crate::NoFootage, RenderOpts { scene_output, ..Default::default() }).comp_frame(cid, Tick::ZERO) + }; + let clip = frame(&p, false); + assert_eq!(clip.data, frame(&p, true).data, "{depth:?} Clip"); + for mode in [HdrMode::Compand, HdrMode::ToneMap] { + p.settings.hdr = mode; + assert_eq!(frame(&p, true).data, clip.data, "{depth:?} {mode:?}"); + assert_ne!(frame(&p, false).data, clip.data, "{depth:?} {mode:?} keeps the HDR step"); + } + } +} + +/// `Pipe::output_linear` decodes with the output space's curve (the Working Gamma's when +/// unmanaged) and undoes `Pipe::output` exactly where the primaries match. +#[test] +fn output_linear_decodes_with_the_output_curve() { + use effectcraft_color::Conversion; + use effectcraft_project::ProjectSettings; + let pipe = |ws: Option, out: Option, gamma: f64| { + crate::color::Pipe::of(&ProjectSettings { working_space: ws, output_space: out, working_gamma: gamma, ..Default::default() }) + }; + assert_eq!(pipe(None, None, 2.2).output_linear(), Conversion::linearize(ColorSpace::Srgb)); + assert_eq!(pipe(None, Some(ColorSpace::DisplayP3), 2.4).output_linear(), Conversion::linearize(ColorSpace::Rec709), "unmanaged: the Working Gamma"); + assert_eq!(pipe(Some(ColorSpace::Srgb), None, 2.4).output_linear(), Conversion::linearize(ColorSpace::Srgb)); + assert_eq!(pipe(Some(ColorSpace::Srgb), Some(ColorSpace::Rec709), 2.2).output_linear(), Conversion::linearize(ColorSpace::Rec709)); + // Linear light in, linear light out: 4.0 stays 4.0 through output + output_linear. + let p = pipe(Some(ColorSpace::Rec709), Some(ColorSpace::Srgb), 2.2); + let p = crate::color::Pipe { linear: true, ..p }; + let enc = p.output().unwrap().apply([4.0, 1.0, 0.25]); + let lin = p.output_linear().apply(enc); + for (got, want) in lin.into_iter().zip([4.0, 1.0, 0.25]) { + assert!((got - want).abs() <= 1e-5 * want, "{lin:?}"); + } +} diff --git a/crates/ui-egui/src/app/viewer_frames.rs b/crates/ui-egui/src/app/viewer_frames.rs index f87db615c..241951c54 100644 --- a/crates/ui-egui/src/app/viewer_frames.rs +++ b/crates/ui-egui/src/app/viewer_frames.rs @@ -164,6 +164,7 @@ impl EffectcraftApp { backend: effectcraft_engine::render::Backend::Auto, nested_switches: self.session.prefs.general.switches_affect_nested_comps, draft_shadows: self.session.prefs.three_d.realtime_shadows, + scene_output: false, proxy: Default::default(), } } diff --git a/crates/ui-egui/src/frames/tests.rs b/crates/ui-egui/src/frames/tests.rs index d04793f50..77584e2df 100644 --- a/crates/ui-egui/src/frames/tests.rs +++ b/crates/ui-egui/src/frames/tests.rs @@ -6,6 +6,14 @@ fn converting_a_frame_with_nan_alpha_keeps_the_viewer_alive() { assert_eq!(to_color_image(&image).pixels, [egui::Color32::TRANSPARENT]); } +/// A float output's frames (`RenderOpts::scene_output`, no HDR step) never share a RAM or disk +/// cache key with the HDR-mapped frames of the same options. +#[test] +fn scene_output_frames_have_keys_of_their_own() { + let opts = RenderOpts::default(); + assert_ne!(opts_hash(&opts), opts_hash(&RenderOpts { scene_output: true, ..opts })); +} + struct ProbeDisk { keys: Mutex>, probes: std::sync::atomic::AtomicUsize, diff --git a/docs/architecture.md b/docs/architecture.md index c4bebd9ed..226f93a14 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -407,7 +407,11 @@ inputs in 32 bpc. With a working space (sRGB, Rec. 709, Rec. 2020, Display P3; m from the published primaries), footage is converted from its colour profile (stream metadata or Interpret Footage, else sRGB) and the top-level comp is converted to the sRGB display. Linearize Working Space runs everything in linear light; Blend Colors Using 1.0 Gamma linearises only for -blending. Layer cache keys include these settings. +blending. Layer cache keys include these settings. Project Settings ▸ HDR (compand / tone +mapping) is the last top-level step, for standard-dynamic-range displays and outputs; OpenEXR +renders (sequences and ProEXR) skip it (`RenderOpts::scene_output`) and decode the frame with the +output space's own curve (`Pipe::output_linear`), so they hold linear light in the output space's +primaries. **Frame blending**: footage and precomp layers whose source time falls between source frames (rate conform, stretch, remap) blend the two neighbouring frames, by cross-fade (Frame Mix) or by From 406c9a9648d1e4d1eedc7b594d9d65efa9238a55 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 01:57:53 +0100 Subject: [PATCH 03/25] EXR export: raw alpha for Alpha-only files, skip_hdr_step name, more tests - Rename RenderOpts::scene_output to skip_hdr_step (Pipe::without_hdr_step): the consumer decodes with Pipe::output_linear or applies the step itself. - Alpha-only EXRs hold the alpha as it is in every output space (it is coverage, not encoded colour; it used to be sRGB-decoded). - Tests: a half-opacity layer over a coloured background (premultiplied and RGB-over-background paths, golden hashes taken before the change), Rec. 2100 PQ and HLG EXR outputs, raw alpha, and a visible skip of the GPU test without an adapter. - docs/architecture.md: the Working Gamma curve for unmanaged projects, and no HDR step for unmanaged projects or PQ / HLG outputs. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/frame_export.rs | 15 +-- crates/export/src/lib.rs | 8 +- crates/export/src/pipeline.rs | 2 +- crates/export/tests/it/exr_scene_linear.rs | 148 +++++++++++++++++++-- crates/gpu/src/tests.rs | 14 +- crates/render/src/accel.rs | 7 +- crates/render/src/color.rs | 8 +- crates/render/src/opts.rs | 12 +- crates/render/src/tests/color.rs | 8 +- crates/ui-egui/src/app/viewer_frames.rs | 2 +- crates/ui-egui/src/frames/tests.rs | 6 +- docs/architecture.md | 5 +- 12 files changed, 182 insertions(+), 53 deletions(-) diff --git a/crates/engine/src/cmd/frame_export.rs b/crates/engine/src/cmd/frame_export.rs index 24b6f0c3e..7347b84bb 100644 --- a/crates/engine/src/cmd/frame_export.rs +++ b/crates/engine/src/cmd/frame_export.rs @@ -27,15 +27,14 @@ struct Isolated { image: Image, } -/// `scene_output`: a float output (ProEXR), rendered without the HDR step -/// ([`RenderOpts::scene_output`]). -fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64, scene_output: bool) -> Image { +/// `skip_hdr_step`: no HDR step, for a float output (ProEXR; [`RenderOpts::skip_hdr_step`]). +fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64, skip_hdr_step: bool) -> Image { let opts = RenderOpts { scale, backend: Backend::Auto, nested_switches: s.prefs.general.switches_affect_nested_comps, draft_shadows: s.prefs.three_d.realtime_shadows, - scene_output, + skip_hdr_step, ..Default::default() }; let mut r = Renderer::new(project, s.footage.as_ref(), opts); @@ -44,11 +43,11 @@ fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64 r.comp_frame(comp, t) } -/// The comp frame and each visible layer rendered alone (top layer first). `scene_output`: as +/// The comp frame and each visible layer rendered alone (top layer first). `skip_hdr_step`: as /// [`render_with`]. -fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64, scene_output: bool) -> Result<(Image, Vec)> { +fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64, skip_hdr_step: bool) -> Result<(Image, Vec)> { let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; - let full = render_with(s, &s.project, cid, t, scale, scene_output); + let full = render_with(s, &s.project, cid, t, scale, skip_hdr_step); let any_solo = comp.layers.iter().any(|l| l.switches.solo && l.has_video()); let ids: Vec = comp .layers @@ -76,7 +75,7 @@ fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64, scene_output: bool) -> } } let Some((name, blend, opacity)) = info else { continue }; - let image = render_with(s, &p, cid, t, scale, scene_output); + let image = render_with(s, &p, cid, t, scale, skip_hdr_step); out.push(Isolated { name, blend, opacity, image }); } Ok((full, out)) diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 2d0d38071..9a9e66d0f 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -302,7 +302,7 @@ fn write_still(cx: &Cx, file: &mut W, img: &I OutputFormat::ExrSequence => { // Linear light, premultiplied (the EXR convention); RGB keeps alpha = 1 over the background. // Colour is floored at 0 but not capped at 1: a 32 bpc frame's over-range values survive (#339). - // The frame was rendered without the HDR step (`RenderOpts::scene_output`) and is + // The frame was rendered without the HDR step (`RenderOpts::skip_hdr_step`) and is // decoded with the output space's curve, in the output space's primaries. let dec = effectcraft_render::color::Pipe::of(&cx.project.settings).output_linear(); let lin3 = |c: [f32; 3]| dec.apply(c.map(|v| v.max(0.0))); @@ -311,10 +311,8 @@ fn write_still(cx: &Cx, file: &mut W, img: &I for x in 0..w { let p = if x < img.width && y < img.height { img.data[img.idx(x, y)] } else { [0.0; 4] }; let (c, a) = match channels { - Channels::Alpha => { - let a = p[3].clamp(0.0, 1.0); - (lin3([a; 3]), 1.0) - } + // Alpha is coverage, not encoded colour: written as it is. + Channels::Alpha => ([p[3].clamp(0.0, 1.0); 3], 1.0), Channels::Rgba => { let a = p[3].clamp(0.0, 1.0); let c = if a > 0.0 { lin3([p[0] / a, p[1] / a, p[2] / a]).map(|v| v * a) } else { [0.0; 3] }; diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index 3742ae0a2..01e3f3faf 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -252,7 +252,7 @@ impl<'a> Cx<'a> { nested_switches: self.job.nested_switches, draft_shadows: true, // OpenEXR keeps over-range values: no HDR compand / tone mapping on its frames. - scene_output: self.job.output.format == effectcraft_project::render_queue::OutputFormat::ExrSequence, + skip_hdr_step: self.job.output.format == effectcraft_project::render_queue::OutputFormat::ExrSequence, proxy: s.proxy_use, }; let mut r = Renderer::new(&self.project, self.job.footage, opts); diff --git a/crates/export/tests/it/exr_scene_linear.rs b/crates/export/tests/it/exr_scene_linear.rs index 99a419894..b4b8c027c 100644 --- a/crates/export/tests/it/exr_scene_linear.rs +++ b/crates/export/tests/it/exr_scene_linear.rs @@ -43,6 +43,34 @@ fn project(depth: BitDepth, hdr: HdrMode, ws: Option, linearize: boo (p, cid) } +/// A comp builder: bit depth, HDR mode, working space, Linearize Working Space, output space. +type Build = fn(BitDepth, HdrMode, Option, bool, Option) -> (Project, ItemId); + +/// The half-opacity layer of [`half_project`]: over range in red. +const HALF: [f32; 3] = [4.0, 0.6, 0.3]; + +/// A one-frame comp over a coloured background whose only layer, [`HALF`] at 50% opacity, +/// covers the left two thirds: semi-transparent and transparent pixels (the premultiplied and +/// RGB-over-background paths of the writers). +fn half_project(depth: BitDepth, hdr: HdrMode, ws: Option, linearize: bool, out: Option) -> (Project, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = depth; + p.settings.hdr = hdr; + p.settings.working_space = ws; + p.settings.linearize = linearize; + p.settings.output_space = out; + let mut comp = Comp::new(W, H, FrameRate::new(10, 1), Tick::from_seconds_f64(0.1)); + comp.background = [0.2, 0.3, 0.4]; + let w = W / 3 * 2; + let sid = p.add_item("Half", Label::Red, None, ItemKind::Solid(Solid { color: HALF, width: w, height: H, pixel_aspect: 1.0 })); + let mut l = build::layer(&mut p, &comp, "Half", LayerSource::Solid { item: sid }, (w, H), None); + l.props.prop_mut("transform/position").expect("position").value = effectcraft_keyframe::Value::Vec3([w as f64 / 2.0, H as f64 / 2.0, 0.0]); + l.props.prop_mut("transform/opacity").expect("opacity").value = effectcraft_keyframe::Value::Scalar(50.0); + comp.layers.push(l); + let cid = p.add_item("Comp 1", Label::Sandstone, None, ItemKind::Comp(comp.into())); + (p, cid) +} + /// Export the comp's one frame as a `fmt` sequence; the frame file's path. fn render(p: &Project, cid: ItemId, fmt: OutputFormat, channels: Channels, dir: &Path, ext: &str) -> PathBuf { let mut om = OutputModule::for_format(fmt); @@ -80,11 +108,11 @@ const STILLS: [(OutputFormat, Channels, &str); 3] = [ /// The bytes of every 8-bit still for each bit depth × HDR mode, in a managed linear sRGB /// project with a Display P3 output. -fn still_hashes() -> Vec { +fn still_hashes(build: Build) -> Vec { let mut out = vec![]; for depth in DEPTHS { for mode in MODES { - let (p, cid) = project(depth, mode, Some(ColorSpace::Srgb), true, Some(ColorSpace::DisplayP3)); + let (p, cid) = build(depth, mode, Some(ColorSpace::Srgb), true, Some(ColorSpace::DisplayP3)); for (fmt, ch, ext) in STILLS { let d = out_dir(&format!("golden-{depth:?}-{mode:?}-{ext}")); let bytes = std::fs::read(render(&p, cid, fmt, ch, &d, ext)).expect("read"); @@ -130,9 +158,41 @@ const STILL_GOLDEN: [u64; 27] = [ #[test] fn eight_bit_stills_are_unchanged() { - assert_eq!(still_hashes(), STILL_GOLDEN); + assert_eq!(still_hashes(project), STILL_GOLDEN); + assert_eq!(still_hashes(half_project), HALF_STILL_GOLDEN); } +/// [`still_hashes`] of [`half_project`] before: premultiplied and RGB-over-background stills. +const HALF_STILL_GOLDEN: [u64; 27] = [ + 0x3abd611ee63b7edd, + 0x51e7ec0426773cdb, + 0xbee34abe9f0a3d87, + 0x554c03dba915d290, + 0xe98490fc826a7adb, + 0x0a0e77927bf1a187, + 0x4f1c076634822d50, + 0xe790f474e05b3adb, + 0xe517c3136471a187, + 0x07cd5e73765539f9, + 0x68e51bbd8d442a64, + 0x844a5f23a9cc9a3d, + 0xc302b2b86f4cdd70, + 0xb7247795d204a2c5, + 0x3b18867537c69265, + 0x9a5f7679bb5a2399, + 0x868b729dcc804656, + 0x2e0bc64a3fb562f0, + 0x6dc9e1a1eac1220d, + 0x6e101cd849e78f32, + 0x7b7a2a776f73d70b, + 0x6b3244f808c65d4e, + 0xbfa1b35fada2c77d, + 0x8c3a31415ed47e33, + 0xac787507caf1c896, + 0x4a38297060de19ff, + 0x300d6e4222eaa5b5, +]; + /// The decoded samples of a still EXR frame, row-major RGBA. fn exr_samples(path: &Path) -> Vec { image::open(path).expect("exr").to_rgba32f().into_raw() @@ -143,12 +203,12 @@ fn bits_hash(v: &[f32]) -> u64 { } /// Clip-mode EXR samples with an sRGB output, for each colour setup × bit depth × channels. -fn exr_clip_hashes() -> Vec { +fn exr_clip_hashes(build: Build, channels: &[Channels]) -> Vec { let mut out = vec![]; for (ws, linearize) in [(None, false), (Some(ColorSpace::Srgb), false), (Some(ColorSpace::Srgb), true), (Some(ColorSpace::AcesCg), true)] { for depth in DEPTHS { - let (p, cid) = project(depth, HdrMode::Clip, ws, linearize, None); - for ch in [Channels::Rgba, Channels::Rgb, Channels::Alpha] { + let (p, cid) = build(depth, HdrMode::Clip, ws, linearize, None); + for &ch in channels { let d = out_dir(&format!("clip-{ws:?}-{linearize}-{depth:?}-{ch:?}")); out.push(bits_hash(&exr_samples(&render(&p, cid, OutputFormat::ExrSequence, ch, &d, "exr")))); let _ = std::fs::remove_dir_all(&d); @@ -200,7 +260,54 @@ const EXR_CLIP_GOLDEN: [u64; 36] = [ #[test] fn exr_clip_with_srgb_output_is_unchanged() { - assert_eq!(exr_clip_hashes(), EXR_CLIP_GOLDEN); + // The bands are opaque everywhere, so the Alpha-only files (alpha 1.0) are unchanged too. + assert_eq!(exr_clip_hashes(project, &[Channels::Rgba, Channels::Rgb, Channels::Alpha]), EXR_CLIP_GOLDEN); + assert_eq!(exr_clip_hashes(half_project, &[Channels::Rgba, Channels::Rgb]), HALF_EXR_CLIP_GOLDEN); +} + +/// [`exr_clip_hashes`] of [`half_project`] before (RGBA premultiplied, RGB over the background). +const HALF_EXR_CLIP_GOLDEN: [u64; 24] = [ + 0xe7f7d517df7baf25, + 0xc9d6dcb2671f9925, + 0x2fe5733c26d16325, + 0x8b267197d2645925, + 0x6fe736332cb80f25, + 0x30d5f66395d92d25, + 0xe7f7d517df7baf25, + 0xc9d6dcb2671f9925, + 0x2fe5733c26d16325, + 0x8b267197d2645925, + 0x6fe736332cb80f25, + 0x30d5f66395d92d25, + 0x8c261c1cd9674b25, + 0xace550c3a8e72925, + 0xb9bd0b7863b78325, + 0xdf8e948c5e8af125, + 0x6fe736332cb80f25, + 0x30d5f66395d92d25, + 0xe4aedfefacc80725, + 0x95ddfccddc885525, + 0x81fa3c39c1d09725, + 0x49cc0014f05a1d25, + 0x4ed1b417cc847725, + 0x2df6df5ce394d925, +]; + +/// An Alpha-only EXR holds the alpha (coverage) as it is in every output space: no transfer +/// curve is decoded from it (it used to be sRGB-decoded, 0.5 → 0.214). +#[test] +fn exr_alpha_only_is_raw_alpha() { + for out in [None, Some(ColorSpace::Rec709), Some(ColorSpace::Rec2100Pq), Some(ColorSpace::AcesCg)] { + for ws in [None, Some(ColorSpace::Srgb)] { + let (p, cid) = half_project(BitDepth::Bpc32, HdrMode::Clip, ws, false, out); + let d = out_dir(&format!("alpha-{ws:?}-{out:?}")); + let v = exr_samples(&render(&p, cid, OutputFormat::ExrSequence, Channels::Alpha, &d, "exr")); + let _ = std::fs::remove_dir_all(&d); + let at = |x: u32| &v[((H / 2 * W + x) * 4) as usize..][..4]; + assert_eq!(at(2), [0.5, 0.5, 0.5, 1.0], "{ws:?} {out:?} covered"); + assert_eq!(at(W - 2), [0.0, 0.0, 0.0, 1.0], "{ws:?} {out:?} transparent"); + } + } } /// Compand and tone mapping leave EXR output alone: the file holds the Clip mode's samples, for @@ -215,17 +322,19 @@ fn exr_ignores_compand_and_tone_mapping() { ]; for (ws, linearize, out) in setups { for depth in DEPTHS { - for ch in [Channels::Rgba, Channels::Rgb] { + for (ch, (name, build)) in + [Channels::Rgba, Channels::Rgb].into_iter().flat_map(|c| [(c, ("bands", project as Build)), (c, ("half", half_project as Build))]) + { let samples = |mode: HdrMode| { - let (p, cid) = project(depth, mode, ws, linearize, out); - let d = out_dir(&format!("modes-{ws:?}-{linearize}-{out:?}-{depth:?}-{ch:?}-{mode:?}")); + let (p, cid) = build(depth, mode, ws, linearize, out); + let d = out_dir(&format!("modes-{name}-{ws:?}-{linearize}-{out:?}-{depth:?}-{ch:?}-{mode:?}")); let v = exr_samples(&render(&p, cid, OutputFormat::ExrSequence, ch, &d, "exr")); let _ = std::fs::remove_dir_all(&d); v }; let clip = samples(HdrMode::Clip); for mode in [HdrMode::Compand, HdrMode::ToneMap] { - assert!(samples(mode) == clip, "{ws:?} {linearize} {out:?} {depth:?} {ch:?} {mode:?}"); + assert!(samples(mode) == clip, "{name} {ws:?} {linearize} {out:?} {depth:?} {ch:?} {mode:?}"); } } } @@ -275,6 +384,23 @@ fn exr_holds_scene_linear_values_in_output_primaries() { assert_eq!(exr_centre(&p, cid, "lin-exact")[..3], band, "4.0 is exactly 4.0"); } +/// Rec. 2100 PQ and HLG outputs (no HDR step there in any mode) are decoded with their own +/// curves: linear light in Rec. 2020 primaries, 1.0 = reference white (203 cd/m²). +#[test] +fn exr_decodes_pq_and_hlg_outputs() { + let [_, _, band] = BANDS; + let want = Conversion::new(ColorSpace::AcesCg, true, ColorSpace::Rec2020, true).expect("matrix").apply(band).map(|v| v.max(0.0)); + for out in [ColorSpace::Rec2100Pq, ColorSpace::Rec2100Hlg] { + for mode in MODES { + let (p, cid) = project(BitDepth::Bpc32, mode, Some(ColorSpace::AcesCg), true, Some(out)); + let got = exr_centre(&p, cid, &format!("hdr-out-{out:?}-{mode:?}")); + for i in 0..3 { + assert!((got[i] - want[i]).abs() <= 1e-3 * want[i].max(1.0), "{out:?} {mode:?} channel {i}: got {got:?}, want {want:?}"); + } + } + } +} + /// The EXR is decoded with the output space's curve, not always sRGB's: a Rec. 709 output (pure /// 2.4 power curve) and an unmanaged project at Working Gamma 2.4 give back the linear values. #[test] diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index 4c09faa29..0d27da63d 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -1316,11 +1316,16 @@ fn owned_devices_favour_memory_usage() { assert_eq!(d.required_limits.max_texture_dimension_2d, wgpu::Limits::default().max_texture_dimension_2d); } -/// A float output (`RenderOpts::scene_output`) skips Project Settings ▸ HDR on the GPU as on the +/// `RenderOpts::skip_hdr_step` (float outputs) skips Project Settings ▸ HDR on the GPU as on the /// CPU: both compositors agree, and the frame is the Clip frame with its over-range values. #[test] -fn scene_output_skips_the_hdr_step_on_the_gpu() { +fn skipping_the_hdr_step_on_the_gpu_matches_clip() { use effectcraft_project::HdrMode; + let Some(g) = gpu() else { + // Visible skip (EFFECTCRAFT_REQUIRE_GPU=1 makes a missing adapter fail instead). + eprintln!("skipping_the_hdr_step_on_the_gpu_matches_clip: skipped: no GPU adapter"); + return; + }; for mode in [HdrMode::Compand, HdrMode::ToneMap] { let mut s = blend_scene(BitDepth::Bpc32, BlendMode::Normal); let mut bright = s.solid([3.0, 1.5, 0.5], 30, 20); @@ -1329,9 +1334,8 @@ fn scene_output_skips_the_hdr_step_on_the_gpu() { s.p.settings.working_space = Some(ColorSpace::Srgb); s.p.settings.linearize = true; s.p.settings.hdr = mode; - let scene = RenderOpts { scene_output: true, ..opts() }; - check(&format!("scene output {mode:?}"), compare_at(&s, scene, Tick::ZERO), 0.0); - let Some(g) = gpu() else { continue }; + let scene = RenderOpts { skip_hdr_step: true, ..opts() }; + check(&format!("no HDR step {mode:?}"), compare_at(&s, scene, Tick::ZERO), 0.0); let mut r = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Gpu, ..scene }); r.accel = Some(g); let img = g.render(&r, s.cid, Tick::ZERO).expect("the GPU renders the frame"); diff --git a/crates/render/src/accel.rs b/crates/render/src/accel.rs index 613853e69..52cd2be4a 100644 --- a/crates/render/src/accel.rs +++ b/crates/render/src/accel.rs @@ -174,10 +174,11 @@ impl<'a> Renderer<'a> { self.pipe } - /// The colour pipeline of the top-level output steps: the project's, without the HDR step - /// for a float output ([`crate::RenderOpts::scene_output`]). + /// The colour pipeline of the top-level output steps: the project's, with no HDR step under + /// [`crate::RenderOpts::skip_hdr_step`] (the consumer decodes with + /// [`color::Pipe::output_linear`] or applies the step itself). pub fn output_pipe(&self) -> color::Pipe { - if self.opts.scene_output { self.pipe.scene_output() } else { self.pipe } + if self.opts.skip_hdr_step { self.pipe.without_hdr_step() } else { self.pipe } } /// Nesting depth (0 = the top-level comp). diff --git a/crates/render/src/color.rs b/crates/render/src/color.rs index 54d6d82e0..16760c95a 100644 --- a/crates/render/src/color.rs +++ b/crates/render/src/color.rs @@ -117,13 +117,13 @@ impl Pipe { Conversion::new(ws, self.linear, self.out, false) } - /// The pipe of a float output ([`crate::RenderOpts::scene_output`]): the same, without - /// the HDR step (Clip), so [`Pipe::output`] converts to the output space as it is. - pub fn scene_output(self) -> Pipe { + /// The pipe without the HDR step (Clip), as [`crate::RenderOpts::skip_hdr_step`] renders + /// with: [`Pipe::output`] converts to the output space as it is. + pub fn without_hdr_step(self) -> Pipe { Pipe { hdr: HdrMode::Clip, ..self } } - /// The top-level frame of a [`Pipe::scene_output`] render (output-encoded, straight colour) + /// The top-level frame of a [`Pipe::without_hdr_step`] render (output-encoded, straight colour) /// → linear light in the output space's primaries: the output space's curve, or the Working /// Gamma's when unmanaged (sRGB at 2.2, a pure 2.4 power curve at 2.4). The identity for a /// linear output (ACES). diff --git a/crates/render/src/opts.rs b/crates/render/src/opts.rs index ae8d3435f..352f85983 100644 --- a/crates/render/src/opts.rs +++ b/crates/render/src/opts.rs @@ -28,11 +28,11 @@ pub struct RenderOpts { pub nested_switches: bool, /// Realtime Shadows in Draft (Settings ▸ 3D): draft renders (`draft`) still cast shadows. pub draft_shadows: bool, - /// A float output (OpenEXR): the top-level comp leaves the working space for the output - /// space without Project Settings ▸ HDR (compand / tone mapping are for standard-dynamic-range - /// displays and outputs), so over-range values reach the file as they are. The writer - /// decodes the frame with [`crate::color::Pipe::output_linear`]. - pub scene_output: bool, + /// No HDR step: the top-level comp leaves the working space for the output space without + /// Project Settings ▸ HDR (compand / tone mapping), so over-range values stay. The consumer + /// decodes the frame with [`crate::color::Pipe::output_linear`] (float outputs: OpenEXR) + /// or applies the step itself. Part of the viewer's frame keys (`opts_hash`). + pub skip_hdr_step: bool, } impl RenderOpts { @@ -54,7 +54,7 @@ impl Default for RenderOpts { backend: Backend::Cpu, nested_switches: true, draft_shadows: true, - scene_output: false, + skip_hdr_step: false, proxy: effectcraft_project::render_queue::ProxyUse::CurrentSettings, } } diff --git a/crates/render/src/tests/color.rs b/crates/render/src/tests/color.rs index de3ad19c6..4e5cff574 100644 --- a/crates/render/src/tests/color.rs +++ b/crates/render/src/tests/color.rs @@ -340,18 +340,18 @@ fn float_footage_is_encoded_once_with_the_working_gamma() { } } -/// A float output (`RenderOpts::scene_output`) skips Project Settings ▸ HDR: its top-level frame +/// `RenderOpts::skip_hdr_step` (float outputs) skips Project Settings ▸ HDR: its top-level frame /// is the Clip frame in every mode, while other renders keep compand / tone mapping. #[test] -fn scene_output_skips_the_hdr_step() { +fn skip_hdr_step_frames_are_the_clip_frames() { use effectcraft_project::HdrMode; for depth in [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32] { let (mut p, cid, _) = setup(depth); p.settings.working_space = Some(ColorSpace::Srgb); p.settings.linearize = true; add_solid(&mut p, cid, [4.0, 2.0, 0.5], BlendMode::Normal, 100.0); - let frame = |p: &Project, scene_output: bool| { - Renderer::new(p, &crate::NoFootage, RenderOpts { scene_output, ..Default::default() }).comp_frame(cid, Tick::ZERO) + let frame = |p: &Project, skip_hdr_step: bool| { + Renderer::new(p, &crate::NoFootage, RenderOpts { skip_hdr_step, ..Default::default() }).comp_frame(cid, Tick::ZERO) }; let clip = frame(&p, false); assert_eq!(clip.data, frame(&p, true).data, "{depth:?} Clip"); diff --git a/crates/ui-egui/src/app/viewer_frames.rs b/crates/ui-egui/src/app/viewer_frames.rs index 241951c54..7277a7cb5 100644 --- a/crates/ui-egui/src/app/viewer_frames.rs +++ b/crates/ui-egui/src/app/viewer_frames.rs @@ -164,7 +164,7 @@ impl EffectcraftApp { backend: effectcraft_engine::render::Backend::Auto, nested_switches: self.session.prefs.general.switches_affect_nested_comps, draft_shadows: self.session.prefs.three_d.realtime_shadows, - scene_output: false, + skip_hdr_step: false, proxy: Default::default(), } } diff --git a/crates/ui-egui/src/frames/tests.rs b/crates/ui-egui/src/frames/tests.rs index 77584e2df..3716db953 100644 --- a/crates/ui-egui/src/frames/tests.rs +++ b/crates/ui-egui/src/frames/tests.rs @@ -6,12 +6,12 @@ fn converting_a_frame_with_nan_alpha_keeps_the_viewer_alive() { assert_eq!(to_color_image(&image).pixels, [egui::Color32::TRANSPARENT]); } -/// A float output's frames (`RenderOpts::scene_output`, no HDR step) never share a RAM or disk +/// Frames without the HDR step (`RenderOpts::skip_hdr_step`) never share a RAM or disk /// cache key with the HDR-mapped frames of the same options. #[test] -fn scene_output_frames_have_keys_of_their_own() { +fn frames_without_the_hdr_step_have_keys_of_their_own() { let opts = RenderOpts::default(); - assert_ne!(opts_hash(&opts), opts_hash(&RenderOpts { scene_output: true, ..opts })); + assert_ne!(opts_hash(&opts), opts_hash(&RenderOpts { skip_hdr_step: true, ..opts })); } struct ProbeDisk { diff --git a/docs/architecture.md b/docs/architecture.md index 226f93a14..48f3ad442 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -409,9 +409,10 @@ Interpret Footage, else sRGB) and the top-level comp is converted to the sRGB di Working Space runs everything in linear light; Blend Colors Using 1.0 Gamma linearises only for blending. Layer cache keys include these settings. Project Settings ▸ HDR (compand / tone mapping) is the last top-level step, for standard-dynamic-range displays and outputs; OpenEXR -renders (sequences and ProEXR) skip it (`RenderOpts::scene_output`) and decode the frame with the +renders (sequences and ProEXR) skip it (`RenderOpts::skip_hdr_step`) and decode the frame with the output space's own curve (`Pipe::output_linear`), so they hold linear light in the output space's -primaries. +primaries; unmanaged projects decode with the Working Gamma's curve (sRGB at 2.2, a pure 2.4 power +curve at 2.4). Unmanaged projects and Rec. 2100 PQ / HLG outputs never get the HDR step. **Frame blending**: footage and precomp layers whose source time falls between source frames (rate conform, stretch, remap) blend the two neighbouring frames, by cross-fade (Frame Mix) or by From 8add8ac7b401634fc2af0c6d2cde9ce96d1d3f08 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 00:51:59 +0100 Subject: [PATCH 04/25] #474: decode a multi-layer EXR once per frame, and only its picture when nothing else reads it - The channel cache decodes a file once however many callers ask for it at the same time (the others wait for that decode), and is bounded by the bytes of its planes (1 GiB, least recently used out first, a file counting as used when its decode finishes, the newest file always kept) instead of four files of any size. - Once an effect has read a footage item's channels, its pictures are made of the cached planes, also for files with an unnamed RGB layer (bit for bit what the image crate shows, which still decodes the files the planes can't stand in for); footage alone decodes only its picture's channels (a multi-layer file's were all decoded and cached for every frame). - Channels are decoded straight into their display-window planes, and the picture is converted from the planes without two interleaved copies. Measured on a 30-frame 2560x1440 sequence of 32 channels (DWAA), one frame at a time: footage alone 0.185 -> 0.073 s per frame and 4.4 -> 1.8 GB peak; with an unnamed RGB layer plus Cryptomatte 0.244 -> 0.159 s; multi-part plus EXtractoR 0.194 -> 0.127 s. Co-Authored-By: Claude Opus 5.5 --- crates/media/src/convert.rs | 52 +- crates/media/src/exr_channels.rs | 816 +++++++++++++++++++++++++------ crates/media/src/pool.rs | 30 +- docs/effects-parity.md | 2 +- docs/file-format-parity.md | 2 +- docs/footage.md | 5 + docs/gaps.md | 2 +- 7 files changed, 742 insertions(+), 167 deletions(-) diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index 9e88ab4dd..a2b24b3ae 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -376,30 +376,20 @@ fn convert_yuv(img: &mut Image, source: YuvSource<'_, T>, k: &Yuv, op pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image { use image::DynamicImage as D; let (w, h) = (img.width(), img.height()); + if let D::ImageRgb32F(_) | D::ImageRgba32F(_) = img { + let px = img.to_rgba32f(); + let raw = px.as_raw(); + return float_to_image(w, h, op, |i| { + let s = &raw[i * 4..i * 4 + 4]; + [s[0], s[1], s[2], s[3]] + }); + } let mut out = Image::new(w, h); let wu = w as usize; if wu == 0 || h == 0 { return out; } match img { - D::ImageRgb32F(_) | D::ImageRgba32F(_) => { - let px = img.to_rgba32f(); - let raw = px.as_raw(); - let premul = op.mode == AlphaMode::Premultiplied; - out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { - for (x, o) in row.iter_mut().enumerate() { - let s = &raw[(y * wu + x) * 4..(y * wu + x) * 4 + 4]; - let a = s[3]; - let c = if premul { - let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; - [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] - } else { - [op.float(s[0]), op.float(s[1]), op.float(s[2])] - }; - *o = op.apply(c, a.clamp(0.0, 1.0)); - } - }); - } D::ImageLuma16(_) | D::ImageLumaA16(_) | D::ImageRgb16(_) | D::ImageRgba16(_) => { let px = img.to_rgba16(); let raw = px.as_raw(); @@ -425,6 +415,32 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image out } +/// Float RGBA (linear light, alpha straight or premultiplied as the footage says), `px(i)` for +/// pixel `i` in rows, to an [`Image`]: how [`dynamic_to_image`] converts float stills, also used +/// for OpenEXR pictures made from decoded channel planes without an interleaved copy (#474). +pub(crate) fn float_to_image(w: u32, h: u32, op: AlphaOp, px: impl Fn(usize) -> [f32; 4] + Sync) -> Image { + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + let premul = op.mode == AlphaMode::Premultiplied; + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = px(y * wu + x); + let a = s[3]; + let c = if premul { + let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; + [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] + } else { + [op.float(s[0]), op.float(s[1]), op.float(s[2])] + }; + *o = op.apply(c, a.clamp(0.0, 1.0)); + } + }); + out +} + /// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`]. pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image { let mut out = Image::new(w, h); diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index d372ec849..747882337 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -6,12 +6,25 @@ //! 0, or the background depth for depth channels). Cryptomatte manifests come from the //! `cryptomatte//name` and `cryptomatte//manifest` header attributes (the published //! Cryptomatte metadata convention: the manifest is a JSON object of name → hex hash). +//! +//! Decoded files are kept in a small cache ([`CACHE_BUDGET`]) that the picture of an EXR +//! footage item shares with EXtractoR / Cryptomatte once an effect has read its channels +//! (#474); a file is decoded once however many callers ask for it at the same time. use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError}; -use effectcraft_raster::AuxChannels; use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, split_channel}; +use effectcraft_raster::{AuxChannels, Image}; +use exr::block::UncompressedBlock; +use exr::block::chunk::TileCoordinates; +use exr::image::read::image::ReadImage; +use exr::image::read::layers::{ChannelsReader, ReadAllLayers, ReadChannels}; +use exr::meta::attribute::SampleType; +use exr::meta::header::Header; + +use crate::convert::{AlphaOp, float_to_image}; /// The largest display window decoded (16384²; vector footage and Continuously Rasterize /// stop at the same size). @@ -33,6 +46,11 @@ fn channel_name(part: Option<&str>, channel: &str) -> String { } } +/// The part name channels are prefixed with (`None`: unnamed). +fn part_name(header: &Header) -> Option { + header.own_attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()) +} + /// Parse a Cryptomatte manifest (`{"name": "hexhash", …}`). pub fn parse_manifest(json: &str) -> Vec<(String, u32)> { let Ok(serde_json::Value::Object(m)) = serde_json::from_str::(json) else { return vec![] }; @@ -53,23 +71,209 @@ pub fn unsupported_compression(bytes: &[u8]) -> Option<&'static str> { .then_some("OpenEXR HTJ2K compression isn't supported yet: re-save the file with ZIP, PIZ or DWAA compression") } -/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. -pub fn read_exr_channels(bytes: &[u8]) -> Option { - use exr::prelude::*; - let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_buffered(std::io::Cursor::new(bytes)).ok()?; - let dw = img.attributes.display_window; - let (w, h) = (dw.size.0, dw.size.1); - if w == 0 || h == 0 { - return None; +/// Which channels a decode keeps, by plane name, and the display window they are placed in +/// (the first part's, as the `exr` crate reports the image's). +struct Planes<'k> { + keep: &'k dyn Fn(&str) -> bool, + display: exr::meta::attribute::IntegerBounds, +} + +/// One part's kept channels while it is decoded: each line of samples goes straight into its +/// channel's display-window plane (no copy of the part's samples in their own type first). +struct PlanesReader { + /// Per channel of the part: its plane name, sample type and plane (`None`: not kept). + planes: Vec)>>, + /// The data window's position in the display window, and the data window's and display + /// window's sizes. + offset: (i64, i64), + layer: (usize, usize), + display: (usize, usize), + f16s: Vec, + u32s: Vec, + f32s: Vec, + /// The part's channel indices of exact `R`, `G`, `B` and `A`, when it has R, G and B. + rgb: Option<([usize; 3], Option)>, + /// The part has a channel that doesn't sample every pixel. + subsampled: bool, + /// The part's display window differs from the first part's. + own_display: bool, +} + +/// A decoded part: its kept planes (in channel order) and what [`PlanesReader`] found. +struct PartPlanes { + planes: Vec<(String, Vec)>, + /// Positions in `planes` of the part's R, G, B (and A) channels, when it has them and keeps + /// them all. + rgb: Option<([usize; 3], Option)>, + has_rgb: bool, + exact: bool, +} + +impl<'s> ReadChannels<'s> for Planes<'_> { + type Reader = PlanesReader; + + fn create_channels_reader(&'s self, header: &Header) -> exr::error::Result { + if header.deep { + return Err(exr::error::Error::NotSupported("deep data".into())); + } + let dw = self.display; + let display = (dw.size.0, dw.size.1); + // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a + // multi-gigabyte allocation per channel (#2141). + let n = display.0.checked_mul(display.1).filter(|n| *n > 0 && *n <= MAX_DISPLAY_PIXELS).ok_or(exr::error::Error::Invalid("display window".into()))?; + let prefix = part_name(header); + let pos = header.own_attributes.layer_position; + let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); + let planes = header + .channels + .list + .iter() + .map(|ch| { + let name = channel_name(prefix.as_deref(), &ch.name.to_string()); + let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; + (self.keep)(&name).then(|| (name, ch.sample_type, vec![fill; n])) + }) + .collect(); + let find = |c: &str| header.channels.find_index_of_channel(&exr::meta::attribute::Text::from(c)); + let rgb = match (find("R"), find("G"), find("B")) { + (Some(r), Some(g), Some(b)) => Some(([r, g, b], find("A"))), + _ => None, + }; + Ok(PlanesReader { + planes, + offset, + layer: (header.layer_size.0, header.layer_size.1), + display, + f16s: vec![], + u32s: vec![], + f32s: vec![], + rgb, + subsampled: header.channels.list.iter().any(|c| c.sampling != exr::math::Vec2(1, 1)), + own_display: header.shared_attributes.display_window != dw, + }) } - let mut aux = AuxChannels::new(w as u32, h as u32, 1.0); +} + +impl ChannelsReader for PlanesReader { + type Channels = PartPlanes; + + fn filter_block(&self, tile: TileCoordinates) -> bool { + tile.is_largest_resolution_level() && self.planes.iter().any(Option::is_some) + } + + fn read_block(&mut self, header: &Header, block: UncompressedBlock) -> exr::error::UnitResult { + let ((lw, lh), (w, h)) = (self.layer, self.display); + for line in block.lines(&header.channels) { + let Some(Some((_, kind, plane))) = self.planes.get_mut(line.location.channel) else { continue }; + let (x0, y, count) = (line.location.position.0, line.location.position.1, line.location.sample_count); + // The samples of data-window row `y`, columns `x0..x0 + count`, at display row `ty` + // from display column `tx0`; clipped to the display window. + if y >= lh || x0.checked_add(count).is_none_or(|end| end > lw) { + continue; + } + let Some(ty) = usize::try_from(y as i64 + self.offset.1).ok().filter(|ty| *ty < h) else { continue }; + let tx0 = x0 as i64 + self.offset.0; + let skip = usize::try_from(-tx0).unwrap_or(0).min(count); + let tx = usize::try_from(tx0.max(0)).unwrap_or(usize::MAX); + let take = (count - skip).min(w.saturating_sub(tx)); + let start = ty.checked_mul(w).and_then(|row| row.checked_add(tx)); + let Some(dst) = start.filter(|_| take > 0).and_then(|s| plane.get_mut(s..s.checked_add(take)?)) else { continue }; + match kind { + SampleType::F16 => { + self.f16s.resize(count, exr::prelude::f16::ZERO); + line.read_samples_into_slice(&mut self.f16s)?; + for (d, s) in dst.iter_mut().zip(self.f16s.iter().skip(skip)) { + *d = s.to_f32(); + } + } + SampleType::F32 => { + self.f32s.resize(count, 0.0); + line.read_samples_into_slice(&mut self.f32s)?; + for (d, s) in dst.iter_mut().zip(self.f32s.iter().skip(skip)) { + *d = *s; + } + } + SampleType::U32 => { + self.u32s.resize(count, 0); + line.read_samples_into_slice(&mut self.u32s)?; + for (d, s) in dst.iter_mut().zip(self.u32s.iter().skip(skip)) { + *d = *s as f32; + } + } + } + } + Ok(()) + } + + fn into_channels(self) -> PartPlanes { + // Positions of the kept channels among the kept planes. + let mut at = vec![None; self.planes.len()]; + let mut planes = Vec::with_capacity(self.planes.len()); + for (i, p) in self.planes.into_iter().enumerate() { + if let Some((name, _, data)) = p { + if let Some(slot) = at.get_mut(i) { + *slot = Some(planes.len()); + } + planes.push((name, data)); + } + } + let kept = |i: usize| at.get(i).copied().flatten(); + let has_rgb = self.rgb.is_some(); + let rgb = self.rgb.and_then(|([r, g, b], a)| { + let a = match a { + Some(a) => Some(kept(a)?), + None => None, + }; + Some(([kept(r)?, kept(g)?, kept(b)?], a)) + }); + PartPlanes { planes, rgb, has_rgb, exact: !self.subsampled && !self.own_display } + } +} + +/// How the `image` crate's OpenEXR decoder would show a file: the first part with exact `R`, `G` +/// and `B` channels (alpha `A` when it has one, else opaque), placed in the display window with +/// 0 outside the data window. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Rgb { + /// No part has R, G and B: the decoder refuses the file (a multi-layer file, shown with + /// [`layered_image`]'s rule). + None, + /// The channels (indices in [`AuxChannels::channels`]) hold exactly what it would show. + Planes([usize; 3], Option), + /// The planes can't stand in for it (subsampled channels, a depth-filled plane, another + /// display window): decode with the `image` crate. + Unsupported, +} + +/// A decoded file: its channels and how its picture is made of them. +pub(crate) struct Decoded { + pub(crate) aux: Arc, + rgb: Rgb, +} + +impl Decoded { + /// Bytes held (the planes; names and manifests are small). + fn bytes(&self) -> usize { + self.aux.channels.iter().map(|(n, p)| n.len() + std::mem::size_of_val(p.as_slice())).sum() + } +} + +/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). `None` when it +/// is not a readable EXR. +fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let display = meta.headers.first()?.shared_attributes.display_window; + let read = ReadImage::new(ReadAllLayers { read_channels: Planes { keep, display } }, |_| {}); + let img = read.from_buffered(std::io::Cursor::new(bytes)).ok()?; + let dw = img.attributes.display_window; + let mut aux = AuxChannels::new(dw.size.0 as u32, dw.size.1 as u32, 1.0); let mut crypto: HashMap, Option)> = HashMap::new(); - let mut collect_meta = |other: &HashMap| { + let mut collect_meta = |other: &HashMap| { for (k, v) in other { let key = k.to_string(); let Some(rest) = key.strip_prefix("cryptomatte/") else { continue }; let Some((id, field)) = rest.split_once('/') else { continue }; - let AttributeValue::Text(t) = v else { continue }; + let exr::meta::attribute::AttributeValue::Text(t) = v else { continue }; let e = crypto.entry(id.to_string()).or_default(); match field { "name" => e.0 = Some(t.to_string()), @@ -79,24 +283,22 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } }; collect_meta(&img.attributes.other); - // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a - // multi-gigabyte allocation per channel (#2141). - let n = w.checked_mul(h).filter(|n| *n <= MAX_DISPLAY_PIXELS)?; - // The decoded image is consumed channel by channel, so each channel's samples are freed (or - // moved) as its plane is made instead of the whole file being held twice (#474). + let mut rgb = Rgb::None; for layer in img.layer_data { collect_meta(&layer.attributes.other); - let prefix = layer.attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()); - let size = (layer.size.0, layer.size.1); - let pos = layer.attributes.layer_position; - let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); - for ch in layer.channel_data.list { - let name = channel_name(prefix.as_deref(), &ch.name.to_string()); - let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; - if let Some(plane) = place(ch.sample_data, size, offset, (w, h), n, fill) { - aux.channels.push((name, plane)); - } + let part = layer.channel_data; + if rgb == Rgb::None && part.has_rgb { + let base = aux.channels.len(); + rgb = match part.rgb { + Some(([r, g, b], a)) if part.exact => { + let picked = [Some(r), Some(g), Some(b), a]; + let depth = picked.iter().flatten().any(|i| part.planes.get(*i).is_some_and(|(n, _)| is_depth(n))); + if depth { Rgb::Unsupported } else { Rgb::Planes([base + r, base + g, base + b], a.map(|a| base + a)) } + } + _ => Rgb::Unsupported, + }; } + aux.channels.extend(part.planes); } for (_, (name, manifest)) in crypto { if let (Some(n), Some(m)) = (name, manifest) { @@ -104,46 +306,13 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } } aux.manifests.sort(); - Some(aux) + Some(Decoded { aux: Arc::new(aux), rgb }) } -/// One channel's samples (`size`, at `offset` in the `display` window of `n` pixels) as a -/// display-window plane, `fill` outside its data window. A channel covering exactly the display -/// window is converted in one pass, and an f32 one is moved without a copy. `None` when the -/// channel has fewer samples than its size. -fn place(samples: exr::image::FlatSamples, size: (usize, usize), offset: (i64, i64), display: (usize, usize), n: usize, fill: f32) -> Option> { - use exr::image::FlatSamples; - let (lw, lh) = size; - if samples.len() < lw.checked_mul(lh)? { - return None; - } - if size == display && offset == (0, 0) { - return Some(match samples { - FlatSamples::F32(mut v) => { - v.truncate(n); - v - } - FlatSamples::F16(v) => v.iter().take(n).map(|s| s.to_f32()).collect(), - FlatSamples::U32(v) => v.iter().take(n).map(|s| *s as f32).collect(), - }); - } - let value = |i: usize| match &samples { - FlatSamples::F32(v) => v.get(i).copied(), - FlatSamples::F16(v) => v.get(i).map(|s| s.to_f32()), - FlatSamples::U32(v) => v.get(i).map(|s| *s as f32), - }; - let (w, h) = display; - let mut plane = vec![fill; n]; - for y in 0..lh { - let Some(ty) = usize::try_from(y as i64 + offset.1).ok().filter(|ty| *ty < h) else { continue }; - for x in 0..lw { - let Some(tx) = usize::try_from(x as i64 + offset.0).ok().filter(|tx| *tx < w) else { continue }; - if let (Some(v), Some(p)) = (value(y * lw + x), plane.get_mut(ty * w + tx)) { - *p = v; - } - } - } - Some(plane) +/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. +pub fn read_exr_channels(bytes: &[u8]) -> Option { + let aux = decode(bytes, &|_| true)?.aux; + Some(Arc::try_unwrap(aux).unwrap_or_else(|a| (*a).clone())) } /// Names of the beauty pass, most preferred first, matched against the last part of a layer's @@ -186,6 +355,36 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { .copied() } +/// The planes a multi-layer file's picture is made of (see [`layered_image`]): red, green, blue +/// (one channel thrice for grey) and alpha, by name. Only the names of `aux` are read. +fn picture_names(aux: &AuxChannels) -> Option<([String; 3], Option)> { + let layers = aux.layers(); + let colour = picture_layer(aux, &layers)?; + let names = aux.layer_rgba(colour); + // A layer without R, G and B channels shows its first channel as grey. + let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); + let pick = |k: usize| names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first).to_string(); + let a = names.get(3).filter(|n| !n.is_empty() && aux.get(n).is_some()).cloned(); + Some(([pick(0), pick(1), pick(2)], a)) +} + +/// The channels of the picture of a multi-layer file and nothing else (names first from the +/// headers, then only those channels decoded; in a multi-part file the other parts' pixels are +/// not even decompressed), with the names of its planes. +fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let mut names = AuxChannels::new(0, 0, 1.0); + for header in meta.headers.iter() { + let prefix = part_name(header); + names.channels.extend(header.channels.list.iter().map(|ch| (channel_name(prefix.as_deref(), &ch.name.to_string()), vec![]))); + } + let (rgb, a) = picture_names(&names)?; + // Planes are looked up by name ignoring case, so every channel that could answer is kept. + let keep = |n: &str| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(n)); + let decoded = decode(bytes, &keep)?; + Some((decoded, rgb, a)) +} + /// The picture of a multi-layer OpenEXR file without an unnamed RGB layer (Blender's /// `ViewLayer.Combined.R`, the compositor's `Image.R`, Nuke's `beauty.R`, …), which the image /// decoder rejects. After Effects shows a file's main RGBA channels and leaves the other layers @@ -193,26 +392,13 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { /// that is not a data pass (never Cryptomatte, depth or normals, #412), else the first channel /// as grey. Linear, straight RGBA as the file stores it. `None` when it is not a readable EXR. pub fn layered_image(bytes: &[u8]) -> Option { - picture_of(&read_exr_channels(bytes)?) -} - -/// [`layered_image`] of the EXR at `path` from the channel cache, so the picture and EXtractoR / -/// Cryptomatte share one decode of the file (#474). -pub(crate) fn cached_layered_image(path: &str, bytes: Arc<[u8]>) -> Option { - let aux = cached(path, || Some(bytes))?; - picture_of(&aux) -} - -/// The picture of decoded channels (see [`layered_image`]). -fn picture_of(aux: &AuxChannels) -> Option { - let layers = aux.layers(); - let colour = picture_layer(aux, &layers)?; - let names = aux.layer_rgba(colour); - // A layer without R, G and B channels shows its first channel as grey. - let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); - let plane = |k: usize| -> Option<&[f32]> { aux.get(names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first)) }; - let (r, g, b) = (plane(0)?, plane(1)?, plane(2)?); - let a = names.get(3).filter(|n| !n.is_empty()).and_then(|n| aux.get(n)); + let (d, rgb, a) = picture_only(bytes)?; + let aux = &d.aux; + let [r, g, b] = [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?]; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; let n = (aux.width as usize).checked_mul(aux.height as usize)?; let mut data = Vec::with_capacity(n.checked_mul(4)?); for i in 0..n { @@ -227,40 +413,214 @@ fn picture_of(aux: &AuxChannels) -> Option { } } -type Cache = Mutex>)>>; +/// The picture made of planes `rgb` and `a` (opaque without one) of `aux`, as +/// [`crate::convert::dynamic_to_image`] makes it of the same values. `None` when a plane is +/// missing or short. +fn planes_picture(aux: &AuxChannels, rgb: [&[f32]; 3], a: Option<&[f32]>, op: AlphaOp) -> Option { + let n = (aux.width as usize).checked_mul(aux.height as usize)?; + if rgb.iter().chain(&a).any(|p| p.len() < n) { + return None; + } + let [r, g, b] = rgb; + let at = |p: &[f32], i: usize| p.get(i).copied().unwrap_or(0.0); + Some(float_to_image(aux.width, aux.height, op, |i| [at(r, i), at(g, i), at(b, i), a.map_or(1.0, |a| at(a, i))])) +} -fn cache() -> &'static Cache { - static CACHE: OnceLock = OnceLock::new(); - CACHE.get_or_init(|| Mutex::new(Vec::new())) +/// The picture of a decoded file by [`layered_image`]'s rule. +fn layered_picture_of(aux: &AuxChannels, op: AlphaOp) -> Option { + let (rgb, a) = picture_names(aux)?; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the multi-layer EXR at `path` ([`layered_image`]'s rule) as an [`Image`]: +/// from the channel cache when `shared` (an effect reads the footage's channels, so they are +/// decoded anyway), else of the picture's channels alone. +pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Option { + if shared { + let d = cache().get(path, || Some(bytes))?; + return layered_picture_of(&d.aux, op); + } + let (d, rgb, a) = picture_only(&bytes)?; + let aux = &d.aux; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the EXR at `path` from the channel cache (#474: EXtractoR / Cryptomatte read +/// this footage's channels, so the picture shares their decode): what the `image` crate's +/// decoder shows for a file it reads, [`layered_image`]'s rule for one it refuses. `None` when +/// the planes can't stand in for the `image` crate's picture (decode with it), or the file is +/// not a readable EXR. +pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Option { + let d = cache().get(path, || Some(bytes))?; + let aux = &d.aux; + match d.rgb { + Rgb::None => layered_picture_of(aux, op), + Rgb::Planes([r, g, b], a) => { + let plane = |i: usize| aux.channels.get(i).map(|(_, p)| p.as_slice()); + let a = match a { + Some(a) => Some(plane(a)?), + None => None, + }; + planes_picture(aux, [plane(r)?, plane(g)?, plane(b)?], a, op) + } + Rgb::Unsupported => None, + } +} + +/// How many bytes of decoded channels the cache keeps, like the frame cache's default +/// ([`crate::DEFAULT_BUDGET`]): two 2560×1440 files of 32 channels (470 MB each, as every plane +/// is f32). The newest file is kept whatever its size, so a frame's picture and its effects +/// share its decode. (Measured on such a sequence decoded one frame at a time: 1 GiB held peak +/// memory about 1 GB lower than 2 GiB, at the same speed.) +#[cfg(not(target_arch = "wasm32"))] +pub const CACHE_BUDGET: usize = 1 << 30; +#[cfg(target_arch = "wasm32")] +pub const CACHE_BUDGET: usize = 512 << 20; + +/// At most this many files are cached (unreadable ones hold no bytes). +const CACHE_FILES: usize = 64; + +/// A file's decode: done once, by the first caller; the others wait for it. +type Slot = Arc>>>; + +struct Entry { + path: String, + slot: Slot, + /// Bytes of the finished decode (0 while decoding). + bytes: usize, + /// When it was last asked for (a counter). + used: u64, +} + +/// Decoded EXR files by path, bounded by bytes, least recently used out first. +pub(crate) struct ChannelCache { + entries: Mutex<(Vec, u64)>, + budget: usize, + /// Files decoded (for tests). + decodes: AtomicU64, +} + +impl ChannelCache { + pub(crate) const fn new(budget: usize) -> Self { + Self { entries: Mutex::new((Vec::new(), 0)), budget, decodes: AtomicU64::new(0) } + } + + /// The decoded file at `path`, read with `read` by the first caller; callers asking for the + /// same file meanwhile wait for that decode instead of decoding it again. + pub(crate) fn get(&self, path: &str, read: impl FnOnce() -> Option>) -> Option> { + let slot = { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + match entries.iter_mut().find(|e| e.path == path) { + Some(e) => { + e.used = *clock; + e.slot.clone() + } + None => { + let slot = Slot::default(); + entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: *clock }); + slot + } + } + }; + let mut decoded_here = false; + let v = slot + .get_or_init(|| { + decoded_here = true; + self.decodes.fetch_add(1, Ordering::Relaxed); + read().and_then(|b| decode(&b, &|_| true)).map(Arc::new) + }) + .clone(); + if decoded_here { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + // (gone when the file was forgotten meanwhile) Used as of now: a file asked for + // before another but decoded after it is not the older one, or concurrent frames + // would evict each other's files before their effects read them. + if let Some(e) = entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, &slot)) { + e.bytes = v.as_ref().map_or(0, |d| d.bytes()); + e.used = *clock; + } + Self::evict(entries, self.budget, &slot); + } + v + } + + /// Drop least recently used finished files until the rest fit the budget, keeping `newest`. + fn evict(entries: &mut Vec, budget: usize, newest: &Slot) { + while entries.iter().map(|e| e.bytes).sum::() > budget || entries.len() > CACHE_FILES { + let oldest = entries + .iter() + .enumerate() + .filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && e.slot.get().is_some()) + .min_by_key(|(_, e)| e.used) + .map(|(i, _)| i); + let Some(i) = oldest else { break }; + entries.remove(i); + } + } + + pub(crate) fn forget(&self, path: &str) { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.retain(|e| e.path != path); + } + + #[cfg(test)] + fn decodes(&self) -> u64 { + self.decodes.load(Ordering::Relaxed) + } + + #[cfg(test)] + fn cached(&self) -> Vec { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.iter().map(|e| e.path.clone()).collect() + } +} + +fn cache() -> &'static ChannelCache { + static CACHE: ChannelCache = ChannelCache::new(CACHE_BUDGET); + &CACHE } /// Drop the cached channels of `path` (the file changed on disk). pub(crate) fn forget(path: &str) { - if let Ok(mut c) = cache().lock() { - c.retain(|(p, _)| p != path); - } + cache().forget(path); } -/// Read (and cache, a few files) the channels of the EXR at `path` via `read`. +/// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { - let cache = cache(); - if let Some(v) = cache.lock().ok().and_then(|c| c.iter().find(|(p, _)| p == path).map(|(_, v)| v.clone())) { - return v; - } - let v = read().and_then(|b| read_exr_channels(&b)).map(Arc::new); - if let Ok(mut c) = cache.lock() { - if c.len() >= 4 { - c.remove(0); - } - c.push((path.to_string(), v.clone())); - } - v + cache().get(path, read).map(|d| d.aux.clone()) } #[cfg(test)] mod tests { use super::*; + /// An EXR file of `layers` (with their display window `display`). + fn file(layers: Vec>>, display: exr::prelude::IntegerBounds) -> Vec { + use exr::prelude::*; + let image = Image::from_layers(ImageAttributes::new(display), layers); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() + } + + /// A single-part file of float channels named `names`, `w`×`h`, every sample `v`. + fn flat(names: &[&str], (w, h): (usize, usize), v: f32) -> Vec { + use exr::prelude::*; + let list: Vec> = names.iter().map(|n| AnyChannel::new(*n, FlatSamples::F32(vec![v; w * h]))).collect(); + let layer = Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())); + file(vec![layer], IntegerBounds::from_dimensions((w, h))) + } + #[test] fn reads_layered_channels_and_cryptomatte_metadata() { use exr::prelude::*; @@ -294,53 +654,229 @@ mod tests { assert!(read_exr_channels(b"not an exr").is_none()); } - /// #474: a full-window f32 channel (Cryptomatte ranks, …) becomes its plane without a copy, - /// so a decode no longer holds the file's samples twice; other channels are placed in the - /// display window as before. + /// #474: each line of samples goes straight into its display-window plane: a data window + /// inside the display window is placed with the fill around it (the background depth for + /// depth), one partly outside is clipped, never out of bounds; half and uint samples are + /// converted as the file stores them. #[test] - fn channels_become_planes_without_an_extra_copy() { - use exr::image::FlatSamples; - let v = vec![0.25f32; 6]; - let ptr = v.as_ptr(); - let plane = place(FlatSamples::F32(v), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(plane.as_ptr(), ptr, "moved, not copied"); - assert_eq!(plane, [0.25; 6]); - let half = |x: f32| exr::prelude::f16::from_f32(x); - let full = place(FlatSamples::F16(vec![half(0.5); 6]), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(full, [0.5; 6]); - // A 2×1 data window at (1, 1) of a 3×2 display window; the rest is the fill. - let part = place(FlatSamples::F16(vec![half(1.0), half(2.0)]), (2, 1), (1, 1), (3, 2), 6, 9.0).unwrap(); - assert_eq!(part, [9.0, 9.0, 9.0, 9.0, 1.0, 2.0]); - // Partly outside the display window: clipped, never out of bounds. - let off = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, -1), (3, 2), 6, 0.0).unwrap(); - assert_eq!(off, [0.0; 6]); - let edge = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(edge, [0.0, 0.0, 7.0, 0.0, 0.0, 0.0]); - assert!(place(FlatSamples::F32(vec![1.0; 5]), (3, 2), (0, 0), (3, 2), 6, 0.0).is_none(), "too few samples"); + fn channels_are_placed_in_the_display_window() { + use exr::prelude::*; + let half = |x: f32| f16::from_f32(x); + let part = |pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = LayerAttributes::default(); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::from_dimensions((3, 2)); + // A 2×1 data window at (1, 1). + let inside = file( + vec![part((1, 1), (2, 1), vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))])], + display, + ); + let aux = read_exr_channels(&inside).unwrap(); + assert_eq!(aux.get("G").unwrap(), [0.0, 0.0, 0.0, 0.0, 1.0, 2.0]); + let bg = BACKGROUND_DEPTH; + assert_eq!(aux.get("Z").unwrap(), [bg, bg, bg, bg, 7.0, 8.0]); + // A 2×2 data window at (2, -1): only its bottom-left sample lands at (2, 0). + let off = file(vec![part((2, -1), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0, 2.0, 3.0, 4.0]))])], display); + assert_eq!(read_exr_channels(&off).unwrap().get("G").unwrap(), [0.0, 0.0, 3.0, 0.0, 0.0, 0.0]); + // Entirely outside: all fill. + let away = file(vec![part((-5, 0), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0; 4]))])], display); + assert_eq!(read_exr_channels(&away).unwrap().get("G").unwrap(), [0.0; 6]); + } + + /// #474: concurrent requests for a file that isn't cached decode it once; the others wait + /// for that decode and share its planes. + #[test] + fn concurrent_requests_decode_a_file_once() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "Z"], (64, 64), 0.5).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let reads = AtomicU64::new(0); + let barrier = std::sync::Barrier::new(8); + let got: Vec> = std::thread::scope(|s| { + let jobs: Vec<_> = (0..8) + .map(|_| { + s.spawn(|| { + barrier.wait(); + cache + .get("/474-concurrent.exr", || { + reads.fetch_add(1, Ordering::Relaxed); + // Long enough for the others to arrive while this one decodes. + std::thread::sleep(std::time::Duration::from_millis(50)); + Some(bytes.clone()) + }) + .unwrap() + }) + }) + .collect(); + jobs.into_iter().map(|j| j.join().unwrap()).collect() + }); + assert_eq!(cache.decodes(), 1, "one decode"); + assert_eq!(reads.load(Ordering::Relaxed), 1, "the file is read once"); + assert!(got.iter().all(|d| Arc::ptr_eq(d, &got[0])), "every caller has the same planes"); + assert_eq!(got[0].aux.get("Z").and_then(|z| z.first()).copied(), Some(0.5)); + } + + /// #474: the cache is bounded by the bytes of its planes, least recently used out first; the + /// newest file stays even when it alone is over the budget, and unreadable files are cached + /// (as such) without counting bytes. + #[test] + fn cache_evicts_least_recently_used_by_bytes() { + // 4 float planes of 32×32: 16 KiB of samples (plus a few bytes of names) per file. + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + let get = |p: &str| cache.get(p, || Some(bytes.clone())).unwrap(); + assert_eq!(get("/a.exr").bytes(), one); + get("/b.exr"); + get("/a.exr"); // a is now more recent than b + get("/c.exr"); + assert_eq!(cache.cached(), ["/a.exr", "/c.exr"], "b was least recently used"); + assert_eq!(cache.decodes(), 3); + get("/a.exr"); + assert_eq!(cache.decodes(), 3, "a hit"); + // A file over the budget on its own is still kept while it is the newest. + let small = ChannelCache::new(one / 2); + assert!(small.get("/big.exr", || Some(bytes.clone())).is_some()); + assert_eq!(small.cached(), ["/big.exr"]); + small.get("/big2.exr", || Some(bytes.clone())); + assert_eq!(small.cached(), ["/big2.exr"]); + // An unreadable file is remembered (not read again) without bytes, and forgotten on request. + assert!(cache.get("/broken.exr", || Some(Arc::from(&b"not an exr"[..]))).is_none()); + assert!(cache.get("/broken.exr", || panic!("not read again")).is_none()); + cache.forget("/broken.exr"); + assert!(!cache.cached().contains(&"/broken.exr".to_string())); + // At most CACHE_FILES files, however small. + for i in 0..CACHE_FILES + 10 { + cache.get(&format!("/none{i}.exr"), || None); + } + assert!(cache.cached().len() <= CACHE_FILES); + } + + /// #474: a file counts as used when its decode finishes: one asked for first but decoded + /// after another is the newer of the two, so concurrent frames don't evict each other's + /// files before their effects read them. + #[test] + fn a_file_is_used_as_of_its_decode() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + cache.get("/first.exr", || { + // Another frame's file is asked for and decoded while this one decodes. + cache.get("/second.exr", || Some(bytes.clone())); + Some(bytes.clone()) + }); + cache.get("/third.exr", || Some(bytes.clone())); + assert_eq!(cache.cached(), ["/first.exr", "/third.exr"], "the file decoded first went first"); + } + + /// #474: a file forgotten (changed on disk) while it decodes isn't cached under its path when + /// that decode finishes. + #[test] + fn a_file_forgotten_while_it_decodes_is_not_cached() { + let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let d = cache.get("/changed.exr", || { + cache.forget("/changed.exr"); + Some(bytes.clone()) + }); + assert!(d.is_some()); + assert!(cache.cached().is_empty()); } /// #295: a multi-layer EXR whose colour is only in named layers (`diffuse.R`, as Blender and /// Nuke write them) failed to import ("does not contain non-deep rgb channels"); it now /// shows its colour layer and keeps every channel for EXtractoR. - /// #474: the picture and EXtractoR / Cryptomatte share one decode of a multi-layer file: - /// after the frame is decoded, the aux channels come from the cache without reading the file. + /// #474: once an effect has read a footage item's channels, the picture of each later frame + /// shares one decode with them: the frame's decode is cached and the channels come from it + /// without reading the file again. #[test] fn picture_and_aux_channels_share_one_decode() { use exr::prelude::*; let (w, h) = (2usize, 2usize); - let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); - let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", 0.25)].into()); - let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); - let mut bytes = std::io::Cursor::new(Vec::new()); - image.write().to_buffered(&mut bytes).unwrap(); - let b: Arc<[u8]> = bytes.into_inner().into(); - let path = "/474-shared-decode.exr"; + let frame = |v: f32| -> Arc<[u8]> { + let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); + let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", v)].into()); + let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner().into() + }; + let paths = ["/474-shared-decode_0001.exr", "/474-shared-decode_0002.exr"]; let pool = crate::MediaPool::new(); - pool.add_bytes(path, b.clone()); - let f = crate::probe_bytes(path, b).unwrap(); + pool.add_bytes(paths[0], frame(0.25)); + pool.add_bytes(paths[1], frame(0.75)); + let mut f = crate::probe_bytes(paths[0], frame(0.25)).unwrap(); + f.kind = effectcraft_project::FootageKind::Sequence; + f.sequence = paths.iter().map(|p| p.to_string()).collect(); + let t1 = f.frame_rate.tick_of(1); + use effectcraft_render::FootageSource; + // The first frame: the picture, then an effect reads the channels. pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap(); - let aux = cached(path, || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + let aux = pool.aux(effectcraft_project::ItemId(1), &f, effectcraft_time::Tick::ZERO).unwrap(); assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); + // The next frame's picture decodes the file into the cache, so its channels are not read. + let px = pool.frame_at(&f, t1).unwrap().get(0, 0); + let aux = cached(paths[1], || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.75)); + assert_eq!(px, pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap().get(0, 0), "the same picture"); + } + + /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the + /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 + /// around it), in every alpha interpretation and float curve; a file it can't stand in for + /// (a subsampled channel) is left to the `image` crate. + #[test] + fn picture_from_planes_is_the_image_crates() { + use effectcraft_project::{AlphaMode, Footage}; + use effectcraft_render::FloatCurve; + use exr::prelude::*; + let half = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F16((0..len).map(|i| f16::from_f32(v + i as f32 * 0.37 - 1.0)).collect())); + let float = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F32((0..len).map(|i| ((i * 7) % 11) as f32 * v).collect())); + let part = |name: Option<&str>, pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::SMALL_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::new((1, -1), (5, 4)); + let n = 20; + let files = [ + file(vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], display), + file(vec![part(None, (0, 0), (3, 2), vec![float("R", 0.4, 6), float("G", 0.5, 6), float("B", 0.6, 6), half("Image.R", 0.1, 6)])], display), + file( + vec![ + part(Some("diffuse"), (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n)]), + part(Some("beauty"), (2, 0), (2, 2), vec![half("R", 0.4, 4), half("G", 0.5, 4), half("B", 0.6, 4), half("A", 0.5, 4)]), + ], + display, + ), + ]; + let bits = |i: &effectcraft_raster::Image| i.data.iter().flat_map(|p| p.map(f32::to_bits)).collect::>(); + for (k, bytes) in files.iter().enumerate() { + let path = format!("/474-planes-{k}.exr"); + let d = cache().get(&path, || Some(bytes.as_slice().into())).unwrap(); + assert!(matches!(d.rgb, Rgb::Planes(..)), "file {k}: {:?}", d.rgb); + let img = crate::probe::load_oriented(bytes, Some(image::ImageFormat::OpenExr)).unwrap(); + for alpha in [AlphaMode::Straight, AlphaMode::Premultiplied, AlphaMode::Ignore] { + for curve in [FloatCurve::Srgb, FloatCurve::Linear, FloatCurve::Gamma24] { + let op = AlphaOp::of(&Footage { alpha, premul_color: [0.2, 0.4, 0.6], ..Default::default() }, curve); + let want = crate::convert::dynamic_to_image(&img, op); + let got = cached_picture(&path, bytes.as_slice().into(), op).unwrap(); + assert_eq!((got.width, got.height), (want.width, want.height)); + assert_eq!(bits(&got), bits(&want), "file {k} {alpha:?} {curve:?}"); + } + } + } + // A subsampled channel in the picture's part: the `image` crate decodes it. + let mut sub = part(None, (0, 0), (4, 2), vec![half("R", 0.1, 8), half("G", 0.2, 8), half("B", 0.3, 8), half("C", 0.3, 2)]); + if let Some(c) = sub.channel_data.list.iter_mut().find(|c| c.name == *"C") { + c.sampling = Vec2(2, 2); + } + let mut bytes = std::io::Cursor::new(Vec::new()); + if Image::from_layer(sub).write().to_buffered(&mut bytes).is_ok() { + let d = cache().get("/474-subsampled.exr", || Some(bytes.into_inner().into())).map(|d| d.rgb); + assert!(d.is_none_or(|r| r == Rgb::Unsupported), "{d:?}"); + } } /// #2141: one pixel of data in a 60000² display window is refused, not allocated per channel. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 655ca4ed8..9f1f2aa23 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -245,6 +245,9 @@ struct Inner { /// Movies that failed to open or decode (path to a readable reason; see /// [`FootageSource::problem`]). The first failure of a file is logged, later ones are not. problems: Mutex, String>>, + /// OpenEXR footage (by path) whose channels an effect has read (EXtractoR, Cryptomatte, the 3D + /// Channel effects): its pictures come from the same cached decode (#474). + aux_footage: Mutex>, } /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus @@ -315,6 +318,7 @@ impl MediaPool { conforming: Mutex::default(), conform_failed: Mutex::default(), problems: Mutex::default(), + aux_footage: Mutex::default(), }), } } @@ -856,13 +860,19 @@ impl Inner { { return Err(MediaError::Decode(format!("{path}: {why}"))); } - let loaded = crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)); - let img = match loaded { - Ok(img) => img, + // An effect reads this footage's channels: the picture is made of the same decode (#474). + let shared = lock(&self.aux_footage).contains(&footage.path); + if shared + && path.to_ascii_lowercase().ends_with(".exr") + && let Some(img) = crate::exr_channels::cached_picture(path, bytes.clone(), op) + { + return Ok(img); + } + match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { + Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. - Err(e) => crate::exr_channels::cached_layered_image(path, bytes).ok_or_else(|| MediaError::Decode(format!("{path}: {e}")))?, - }; - Ok(dynamic_to_image(&img, op)) + Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared).ok_or_else(|| MediaError::Decode(format!("{path}: {e}"))), + } } fn decode_media(&self, loc: &Loc, footage: &Footage) -> Result { @@ -962,6 +972,14 @@ impl FootageSource for MediaPool { return None; } let path = loc.key.path.clone(); + { + let mut wanted = lock(&self.inner.aux_footage); + // (a bound: names of footage items, never cleared otherwise) + if wanted.len() >= 4096 { + wanted.clear(); + } + wanted.insert(footage.path.clone()); + } crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) } diff --git a/docs/effects-parity.md b/docs/effects-parity.md index bd81a09df..ce4debd01 100644 --- a/docs/effects-parity.md +++ b/docs/effects-parity.md @@ -26,7 +26,7 @@ corpus compares renders, each effect is assumed to differ somewhere, and users k | Category | Effects | On the GPU | Known differences or bugs | |---|---:|---:|---| -| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR decode shared (#474 open: f16 storage and cache sizing remain) | +| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache bounded at 1 GB (#474 open: f16 storage remains) | | Audio | 13 | 0 | CPU by design (applied in the mixdown) | | Blur & Sharpen | 16 | 15 | Radial Blur keeps the layer edges as After Effects 2026 (713 now pixel-identical); Gaussian Blur interaction and output to be measured first ([fidelity/ae-2026-acceptance.md](fidelity/ae-2026-acceptance.md)) | | Channel | 13 | 13 | — | diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 6eeef1719..741582d9e 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); EXRs without an unnamed RGB layer decode twice (#474) | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, footage alone only its picture's channels (#474; decoded channels are held as f32) | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index f7f1df21b..0ebbeeecc 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -120,6 +120,11 @@ Nuke's `beauty`) show the beauty pass: the layer named Combined, Beauty, Image, Color, else the first colour layer that isn't a data pass. Cryptomatte, depth, mist, normals, vectors, positions and IDs are never the picture. EXtractoR reads every other layer and channel. +Footage on its own decodes only the channels of its picture. Once an effect reads an item's +channels (EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for +the picture and the effects together, and kept for a while (up to 1 GB of decoded channels); +several frames asking for the same file at once wait for one decode. + ## Replacing and reloading Importing a layered file as a composition refreshes the decoded sources used by its layers, diff --git a/docs/gaps.md b/docs/gaps.md index 016aeff22..46a96983a 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -65,7 +65,7 @@ are grouped by area; the issue tracker has the rest. | 6 | **Linux and Windows interaction bugs** | Open: #488 rename arrows (hardening landed, reporter asked for details), #492 anchor drag, #268 file drops on Wayland (winit), #613 Direct3D 12 compositor. Now closed: #491, #493, #467, #320, #302, #596 | Many reports come from Linux; some Windows GPUs fail to start the compositor | 25–45 h | G2 · [ui-parity.md](ui-parity.md) | | 7 | **Animation presets: 76 against 621** | After Effects ships 621 `.ffx` presets (Behaviors, Backgrounds, Shapes, Synthetics, Text, Transitions…); we ship 76 original presets (8 text-animator; 23 Transitions: wipes, clock and fan reveals, blinds, dissolves, card flips, CC wipes, fades, #757; 23 Backgrounds: drifting gradients, fog, cloud and smoke, plasma and magma, cells, hexagons, scrolling grids and checks, pulse rings, bokeh, light sweep, #1096, driven by expressions so they keep moving, not seamless loops; 22 Behaviors: wiggle, shake, float, orbit, bounce, drift, pendulum, spin, pulse, heartbeat, blink, strobe, flicker on transform and opacity, #1510) and read only our `.ecpreset`. Missing: Shapes, Synthetics, Image and more Text | Presets are how many designers start; tutorials assume them | 15–35 h (original presets; `.ffx` reading is out of clean-room scope) | G8 · [effects-parity.md](effects-parity.md) | | 8 | **Mask and shape bugs** | Open: #1004 mask vertex multi-select, #999 auto-pan, #1042 / #1033 vertex nodes under the bounding box, #959 / #960 shape group toggles and operators. Now closed: #510, #513, #527, #509, #303, #553, #932 (Convert To Bezier Path on a shape layer's parametric path) | Masks are core compositing | 10–20 h | G2 · [target-app-parity.md](target-app-parity.md) | -| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (decode shared, faffad3a; f16 storage and cache sizing open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | +| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | | 10 | **Colour pipeline differences** | #340 ProRes 4444 tagged bt709 but sRGB-encoded (open); ACES 2.0 / OCIO 2.5 configs (26.5) missing; now closed: #494 32-bpc blend modes, #495 and #750 Working Gamma 2.4, #546 / #547 viewer colour | Wrong colour in delivery is a client-visible failure | 15–30 h | G1 · [file-format-parity.md](file-format-parity.md) | | 11 | **Text rendering and layout** | #453 CJK text in comps: the Han fallback now reaches the installed Chinese faces; open: #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | From 2939b3769cb98274af5c7c83a2e302656fba820e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 02:18:30 +0100 Subject: [PATCH 05/25] #474: format the OpenEXR golden test Formatting only; no hash changes. Co-Authored-By: Claude Opus 5.5 --- crates/media/tests/it/exr_golden.rs | 34 ++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs index 498944882..f94b3cac0 100644 --- a/crates/media/tests/it/exr_golden.rs +++ b/crates/media/tests/it/exr_golden.rs @@ -95,7 +95,14 @@ fn files() -> Vec<(&'static str, Vec, bool)> { let origin = ((0, 0), full); let cat = |a: &[(&'static str, Kind)], b: &[(&'static str, Kind)]| -> Vec<(&'static str, Kind)> { a.iter().chain(b).copied().collect() }; let layered: Vec<(&str, Kind)> = cat( - &[("Image.R", Kind::Half), ("Image.G", Kind::Half), ("Image.B", Kind::Half), ("Image.A", Kind::Half), ("depth.Z", Kind::Float), ("ObjectID", Kind::Uint)], + &[ + ("Image.R", Kind::Half), + ("Image.G", Kind::Half), + ("Image.B", Kind::Half), + ("Image.A", Kind::Half), + ("depth.Z", Kind::Float), + ("ObjectID", Kind::Uint), + ], &CRYPTO, ); let unnamed_plus: Vec<(&str, Kind)> = @@ -103,7 +110,11 @@ fn files() -> Vec<(&'static str, Vec, bool)> { let rgb_float = [("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("depth.Z", Kind::Float)]; vec![ ("rgba_half_zip16", write(vec![part(None, &RGBA_HALF, full, (0, 0), Compression::ZIP16)], origin), true), - ("rgb_float_piz", write(vec![part(None, &[("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("Z", Kind::Float)], full, (0, 0), Compression::PIZ)], origin), true), + ( + "rgb_float_piz", + write(vec![part(None, &[("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("Z", Kind::Float)], full, (0, 0), Compression::PIZ)], origin), + true, + ), ("layered_rle", write(vec![part(None, &layered, full, (0, 0), Compression::RLE)], origin), true), ("unnamed_rgb_plus_layers_zip1", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::ZIP1)], origin), true), ( @@ -112,7 +123,13 @@ fn files() -> Vec<(&'static str, Vec, bool)> { vec![ part(Some("Image"), &RGBA_HALF, full, (0, 0), Compression::ZIP16), part(Some("depth"), &[("Z", Kind::Float)], full, (0, 0), Compression::ZIP16), - part(Some("CryptoObject00"), &[("r", Kind::Float), ("g", Kind::Float), ("b", Kind::Float), ("a", Kind::Float)], full, (0, 0), Compression::ZIP16), + part( + Some("CryptoObject00"), + &[("r", Kind::Float), ("g", Kind::Float), ("b", Kind::Float), ("a", Kind::Float)], + full, + (0, 0), + Compression::ZIP16, + ), ], origin, ), @@ -138,7 +155,12 @@ fn files() -> Vec<(&'static str, Vec, bool)> { part(Some("ViewLayer.Depth"), &[("ViewLayer.Depth.Z", Kind::Float)], full, (0, 0), Compression::RLE), part( Some("ViewLayer.Combined"), - &[("ViewLayer.Combined.R", Kind::Half), ("ViewLayer.Combined.G", Kind::Half), ("ViewLayer.Combined.B", Kind::Half), ("ViewLayer.Combined.A", Kind::Half)], + &[ + ("ViewLayer.Combined.R", Kind::Half), + ("ViewLayer.Combined.G", Kind::Half), + ("ViewLayer.Combined.B", Kind::Half), + ("ViewLayer.Combined.A", Kind::Half), + ], full, (0, 0), Compression::RLE, @@ -194,7 +216,9 @@ fn aux_hash(pool: &MediaPool, f: &Footage) -> u64 { /// The interpretations a picture is decoded in: (label, footage, curve, size, pinned). fn variants(f: &Footage) -> Vec<(String, Footage, FloatCurve, Option<(u32, u32)>, bool)> { let mut out = vec![]; - for (mode, label) in [(None, "probed"), (Some(AlphaMode::Straight), "straight"), (Some(AlphaMode::Premultiplied), "premul"), (Some(AlphaMode::Ignore), "ignore")] { + for (mode, label) in + [(None, "probed"), (Some(AlphaMode::Straight), "straight"), (Some(AlphaMode::Premultiplied), "premul"), (Some(AlphaMode::Ignore), "ignore")] + { let mut g = f.clone(); if let Some(m) = mode { g.alpha = m; From d4816719474e4b1d5fa3ae6272f781333b421b57 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 02:20:15 +0100 Subject: [PATCH 06/25] #474: golden OpenEXR files for tiles, mip levels, line order, uncompressed and uint RGB Adds tiled (ZIP16, RLE), tiled mip-mapped (ZIP16, PIZ), decreasing line order, uncompressed, a uint R/G/B picture and a multi-part file whose first R/G/B part is depth-named (the planes can't stand in for the image crate's picture there). The new hashes were taken with main's crates/media/src at a646bdcc (extracted with git archive) and the same test, before the decode rework. Co-Authored-By: Claude Opus 5.5 --- crates/media/tests/it/exr_golden.rs | 156 +++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs index f94b3cac0..d99de635a 100644 --- a/crates/media/tests/it/exr_golden.rs +++ b/crates/media/tests/it/exr_golden.rs @@ -70,9 +70,36 @@ fn with_manifest(mut attrs: LayerAttributes) -> LayerAttributes { /// One part: `list` channels over a data window of `size` at `pos`. fn part(name: Option<&str>, list: &[(&str, Kind)], size: (usize, usize), pos: (i32, i32), compression: Compression) -> Layer> { + part_with(name, list, size, pos, encoding(compression)) +} + +/// [`part`] with its own block layout (tiles) and line order. +fn part_with(name: Option<&str>, list: &[(&str, Kind)], size: (usize, usize), pos: (i32, i32), encoding: Encoding) -> Layer> { let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); attrs.layer_position = Vec2(pos.0, pos.1); - Layer::new(size, with_manifest(attrs), encoding(compression), channels(list, size)) + Layer::new(size, with_manifest(attrs), encoding, channels(list, size)) +} + +/// A tiled single-part file of `list` with mip levels down to 1×1 (each level its own values); +/// the picture and the channels are its largest level. +fn mipmapped(list: &[(&str, Kind)], compression: Compression) -> Vec { + let levels = |name: &str, kind: Kind, c: usize| { + let (mut data, mut size) = (vec![], (W, H)); + loop { + data.push(channel(name, kind, size, c + data.len()).sample_data); + if size == (1, 1) { + break; + } + size = ((size.0 / 2).max(1), (size.1 / 2).max(1)); + } + AnyChannel::new(name, Levels::Mip { rounding_mode: exr::math::RoundingMode::Down, level_data: data.into() }) + }; + let list: Vec>> = list.iter().enumerate().map(|(c, (n, k))| levels(n, *k, c)).collect(); + let encoding = Encoding { compression, blocks: Blocks::Tiles(Vec2(16, 16)), line_order: LineOrder::Increasing }; + let layer = Layer::new((W, H), with_manifest(LayerAttributes::default()), encoding, AnyChannels::sort(list.into())); + let mut bytes = std::io::Cursor::new(Vec::new()); + Image::from_layer(layer).write().non_parallel().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() } /// Encode `layers` with the display window `display` (position, size). @@ -176,6 +203,63 @@ fn files() -> Vec<(&'static str, Vec, bool)> { ("layered_window_larger_piz", write(vec![part(None, &layered, (45, 30), (-4, -3), Compression::PIZ)], origin), true), ("unnamed_rgb_plus_layers_dwaa", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::DWAA(None))], origin), false), ("layered_dwaa", write(vec![part(None, &layered, full, (0, 0), Compression::DWAA(None))], origin), false), + ( + "unnamed_rgb_plus_layers_tiled_zip16", + write( + vec![part_with(None, &unnamed_plus, (30, 20), (3, 2), Encoding { blocks: Blocks::Tiles(Vec2(16, 8)), ..encoding(Compression::ZIP16) })], + origin, + ), + true, + ), + ( + "layered_tiled_rle", + write(vec![part_with(None, &layered, full, (0, 0), Encoding { blocks: Blocks::Tiles(Vec2(8, 8)), ..encoding(Compression::RLE) })], origin), + true, + ), + ("unnamed_rgb_plus_layers_mipmapped_zip16", mipmapped(&unnamed_plus, Compression::ZIP16), true), + ("layered_mipmapped_piz", mipmapped(&layered, Compression::PIZ), true), + ( + "unnamed_rgb_plus_layers_decreasing_zip1", + write( + vec![part_with(None, &unnamed_plus, (20, 10), (5, 7), Encoding { line_order: LineOrder::Decreasing, ..encoding(Compression::ZIP1) })], + origin, + ), + true, + ), + ( + "layered_decreasing_rle", + write(vec![part_with(None, &layered, full, (0, 0), Encoding { line_order: LineOrder::Decreasing, ..encoding(Compression::RLE) })], origin), + true, + ), + ("unnamed_rgb_plus_layers_uncompressed", write(vec![part(None, &unnamed_plus, (20, 10), (-3, 4), Compression::Uncompressed)], origin), true), + ("layered_uncompressed", write(vec![part(None, &layered, full, (0, 0), Compression::Uncompressed)], origin), true), + ( + "uint_rgb_zip16", + write( + vec![part( + None, + &[("R", Kind::Uint), ("G", Kind::Uint), ("B", Kind::Uint), ("A", Kind::Half), ("depth.Z", Kind::Float)], + full, + (0, 0), + Compression::ZIP16, + )], + origin, + ), + true, + ), + ( + // The image crate shows the part "depth" (the first with R, G and B), whose planes are + // filled with the background depth outside its data window: decoded by the image crate. + "multipart_depth_named_rgb_zip16", + write( + vec![ + part(Some("depth"), &[("R", Kind::Half), ("G", Kind::Half), ("B", Kind::Half)], (20, 10), (5, 7), Compression::ZIP16), + part(Some("beauty"), &RGBA_HALF, full, (0, 0), Compression::ZIP16), + ], + origin, + ), + true, + ), ] } @@ -331,6 +415,76 @@ const PINNED: &[(&str, &str, u64)] = &[ ("layered_window_larger_piz", "ignore/linear", 0xaa01706accce2cc4), ("layered_window_larger_piz", "preserve_rgb", 0xb2cbb40b10bc56b4), ("layered_window_larger_piz", "aux", 0xb7d91c8eea56a772), + ("unnamed_rgb_plus_layers_tiled_zip16", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_tiled_zip16", "probed/linear", 0x7612f20634fe957e), + ("unnamed_rgb_plus_layers_tiled_zip16", "straight/linear", 0xaa5f42cb6a31c78a), + ("unnamed_rgb_plus_layers_tiled_zip16", "premul/linear", 0x7612f20634fe957e), + ("unnamed_rgb_plus_layers_tiled_zip16", "ignore/linear", 0xbbf236635dcaac90), + ("unnamed_rgb_plus_layers_tiled_zip16", "preserve_rgb", 0x15b555a243d79613), + ("unnamed_rgb_plus_layers_tiled_zip16", "aux", 0x3d1439eed2202e41), + ("layered_tiled_rle", "probe", 0x0a32e074a28c1f66), + ("layered_tiled_rle", "probed/linear", 0xa84600283dce98f6), + ("layered_tiled_rle", "straight/linear", 0xf028e1367a2fbb30), + ("layered_tiled_rle", "premul/linear", 0xa84600283dce98f6), + ("layered_tiled_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_tiled_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_tiled_rle", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "probed/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "straight/linear", 0xf028e1367a2fbb30), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "premul/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "ignore/linear", 0xa90a57f9a67f7468), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "preserve_rgb", 0xc8e0050f5efe75b7), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "aux", 0x27cc9b53d0862db1), + ("layered_mipmapped_piz", "probe", 0x0a32e074a28c1f66), + ("layered_mipmapped_piz", "probed/linear", 0xa84600283dce98f6), + ("layered_mipmapped_piz", "straight/linear", 0xf028e1367a2fbb30), + ("layered_mipmapped_piz", "premul/linear", 0xa84600283dce98f6), + ("layered_mipmapped_piz", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_mipmapped_piz", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_mipmapped_piz", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_decreasing_zip1", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_decreasing_zip1", "probed/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_plus_layers_decreasing_zip1", "straight/linear", 0x2c73f9acaa936420), + ("unnamed_rgb_plus_layers_decreasing_zip1", "premul/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_plus_layers_decreasing_zip1", "ignore/linear", 0x3b86057d18a16bf0), + ("unnamed_rgb_plus_layers_decreasing_zip1", "preserve_rgb", 0x5b999014e4a64bb2), + ("unnamed_rgb_plus_layers_decreasing_zip1", "aux", 0x8f78d3233450acf3), + ("layered_decreasing_rle", "probe", 0x0a32e074a28c1f66), + ("layered_decreasing_rle", "probed/linear", 0xa84600283dce98f6), + ("layered_decreasing_rle", "straight/linear", 0xf028e1367a2fbb30), + ("layered_decreasing_rle", "premul/linear", 0xa84600283dce98f6), + ("layered_decreasing_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_decreasing_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_decreasing_rle", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_uncompressed", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_uncompressed", "probed/linear", 0xa27668bf31d9426b), + ("unnamed_rgb_plus_layers_uncompressed", "straight/linear", 0xfc73fe89457ce029), + ("unnamed_rgb_plus_layers_uncompressed", "premul/linear", 0xa27668bf31d9426b), + ("unnamed_rgb_plus_layers_uncompressed", "ignore/linear", 0xb3276cc5126776d4), + ("unnamed_rgb_plus_layers_uncompressed", "preserve_rgb", 0xdee1105fa6e6822a), + ("unnamed_rgb_plus_layers_uncompressed", "aux", 0x854a5e84e7aaa5c2), + ("layered_uncompressed", "probe", 0x0a32e074a28c1f66), + ("layered_uncompressed", "probed/linear", 0xa84600283dce98f6), + ("layered_uncompressed", "straight/linear", 0xf028e1367a2fbb30), + ("layered_uncompressed", "premul/linear", 0xa84600283dce98f6), + ("layered_uncompressed", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_uncompressed", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_uncompressed", "aux", 0xa6b9bd9f02c192ad), + ("uint_rgb_zip16", "probe", 0x0a32e074a28c1f66), + ("uint_rgb_zip16", "probed/linear", 0xed89310ef0cdb9fd), + ("uint_rgb_zip16", "straight/linear", 0xf9fe11ab8ed97cc8), + ("uint_rgb_zip16", "premul/linear", 0xed89310ef0cdb9fd), + ("uint_rgb_zip16", "ignore/linear", 0xf78cd263c0d74f12), + ("uint_rgb_zip16", "preserve_rgb", 0x2a19d67b07cf4359), + ("uint_rgb_zip16", "aux", 0x03e42411f8521856), + ("multipart_depth_named_rgb_zip16", "probe", 0x6a38346c4c5662d5), + ("multipart_depth_named_rgb_zip16", "probed/linear", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "straight/linear", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "premul/linear", 0x869374e9b0384cea), + ("multipart_depth_named_rgb_zip16", "ignore/linear", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "preserve_rgb", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "aux", 0x0fa56884e83b4417), ]; #[test] From eedd64906c5859b9118bc3b0264646c3719bb699 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 02:25:07 +0100 Subject: [PATCH 07/25] #474: keep files in use in the channel cache, cap plane memory, fall back on a missing cache entry Review follow-ups: - Files in use are never dropped from the channel cache: planes an effect holds, the files of the last 4 pictures made of their planes until their effects ask for the channels, and decodes in progress, which now count the planes they will allocate (from the header) against the budget. With 4 frames at once over a budget of one file, each file decodes once (test). - A decode that panics removes its entry instead of leaving it decoding. - One decode allocates at most 8 GiB of planes (512 MiB on 32-bit and wasm); a header asking for more is refused with the reason. - A shared picture whose file has no cached channels (a failed read, a broken part it doesn't use) is decoded from its own channels. - Comment why waiting on a decode may block a rayon worker; drop the subsampling check and test (the exr crate refuses subsampled files). - Docs: the cache is a soft bound; a single-part file still decompresses every channel; the first frame and the image crate's route decode twice. Co-Authored-By: Claude Opus 5.5 --- crates/media/src/exr_channels.rs | 389 +++++++++++++++++++++------- crates/media/src/pool.rs | 3 +- crates/media/tests/it/exr_golden.rs | 2 +- docs/effects-parity.md | 2 +- docs/file-format-parity.md | 2 +- docs/footage.md | 10 +- docs/gaps.md | 2 +- 7 files changed, 309 insertions(+), 101 deletions(-) diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index 747882337..8a8f5edcc 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -93,8 +93,6 @@ struct PlanesReader { f32s: Vec, /// The part's channel indices of exact `R`, `G`, `B` and `A`, when it has R, G and B. rgb: Option<([usize; 3], Option)>, - /// The part has a channel that doesn't sample every pixel. - subsampled: bool, /// The part's display window differs from the first part's. own_display: bool, } @@ -148,7 +146,6 @@ impl<'s> ReadChannels<'s> for Planes<'_> { u32s: vec![], f32s: vec![], rgb, - subsampled: header.channels.list.iter().any(|c| c.sampling != exr::math::Vec2(1, 1)), own_display: header.shared_attributes.display_window != dw, }) } @@ -226,7 +223,7 @@ impl ChannelsReader for PlanesReader { }; Some(([kept(r)?, kept(g)?, kept(b)?], a)) }); - PartPlanes { planes, rgb, has_rgb, exact: !self.subsampled && !self.own_display } + PartPlanes { planes, rgb, has_rgb, exact: !self.own_display } } } @@ -240,8 +237,8 @@ enum Rgb { None, /// The channels (indices in [`AuxChannels::channels`]) hold exactly what it would show. Planes([usize; 3], Option), - /// The planes can't stand in for it (subsampled channels, a depth-filled plane, another - /// display window): decode with the `image` crate. + /// The planes can't stand in for it (a depth-named channel, whose plane is filled with the + /// background depth, or another display window): decode with the `image` crate. Unsupported, } @@ -258,13 +255,52 @@ impl Decoded { } } -/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). `None` when it -/// is not a readable EXR. -fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { - let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; - let display = meta.headers.first()?.shared_attributes.display_window; +/// The most bytes of planes one decode allocates (every kept channel is a display-window plane +/// of f32, allocated before any pixels are read): a header can ask for far more than the file +/// holds. +#[cfg(target_pointer_width = "64")] +const MAX_DECODE_BYTES: usize = 8 << 30; +#[cfg(not(target_pointer_width = "64"))] +const MAX_DECODE_BYTES: usize = 512 << 20; + +/// Why a file's channels weren't decoded. +#[derive(Debug, PartialEq)] +enum Refused { + /// Not a readable OpenEXR file. + Unreadable, + /// Its kept channels need more memory than [`MAX_DECODE_BYTES`] (the reason, to show). + TooLarge(String), +} + +/// Bytes the planes of the channels `keep` names would take (`None`: not a readable header). +fn planes_bytes(meta: &exr::meta::MetaData, keep: &dyn Fn(&str) -> bool) -> Option { + let dw = meta.headers.first()?.shared_attributes.display_window; + let n = dw.size.0.checked_mul(dw.size.1)?; + let mut kept = 0usize; + for header in meta.headers.iter() { + let prefix = part_name(header); + kept = kept.saturating_add(header.channels.list.iter().filter(|ch| keep(&channel_name(prefix.as_deref(), &ch.name.to_string()))).count()); + } + Some(n.saturating_mul(kept).saturating_mul(std::mem::size_of::())) +} + +/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). +fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Result { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).map_err(|_| Refused::Unreadable)?; + let display = meta.headers.first().ok_or(Refused::Unreadable)?.shared_attributes.display_window; + let need = planes_bytes(&meta, keep).ok_or(Refused::Unreadable)?; + if need > MAX_DECODE_BYTES { + let gb = |b: usize| b as f64 / f64::from(1u32 << 30); + return Err(Refused::TooLarge(format!( + "its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", + display.size.0, + display.size.1, + gb(need), + gb(MAX_DECODE_BYTES) + ))); + } let read = ReadImage::new(ReadAllLayers { read_channels: Planes { keep, display } }, |_| {}); - let img = read.from_buffered(std::io::Cursor::new(bytes)).ok()?; + let img = read.from_buffered(std::io::Cursor::new(bytes)).map_err(|_| Refused::Unreadable)?; let dw = img.attributes.display_window; let mut aux = AuxChannels::new(dw.size.0 as u32, dw.size.1 as u32, 1.0); let mut crypto: HashMap, Option)> = HashMap::new(); @@ -306,12 +342,12 @@ fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { } } aux.manifests.sort(); - Some(Decoded { aux: Arc::new(aux), rgb }) + Ok(Decoded { aux: Arc::new(aux), rgb }) } /// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. pub fn read_exr_channels(bytes: &[u8]) -> Option { - let aux = decode(bytes, &|_| true)?.aux; + let aux = decode(bytes, &|_| true).ok()?.aux; Some(Arc::try_unwrap(aux).unwrap_or_else(|a| (*a).clone())) } @@ -371,18 +407,18 @@ fn picture_names(aux: &AuxChannels) -> Option<([String; 3], Option)> { /// The channels of the picture of a multi-layer file and nothing else (names first from the /// headers, then only those channels decoded; in a multi-part file the other parts' pixels are /// not even decompressed), with the names of its planes. -fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> { - let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; +fn picture_only(bytes: &[u8]) -> Result<(Decoded, [String; 3], Option), Refused> { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).map_err(|_| Refused::Unreadable)?; let mut names = AuxChannels::new(0, 0, 1.0); for header in meta.headers.iter() { let prefix = part_name(header); names.channels.extend(header.channels.list.iter().map(|ch| (channel_name(prefix.as_deref(), &ch.name.to_string()), vec![]))); } - let (rgb, a) = picture_names(&names)?; + let (rgb, a) = picture_names(&names).ok_or(Refused::Unreadable)?; // Planes are looked up by name ignoring case, so every channel that could answer is kept. let keep = |n: &str| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(n)); let decoded = decode(bytes, &keep)?; - Some((decoded, rgb, a)) + Ok((decoded, rgb, a)) } /// The picture of a multi-layer OpenEXR file without an unnamed RGB layer (Blender's @@ -392,7 +428,7 @@ fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> /// that is not a data pass (never Cryptomatte, depth or normals, #412), else the first channel /// as grey. Linear, straight RGBA as the file stores it. `None` when it is not a readable EXR. pub fn layered_image(bytes: &[u8]) -> Option { - let (d, rgb, a) = picture_only(bytes)?; + let (d, rgb, a) = picture_only(bytes).ok()?; let aux = &d.aux; let [r, g, b] = [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?]; let a = match &a { @@ -438,19 +474,24 @@ fn layered_picture_of(aux: &AuxChannels, op: AlphaOp) -> Option { /// The picture of the multi-layer EXR at `path` ([`layered_image`]'s rule) as an [`Image`]: /// from the channel cache when `shared` (an effect reads the footage's channels, so they are -/// decoded anyway), else of the picture's channels alone. -pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Option { - if shared { - let d = cache().get(path, || Some(bytes))?; - return layered_picture_of(&d.aux, op); +/// decoded anyway), else of the picture's channels alone (also when the cache has no channels +/// for the file: a read that failed, or a broken part the picture doesn't use). `Err(Some(why))` +/// when the file is too large to decode, `Err(None)` when it isn't a readable multi-layer EXR. +pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Result> { + if shared && let Some(d) = cache().get(path, Use::Picture, || Some(bytes.clone())) { + return layered_picture_of(&d.aux, op).ok_or(None); } - let (d, rgb, a) = picture_only(&bytes)?; + let (d, rgb, a) = picture_only(&bytes).map_err(|r| match r { + Refused::TooLarge(why) => Some(why), + Refused::Unreadable => None, + })?; let aux = &d.aux; + let plane = |n: &str| aux.get(n).ok_or(None); let a = match &a { - Some(n) => Some(aux.get(n)?), + Some(n) => Some(plane(n)?), None => None, }; - planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) + planes_picture(aux, [plane(&rgb[0])?, plane(&rgb[1])?, plane(&rgb[2])?], a, op).ok_or(None) } /// The picture of the EXR at `path` from the channel cache (#474: EXtractoR / Cryptomatte read @@ -459,7 +500,7 @@ pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: /// the planes can't stand in for the `image` crate's picture (decode with it), or the file is /// not a readable EXR. pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Option { - let d = cache().get(path, || Some(bytes))?; + let d = cache().get(path, Use::Picture, || Some(bytes))?; let aux = &d.aux; match d.rgb { Rgb::None => layered_picture_of(aux, op), @@ -477,9 +518,11 @@ pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Optio /// How many bytes of decoded channels the cache keeps, like the frame cache's default /// ([`crate::DEFAULT_BUDGET`]): two 2560×1440 files of 32 channels (470 MB each, as every plane -/// is f32). The newest file is kept whatever its size, so a frame's picture and its effects -/// share its decode. (Measured on such a sequence decoded one frame at a time: 1 GiB held peak -/// memory about 1 GB lower than 2 GiB, at the same speed.) +/// is f32). A soft bound: the newest file is kept whatever its size (so a frame's picture and its +/// effects share its decode), and files still in use are never dropped (planes an effect holds, +/// [`HELD_FILES`] pictures waiting for their effects, decodes in progress), so memory can go +/// above it while they are. (Measured on such a sequence decoded one frame at a time: 1 GiB held +/// peak memory about 1 GB lower than 2 GiB, at the same speed.) #[cfg(not(target_arch = "wasm32"))] pub const CACHE_BUDGET: usize = 1 << 30; #[cfg(target_arch = "wasm32")] @@ -488,90 +531,178 @@ pub const CACHE_BUDGET: usize = 512 << 20; /// At most this many files are cached (unreadable ones hold no bytes). const CACHE_FILES: usize = 64; +/// At most this many files whose picture was made of their planes are kept for their effects +/// (the renderer asks for a frame's picture, then its channels): about as many frames as are +/// rendered at once. The oldest stops being kept when another comes. +const HELD_FILES: usize = 4; + /// A file's decode: done once, by the first caller; the others wait for it. type Slot = Arc>>>; struct Entry { path: String, slot: Slot, - /// Bytes of the finished decode (0 while decoding). + /// Bytes of the finished decode, or the planes it will allocate while it decodes. bytes: usize, - /// When it was last asked for (a counter). + /// When it was last asked for or finished decoding (a counter). used: u64, } +impl Entry { + /// Dropping it would free nothing or lose a decode in progress: an effect (or a picture + /// waiting for its effects) still holds its planes, or it is decoding. + fn in_use(&self) -> bool { + match self.slot.get() { + None => true, + Some(None) => false, + Some(Some(d)) => Arc::strong_count(d) > 1 || Arc::strong_count(&d.aux) > 1, + } + } +} + +struct State { + entries: Vec, + clock: u64, + /// Files whose picture was made of their planes and whose channels haven't been asked for + /// since, oldest first (at most [`HELD_FILES`]). + held: std::collections::VecDeque<(String, Arc)>, +} + +/// Who asks for a file's channels. +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum Use { + /// The picture of a footage item whose effects read its channels next: kept for them. + Picture, + /// The channels themselves (EXtractoR, Cryptomatte, the 3D Channel effects). + Channels, +} + /// Decoded EXR files by path, bounded by bytes, least recently used out first. pub(crate) struct ChannelCache { - entries: Mutex<(Vec, u64)>, + state: Mutex, budget: usize, /// Files decoded (for tests). decodes: AtomicU64, } +/// Removes a decode's entry if the decode panics, so the file doesn't stay "decoding" (never +/// dropped, its estimated bytes counted) until someone asks for it again. +struct Abandon<'a> { + cache: &'a ChannelCache, + slot: &'a Slot, +} + +impl Drop for Abandon<'_> { + fn drop(&mut self) { + if std::thread::panicking() { + self.cache.lock().entries.retain(|e| !Arc::ptr_eq(&e.slot, self.slot)); + } + } +} + impl ChannelCache { pub(crate) const fn new(budget: usize) -> Self { - Self { entries: Mutex::new((Vec::new(), 0)), budget, decodes: AtomicU64::new(0) } + Self { state: Mutex::new(State { entries: Vec::new(), clock: 0, held: std::collections::VecDeque::new() }), budget, decodes: AtomicU64::new(0) } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Set the bytes of the entry of `slot` (if it's still cached), mark it used now and drop + /// least recently used files until the rest fit the budget. + fn account(&self, slot: &Slot, bytes: usize) { + let mut state = self.lock(); + state.clock += 1; + let clock = state.clock; + // (gone when the file was forgotten meanwhile) Used as of now: a file asked for before + // another but decoded after it is not the older one. + if let Some(e) = state.entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, slot)) { + e.bytes = bytes; + e.used = clock; + } + Self::evict(&mut state.entries, self.budget, slot); } /// The decoded file at `path`, read with `read` by the first caller; callers asking for the /// same file meanwhile wait for that decode instead of decoding it again. - pub(crate) fn get(&self, path: &str, read: impl FnOnce() -> Option>) -> Option> { + pub(crate) fn get(&self, path: &str, by: Use, read: impl FnOnce() -> Option>) -> Option> { let slot = { - let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); - let (entries, clock) = &mut *guard; - *clock += 1; - match entries.iter_mut().find(|e| e.path == path) { + let mut state = self.lock(); + state.clock += 1; + let clock = state.clock; + match state.entries.iter_mut().find(|e| e.path == path) { Some(e) => { - e.used = *clock; + e.used = clock; e.slot.clone() } None => { let slot = Slot::default(); - entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: *clock }); + state.entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: clock }); slot } } }; let mut decoded_here = false; + // Waiting here can block a rayon worker (which `Inner::frame_at` avoids for frames): that + // is safe because the decode never waits on rayon's global pool (the exr crate + // decompresses on a pool of its own and this thread only receives the blocks), so the + // decoding thread can't be waiting for this one. let v = slot .get_or_init(|| { decoded_here = true; self.decodes.fetch_add(1, Ordering::Relaxed); - read().and_then(|b| decode(&b, &|_| true)).map(Arc::new) + let _abandon = Abandon { cache: self, slot: &slot }; + let bytes = read()?; + // The planes it will allocate count against the budget while it decodes. + if let Ok(meta) = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(&bytes[..]), false) + && let Some(need) = planes_bytes(&meta, &|_| true) + { + self.account(&slot, need.min(MAX_DECODE_BYTES)); + } + match decode(&bytes, &|_| true) { + Ok(d) => Some(Arc::new(d)), + Err(Refused::TooLarge(why)) => { + log::warn!("media: {path}: {why}"); + None + } + Err(Refused::Unreadable) => None, + } }) .clone(); if decoded_here { - let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); - let (entries, clock) = &mut *guard; - *clock += 1; - // (gone when the file was forgotten meanwhile) Used as of now: a file asked for - // before another but decoded after it is not the older one, or concurrent frames - // would evict each other's files before their effects read them. - if let Some(e) = entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, &slot)) { - e.bytes = v.as_ref().map_or(0, |d| d.bytes()); - e.used = *clock; - } - Self::evict(entries, self.budget, &slot); + self.account(&slot, v.as_ref().map_or(0, |d| d.bytes())); } + let mut state = self.lock(); + match (by, &v) { + (Use::Picture, Some(d)) => { + if !state.held.iter().any(|(p, _)| p == path) { + state.held.push_back((path.to_string(), d.clone())); + if state.held.len() > HELD_FILES { + state.held.pop_front(); + } + } + } + (Use::Channels, _) => state.held.retain(|(p, _)| p != path), + (Use::Picture, None) => {} + } + drop(state); v } - /// Drop least recently used finished files until the rest fit the budget, keeping `newest`. + /// Drop least recently used files not in use until the rest fit the budget, keeping `newest`. fn evict(entries: &mut Vec, budget: usize, newest: &Slot) { while entries.iter().map(|e| e.bytes).sum::() > budget || entries.len() > CACHE_FILES { - let oldest = entries - .iter() - .enumerate() - .filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && e.slot.get().is_some()) - .min_by_key(|(_, e)| e.used) - .map(|(i, _)| i); + let oldest = entries.iter().enumerate().filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && !e.in_use()).min_by_key(|(_, e)| e.used).map(|(i, _)| i); let Some(i) = oldest else { break }; entries.remove(i); } } pub(crate) fn forget(&self, path: &str) { - self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.retain(|e| e.path != path); + let mut state = self.lock(); + state.entries.retain(|e| e.path != path); + state.held.retain(|(p, _)| p != path); } #[cfg(test)] @@ -581,7 +712,7 @@ impl ChannelCache { #[cfg(test)] fn cached(&self) -> Vec { - self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.iter().map(|e| e.path.clone()).collect() + self.lock().entries.iter().map(|e| e.path.clone()).collect() } } @@ -597,7 +728,7 @@ pub(crate) fn forget(path: &str) { /// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { - cache().get(path, read).map(|d| d.aux.clone()) + cache().get(path, Use::Channels, read).map(|d| d.aux.clone()) } #[cfg(test)] @@ -663,14 +794,17 @@ mod tests { use exr::prelude::*; let half = |x: f32| f16::from_f32(x); let part = |pos: (i32, i32), size: (usize, usize), list: Vec>| { - let mut attrs = LayerAttributes::default(); - attrs.layer_position = Vec2(pos.0, pos.1); + let attrs = LayerAttributes { layer_position: Vec2(pos.0, pos.1), ..Default::default() }; Layer::new(size, attrs, Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())) }; let display = IntegerBounds::from_dimensions((3, 2)); // A 2×1 data window at (1, 1). let inside = file( - vec![part((1, 1), (2, 1), vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))])], + vec![part( + (1, 1), + (2, 1), + vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))], + )], display, ); let aux = read_exr_channels(&inside).unwrap(); @@ -699,7 +833,7 @@ mod tests { s.spawn(|| { barrier.wait(); cache - .get("/474-concurrent.exr", || { + .get("/474-concurrent.exr", Use::Channels, || { reads.fetch_add(1, Ordering::Relaxed); // Long enough for the others to arrive while this one decodes. std::thread::sleep(std::time::Duration::from_millis(50)); @@ -726,7 +860,7 @@ mod tests { let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); let one = 4 * 32 * 32 * 4 + 4; let cache = ChannelCache::new(2 * one); - let get = |p: &str| cache.get(p, || Some(bytes.clone())).unwrap(); + let get = |p: &str| cache.get(p, Use::Channels, || Some(bytes.clone())).unwrap(); assert_eq!(get("/a.exr").bytes(), one); get("/b.exr"); get("/a.exr"); // a is now more recent than b @@ -737,18 +871,18 @@ mod tests { assert_eq!(cache.decodes(), 3, "a hit"); // A file over the budget on its own is still kept while it is the newest. let small = ChannelCache::new(one / 2); - assert!(small.get("/big.exr", || Some(bytes.clone())).is_some()); + assert!(small.get("/big.exr", Use::Channels, || Some(bytes.clone())).is_some()); assert_eq!(small.cached(), ["/big.exr"]); - small.get("/big2.exr", || Some(bytes.clone())); + small.get("/big2.exr", Use::Channels, || Some(bytes.clone())); assert_eq!(small.cached(), ["/big2.exr"]); // An unreadable file is remembered (not read again) without bytes, and forgotten on request. - assert!(cache.get("/broken.exr", || Some(Arc::from(&b"not an exr"[..]))).is_none()); - assert!(cache.get("/broken.exr", || panic!("not read again")).is_none()); + assert!(cache.get("/broken.exr", Use::Channels, || Some(Arc::from(&b"not an exr"[..]))).is_none()); + assert!(cache.get("/broken.exr", Use::Channels, || panic!("not read again")).is_none()); cache.forget("/broken.exr"); assert!(!cache.cached().contains(&"/broken.exr".to_string())); // At most CACHE_FILES files, however small. for i in 0..CACHE_FILES + 10 { - cache.get(&format!("/none{i}.exr"), || None); + cache.get(&format!("/none{i}.exr"), Use::Channels, || None); } assert!(cache.cached().len() <= CACHE_FILES); } @@ -761,22 +895,101 @@ mod tests { let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); let one = 4 * 32 * 32 * 4 + 4; let cache = ChannelCache::new(2 * one); - cache.get("/first.exr", || { + cache.get("/first.exr", Use::Channels, || { // Another frame's file is asked for and decoded while this one decodes. - cache.get("/second.exr", || Some(bytes.clone())); + cache.get("/second.exr", Use::Channels, || Some(bytes.clone())); Some(bytes.clone()) }); - cache.get("/third.exr", || Some(bytes.clone())); + cache.get("/third.exr", Use::Channels, || Some(bytes.clone())); assert_eq!(cache.cached(), ["/first.exr", "/third.exr"], "the file decoded first went first"); } + /// #474: frames rendered at once (each asks for its picture, then its effects for the + /// channels while other frames decode) decode each file once, though the budget holds only + /// one: a picture's file is kept for its effects, and files in use are never dropped. + #[test] + fn concurrent_frames_decode_each_file_once() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "Z"], (64, 64), 0.5).into(); + let cache = ChannelCache::new(4 * 64 * 64 * 4 + 4); + let (next, reads, frames) = (std::sync::atomic::AtomicUsize::new(0), AtomicU64::new(0), 24); + let read = || { + reads.fetch_add(1, Ordering::Relaxed); + Some(bytes.clone()) + }; + std::thread::scope(|s| { + for _ in 0..4 { + s.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= frames { + break; + } + let path = format!("/474-frames/{i:04}.exr"); + drop(cache.get(&path, Use::Picture, read).unwrap()); + std::thread::sleep(std::time::Duration::from_millis(3)); + let aux = cache.get(&path, Use::Channels, read).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(2)); + assert_eq!(aux.aux.get("Z").and_then(|z| z.first()).copied(), Some(0.5)); + } + }); + } + }); + assert_eq!(cache.decodes(), frames as u64, "one decode per frame"); + assert_eq!(reads.load(Ordering::Relaxed), frames as u64); + } + + /// #474: a decode that panics doesn't leave its file "decoding" in the cache (never dropped, + /// its planes counted): the entry goes, and the next request decodes the file. + #[test] + fn a_decode_that_panics_is_not_left_in_the_cache() { + let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| cache.get("/474-panics.exr", Use::Channels, || panic!("the read failed")))); + assert!(r.is_err()); + assert!(cache.cached().is_empty()); + assert!(cache.get("/474-panics.exr", Use::Channels, || Some(bytes.clone())).is_some()); + assert_eq!(cache.cached(), ["/474-panics.exr"]); + } + + /// #474: the picture of a multi-layer file whose effects read its channels comes from the + /// cache, and from the picture's own channels when the cache has none for the file (a read + /// that failed, or a broken part the picture doesn't use). + #[test] + fn a_shared_picture_without_cached_channels_is_still_decoded() { + use effectcraft_project::Footage; + use effectcraft_render::FloatCurve; + let bytes: Arc<[u8]> = flat(&["Image.R", "Image.G", "Image.B", "depth.Z"], (4, 2), 0.5).into(); + let path = "/474-shared-fallback.exr"; + assert!(cache().get(path, Use::Channels, || None).is_none(), "a read that failed"); + let op = AlphaOp::of(&Footage::default(), FloatCurve::Linear); + let img = layered_picture(path, bytes, op, true).unwrap(); + assert_eq!(img.get(1, 1), [0.5, 0.5, 0.5, 1.0]); + } + + /// #474: a header asking for more plane memory than one decode may allocate (16384² display + /// window, 40 channels: 43 GB) is refused with a reason before anything is allocated. + #[test] + fn a_header_asking_for_too_much_memory_is_refused() { + use exr::prelude::*; + let list: Vec> = (0..40).map(|i| AnyChannel::new(format!("pass{i:02}.Y").as_str(), FlatSamples::F32(vec![0.5]))).collect(); + let layer = Layer::new((1, 1), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())); + let bytes = file(vec![layer], IntegerBounds::new((0, 0), (16384, 16384))); + match decode(&bytes, &|_| true) { + Err(Refused::TooLarge(why)) => assert!(why.contains("16384×16384") && why.contains("GB"), "{why}"), + Err(Refused::Unreadable) => panic!("readable"), + Ok(_) => panic!("refused"), + } + assert!(read_exr_channels(&bytes).is_none()); + assert!(cached("/474-too-large.exr", || Some(bytes.as_slice().into())).is_none()); + } + /// #474: a file forgotten (changed on disk) while it decodes isn't cached under its path when /// that decode finishes. #[test] fn a_file_forgotten_while_it_decodes_is_not_cached() { let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); let cache = ChannelCache::new(CACHE_BUDGET); - let d = cache.get("/changed.exr", || { + let d = cache.get("/changed.exr", Use::Channels, || { cache.forget("/changed.exr"); Some(bytes.clone()) }); @@ -824,8 +1037,7 @@ mod tests { /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 - /// around it), in every alpha interpretation and float curve; a file it can't stand in for - /// (a subsampled channel) is left to the `image` crate. + /// around it), in every alpha interpretation and float curve. #[test] fn picture_from_planes_is_the_image_crates() { use effectcraft_project::{AlphaMode, Footage}; @@ -841,7 +1053,10 @@ mod tests { let display = IntegerBounds::new((1, -1), (5, 4)); let n = 20; let files = [ - file(vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], display), + file( + vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], + display, + ), file(vec![part(None, (0, 0), (3, 2), vec![float("R", 0.4, 6), float("G", 0.5, 6), float("B", 0.6, 6), half("Image.R", 0.1, 6)])], display), file( vec![ @@ -854,7 +1069,7 @@ mod tests { let bits = |i: &effectcraft_raster::Image| i.data.iter().flat_map(|p| p.map(f32::to_bits)).collect::>(); for (k, bytes) in files.iter().enumerate() { let path = format!("/474-planes-{k}.exr"); - let d = cache().get(&path, || Some(bytes.as_slice().into())).unwrap(); + let d = cache().get(&path, Use::Channels, || Some(bytes.as_slice().into())).unwrap(); assert!(matches!(d.rgb, Rgb::Planes(..)), "file {k}: {:?}", d.rgb); let img = crate::probe::load_oriented(bytes, Some(image::ImageFormat::OpenExr)).unwrap(); for alpha in [AlphaMode::Straight, AlphaMode::Premultiplied, AlphaMode::Ignore] { @@ -867,16 +1082,6 @@ mod tests { } } } - // A subsampled channel in the picture's part: the `image` crate decodes it. - let mut sub = part(None, (0, 0), (4, 2), vec![half("R", 0.1, 8), half("G", 0.2, 8), half("B", 0.3, 8), half("C", 0.3, 2)]); - if let Some(c) = sub.channel_data.list.iter_mut().find(|c| c.name == *"C") { - c.sampling = Vec2(2, 2); - } - let mut bytes = std::io::Cursor::new(Vec::new()); - if Image::from_layer(sub).write().to_buffered(&mut bytes).is_ok() { - let d = cache().get("/474-subsampled.exr", || Some(bytes.into_inner().into())).map(|d| d.rgb); - assert!(d.is_none_or(|r| r == Rgb::Unsupported), "{d:?}"); - } } /// #2141: one pixel of data in a 60000² display window is refused, not allocated per channel. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 9f1f2aa23..910f8238f 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -871,7 +871,8 @@ impl Inner { match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. - Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared).ok_or_else(|| MediaError::Decode(format!("{path}: {e}"))), + Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared) + .map_err(|why| MediaError::Decode(format!("{path}: {}", why.unwrap_or_else(|| e.to_string())))), } } diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs index d99de635a..f760ed7c2 100644 --- a/crates/media/tests/it/exr_golden.rs +++ b/crates/media/tests/it/exr_golden.rs @@ -92,7 +92,7 @@ fn mipmapped(list: &[(&str, Kind)], compression: Compression) -> Vec { } size = ((size.0 / 2).max(1), (size.1 / 2).max(1)); } - AnyChannel::new(name, Levels::Mip { rounding_mode: exr::math::RoundingMode::Down, level_data: data.into() }) + AnyChannel::new(name, Levels::Mip { rounding_mode: exr::math::RoundingMode::Down, level_data: data }) }; let list: Vec>> = list.iter().enumerate().map(|(c, (n, k))| levels(n, *k, c)).collect(); let encoding = Encoding { compression, blocks: Blocks::Tiles(Vec2(16, 16)), line_order: LineOrder::Increasing }; diff --git a/docs/effects-parity.md b/docs/effects-parity.md index ce4debd01..d80a874bd 100644 --- a/docs/effects-parity.md +++ b/docs/effects-parity.md @@ -26,7 +26,7 @@ corpus compares renders, each effect is assumed to differ somewhere, and users k | Category | Effects | On the GPU | Known differences or bugs | |---|---:|---:|---| -| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache bounded at 1 GB (#474 open: f16 storage remains) | +| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache softly bounded at 1 GB (#474; holding channels as f16 is still to do) | | Audio | 13 | 0 | CPU by design (applied in the mixdown) | | Blur & Sharpen | 16 | 15 | Radial Blur keeps the layer edges as After Effects 2026 (713 now pixel-identical); Gaussian Blur interaction and output to be measured first ([fidelity/ae-2026-acceptance.md](fidelity/ae-2026-acceptance.md)) | | Channel | 13 | 13 | — | diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 741582d9e..648eadcee 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, footage alone only its picture's channels (#474; decoded channels are held as f32) | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, except the first frame an effect reads and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index 0ebbeeecc..50bd4e7d1 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -120,10 +120,12 @@ Nuke's `beauty`) show the beauty pass: the layer named Combined, Beauty, Image, Color, else the first colour layer that isn't a data pass. Cryptomatte, depth, mist, normals, vectors, positions and IDs are never the picture. EXtractoR reads every other layer and channel. -Footage on its own decodes only the channels of its picture. Once an effect reads an item's -channels (EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for -the picture and the effects together, and kept for a while (up to 1 GB of decoded channels); -several frames asking for the same file at once wait for one decode. +Footage on its own keeps only the channels of its picture (a single-part file still +decompresses all of its pixels to read them). Once an effect reads an item's channels +(EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for the +picture and the effects together, and kept for a while: about 1 GB of decoded channels (512 MB +in the browser), a soft bound, as the newest file and files still in use stay whatever their +size. Several frames asking for the same file at once wait for one decode. ## Replacing and reloading diff --git a/docs/gaps.md b/docs/gaps.md index 46a96983a..75e49a1d6 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -65,7 +65,7 @@ are grouped by area; the issue tracker has the rest. | 6 | **Linux and Windows interaction bugs** | Open: #488 rename arrows (hardening landed, reporter asked for details), #492 anchor drag, #268 file drops on Wayland (winit), #613 Direct3D 12 compositor. Now closed: #491, #493, #467, #320, #302, #596 | Many reports come from Linux; some Windows GPUs fail to start the compositor | 25–45 h | G2 · [ui-parity.md](ui-parity.md) | | 7 | **Animation presets: 76 against 621** | After Effects ships 621 `.ffx` presets (Behaviors, Backgrounds, Shapes, Synthetics, Text, Transitions…); we ship 76 original presets (8 text-animator; 23 Transitions: wipes, clock and fan reveals, blinds, dissolves, card flips, CC wipes, fades, #757; 23 Backgrounds: drifting gradients, fog, cloud and smoke, plasma and magma, cells, hexagons, scrolling grids and checks, pulse rings, bokeh, light sweep, #1096, driven by expressions so they keep moving, not seamless loops; 22 Behaviors: wiggle, shake, float, orbit, bounce, drift, pendulum, spin, pulse, heartbeat, blink, strobe, flicker on transform and opacity, #1510) and read only our `.ecpreset`. Missing: Shapes, Synthetics, Image and more Text | Presets are how many designers start; tutorials assume them | 15–35 h (original presets; `.ffx` reading is out of clean-room scope) | G8 · [effects-parity.md](effects-parity.md) | | 8 | **Mask and shape bugs** | Open: #1004 mask vertex multi-select, #999 auto-pan, #1042 / #1033 vertex nodes under the bounding box, #959 / #960 shape group toggles and operators. Now closed: #510, #513, #527, #509, #303, #553, #932 (Convert To Bezier Path on a shape layer's parametric path) | Masks are core compositing | 10–20 h | G2 · [target-app-parity.md](target-app-parity.md) | -| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | +| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage still to do). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | | 10 | **Colour pipeline differences** | #340 ProRes 4444 tagged bt709 but sRGB-encoded (open); ACES 2.0 / OCIO 2.5 configs (26.5) missing; now closed: #494 32-bpc blend modes, #495 and #750 Working Gamma 2.4, #546 / #547 viewer colour | Wrong colour in delivery is a client-visible failure | 15–30 h | G1 · [file-format-parity.md](file-format-parity.md) | | 11 | **Text rendering and layout** | #453 CJK text in comps: the Han fallback now reaches the installed Chinese faces; open: #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | From 5b31b96cde1187eff124da080fb0755dfac2813d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 03:18:16 +0100 Subject: [PATCH 08/25] perf(media): tell the media pool which EXR footage effects read (#474) The pool learned that an effect reads a footage item's channels only at the first aux() call, so the first frame was decoded twice (picture, then in full), and the hint was never cleared after the effect was removed. The session now gives the footage source the whole set of files read by enabled 3D Channel effects (FootageSource::set_aux_footage) with every project change (Session::bump: edits, undo, redo, history jumps, rollbacks, project open), sending it only when it changed. The pool keeps that set and, as a safety net, what aux() learns; a new set drops what was learned. Refs #474 Co-Authored-By: Claude Opus 5.5 --- crates/automation/src/confine.rs | 3 + crates/effects/src/channel3d.rs | 19 +++ crates/effects/src/channel3d/tests.rs | 24 ++++ crates/effects/src/lib.rs | 2 +- crates/engine/src/lib.rs | 4 + crates/engine/src/session/edit.rs | 3 + crates/engine/src/session/render.rs | 39 ++++++ crates/engine/src/tests.rs | 1 + crates/engine/src/tests/aux_footage.rs | 175 +++++++++++++++++++++++++ crates/media/src/exr_channels.rs | 91 +++++++++++++ crates/media/src/pool.rs | 42 +++++- crates/render/src/footage.rs | 9 ++ docs/file-format-parity.md | 2 +- docs/footage.md | 10 +- 14 files changed, 412 insertions(+), 12 deletions(-) create mode 100644 crates/engine/src/tests/aux_footage.rs diff --git a/crates/automation/src/confine.rs b/crates/automation/src/confine.rs index 5f0a7a51a..7cfe11ac2 100644 --- a/crates/automation/src/confine.rs +++ b/crates/automation/src/confine.rs @@ -349,6 +349,9 @@ impl FootageSource for ConfinedFootage { fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { self.inside(footage).then(|| self.inner.aux(item, footage, t)).flatten() } + fn set_aux_footage(&self, paths: &[String]) { + self.inner.set_aux_footage(paths) + } fn model(&self, item: ItemId, footage: &Footage) -> Option> { self.inside(footage).then(|| self.inner.model(item, footage)).flatten() } diff --git a/crates/effects/src/channel3d.rs b/crates/effects/src/channel3d.rs index dc65c3255..59128eb7e 100644 --- a/crates/effects/src/channel3d.rs +++ b/crates/effects/src/channel3d.rs @@ -30,6 +30,25 @@ fn spec(id: &'static str, name: &'static str, params: Vec, ren EffectSpec { id, name, category: "3D Channel", params, render, gpu: false, float: true } } +/// The effects that read their layer's auxiliary channels ([`crate::EffectHost::aux`]): every +/// 3D Channel effect but 3D Layer Instance. +const READS_AUX: [&str; 8] = [ + "ec.3d.channelextract", + "ec.3d.cryptomatte", + "ec.3d.depthmatte", + "ec.3d.depthoffield", + "ec.3d.extractor", + "ec.3d.fog3d", + "ec.3d.idmatte", + "ec.3d.identifier", +]; + +/// Does effect `id` read its layer's auxiliary channels ([`crate::EffectHost::aux`])? The media +/// layer then decodes the footage's picture together with its channels (#474). +pub fn reads_aux(id: &str) -> bool { + READS_AUX.contains(&id) +} + fn aux(ctx: &EffectCtx) -> Option> { ctx.env.host?.aux() } diff --git a/crates/effects/src/channel3d/tests.rs b/crates/effects/src/channel3d/tests.rs index e69c2a6aa..9d9342a95 100644 --- a/crates/effects/src/channel3d/tests.rs +++ b/crates/effects/src/channel3d/tests.rs @@ -399,3 +399,27 @@ fn instance3d_replaces_the_pixels_with_the_referenced_layer() { let out = run_fx("ec.3d.instance3d", &[], solid(), 0.0, EffectEnv::default()); assert_eq!(out.img.data, solid().data); } + +/// #474: [`reads_aux`] names exactly the 3D Channel effects that ask the host for channels. +#[test] +fn reads_aux_names_the_effects_that_read_channels() { + struct Counting(AuxHost, std::sync::atomic::AtomicUsize); + impl EffectHost for Counting { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn aux(&self) -> Option> { + self.1.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.0.aux() + } + } + for s in specs() { + let host = Counting(scene(), Default::default()); + run_fx(s.id, &[], img(), 0.0, EffectEnv { host: Some(&host), ..Default::default() }); + assert_eq!(host.1.into_inner() > 0, reads_aux(s.id), "{}", s.id); + } + assert!(READS_AUX.iter().all(|id| specs().iter().any(|s| s.id == *id)), "every listed effect exists"); +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c865d4f96..c574509f3 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -115,7 +115,7 @@ pub use time_fx::{PixelMotion, TimewarpPlan, TwStep, pmb_plan, posterized_time, // effectcraft-gpu fx_tone. pub use ocio::color_stabilizer_maps; // GPU effects, part B (3D Channel, Immersive Video, colour management, simulation render passes). -pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, selection_hashes}; +pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, reads_aux, selection_hashes}; pub use ocio::{ColorOp, Straight, Tf, color_program}; pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, bubble_sample_bounds, drizzle_drops, mercury_blobs, sprite_plan}; pub use sim2::{ diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 0cc9a7189..54616fdbf 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -244,6 +244,9 @@ pub struct Session { pub footage_stamps: std::collections::HashMap, /// The disk cache's footage salt and the footage it was taken for ([`Session::refresh_disk_salt`]). pub disk_salt: Option<(u64, u64)>, + /// The footage files effects read the channels of, as last given to the footage source + /// ([`Session::refresh_aux_footage`]); `None` = not given yet. + pub aux_footage: Option>, /// How many script-issued commands are running ([`Session::execute_for_script`]): while one /// is, settings that guard what scripts may do can't change (see /// [`prefs::Scripting::same_permissions`]). @@ -350,6 +353,7 @@ impl Default for Session { stored_paths: Default::default(), footage_stamps: Default::default(), disk_salt: None, + aux_footage: None, script_calls: ScriptCalls::default(), } } diff --git a/crates/engine/src/session/edit.rs b/crates/engine/src/session/edit.rs index c2590abc3..d63ec5ead 100644 --- a/crates/engine/src/session/edit.rs +++ b/crates/engine/src/session/edit.rs @@ -98,6 +98,7 @@ impl Session { self.saved_revision = self.revision; } self.refresh_disk_salt(false); + self.refresh_aux_footage(); self.events.push(Event::ProjectChanged { revision: self.revision }); } @@ -252,6 +253,8 @@ impl Session { self.project = Arc::new(p); // Its files may have changed since they were last stamped. self.disk_salt = None; + // The set of footage effects read the channels of is sent again, even when it is empty. + self.aux_footage = None; self.history = History::default(); self.state = EditorState { snapping: true, view_layout: 1, ..Default::default() }; self.path = path; diff --git a/crates/engine/src/session/render.rs b/crates/engine/src/session/render.rs index 86127a510..24f6552dd 100644 --- a/crates/engine/src/session/render.rs +++ b/crates/engine/src/session/render.rs @@ -137,4 +137,43 @@ impl Session { self.layer_cache.clear(); self.drop_crypto_probe(); } + + /// Tell the footage source which files effects read the channels of ([`aux_footage`]), so + /// it can decode their pictures and channels at once from the first frame (#474). Called with + /// every project change ([`Session::bump`]); the set is only sent when it changed, or after + /// [`Session::replace_project`] cleared what was sent. + pub(crate) fn refresh_aux_footage(&mut self) { + let paths = aux_footage(&self.project); + if self.aux_footage.as_ref() != Some(&paths) { + self.footage.set_aux_footage(&paths); + self.aux_footage = Some(paths); + } + } +} + +/// The files (each footage item's [`effectcraft_project::Footage::path`], sorted, once each) of +/// the footage layers that have an enabled effect reading their channels +/// ([`effectcraft_effects::reads_aux`]), with the layer's Effects switch on. Several items can +/// use one file: it stays in while any of them is read. +pub(crate) fn aux_footage(project: &effectcraft_project::Project) -> Vec { + use effectcraft_project::{GroupKind, ItemKind, LayerSource}; + let mut paths: Vec = project + .comps() + .flat_map(|(_, c)| c.layers.iter()) + .filter(|l| l.switches.effects) + .filter(|l| { + l.effects() + .is_some_and(|fx| fx.groups().any(|g| g.enabled && matches!(&g.kind, GroupKind::Effect { effect } if effectcraft_effects::reads_aux(effect)))) + }) + .filter_map(|l| match &l.source { + LayerSource::Footage { item } => match &project.item(*item)?.kind { + ItemKind::Footage(f) if !f.path.is_empty() => Some(f.path.clone()), + _ => None, + }, + _ => None, + }) + .collect(); + paths.sort_unstable(); + paths.dedup(); + paths } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 013d2f24c..85d9353d3 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1,6 +1,7 @@ mod add_item_options; mod anim_presets; mod anim_tools; +mod aux_footage; mod blender; mod camera_track; mod codec_options; diff --git a/crates/engine/src/tests/aux_footage.rs b/crates/engine/src/tests/aux_footage.rs new file mode 100644 index 000000000..5a74d3b77 --- /dev/null +++ b/crates/engine/src/tests/aux_footage.rs @@ -0,0 +1,175 @@ +//! The footage files effects read the channels of (#474): the session gives the footage source +//! the whole set with every project change ([`Session::bump`]), so a multi-layer OpenEXR read by +//! EXtractoR or Cryptomatte decodes its picture and channels at once from the first frame, and +//! goes back to its picture alone once no effect reads it. + +use std::sync::{Arc, Mutex}; + +use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, Project}; +use effectcraft_raster::Image; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::json; + +use crate::Session; +use crate::render::{FootageSource, RenderOpts}; + +/// Records every set it is given, and the set it had when it was first asked for a frame. +#[derive(Default)] +struct Recorder { + sets: Mutex>>, + at_first_frame: Mutex>>>, +} + +impl Recorder { + fn last(&self) -> Option> { + self.sets.lock().unwrap().last().cloned() + } + fn calls(&self) -> usize { + self.sets.lock().unwrap().len() + } +} + +impl FootageSource for Recorder { + fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option> { + self.at_first_frame.lock().unwrap().get_or_insert_with(|| self.last()); + Some(Arc::new(Image::new(f.width, f.height))) + } + fn set_aux_footage(&self, paths: &[String]) { + self.sets.lock().unwrap().push(paths.to_vec()); + } +} + +struct Probe; +impl crate::Importer for Probe { + fn probe(&self, path: &str) -> Result { + Ok(Footage { + path: path.into(), + kind: FootageKind::Still, + width: 8, + height: 4, + has_video: true, + frame_rate: FrameRate::from_f64(24.0), + duration: Tick::from_seconds_f64(2.0), + ..Default::default() + }) + } +} + +fn session(rec: &Arc) -> Session { + let mut s = Session { footage: rec.clone(), importer: Some(Arc::new(Probe)), ..Default::default() }; + s.execute("comp.new", json!({"name": "C", "width": 8, "height": 4, "frameRate": 24, "duration": 2})).unwrap(); + s +} + +fn import(s: &mut Session, path: &str) -> serde_json::Value { + s.execute("file.import", json!({"paths": [path]})).unwrap()["items"][0].clone() +} + +/// Add a layer of `item` to the active comp; its layer id. +fn add_layer(s: &mut Session, item: &serde_json::Value) -> u64 { + s.execute("layer.addItem", json!({"item": item})).unwrap(); + s.state.selected_layers[0].0 +} + +fn paths(v: &[&str]) -> Option> { + Some(v.iter().map(|p| p.to_string()).collect()) +} + +/// Adding, disabling, removing an effect and undoing or redoing each change the set; edits that +/// don't change it don't send it again. +#[test] +fn effect_edits_undo_and_redo_update_the_set() { + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let item = import(&mut s, "/renders/a.exr"); + let l = add_layer(&mut s, &item); + assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "no effect reads it yet"); + s.execute_checked("effect.apply", json!({"effect": "Gaussian Blur", "layers": [l]})).unwrap(); + assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "a blur doesn't read channels"); + s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [l]})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"])); + assert!(s.undo()); + assert_eq!(rec.last(), paths(&[]), "undoing the add"); + assert!(s.redo()); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "redoing it"); + let calls = rec.calls(); + s.execute_checked("effect.toggle", json!({"layer": l, "effect": "Gaussian Blur", "value": false})).unwrap(); + assert_eq!(rec.calls(), calls, "the set didn't change: it isn't sent again"); + s.execute_checked("effect.toggle", json!({"layer": l, "effect": "EXtractoR", "value": false})).unwrap(); + assert_eq!(rec.last(), paths(&[]), "a disabled effect reads nothing"); + s.execute_checked("effect.toggle", json!({"layer": l, "effect": "EXtractoR", "value": true})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"])); + s.execute_checked("effect.remove", json!({"layer": l, "effect": "EXtractoR"})).unwrap(); + assert_eq!(rec.last(), paths(&[]), "removed"); + assert!(s.undo()); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "undoing the removal"); + assert!(s.redo()); + assert_eq!(rec.last(), paths(&[]), "redoing it"); + assert!(s.undo()); + s.execute_checked("effect.duplicate", json!({"layer": l, "effect": "EXtractoR"})).unwrap(); + s.execute_checked("effect.remove", json!({"layer": l, "effect": "EXtractoR"})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "the duplicate still reads it"); +} + +/// Opening a project gives the set before the first frame is rendered; opening another sends +/// it again even when it is empty. +#[test] +fn opening_a_project_gives_the_set_before_the_first_frame() { + let made = Arc::new(Recorder::default()); + let mut s = session(&made); + let item = import(&mut s, "/renders/crypto.exr"); + let l = add_layer(&mut s, &item); + s.execute_checked("effect.apply", json!({"effect": "Cryptomatte", "layers": [l]})).unwrap(); + let project: Project = (*s.project).clone(); + let comp = s.state.active_comp.unwrap(); + + let rec = Arc::new(Recorder::default()); + let mut s = Session { footage: rec.clone(), ..Default::default() }; + s.replace_project(project, None); + assert_eq!(rec.last(), paths(&["/renders/crypto.exr"])); + s.render(comp, Tick::ZERO, RenderOpts::default()); + assert_eq!(*rec.at_first_frame.lock().unwrap(), Some(paths(&["/renders/crypto.exr"])), "given before the first frame"); + s.replace_project(Project::default(), None); + assert_eq!(rec.last(), paths(&[])); + let calls = rec.calls(); + s.replace_project(Project::default(), None); + assert_eq!(rec.calls(), calls + 1, "each opened project gets its set"); +} + +/// Two footage items of one file: the file stays in while an effect reads either of them. +#[test] +fn two_items_of_one_file() { + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let a = import(&mut s, "/renders/shared.exr"); + let f = match &s.project.item(ItemId(a.as_u64().unwrap())).unwrap().kind { + ItemKind::Footage(f) => f.clone(), + k => panic!("{k:?}"), + }; + let b = s.edit("Add", None, |p, _| Ok(p.add_item("shared copy", crate::color::Label::None, None, ItemKind::Footage(f)))).unwrap(); + let la = add_layer(&mut s, &a); + let lb = add_layer(&mut s, &json!(b.0)); + s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [la]})).unwrap(); + s.execute_checked("effect.apply", json!({"effect": "ID Matte", "layers": [lb]})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/shared.exr"]), "once"); + s.execute_checked("effect.remove", json!({"layer": la, "effect": "EXtractoR"})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/shared.exr"]), "the other item still reads it"); + s.execute_checked("effect.remove", json!({"layer": lb, "effect": "ID Matte"})).unwrap(); + assert_eq!(rec.last(), paths(&[])); +} + +/// Replacing a layer's source or an item's file moves the set to the new file. +#[test] +fn source_and_footage_replacement_follow() { + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let a = import(&mut s, "/renders/a.exr"); + let b = import(&mut s, "/renders/b.exr"); + let l = add_layer(&mut s, &a); + s.execute_checked("effect.apply", json!({"effect": "Depth Matte", "layers": [l]})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"])); + s.execute_checked("layer.replaceSource", json!({"layers": [l], "item": b})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/b.exr"]), "the layer's new source"); + s.execute_checked("file.replaceFootage", json!({"item": b, "path": "/renders/c.exr"})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/c.exr"]), "the item's new file"); +} diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index 8a8f5edcc..3b5ee6084 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -652,6 +652,8 @@ impl ChannelCache { .get_or_init(|| { decoded_here = true; self.decodes.fetch_add(1, Ordering::Relaxed); + #[cfg(test)] + count_decode(); let _abandon = Abandon { cache: self, slot: &slot }; let bytes = read()?; // The planes it will allocate count against the budget while it decodes. @@ -726,11 +728,37 @@ pub(crate) fn forget(path: &str) { cache().forget(path); } +/// Stop keeping the decodes of the last pictures for their effects ([`HELD_FILES`]): which +/// footage effects read changed, so they may wait for effects that are gone. Their entries stay +/// cached as any other, least recently used out first. +pub(crate) fn release_held() { + cache().lock().held.clear(); +} + /// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { cache().get(path, Use::Channels, read).map(|d| d.aux.clone()) } +#[cfg(test)] +thread_local! { + /// Decodes of a still's pixels on this thread: the channel cache's and the `image` crate's + /// ([`count_decode`]). + static DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Count a decode of a file's pixels on this thread (tests). +#[cfg(test)] +pub(crate) fn count_decode() { + DECODES.with(|d| d.set(d.get() + 1)); +} + +/// The decodes [`count_decode`] counted on this thread since the last call. +#[cfg(test)] +pub(crate) fn take_decodes() -> u64 { + DECODES.with(|d| d.replace(0)) +} + #[cfg(test)] mod tests { use super::*; @@ -1035,6 +1063,69 @@ mod tests { assert_eq!(px, pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap().get(0, 0), "the same picture"); } + /// An EXR file with an unnamed RGB layer and a named `GlossDir.R` of `v` (2×2). + fn rgb_and_gloss(v: f32) -> Arc<[u8]> { + flat(&["R", "G", "B", "GlossDir.R"], (2, 2), v).into() + } + + /// #474: footage the project says effects read the channels of + /// ([`FootageSource::set_aux_footage`]) costs one decode from its first frame on: a single + /// still with EXtractoR decodes once. Without it the first frame decodes twice (the picture, + /// then the channels once `aux()` asks). + #[test] + fn given_aux_footage_decodes_once_from_the_first_frame() { + use effectcraft_render::FootageSource; + let (given, learned) = ("/474-given-still.exr", "/474-learned-still.exr"); + let pool = crate::MediaPool::new(); + let mut f = std::collections::HashMap::new(); + for p in [given, learned] { + pool.add_bytes(p, rgb_and_gloss(0.25)); + f.insert(p, crate::probe_bytes(p, rgb_and_gloss(0.25)).unwrap()); + } + pool.set_aux_footage(&[given.to_string()]); + take_decodes(); + let t = effectcraft_time::Tick::ZERO; + pool.frame_at(&f[given], t).unwrap(); + let aux = pool.aux(effectcraft_project::ItemId(1), &f[given], t).unwrap(); + assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); + assert_eq!(take_decodes(), 1, "the picture and the channels share one decode"); + // Learned only at the first `aux()`: the picture was decoded on its own before. + pool.frame_at(&f[learned], t).unwrap(); + pool.aux(effectcraft_project::ItemId(2), &f[learned], t).unwrap(); + assert_eq!(take_decodes(), 2); + } + + /// #474: once the project no longer names footage (its effect removed), later frames decode + /// only their picture again; what `aux()` learned is dropped with each new set too. + #[test] + fn a_new_aux_footage_set_returns_to_picture_only() { + use effectcraft_render::FootageSource; + let paths: Vec = (1..=4).map(|i| format!("/474-unset_{i:04}.exr")).collect(); + let pool = crate::MediaPool::new(); + for p in &paths { + pool.add_bytes(p, rgb_and_gloss(0.5)); + } + let mut f = crate::probe_bytes(&paths[0], rgb_and_gloss(0.5)).unwrap(); + f.kind = effectcraft_project::FootageKind::Sequence; + f.sequence = paths.clone(); + let at = |i: i64| f.frame_rate.tick_of(i); + let in_cache = |p: &str| cache().cached().iter().any(|c| c == p); + pool.set_aux_footage(std::slice::from_ref(&f.path)); + pool.frame_at(&f, at(0)).unwrap(); + assert!(in_cache(&paths[0]), "given: the picture comes from the channel decode"); + pool.set_aux_footage(&[]); + assert!(!cache().lock().held.iter().any(|(p, _)| *p == paths[0]), "no longer kept once no effect reads it"); + pool.frame_at(&f, at(1)).unwrap(); + assert!(!in_cache(&paths[1]), "no longer given: the picture alone"); + // Learned by `aux()`, then a new set without it: picture only again. + pool.aux(effectcraft_project::ItemId(1), &f, at(1)).unwrap(); + pool.frame_at(&f, at(2)).unwrap(); + assert!(in_cache(&paths[2]), "learned"); + pool.set_aux_footage(&[]); + pool.frame_at(&f, at(3)).unwrap(); + assert!(!in_cache(&paths[3]), "a new set drops what was learned"); + } + /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 /// around it), in every alpha interpretation and float curve. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 910f8238f..e9157a5ca 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -245,9 +245,25 @@ struct Inner { /// Movies that failed to open or decode (path to a readable reason; see /// [`FootageSource::problem`]). The first failure of a file is logged, later ones are not. problems: Mutex, String>>, - /// OpenEXR footage (by path) whose channels an effect has read (EXtractoR, Cryptomatte, the 3D + /// OpenEXR footage (by path) whose channels effects read (EXtractoR, Cryptomatte, the 3D /// Channel effects): its pictures come from the same cached decode (#474). - aux_footage: Mutex>, + aux_footage: Mutex, +} + +/// Footage whose channels effects read ([`Inner::aux_footage`]), by [`Footage::path`]. +#[derive(Default)] +struct AuxFootage { + /// What the project says ([`FootageSource::set_aux_footage`]). + given: HashSet, + /// Read through [`FootageSource::aux`] but not given (a reader the project doesn't name, or + /// a source that never gives a set). Cleared with each new set. + learned: HashSet, +} + +impl AuxFootage { + fn contains(&self, path: &str) -> bool { + self.given.contains(path) || self.learned.contains(path) + } } /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus @@ -868,6 +884,8 @@ impl Inner { { return Ok(img); } + #[cfg(test)] + crate::exr_channels::count_decode(); match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. @@ -964,6 +982,16 @@ impl FootageSource for MediaPool { crate::layered::rasterize_vector(&footage.path, &bytes, footage.layer.as_ref(), footage.page, scale).map(Arc::new) } + fn set_aux_footage(&self, paths: &[String]) { + let given: HashSet = paths.iter().cloned().collect(); + let mut wanted = lock(&self.inner.aux_footage); + if given != wanted.given || !wanted.learned.is_empty() { + crate::exr_channels::release_held(); + } + wanted.given = given; + wanted.learned.clear(); + } + fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { if footage.missing || !footage.has_video { return None; @@ -975,11 +1003,13 @@ impl FootageSource for MediaPool { let path = loc.key.path.clone(); { let mut wanted = lock(&self.inner.aux_footage); - // (a bound: names of footage items, never cleared otherwise) - if wanted.len() >= 4096 { - wanted.clear(); + if !wanted.contains(&footage.path) { + // (a bound: names of footage items, cleared otherwise only by a new set) + if wanted.learned.len() >= 4096 { + wanted.learned.clear(); + } + wanted.learned.insert(footage.path.clone()); } - wanted.insert(footage.path.clone()); } crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) } diff --git a/crates/render/src/footage.rs b/crates/render/src/footage.rs index db1561ad6..8db3530d3 100644 --- a/crates/render/src/footage.rs +++ b/crates/render/src/footage.rs @@ -48,6 +48,12 @@ pub trait FootageSource: Send + Sync { fn aux(&self, _item: ItemId, _footage: &Footage, _t: Tick) -> Option> { None } + /// The footage files (by [`Footage::path`]) whose channels effects of the project read + /// through [`FootageSource::aux`] (EXtractoR, Cryptomatte and the other 3D Channel effects), + /// replacing the set given before. A source that decodes a file's picture and its channels + /// separately can decode both at once for these from the first frame on. Sources that don't + /// ignore it. + fn set_aux_footage(&self, _paths: &[String]) {} /// The parsed 3D model of a [`effectcraft_project::FootageKind::Model`] item (Advanced 3D). fn model(&self, _item: ItemId, _footage: &Footage) -> Option> { None @@ -140,6 +146,9 @@ impl FootageSource for CancellableFootage<'_> { fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { self.inner.aux(item, footage, t) } + fn set_aux_footage(&self, paths: &[String]) { + self.inner.set_aux_footage(paths) + } fn model(&self, item: ItemId, footage: &Footage) -> Option> { self.inner.model(item, footage) } diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 648eadcee..f56a57319 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, except the first frame an effect reads and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte from the first frame on, except files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index 50bd4e7d1..2c88b15e8 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -122,10 +122,12 @@ vectors, positions and IDs are never the picture. EXtractoR reads every other la Footage on its own keeps only the channels of its picture (a single-part file still decompresses all of its pixels to read them). Once an effect reads an item's channels -(EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for the -picture and the effects together, and kept for a while: about 1 GB of decoded channels (512 MB -in the browser), a soft bound, as the newest file and files still in use stay whatever their -size. Several frames asking for the same file at once wait for one decode. +(EXtractoR, Cryptomatte, the 3D Channel effects), each frame, the first included, is decoded +once for the picture and the effects together, and kept for a while: about 1 GB of decoded +channels (512 MB in the browser), a soft bound, as the newest file and files still in use +stay whatever their size. Several frames asking for the same file at once wait for one +decode. Once no enabled effect reads the item any more, its frames go back to the picture's +channels alone. ## Replacing and reloading From 0d2153e25990cc56e41acb565d18620852174763 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 03:27:25 +0100 Subject: [PATCH 09/25] test(media): review follow-ups for the aux footage set (#474) - A watch-folder render gives the host session's set back after its own project replaced it on the shared footage source. - Guard gpu fx_depth::IDS against reads_aux drifting apart. - Count picture and channel decodes per thread instead of reading the shared channel cache, so parallel tests can't make the checks flaky. - The engine test fails when no set was ever sent. - Docs: Media Replacement items are learned at their first frame. Refs #474 Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/watch_folder.rs | 3 ++ crates/engine/src/tests/aux_footage.rs | 29 +++++++++++++-- crates/gpu/src/tests/fx_depth.rs | 7 ++++ crates/media/src/exr_channels.rs | 49 +++++++++++++++----------- crates/media/src/pool.rs | 2 +- docs/file-format-parity.md | 2 +- 6 files changed, 67 insertions(+), 25 deletions(-) diff --git a/crates/engine/src/cmd/watch_folder.rs b/crates/engine/src/cmd/watch_folder.rs index fd41071ed..6e480a9e6 100644 --- a/crates/engine/src/cmd/watch_folder.rs +++ b/crates/engine/src/cmd/watch_folder.rs @@ -88,6 +88,9 @@ fn render_project(host: &Session, path: &Path) -> Value { let failed = s.project.render_queue.iter().any(|i| matches!(i.status, RenderStatus::Failed(_))); Ok(json!({"state": if failed { "failed" } else { "done" }, "items": items})) })(); + // The project shared `host`'s footage source and gave it its own set of footage effects read + // the channels of (#474): give back the host's. + host.footage.set_aux_footage(host.aux_footage.as_deref().unwrap_or(&[])); let mut v = match result { Ok(v) => v, Err(e) => json!({"state": "failed", "error": e, "items": []}), diff --git a/crates/engine/src/tests/aux_footage.rs b/crates/engine/src/tests/aux_footage.rs index 5a74d3b77..01c3c1c96 100644 --- a/crates/engine/src/tests/aux_footage.rs +++ b/crates/engine/src/tests/aux_footage.rs @@ -83,9 +83,9 @@ fn effect_edits_undo_and_redo_update_the_set() { let mut s = session(&rec); let item = import(&mut s, "/renders/a.exr"); let l = add_layer(&mut s, &item); - assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "no effect reads it yet"); + assert_eq!(rec.last(), paths(&[]), "no effect reads it yet"); s.execute_checked("effect.apply", json!({"effect": "Gaussian Blur", "layers": [l]})).unwrap(); - assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "a blur doesn't read channels"); + assert_eq!(rec.last(), paths(&[]), "a blur doesn't read channels"); s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [l]})).unwrap(); assert_eq!(rec.last(), paths(&["/renders/a.exr"])); assert!(s.undo()); @@ -173,3 +173,28 @@ fn source_and_footage_replacement_follow() { s.execute_checked("file.replaceFootage", json!({"item": b, "path": "/renders/c.exr"})).unwrap(); assert_eq!(rec.last(), paths(&["/renders/c.exr"]), "the item's new file"); } + +/// A watch-folder render opens its project in a session sharing the host's footage source: the +/// host's set is given back afterwards. +#[test] +fn a_watch_folder_render_gives_the_host_set_back() { + let dir = std::env::temp_dir().join(format!("ec-aux-footage-watch-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let mut other = Session::default(); + other.execute("comp.new", json!({"name": "Idle", "width": 8, "height": 8, "duration": 1})).unwrap(); + other.execute("file.saveAs", json!({"path": dir.join("idle.ecproj").to_string_lossy()})).unwrap(); + + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let item = import(&mut s, "/renders/a.exr"); + let l = add_layer(&mut s, &item); + s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [l]})).unwrap(); + let calls = rec.calls(); + let r = s.execute("file.watchFolder", json!({"folder": dir.to_string_lossy()})).unwrap(); + assert_eq!(r["rendered"].as_array().map(Vec::len), Some(1), "{r}"); + assert!(rec.calls() > calls + 1, "the watched project gave its own set"); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "the host's set is back"); + s.execute("file.watchFolder", json!({"stop": true})).unwrap(); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/gpu/src/tests/fx_depth.rs b/crates/gpu/src/tests/fx_depth.rs index 8a6665655..ebc75919d 100644 --- a/crates/gpu/src/tests/fx_depth.rs +++ b/crates/gpu/src/tests/fx_depth.rs @@ -208,6 +208,13 @@ fn on() -> Value { Value::Bool(true) } +/// #474: every effect done here reads aux channels, so the session gives the media layer its +/// footage ([`effectcraft_effects::reads_aux`]). +#[test] +fn every_effect_here_reads_aux() { + assert!(crate::fx_depth::IDS.iter().all(|i| effectcraft_effects::reads_aux(i))); +} + #[test] fn channel_extract() { case("ec.3d.channelextract", &[("blackPoint", n(0.0)), ("whitePoint", n(1200.0))]); diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index 3b5ee6084..c3e7cfc3d 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -653,7 +653,7 @@ impl ChannelCache { decoded_here = true; self.decodes.fetch_add(1, Ordering::Relaxed); #[cfg(test)] - count_decode(); + count_decode(true); let _abandon = Abandon { cache: self, slot: &slot }; let bytes = read()?; // The planes it will allocate count against the budget while it decodes. @@ -742,21 +742,27 @@ pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Op #[cfg(test)] thread_local! { - /// Decodes of a still's pixels on this thread: the channel cache's and the `image` crate's - /// ([`count_decode`]). - static DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Decodes of a still's pixels on this thread ([`count_decode`]): the `image` crate's, then + /// the channel cache's. + static DECODES: std::cell::Cell<[u64; 2]> = const { std::cell::Cell::new([0; 2]) }; } -/// Count a decode of a file's pixels on this thread (tests). +/// Count a decode of a file's pixels on this thread (tests): by the channel cache (`channels`) +/// or by the `image` crate. #[cfg(test)] -pub(crate) fn count_decode() { - DECODES.with(|d| d.set(d.get() + 1)); +pub(crate) fn count_decode(channels: bool) { + DECODES.with(|d| { + let mut n = d.get(); + n[usize::from(channels)] += 1; + d.set(n); + }); } -/// The decodes [`count_decode`] counted on this thread since the last call. +/// The decodes [`count_decode`] counted on this thread since the last call: by the `image` crate +/// and by the channel cache. #[cfg(test)] -pub(crate) fn take_decodes() -> u64 { - DECODES.with(|d| d.replace(0)) +pub(crate) fn take_decodes() -> [u64; 2] { + DECODES.with(|d| d.replace([0; 2])) } #[cfg(test)] @@ -1088,11 +1094,11 @@ mod tests { pool.frame_at(&f[given], t).unwrap(); let aux = pool.aux(effectcraft_project::ItemId(1), &f[given], t).unwrap(); assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); - assert_eq!(take_decodes(), 1, "the picture and the channels share one decode"); + assert_eq!(take_decodes(), [0, 1], "the picture and the channels share one decode"); // Learned only at the first `aux()`: the picture was decoded on its own before. pool.frame_at(&f[learned], t).unwrap(); pool.aux(effectcraft_project::ItemId(2), &f[learned], t).unwrap(); - assert_eq!(take_decodes(), 2); + assert_eq!(take_decodes(), [1, 1]); } /// #474: once the project no longer names footage (its effect removed), later frames decode @@ -1109,21 +1115,22 @@ mod tests { f.kind = effectcraft_project::FootageKind::Sequence; f.sequence = paths.clone(); let at = |i: i64| f.frame_rate.tick_of(i); - let in_cache = |p: &str| cache().cached().iter().any(|c| c == p); + // (counted on this thread, so other tests sharing the channel cache don't change them) + let frame = |i: i64| { + take_decodes(); + pool.frame_at(&f, at(i)).unwrap(); + take_decodes() + }; pool.set_aux_footage(std::slice::from_ref(&f.path)); - pool.frame_at(&f, at(0)).unwrap(); - assert!(in_cache(&paths[0]), "given: the picture comes from the channel decode"); + assert_eq!(frame(0), [0, 1], "given: the picture comes from the channel decode"); pool.set_aux_footage(&[]); assert!(!cache().lock().held.iter().any(|(p, _)| *p == paths[0]), "no longer kept once no effect reads it"); - pool.frame_at(&f, at(1)).unwrap(); - assert!(!in_cache(&paths[1]), "no longer given: the picture alone"); + assert_eq!(frame(1), [1, 0], "no longer given: the picture alone"); // Learned by `aux()`, then a new set without it: picture only again. pool.aux(effectcraft_project::ItemId(1), &f, at(1)).unwrap(); - pool.frame_at(&f, at(2)).unwrap(); - assert!(in_cache(&paths[2]), "learned"); + assert_eq!(frame(2), [0, 1], "learned"); pool.set_aux_footage(&[]); - pool.frame_at(&f, at(3)).unwrap(); - assert!(!in_cache(&paths[3]), "a new set drops what was learned"); + assert_eq!(frame(3), [1, 0], "a new set drops what was learned"); } /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index e9157a5ca..b0b2f2539 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -885,7 +885,7 @@ impl Inner { return Ok(img); } #[cfg(test)] - crate::exr_channels::count_decode(); + crate::exr_channels::count_decode(false); match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index f56a57319..ba8c1db7a 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte from the first frame on, except files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte from the first frame on, except Media Replacement items, which are learned at their first frame, and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | From 958e9dc2f8b4225965edc83062948d8b62350795 Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Thu, 5 Nov 2026 11:44:34 +0600 Subject: [PATCH 10/25] Timeline: a double-click on the composition marker bin adds a marker at the current time (#1826) The bin at the right end of the time ruler only answered a drag, so a marker had to be dragged out to where the current time was. As in After Effects, a double-click on it now adds a composition marker at the current time, one undo step. A single click still adds nothing, and dragging a marker out of the bin works as before. Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/panels/markers_ui.rs | 9 +++- crates/ui-egui/tests/it/ui_timeline_zoom.rs | 46 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/src/panels/markers_ui.rs b/crates/ui-egui/src/panels/markers_ui.rs index 54f06d6b6..c24b8a1ca 100644 --- a/crates/ui-egui/src/panels/markers_ui.rs +++ b/crates/ui-egui/src/panels/markers_ui.rs @@ -428,9 +428,16 @@ pub(crate) fn comp_markers(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &C /// (#275): drag a marker out of it and it follows the pointer along the marker strip `strip` /// (snapping like a dragged marker); dropping it there adds a composition marker at that time, /// one undo step. Dropped back on the bin or outside the time graph's width, nothing is added. +/// A double-click on the bin adds one at the current time (#1826). pub(crate) fn marker_bin(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &Comp, tm: TMap, bin: Rect, strip: Rect) { let t = app.tokens; - let resp = ui.interact(bin, egui::Id::unique("marker-bin"), Sense::drag()).on_hover_text(tr("Composition marker bin: drag a marker to the time ruler")); + let resp = + ui.interact(bin, egui::Id::unique("marker-bin"), Sense::click_and_drag()).on_hover_text(tr("Composition marker bin: drag a marker to the time ruler")); + if resp.double_clicked() + && let Err(e) = app.session.execute("comp.addMarker", json!({})) + { + app.ui.status = e.to_string(); + } app.auto.add("timeline.markerBin", bin, "Composition marker bin"); let hot = resp.hovered() || resp.dragged(); if hot { diff --git a/crates/ui-egui/tests/it/ui_timeline_zoom.rs b/crates/ui-egui/tests/it/ui_timeline_zoom.rs index d05ffae64..8ac40c24c 100644 --- a/crates/ui-egui/tests/it/ui_timeline_zoom.rs +++ b/crates/ui-egui/tests/it/ui_timeline_zoom.rs @@ -376,6 +376,52 @@ fn dragging_out_of_the_marker_bin_adds_a_comp_marker() { assert_eq!(markers(&h), vec![13.0, 20.0]); } +/// #1826: as in After Effects, a double-click on the composition marker bin adds a composition +/// marker at the current time, one undo step; a single click adds none, and a marker dragged +/// out of the bin still lands where it is dropped. +#[test] +fn double_clicking_the_marker_bin_adds_a_comp_marker_at_the_current_time() { + let (mut h, _) = harness(); + h.state_mut().ui.timeline.pps = Some(200.0); + h.state_mut().ui.timeline.start = 10.0; + h.state_mut().session.execute("edit.deselectAll", json!({})).unwrap(); + h.state_mut().session.execute("time.set", json!({"time": 21.5})).unwrap(); + h.run_steps(2); + let markers = |h: &Harness<'_, EffectcraftApp>| -> Vec { h.state().session.active_comp().unwrap().markers.iter().map(|m| m.time.seconds()).collect() }; + let bin = rect(&h, "timeline.markerBin"); + let ruler = rect(&h, "timeline.ruler"); + // Presses and releases in one frame each, so a pair falls inside egui's double-click time. + let clicks = |h: &mut Harness<'_, EffectcraftApp>, n: usize| { + h.event(Event::PointerMoved(bin.center())); + h.step(); + for _ in 0..n { + for pressed in [true, false] { + h.input_mut().events.push(Event::PointerButton { + pos: bin.center(), + button: egui::PointerButton::Primary, + pressed, + modifiers: Modifiers::NONE, + }); + } + } + h.step(); + h.run_steps(3); + }; + clicks(&mut h, 1); + assert!(markers(&h).is_empty(), "a single click: {:?}", markers(&h)); + let undo = h.state().session.history.undo.len(); + clicks(&mut h, 2); + assert_eq!(markers(&h), vec![21.5], "at the current time"); + let s = &h.state().session; + assert_eq!(s.active_comp().unwrap().markers[0].time, s.time(), "exactly the current time"); + let history = &h.state().session.history.undo; + assert_eq!((history.len(), history.last().map(|u| u.0.as_str())), (undo + 1, Some("Add Marker")), "one undo step"); + assert_eq!(h.state().session.time().seconds(), 21.5, "the current time stays"); + // Dragged out 3 s into the visible span (from 10 s at 200 px/s): a marker at 13 s too. + drag(&mut h, bin.center(), pos2(ruler.min.x + 6.0 + 600.0, ruler.center().y)); + assert_eq!(markers(&h), vec![13.0, 21.5]); +} + /// #976: as in After Effects, Numpad * marks the selected layers at the current time; only with /// no layer selected does it add a composition marker. #[test] From 5cb2658e382c5d8aac052b24bae8809da7506272 Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Thu, 5 Nov 2026 17:47:44 +0600 Subject: [PATCH 11/25] Timeline: a marker dragged out of the bin and dropped within one frame is added again (#1826) Sensing clicks on the marker bin as well as drags held the drag back until the pointer had moved, so a drag out that was released in that same frame (a fast drag, or a slow frame) added nothing. The bin senses drags only again, so its drag starts the moment it is pressed, as before #1826, and the double-click is read from the pointer over it. The double-click test now also gives the second press and its release frames of their own, with the bin dragged in between. Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/panels/markers_ui.rs | 11 +++--- crates/ui-egui/tests/it/ui_timeline_zoom.rs | 40 ++++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/crates/ui-egui/src/panels/markers_ui.rs b/crates/ui-egui/src/panels/markers_ui.rs index c24b8a1ca..5bdd04ad4 100644 --- a/crates/ui-egui/src/panels/markers_ui.rs +++ b/crates/ui-egui/src/panels/markers_ui.rs @@ -431,11 +431,12 @@ pub(crate) fn comp_markers(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &C /// A double-click on the bin adds one at the current time (#1826). pub(crate) fn marker_bin(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &Comp, tm: TMap, bin: Rect, strip: Rect) { let t = app.tokens; - let resp = - ui.interact(bin, egui::Id::unique("marker-bin"), Sense::click_and_drag()).on_hover_text(tr("Composition marker bin: drag a marker to the time ruler")); - if resp.double_clicked() - && let Err(e) = app.session.execute("comp.addMarker", json!({})) - { + let resp = ui.interact(bin, egui::Id::unique("marker-bin"), Sense::drag()).on_hover_text(tr("Composition marker bin: drag a marker to the time ruler")); + // The bin senses drags only, so a drag starts as soon as it is pressed: sensing clicks too + // would hold that back until the pointer moved, and a drag out released within that frame + // would add nothing. The double-click is read from the pointer instead. + let double_clicked = resp.contains_pointer() && ui.input(|i| i.pointer.button_double_clicked(egui::PointerButton::Primary)); + if double_clicked && let Err(e) = app.session.execute("comp.addMarker", json!({})) { app.ui.status = e.to_string(); } app.auto.add("timeline.markerBin", bin, "Composition marker bin"); diff --git a/crates/ui-egui/tests/it/ui_timeline_zoom.rs b/crates/ui-egui/tests/it/ui_timeline_zoom.rs index 8ac40c24c..78535e3dd 100644 --- a/crates/ui-egui/tests/it/ui_timeline_zoom.rs +++ b/crates/ui-egui/tests/it/ui_timeline_zoom.rs @@ -417,9 +417,47 @@ fn double_clicking_the_marker_bin_adds_a_comp_marker_at_the_current_time() { let history = &h.state().session.history.undo; assert_eq!((history.len(), history.last().map(|u| u.0.as_str())), (undo + 1, Some("Add Marker")), "one undo step"); assert_eq!(h.state().session.time().seconds(), 21.5, "the current time stays"); + // The second press in one frame and its release in the next (a frame here is 0.25 s, inside + // egui's 0.3 s double-click time): the bin is dragged by then, and still a marker at 22 s. + h.state_mut().session.execute("time.set", json!({"time": 22.0})).unwrap(); + h.event(Event::PointerMoved(bin.center())); + h.step(); + for pressed in [true, false, true] { + h.input_mut().events.push(Event::PointerButton { pos: bin.center(), button: egui::PointerButton::Primary, pressed, modifiers: Modifiers::NONE }); + } + h.step(); + assert!(h.ctx.dragged_id().is_some(), "the second press drags the bin"); + h.event(Event::PointerButton { pos: bin.center(), button: egui::PointerButton::Primary, pressed: false, modifiers: Modifiers::NONE }); + h.step(); + h.run_steps(3); + assert_eq!(markers(&h), vec![21.5, 22.0], "a double-click released while the bin is dragged"); // Dragged out 3 s into the visible span (from 10 s at 200 px/s): a marker at 13 s too. drag(&mut h, bin.center(), pos2(ruler.min.x + 6.0 + 600.0, ruler.center().y)); - assert_eq!(markers(&h), vec![13.0, 21.5]); + assert_eq!(markers(&h), vec![13.0, 21.5, 22.0]); +} + +/// A marker dragged out of the bin and dropped within one frame (a fast drag, or a slow frame) +/// still lands where it is dropped: the bin starts its drag as soon as it is pressed. +#[test] +fn a_marker_dragged_out_of_the_bin_and_dropped_in_one_frame_is_added() { + let (mut h, _) = harness(); + h.state_mut().ui.timeline.pps = Some(200.0); + h.state_mut().ui.timeline.start = 10.0; + h.run_steps(2); + let bin = rect(&h, "timeline.markerBin"); + let ruler = rect(&h, "timeline.ruler"); + let at = pos2(ruler.min.x + 6.0 + 600.0, ruler.center().y); + h.event(Event::PointerMoved(bin.center())); + h.step(); + h.event(Event::PointerButton { pos: bin.center(), button: egui::PointerButton::Primary, pressed: true, modifiers: Modifiers::NONE }); + h.step(); + // Both in one frame (`event` would give each a frame of its own). + h.input_mut().events.push(Event::PointerMoved(at)); + h.input_mut().events.push(Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers: Modifiers::NONE }); + h.step(); + h.run_steps(2); + let markers: Vec = h.state().session.active_comp().unwrap().markers.iter().map(|m| m.time.seconds()).collect(); + assert_eq!(markers, vec![13.0]); } /// #976: as in After Effects, Numpad * marks the selected layers at the current time; only with From 5e96b2a38598d52f1444c59982c4cbb11dd232fc Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Thu, 5 Nov 2026 18:05:43 +0600 Subject: [PATCH 12/25] Timeline: the marker bin's double-click needs both clicks on the bin (#1826) egui takes any click within a few points of the last one for the first click of a double-click, so a click on the ruler right beside the bin and then one on the bin added a marker. The bin now remembers when it was last clicked and adds one only when the first click was on it too (or both came in one frame over it). Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/panels/markers_ui.rs | 37 +++++++++++++++++++-- crates/ui-egui/tests/it/ui_timeline_zoom.rs | 12 +++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/ui-egui/src/panels/markers_ui.rs b/crates/ui-egui/src/panels/markers_ui.rs index 5bdd04ad4..c23a6605c 100644 --- a/crates/ui-egui/src/panels/markers_ui.rs +++ b/crates/ui-egui/src/panels/markers_ui.rs @@ -435,8 +435,9 @@ pub(crate) fn marker_bin(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &Com // The bin senses drags only, so a drag starts as soon as it is pressed: sensing clicks too // would hold that back until the pointer moved, and a drag out released within that frame // would add nothing. The double-click is read from the pointer instead. - let double_clicked = resp.contains_pointer() && ui.input(|i| i.pointer.button_double_clicked(egui::PointerButton::Primary)); - if double_clicked && let Err(e) = app.session.execute("comp.addMarker", json!({})) { + if bin_double_clicked(ui, resp.id, bin, resp.contains_pointer()) + && let Err(e) = app.session.execute("comp.addMarker", json!({})) + { app.ui.status = e.to_string(); } app.auto.add("timeline.markerBin", bin, "Composition marker bin"); @@ -463,6 +464,38 @@ pub(crate) fn marker_bin(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &Com } } +/// Whether the marker bin `bin` (widget `id`, under the pointer when `here`) was double-clicked +/// this frame: egui's double-click, with its first click on the bin too. egui takes any click +/// within a few points of the last for the first one, so a click on the ruler right beside the +/// bin and then one on the bin would count. The bin remembers when it was last clicked; two +/// clicks on it within one frame need no memory. +fn bin_double_clicked(ui: &egui::Ui, id: egui::Id, bin: Rect, here: bool) -> bool { + use egui::{Event, PointerButton}; + if !here { + return false; + } + let (time, clicked, double, releases) = ui.input(|i| { + let releases = i + .events + .iter() + .filter(|e| matches!(e, Event::PointerButton { button: PointerButton::Primary, pressed: false, pos, .. } if bin.contains(*pos))) + .count(); + (i.time, i.pointer.button_clicked(PointerButton::Primary), i.pointer.button_double_clicked(PointerButton::Primary), releases) + }); + if !clicked { + return false; + } + let delay = ui.ctx().options(|o| o.input_options.max_double_click_delay); + let id = id.with("last click"); + let last = ui.data_mut(|d| { + let last = d.get_temp::(id); + d.insert_temp(id, time); + last + }); + let last = last.unwrap_or(f64::NEG_INFINITY); + double && (releases >= 2 || time - last < delay) +} + /// Layer markers on a layer's row `r`; on a precomp layer, its comp's markers too (read-only, /// outlined: hover names them, a double-click opens the nested comp at the marker). pub(crate) fn layer_markers(app: &mut EffectcraftApp, ui: &mut egui::Ui, clip: Rect, comp: &Comp, layer: &Layer, tm: TMap, r: Rect) { diff --git a/crates/ui-egui/tests/it/ui_timeline_zoom.rs b/crates/ui-egui/tests/it/ui_timeline_zoom.rs index 78535e3dd..f8a5783f2 100644 --- a/crates/ui-egui/tests/it/ui_timeline_zoom.rs +++ b/crates/ui-egui/tests/it/ui_timeline_zoom.rs @@ -431,6 +431,18 @@ fn double_clicking_the_marker_bin_adds_a_comp_marker_at_the_current_time() { h.step(); h.run_steps(3); assert_eq!(markers(&h), vec![21.5, 22.0], "a double-click released while the bin is dragged"); + // A click on the ruler right beside the bin and then one on the bin, in one frame: egui + // takes them for a double-click (they are a few points apart), but only one was on the bin. + let (beside, on) = (pos2(bin.min.x - 2.0, bin.center().y), pos2(bin.min.x + 2.0, bin.center().y)); + assert!(ruler.contains(beside) && !bin.contains(beside) && bin.contains(on)); + h.event(Event::PointerMoved(beside)); + h.step(); + for (pos, pressed) in [(beside, true), (beside, false), (on, true), (on, false)] { + h.input_mut().events.push(Event::PointerButton { pos, button: egui::PointerButton::Primary, pressed, modifiers: Modifiers::NONE }); + } + h.step(); + h.run_steps(3); + assert_eq!(markers(&h), vec![21.5, 22.0], "one click beside the bin and one on it"); // Dragged out 3 s into the visible span (from 10 s at 200 px/s): a marker at 13 s too. drag(&mut h, bin.center(), pos2(ruler.min.x + 6.0 + 600.0, ruler.center().y)); assert_eq!(markers(&h), vec![13.0, 21.5, 22.0]); From 133dcddf4de8b2718edc44df0dab514454e0faeb Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Thu, 5 Nov 2026 18:26:39 +0600 Subject: [PATCH 13/25] Timeline: the marker bin pairs its own clicks, in the order they come (#1826) The bin counted a double-click from egui's flag for the whole frame and the number of releases over it, so three clicks in one frame, on the bin, beside it and on it again, still added a marker. It now walks the frame's pointer events itself, by egui's rules for a click and a double-click, and adds one only for two clicks in a row that are both on the bin. It also counts while the bin is dragged or its drag has just stopped, not only while the pointer is over it. Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/panels/markers_ui.rs | 81 ++++++++++++++------- crates/ui-egui/tests/it/ui_timeline_zoom.rs | 10 +++ 2 files changed, 63 insertions(+), 28 deletions(-) diff --git a/crates/ui-egui/src/panels/markers_ui.rs b/crates/ui-egui/src/panels/markers_ui.rs index c23a6605c..61a75e72f 100644 --- a/crates/ui-egui/src/panels/markers_ui.rs +++ b/crates/ui-egui/src/panels/markers_ui.rs @@ -435,7 +435,7 @@ pub(crate) fn marker_bin(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &Com // The bin senses drags only, so a drag starts as soon as it is pressed: sensing clicks too // would hold that back until the pointer moved, and a drag out released within that frame // would add nothing. The double-click is read from the pointer instead. - if bin_double_clicked(ui, resp.id, bin, resp.contains_pointer()) + if bin_double_clicked(ui, resp.id, bin, resp.contains_pointer() || resp.dragged() || resp.drag_stopped()) && let Err(e) = app.session.execute("comp.addMarker", json!({})) { app.ui.status = e.to_string(); @@ -464,36 +464,61 @@ pub(crate) fn marker_bin(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &Com } } -/// Whether the marker bin `bin` (widget `id`, under the pointer when `here`) was double-clicked -/// this frame: egui's double-click, with its first click on the bin too. egui takes any click -/// within a few points of the last for the first one, so a click on the ruler right beside the -/// bin and then one on the bin would count. The bin remembers when it was last clicked; two -/// clicks on it within one frame need no memory. -fn bin_double_clicked(ui: &egui::Ui, id: egui::Id, bin: Rect, here: bool) -> bool { +/// What the marker bin keeps of the primary button between frames: the press being held (when, +/// where, and whether it has moved too far since to be a click), and the last click (when, +/// where, and whether it was on the bin). +#[derive(Clone, Copy, Default)] +struct BinClicks { + press: Option<(f64, Pos2, bool)>, + last: Option<(f64, Pos2, bool)>, +} + +/// Whether the marker bin `bin` (widget `id`; `over` when it, not something above it, has the +/// pointer) was double-clicked this frame: two clicks in a row, both on the bin. egui's own +/// double-click takes any click within a few points of the last for the first one, so a click on +/// the ruler right beside the bin and then one on the bin would count. The bin pairs the clicks +/// itself, in the order they came, by egui's rules: a click is released within `max_click_dist` +/// of where it was pressed and within `max_click_duration`, and the second of a double-click comes +/// within `max_double_click_delay` and `max_click_dist` of the first. A third click starts over. +fn bin_double_clicked(ui: &egui::Ui, id: egui::Id, bin: Rect, over: bool) -> bool { use egui::{Event, PointerButton}; - if !here { - return false; - } - let (time, clicked, double, releases) = ui.input(|i| { - let releases = i - .events - .iter() - .filter(|e| matches!(e, Event::PointerButton { button: PointerButton::Primary, pressed: false, pos, .. } if bin.contains(*pos))) - .count(); - (i.time, i.pointer.button_clicked(PointerButton::Primary), i.pointer.button_double_clicked(PointerButton::Primary), releases) + let (dist, duration, delay) = ui.ctx().options(|o| { + let i = &o.input_options; + (i.max_click_dist, i.max_click_duration, i.max_double_click_delay) }); - if !clicked { - return false; - } - let delay = ui.ctx().options(|o| o.input_options.max_double_click_delay); - let id = id.with("last click"); - let last = ui.data_mut(|d| { - let last = d.get_temp::(id); - d.insert_temp(id, time); - last + let id = id.with("clicks"); + let mut s = ui.data(|d| d.get_temp::(id)).unwrap_or_default(); + let double = ui.input(|i| { + let mut double = false; + for e in &i.events { + match *e { + Event::PointerMoved(p) => { + if let Some((_, at, moved)) = &mut s.press { + *moved |= at.distance(p) > dist; + } + } + Event::PointerButton { button: PointerButton::Primary, pressed: true, pos, .. } => s.press = Some((i.time, pos, false)), + Event::PointerButton { button: PointerButton::Primary, pressed: false, pos, .. } => { + let Some((pressed_at, at, moved)) = s.press.take() else { continue }; + if moved || at.distance(pos) > dist || i.time - pressed_at >= duration { + continue; + } + let on = bin.contains(pos); + let pair = s.last.is_some_and(|(t, p, was_on)| was_on && on && i.time - t < delay && p.distance(pos) < dist); + if pair { + double = true; + s.last = None; + } else { + s.last = Some((i.time, pos, on)); + } + } + _ => {} + } + } + double }); - let last = last.unwrap_or(f64::NEG_INFINITY); - double && (releases >= 2 || time - last < delay) + ui.data_mut(|d| d.insert_temp(id, s)); + double && over } /// Layer markers on a layer's row `r`; on a precomp layer, its comp's markers too (read-only, diff --git a/crates/ui-egui/tests/it/ui_timeline_zoom.rs b/crates/ui-egui/tests/it/ui_timeline_zoom.rs index f8a5783f2..17666e3db 100644 --- a/crates/ui-egui/tests/it/ui_timeline_zoom.rs +++ b/crates/ui-egui/tests/it/ui_timeline_zoom.rs @@ -443,6 +443,16 @@ fn double_clicking_the_marker_bin_adds_a_comp_marker_at_the_current_time() { h.step(); h.run_steps(3); assert_eq!(markers(&h), vec![21.5, 22.0], "one click beside the bin and one on it"); + // On the bin, beside it, on it again: the two in a row are never both on the bin. + h.run_steps(3); + for pos in [on, beside, on] { + for pressed in [true, false] { + h.input_mut().events.push(Event::PointerButton { pos, button: egui::PointerButton::Primary, pressed, modifiers: Modifiers::NONE }); + } + } + h.step(); + h.run_steps(3); + assert_eq!(markers(&h), vec![21.5, 22.0], "on, beside, on"); // Dragged out 3 s into the visible span (from 10 s at 200 px/s): a marker at 13 s too. drag(&mut h, bin.center(), pos2(ruler.min.x + 6.0 + 600.0, ruler.center().y)); assert_eq!(markers(&h), vec![13.0, 21.5, 22.0]); From 0c7cf039b96768d08081f6d7d4fb1f42da5b1fda Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Thu, 5 Nov 2026 18:48:34 +0600 Subject: [PATCH 14/25] Timeline: a bin click is pressed and released on the bin, while the bin has the pointer (#1826) A press on the ruler right beside the marker bin, released a few points away on the bin, is still a click, and it counted as one on the bin; two of those added a marker. A click now counts as the bin's only when both its press and its release are on the bin, in a frame when the bin is enabled and has the pointer (not a popup above it). Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/panels/markers_ui.rs | 13 +++++++------ crates/ui-egui/tests/it/ui_timeline_zoom.rs | 10 ++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/crates/ui-egui/src/panels/markers_ui.rs b/crates/ui-egui/src/panels/markers_ui.rs index 61a75e72f..ee33fc9de 100644 --- a/crates/ui-egui/src/panels/markers_ui.rs +++ b/crates/ui-egui/src/panels/markers_ui.rs @@ -435,7 +435,7 @@ pub(crate) fn marker_bin(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &Com // The bin senses drags only, so a drag starts as soon as it is pressed: sensing clicks too // would hold that back until the pointer moved, and a drag out released within that frame // would add nothing. The double-click is read from the pointer instead. - if bin_double_clicked(ui, resp.id, bin, resp.contains_pointer() || resp.dragged() || resp.drag_stopped()) + if bin_double_clicked(ui, resp.id, bin, resp.enabled() && (resp.contains_pointer() || resp.dragged() || resp.drag_stopped())) && let Err(e) = app.session.execute("comp.addMarker", json!({})) { app.ui.status = e.to_string(); @@ -466,15 +466,16 @@ pub(crate) fn marker_bin(app: &mut EffectcraftApp, ui: &mut egui::Ui, comp: &Com /// What the marker bin keeps of the primary button between frames: the press being held (when, /// where, and whether it has moved too far since to be a click), and the last click (when, -/// where, and whether it was on the bin). +/// where, and whether it was on the bin: both pressed and released there). #[derive(Clone, Copy, Default)] struct BinClicks { press: Option<(f64, Pos2, bool)>, last: Option<(f64, Pos2, bool)>, } -/// Whether the marker bin `bin` (widget `id`; `over` when it, not something above it, has the -/// pointer) was double-clicked this frame: two clicks in a row, both on the bin. egui's own +/// Whether the marker bin `bin` (widget `id`; `over` when it is enabled and has the pointer, not +/// something above it) was double-clicked this frame: two clicks in a row, both on the bin, each +/// in a frame when it was `over`. egui's own /// double-click takes any click within a few points of the last for the first one, so a click on /// the ruler right beside the bin and then one on the bin would count. The bin pairs the clicks /// itself, in the order they came, by egui's rules: a click is released within `max_click_dist` @@ -503,7 +504,7 @@ fn bin_double_clicked(ui: &egui::Ui, id: egui::Id, bin: Rect, over: bool) -> boo if moved || at.distance(pos) > dist || i.time - pressed_at >= duration { continue; } - let on = bin.contains(pos); + let on = over && bin.contains(at) && bin.contains(pos); let pair = s.last.is_some_and(|(t, p, was_on)| was_on && on && i.time - t < delay && p.distance(pos) < dist); if pair { double = true; @@ -518,7 +519,7 @@ fn bin_double_clicked(ui: &egui::Ui, id: egui::Id, bin: Rect, over: bool) -> boo double }); ui.data_mut(|d| d.insert_temp(id, s)); - double && over + double } /// Layer markers on a layer's row `r`; on a precomp layer, its comp's markers too (read-only, diff --git a/crates/ui-egui/tests/it/ui_timeline_zoom.rs b/crates/ui-egui/tests/it/ui_timeline_zoom.rs index 17666e3db..76f55740a 100644 --- a/crates/ui-egui/tests/it/ui_timeline_zoom.rs +++ b/crates/ui-egui/tests/it/ui_timeline_zoom.rs @@ -453,6 +453,16 @@ fn double_clicking_the_marker_bin_adds_a_comp_marker_at_the_current_time() { h.step(); h.run_steps(3); assert_eq!(markers(&h), vec![21.5, 22.0], "on, beside, on"); + // Twice pressed beside the bin and released on it, a few points away: clicks, not on the bin. + h.run_steps(3); + for _ in 0..2 { + for (pos, pressed) in [(beside, true), (on, false)] { + h.input_mut().events.push(Event::PointerButton { pos, button: egui::PointerButton::Primary, pressed, modifiers: Modifiers::NONE }); + } + } + h.step(); + h.run_steps(3); + assert_eq!(markers(&h), vec![21.5, 22.0], "pressed beside, released on"); // Dragged out 3 s into the visible span (from 10 s at 200 px/s): a marker at 13 s too. drag(&mut h, bin.center(), pos2(ruler.min.x + 6.0 + 600.0, ruler.center().y)); assert_eq!(markers(&h), vec![13.0, 21.5, 22.0]); From 1786c24523a5d5a67c01ff41213b66018a69bb04 Mon Sep 17 00:00:00 2001 From: Anubhav-Rai <78999271+Anubhav-Rai@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:51:59 +0530 Subject: [PATCH 15/25] expr: an expression cannot rebind a prelude name for the next one (#2133) (#2176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * expr: an expression cannot rebind a prelude name for the next one (#2133) One runtime serves every expression on a thread and user code runs as a script at global scope, so `linear = function () {}` rebound the name for every later expression, and for every later project in the session. #1387 closed the `__` globals, the model constructors and the prelude's own prototypes; the rest of the global surface is here: the prelude's own 104 helpers, the built-ins that an expression and the prelude both call, and the prototypes those hang off. Freezing a prototype costs an expression the right to shadow an inherited name on its own object: once the inherited data property is non-writable, `o.x = …` is dropped in sloppy mode rather than creating an own property, which is the override mistake. Every name that was shadowable before is therefore replaced by an accessor whose setter writes an own property on the receiver, emulating OrdinarySetWithOwnDescriptor so the substitution does not show through. The repair covers every own writable, configurable data property of every prototype the block reaches, not a list of commonly shadowed names: a list leaves `a.map = …` throwing and `a.push = …` and `a.join = …` silently running the native method, which is worse, because the expression keeps working and quietly computes something else. `name` and `message` on an error's prototype are the one exception, and they are frozen rather than repaired. The host renders a thrown error by reading both off the object with no context to call a getter with, and nowhere safe to call one: the time budget is checked between polls of the evaluation and formatting runs after that returns, so a getter of the expression's own would run outside it, bounded only by the loop and recursion limits at some hundreds of millions of iterations. `e.name = 'MyError'` is dropped in silence as the price of keeping user code out of the failure path. The setup code reads every intrinsic it needs before the first prototype is rewritten, and indexes arrays rather than calling their methods. Repairing `Array.prototype` turns `concat` and `forEach` into accessors, and an earlier attempt at this used both while walking the remaining globals, which left every prototype after `Array.prototype` unfrozen and the prelude failing to load at all. Measured on macOS arm64, release, min of 3 x N, against this base: an inherited-name read costs ~68 ns where it is now an accessor, which is +18% on a loop doing a hundred of them and +2% to +7% on ordinary expressions. Extending the repair past `Object.prototype` is what the array-method read pays for; nothing else moves. Co-Authored-By: Claude Opus 5 * expr: the prelude's lock answers delete and defineProperty too (#2133) Tests for the two routes to a global binding the existing ones did not cover against a prelude function, asked for in review. `writable: false` answers an assignment and a `var`; it is `configurable: false` that answers these, and they fail differently: a `delete` of a non-configurable property evaluates to false in sloppy mode, while a `defineProperty` over one throws, so that is the one route whose refusal the user sees. Also records which mode decides that, because it was asked and the answer is not the obvious one: an expression cannot reach strict mode at all. `Runtime::script` wraps the text in `try{…}catch(__e){throw __e}`, so a `"use strict"` the user writes is an expression statement inside a block rather than a directive prologue and has no effect. A refused write is therefore always the silent kind, never a TypeError. No change to the prelude; these pass as it stands. Co-Authored-By: Claude Opus 5 * expr: freeze Math and JSON plainly instead of repairing them (#2176) Math and JSON are namespaces, not prototypes: nothing inherits sin or parse, so the accessor repair keeps no shadowing there and only puts a getter call on every Math.* read of a per-frame expression. Freeze them directly and lock the two bindings so Math = {} cannot replace them for the next expression either. Prototypes keep the repair. --------- Co-authored-by: Anubhav-Rai Co-authored-by: Claude Opus 5 Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> --- crates/expr/src/prelude.js | 155 ++++++++++++++++ crates/expr/src/tests.rs | 353 +++++++++++++++++++++++++++++++++++++ 2 files changed, 508 insertions(+) diff --git a/crates/expr/src/prelude.js b/crates/expr/src/prelude.js index c6fcbab03..eefacc876 100644 --- a/crates/expr/src/prelude.js +++ b/crates/expr/src/prelude.js @@ -1295,3 +1295,158 @@ function nearestKey(t) { return thisProperty.nearestKey(t); } if ('value' in Object.getOwnPropertyDescriptor(globalThis, k)) Object.defineProperty(globalThis, k, { writable: false, configurable: false }); }); })(); + +// --------------------------------------------------------------------------------------------- +// One runtime serves every expression on a thread, and user code runs as a script at global +// scope, so `linear = …` or a top-level `var clamp` would otherwise rebind a prelude name for +// every later expression, and for every later project in the session (#2133). The block above +// closes the `__` globals, the model constructors and the prelude's own prototypes; what is left +// is the rest of the global surface: the prelude's own helpers (`linear`, `clamp`, `wiggle`), the +// JavaScript built-ins that an expression and the prelude both call, and the prototypes those +// hang off. A non-writable, non-configurable binding makes such a write a silent no-op in sloppy +// mode, and `var` leaves an existing global property alone. A name the user introduces is not +// frozen, so an expression that declares its own functions still works. +(function () { + var g = globalThis; + // Every intrinsic this block needs, read once before the first prototype is rewritten. The + // block turns `Array.prototype.concat` and `forEach` into accessors as it goes, so reaching for + // them through a prototype part-way down the list would be reaching for something it has just + // changed under itself: the loops below index arrays directly for the same reason. + var describe = Object.getOwnPropertyDescriptor, define = Object.defineProperty; + var getNames = Object.getOwnPropertyNames, getSyms = Object.getOwnPropertySymbols; + var freeze = Object.freeze, isFrozen = Object.isFrozen, isExtensible = Object.isExtensible; + var protoOf = Object.getPrototypeOf; + + var lock = function (n) { + var d = describe(g, n); + if (!d || !d.writable) return; + define(g, n, { value: d.value, writable: false, enumerable: d.enumerable, configurable: false }); + }; + + // `name` and `message` on an error's prototype are the two names the repair below has to leave + // alone. The host renders a thrown error by reading both straight off the object, with no + // context to call a getter with, and it has no way to run one safely either: the time budget is + // checked between polls of the evaluation, and formatting happens after that returns, so a + // getter of the expression's own would run outside it — bounded by the loop and recursion limits + // at some hundreds of millions of iterations, which is a hang. Left as data properties they are + // frozen rather than repaired, so `e.name = 'MyError'` on an expression's own error is dropped + // in silence. That is the cost; an unbounded getter in the failure path is not worth paying to + // avoid it. + var HOST_READ = ['name', 'message']; + var hostRead = []; + var errorNames = ['Error', 'EvalError', 'RangeError', 'ReferenceError', 'SyntaxError', 'TypeError', + 'URIError', 'AggregateError']; + for (var e = 0; e < errorNames.length; e++) { + var C = g[errorNames[e]]; + if (typeof C === 'function') { + var cd = describe(C, 'prototype'); + if (cd && 'value' in cd) hostRead.push(cd.value); + } + } + var readByHost = function (p, n) { + for (var i = 0; i < HOST_READ.length; i++) { + if (n !== HOST_READ[i]) continue; + for (var j = 0; j < hostRead.length; j++) if (hostRead[j] === p) return true; + } + return false; + }; + + // Freezing a prototype costs an expression the right to shadow an inherited name on its own + // object: once the inherited data property is non-writable, `o.toString = …` is dropped in + // sloppy mode rather than creating an own property — the "override mistake". That is ordinary + // expression code: `counts[w]` with `w` of "constructor" or "valueOf" uses an object as a + // dictionary, and `a.map = …` gives one array its own version of a method. So every name that + // was shadowable before the freeze becomes an accessor whose setter writes an own property on + // the instance, which leaves the shadowing that worked before this block working after it. + var tolerate = function (p, n) { + var d = describe(p, n); + // An accessor has no value to stand in for, and a name already non-writable or + // non-configurable could not be shadowed before this ran either: no regression to repair. + if (!d || !d.configurable || !('value' in d) || !d.writable) return; + var v = d.value; + define(p, n, { + get: function () { return v; }, + set: function (nv) { + // This setter stands in for the data property it replaced, so it answers a write the way + // that property would: an accessor is handed the receiver and nothing else, while a data + // property has the receiver's own property consulted first + // (OrdinarySetWithOwnDescriptor). Doing that here keeps the repair invisible — an own + // writable property is updated in place and keeps its other attributes, an own + // non-writable one drops the write rather than being overwritten, and a target that + // cannot take a new property is left alone instead of being handed to `defineProperty`, + // which would throw. That last case covers a frozen prototype, an object the expression + // froze itself, and `null` or `undefined`. + if (this === p || this === null || this === undefined) return; + var own = describe(this, n); + if (own) { + if (own.writable) define(this, n, { value: nv }); + return; + } + if (!isExtensible(this)) return; + define(this, n, { value: nv, writable: true, enumerable: true, configurable: true }); + }, + enumerable: d.enumerable, + configurable: false, + }); + }; + + var freezeProto = function (p) { + if (p === null || (typeof p !== 'object' && typeof p !== 'function') || isFrozen(p)) return; + var names = getNames(p); + for (var i = 0; i < names.length; i++) if (!readByHost(p, names[i])) tolerate(p, names[i]); + // Symbol keys go the same way: `a[Symbol.iterator] = …` on an expression's own array is an + // ordinary way to make something iterable, and left frozen it is dropped in silence while + // `for…of` quietly walks the array's own elements instead. + var syms = getSyms(p); + for (var j = 0; j < syms.length; j++) tolerate(p, syms[j]); + freeze(p); + }; + + // Freezing a function also makes its own `prototype` property non-writable, so `Comp.prototype = + // {}` is a no-op; freezing that prototype object closes the methods hanging off it. + var shut = function (o) { + var d = describe(o, 'prototype'); + if (d && 'value' in d) freezeProto(d.value); + freeze(o); + }; + + var globals = getNames(g); + for (var k = 0; k < globals.length; k++) { + var n = globals[k]; + var d = describe(g, n); + if (!d || !('value' in d) || typeof d.value !== 'function') continue; + // The function object as well as the binding: the prelude calls `Array.isArray` and + // `Object.setPrototypeOf` while it answers an expression, so a replacement for one of those + // would not merely shadow a name, it would change what a later expression computes. + shut(d.value); + lock(n); + } + // `Math.sin = …` goes the same way, and the prelude leans on both of these itself. They are the + // `prototype` of no function, so the loop above never reaches them, and they are the prototype + // of no object either: nothing inherits `sin` or `parse`, so there is no shadowing for the + // accessor repair to keep, and a plain freeze closes them. That matters on the hot path, where + // a repaired name costs an accessor call on every read and a per-frame expression reads + // `Math.*` more than anything else. The bindings are locked as well, so `Math = {}` cannot take + // them away from the next expression either. + freeze(Math); + freeze(JSON); + lock('Math'); + lock('JSON'); + // The iterator intrinsics are the `prototype` of no global function either, and `for…of`, + // destructuring and spread all run through them: replacing `next` on the array iterator left + // every later expression on the thread unable to iterate at all. + var walk = function (o) { + for (var p = o; p !== null && p !== undefined; p = protoOf(p)) freezeProto(p); + }; + var probes = [function () { return [][Symbol.iterator](); }, + function () { return ''[Symbol.iterator](); }, + function () { return new Map()[Symbol.iterator](); }, + function () { return new Set()[Symbol.iterator](); }, + function () { return (function* () {})(); }, + function () { return 'a'.matchAll(/a/g); }, + function () { return Int8Array.prototype; }]; + for (var m = 0; m < probes.length; m++) { + // Each one is a probe for an intrinsic this engine may not have at all. + try { walk(protoOf(probes[m]())); } catch (e) { /* not present */ } + } +})(); diff --git a/crates/expr/src/tests.rs b/crates/expr/src/tests.rs index fd5de44fa..48f656e52 100644 --- a/crates/expr/src/tests.rs +++ b/crates/expr/src/tests.rs @@ -193,6 +193,315 @@ fn functions_and_let_are_reusable_across_evaluations() { assert_close!(f.num(e, 2.0), 9.0); } +#[test] +fn an_expression_cannot_rebind_a_prelude_name_for_the_next_one() { + let f = fx(); + // #2133: one runtime serves every expression on a thread and user code runs at global scope, so + // a write to a prelude name used to outlive the expression that made it and poison every later + // one — in this session's other projects too. + assert_close!(f.num("linear = function () { return 777 };\n50", 0.0), 50.0); + assert_close!(f.num("linear(time, 0, 1, 0, 100)", 0.0), 0.0); + assert_close!(f.num("var clamp = 5;\n10", 0.0), 10.0); + assert_close!(f.num("clamp(50, 0, 100)", 0.0), 50.0); + assert_close!(f.num("Math.sin = function () { return 9 };\n1", 0.0), 1.0); + assert_close!(f.num("Math.sin(Math.PI / 2) * 100", 0.0), 100.0); + assert_close!(f.num("Object.prototype.leak = 9;\n2", 0.0), 2.0); + assert_close!(f.num("({}).leak === undefined ? 1 : 0", 0.0), 1.0); + // The runtime is one thread-local singleton, so a second project on this thread meets the same + // objects: the isolation has to hold between projects, not only between the evaluations of one. + let g = fx(); + assert_close!(g.num("linear(time, 0, 1, 0, 100)", 0.0), 0.0); + assert_close!(g.num("clamp(50, 0, 100)", 0.0), 50.0); + assert_close!(g.num("Math.sin(Math.PI / 2) * 100", 0.0), 100.0); + assert_close!(g.num("({}).leak === undefined ? 1 : 0", 0.0), 1.0); +} + +/// The four ways a script can reach a global binding, against a prelude *function* rather than one +/// of the unqualified accessors: assignment, `var`, `delete` and `defineProperty`. The lock is +/// `writable: false, configurable: false`, and it is the second half of that pair that answers the +/// last two — a `delete` of a non-configurable property is false, and a `defineProperty` over one +/// is a `TypeError`. Asked for by a reviewer on #2176. +#[test] +fn a_prelude_function_survives_every_way_a_script_can_reach_its_binding() { + let f = fx(); + // Assignment and `var`: silent, and the next expression still has the original. + assert_close!(f.num("clamp = function () { return 777 };\n1", 0.0), 1.0); + assert_close!(f.num("var clamp = 5;\n2", 0.0), 2.0); + assert_close!(f.num("clamp(50, 0, 100)", 0.0), 50.0); + // `delete` of a non-configurable binding answers false in sloppy mode rather than throwing. + assert_close!(f.num("delete clamp ? 1 : 0", 0.0), 0.0); + assert_close!(f.num("clamp(50, 0, 100)", 0.0), 50.0); + assert_close!(f.num("typeof clamp === 'function' ? 1 : 0", 0.0), 1.0); + // `defineProperty` over one is a `TypeError`, so this is the one route the user does see. The + // expression fails; what it was reaching for is unchanged for the next one. + assert!(f.err("Object.defineProperty(globalThis, 'clamp', { value: 5 })").contains("TypeError")); + assert_close!(f.num("clamp(50, 0, 100)", 0.0), 50.0); + // Reconfiguring it to writable is refused the same way, so the lock cannot be reopened. + assert!(f.err("Object.defineProperty(globalThis, 'clamp', { writable: true })").contains("TypeError")); + assert_close!(f.num("clamp = 7;\n3", 0.0), 3.0); + assert_close!(f.num("clamp(50, 0, 100)", 0.0), 50.0); + // A second project on the same thread-local runtime meets the same bindings. + let g = fx(); + assert_close!(g.num("clamp(50, 0, 100)", 0.0), 50.0); + assert_close!(g.num("typeof clamp === 'function' ? 1 : 0", 0.0), 1.0); +} + +/// Which mode a write is silent in decides what the user sees, and the answer is that an expression +/// cannot reach strict mode at all: `Runtime::script` wraps the text in `try{…}catch(__e){throw __e}`, +/// so a `"use strict"` the user writes is an expression statement inside a block rather than a +/// directive prologue, and has no effect. A failed write is therefore always the silent kind. +/// Asked for by a reviewer on #2176. +#[test] +fn a_use_strict_in_an_expression_is_inert_so_a_refused_write_stays_silent() { + let f = fx(); + assert_close!(f.num("\"use strict\";\nlinear = function () { return 777 };\n50", 0.0), 50.0); + assert_close!(f.num("linear(time, 0, 1, 0, 100)", 0.0), 0.0); + // The directive is inert rather than honoured: in strict mode an assignment to an undeclared + // name is a ReferenceError, and here it quietly creates a global instead. + assert_close!(f.num("\"use strict\";\nundeclared_global = 5;\nundeclared_global", 0.0), 5.0); + // Which is also why the refused write above cannot be made to report itself by the user: the + // only mode an expression runs in is the one where it is dropped. + assert_close!(f.num("\"use strict\";\ntypeof linear === 'function' ? 1 : 0", 0.0), 1.0); +} + +/// The same leak, reached through the objects instead of the names: a prelude constructor keeps its +/// methods on its `prototype`, the prelude itself calls `Array.isArray` and `Object.setPrototypeOf` +/// while it answers an expression, and `thisProject` is reachable surface that holds no function at +/// all (#2133). +#[test] +fn an_expression_cannot_tamper_with_a_prelude_object_for_the_next_one() { + let f = fx(); + // An unqualified name is an accessor, so it takes a `defineProperty` or a `delete` rather than + // an assignment; the setter an assignment goes through writes per-evaluation state. + assert_close!(f.num("try { Object.defineProperty(globalThis, 'width', { value: 777 }) } catch (e) {}\n1", 0.0), 1.0); + assert_close!(f.num("delete width;\n2", 0.0), 2.0); + assert_close!(f.num("width", 0.0), 20.0); + assert_close!(f.num("Comp.prototype = {};\n3", 0.0), 3.0); + assert_close!(f.num("Comp.prototype.leak = 9;\n4", 0.0), 4.0); + assert_close!(f.num("thisComp.width", 0.0), 200.0); + // `add` reads its arguments through `Array.isArray`, and reaching an effect builds a group with + // `Object.setPrototypeOf`, so a replacement for either would change the answers below. + assert_close!(f.num("Array.isArray = function () { return false };\n5", 0.0), 5.0); + assert_close!(f.num("Object.setPrototypeOf = function (o) { return o };\n6", 0.0), 6.0); + assert_close!(f.num("add([1, 2], [3, 4])[0]", 0.0), 4.0); + assert_close!(f.num("effect(\"Slider Control\")(\"Slider\")", 0.0), 42.0); + assert_close!(f.num("thisProject = 5;\nthisProject.leak = 9;\n7", 0.0), 7.0); + assert_close!(f.num("typeof thisProject === 'object' && thisProject.leak === undefined ? 1 : 0", 0.0), 1.0); +} + +/// `Math` and `JSON` are namespaces rather than prototypes: nothing inherits from them, so the +/// prelude freezes them plainly instead of turning every method into a shadowable accessor, which +/// would put a getter call on every `Math.sin` of a per-frame expression for a shadowing no object +/// can do. They still cannot be rewritten, through a member or through the binding (#2176). +#[test] +fn math_and_json_are_frozen_and_cannot_be_replaced_for_the_next_expression() { + let f = fx(); + assert_close!(f.num("Object.isFrozen(Math) && Object.isFrozen(JSON) ? 1 : 0", 0.0), 1.0); + // Plain data properties, not the accessor repair: the hot path reads the native method directly. + assert_close!(f.num("'value' in Object.getOwnPropertyDescriptor(Math, 'sin') ? 1 : 0", 0.0), 1.0); + assert_close!(f.num("'value' in Object.getOwnPropertyDescriptor(JSON, 'stringify') ? 1 : 0", 0.0), 1.0); + // A member write is dropped in silence, and the next expression still has the original. + assert_close!(f.num("Math.sin = function () { return 9 }; Math.max = function () { return -1 };\n1", 0.0), 1.0); + assert_close!(f.num("Math.sin(Math.PI / 2) * 100 + Math.max(3, 7)", 0.0), 107.0); + assert_close!(f.num("JSON.stringify = function () { return 'x' }; JSON.parse = null;\n2", 0.0), 2.0); + assert_close!(f.num("JSON.stringify(JSON.parse('[1,2]')) === '[1,2]' ? 1 : 0", 0.0), 1.0); + assert_close!(f.num("Math.leak = 5; JSON.leak = 6;\n3", 0.0), 3.0); + assert_close!(f.num("Math.leak === undefined && JSON.leak === undefined ? 1 : 0", 0.0), 1.0); + assert_close!(f.num("delete Math.floor ? 1 : 0", 0.0), 0.0); + assert!(f.err("Object.defineProperty(Math, 'floor', { value: 5 })").contains("TypeError")); + // The bindings are locked too: assignment, `var` and `delete` leave both in place. + assert_close!(f.num("Math = { sin: function () { return 9 } }; JSON = null;\n4", 0.0), 4.0); + assert_close!(f.num("var Math = 5; var JSON = 6;\n5", 0.0), 5.0); + assert_close!(f.num("delete Math || delete JSON ? 1 : 0", 0.0), 0.0); + assert_close!(f.num("Math.floor(2.7) + JSON.parse('3')", 0.0), 5.0); + // A second project on the same thread-local runtime meets the same objects. + let g = fx(); + assert_close!(g.num("Math.sin(Math.PI / 2) * 100", 0.0), 100.0); + assert_close!(g.num("JSON.stringify([1]) === '[1]' ? 1 : 0", 0.0), 1.0); +} + +/// Closing the prototypes must not cost an expression its own objects: once an inherited data +/// property is non-writable, an assignment meaning to shadow it on an instance is dropped in sloppy +/// mode rather than creating an own property, and using an object as a dictionary runs into exactly +/// those names (#2133). +#[test] +fn an_expression_can_shadow_an_inherited_name_on_its_own_object() { + let f = fx(); + assert_close!(f.num("var o = {}; o.toString = function () { return 'x' }; o.toString() === 'x' ? 1 : 0", 0.0), 1.0); + assert_close!(f.num("var d = {}; d.constructor = 3; d.constructor", 0.0), 3.0); + assert_close!(f.num("var c = {}; c['valueOf'] = 2; c['valueOf']", 0.0), 2.0); + // The shared prototype itself still cannot be written through, which is the point of freezing. + assert_close!(f.num("Object.prototype.toString = function () { return 'z' };\n1", 0.0), 1.0); + assert_close!(f.num("({}).toString() === 'z' ? 0 : 1", 0.0), 1.0); +} + +/// The names an expression shadows are not only `Object.prototype`'s: giving one array its own +/// `map` or one function its own `call` is ordinary JavaScript, and a prototype frozen without the +/// repair takes it away. The two halves fail differently, which is why both are checked here — +/// `map`, `forEach` and `filter` throw, while `push` and `join` silently run the native method and +/// return its answer, so an expression using them keeps working and quietly computes something +/// else (#2133). +#[test] +fn an_expression_can_shadow_an_inherited_method_on_its_own_array_or_function() { + let f = fx(); + assert_close!(f.num("var a = []; a.map = function () { return 42 }; a.map()", 0.0), 42.0); + assert_close!(f.num("var a = [1, 2]; a.forEach = function () { return 5 }; a.forEach()", 0.0), 5.0); + assert_close!(f.num("var a = [1, 2]; a.filter = function () { return 6 }; a.filter()", 0.0), 6.0); + // `push` returned the native length and `join` the native string before the repair. + assert_close!(f.num("var a = [1]; a.push = function () { return 99 }; a.push(2)", 0.0), 99.0); + assert_close!(f.num("var a = [1]; a.push = function () { return 99 }; a.push(2); a.length", 0.0), 1.0); + assert_close!(f.num("var a = [1]; a.join = function () { return 'J' }; a.join() === 'J' ? 1 : 0", 0.0), 1.0); + assert_close!(f.num("var fn = function () {}; fn.call = function () { return 7 }; fn.call()", 0.0), 7.0); + assert_close!(f.num("var fn = function () {}; fn.apply = function () { return 8 }; fn.apply()", 0.0), 8.0); + // A symbol key on a prototype the loop reaches goes the same way. + assert_close!(f.num("var a = [1]; a[Symbol.toPrimitive] = function () { return 11 }; +a", 0.0), 11.0); + // Reading an unshadowed method still answers with the native one, through the repair. + assert_close!(f.num("[1, 2, 3].map(function (x) { return x * 2 })[2]", 0.0), 6.0); + assert_close!(f.num("[1, 2, 3].filter(function (x) { return x > 1 }).length", 0.0), 2.0); + assert_close!(f.num("[3, 1, 2].sort()[0]", 0.0), 1.0); + assert_close!(f.num("[1, 2].concat([3]).length", 0.0), 3.0); + // The shared prototype itself still cannot be written through, which is the point of freezing. + assert_close!(f.num("Array.prototype.map = function () { return 0 };\n1", 0.0), 1.0); + assert_close!(f.num("[1].map(function (x) { return x + 1 })[0]", 0.0), 2.0); + assert_close!(f.num("Function.prototype.call = function () { return 0 };\n1", 0.0), 1.0); + assert_close!(f.num("(function () { return 4 }).call(null)", 0.0), 4.0); +} + +/// `name` and `message` on an error's prototype are the one exception to the repair, because the +/// host reads both off a thrown object to render it and has no context to call a getter with, nor +/// anywhere safe to call one: formatting happens after the evaluation's time budget stops being +/// checked. What matters is that the message still names the error and the line, and that the +/// assignment the exception costs is dropped in silence rather than failing the expression (#2133). +#[test] +fn an_error_still_renders_with_its_name_and_the_users_line() { + let f = fx(); + // The write the exception costs: a no-op, not a throw, and the next statement still runs. + assert_close!(f.num("var e = new Error('x'); e.name = 'MyError'; 1", 0.0), 1.0); + assert_close!(f.num("var e = new Error('x'); e.name === 'Error' ? 1 : 0", 0.0), 1.0); + // An own `message` is the instance's, so it is untouched by any of this. + assert_close!(f.num("new Error('x').message === 'x' ? 1 : 0", 0.0), 1.0); + // The prototype itself still cannot be rebound for the next expression, which is the point. + assert_close!(f.num("Error.prototype.name = 'Hacked';\n1", 0.0), 1.0); + assert_close!(f.num("new Error('x').name === 'Error' ? 1 : 0", 0.0), 1.0); + // A getter the expression installs on its own error is reached through its own property, so it + // never goes near the host's read of the prototype. + assert_close!(f.num("var e = new Error('x'); Object.defineProperty(e, 'name', { get: function () { return 'Own' } }); e.name === 'Own' ? 1 : 0", 0.0), 1.0); +} + +/// The repair must not turn a dropped assignment into a failed expression: in sloppy mode a write +/// to a frozen target is ignored, and the setter runs for every receiver that inherits the name, +/// including the other frozen prototypes and an object the expression froze itself. +#[test] +fn assigning_a_repaired_name_on_a_frozen_object_is_dropped_not_an_error() { + let f = fx(); + // `hasOwnProperty` is `Object.prototype`'s, so this runs the repaired setter with a frozen + // `Array.prototype` as the receiver. + assert_close!(f.num("Array.prototype.hasOwnProperty = function () { return true };\n1", 0.0), 1.0); + assert_close!(f.num("[].hasOwnProperty('length') ? 1 : 0", 0.0), 1.0); + // The prelude freezes these, and each inherits the repaired names. + assert_close!(f.num("Math.toString = function () { return 'm' };\n2", 0.0), 2.0); + assert_close!(f.num("JSON.valueOf = 3;\n3", 0.0), 3.0); + assert_close!(f.num("thisProject.toLocaleString = 4;\n4", 0.0), 4.0); + // An object the expression froze itself behaves as it does outside the prelude. + assert_close!(f.num("var o = {}; Object.freeze(o); o.toString = function () { return 'x' };\n5", 0.0), 5.0); + assert_close!(f.num("var s = {}; Object.seal(s); s.constructor = 6;\n6", 0.0), 6.0); + // Still a no-op where it has to be, and still working where it has to work. + assert_close!(f.num("({}).toString() === '[object Object]' ? 1 : 0", 0.0), 1.0); + assert_close!(f.num("var k = {}; k.toString = function () { return 'y' }; k.toString() === 'y' ? 1 : 0", 0.0), 1.0); + // A primitive target boxes into a throwaway wrapper, and `null` is not extensible, which is why + // the receiver check alone is enough. + assert_close!(f.num("'abc'.toString = function () { return 'q' };\n7", 0.0), 7.0); + assert_close!(f.num("(5).constructor = 8;\n8", 0.0), 8.0); + assert_close!(f.num("Object.isExtensible(null) === false && Object.isExtensible(undefined) === false ? 1 : 0", 0.0), 1.0); +} + +/// The repaired name is an accessor standing in for a data property, and an accessor is handed the +/// receiver while a data property has the receiver's own property consulted first, so the setter +/// has to do that itself or the substitution shows through. An ordinary assignment finds the own +/// property and never reaches the setter, so these call it with the receiver directly, which is +/// what `Reflect.set(Object.prototype, name, v, o)` would do; `Reflect` is not available to +/// expressions. Every expected value is what that `Reflect.set` answers in node, where the name +/// is still the data property this one stands in for. +#[test] +fn the_repair_answers_a_write_as_the_data_property_it_replaced_would() { + let f = fx(); + let set = "var put = Object.getOwnPropertyDescriptor(Object.prototype, 'toString').set;"; + // An own writable property is updated in place, even when the receiver takes no new ones. + assert_close!(f.num(&format!("{set} var o = {{ toString: 1 }}; Object.preventExtensions(o); put.call(o, 2); o.toString"), 0.0), 2.0); + assert_close!(f.num(&format!("{set} var o = {{ toString: 1 }}; put.call(o, 2); o.toString"), 0.0), 2.0); + // ... and keeps the attributes it had, rather than being redefined as a fresh enumerable one. + assert_close!( + f.num( + &format!( + "{set} var o = {{}}; Object.defineProperty(o, 'toString', {{ value: 1, writable: true, enumerable: false, configurable: true }}); + put.call(o, 2); Object.getOwnPropertyDescriptor(o, 'toString').enumerable ? 1 : 0" + ), + 0.0 + ), + 0.0 + ); + // An own non-writable property drops the write instead of being overwritten. + assert_close!( + f.num( + &format!( + "{set} var o = {{}}; Object.defineProperty(o, 'toString', {{ value: 1, writable: false, enumerable: false, configurable: true }}); + put.call(o, 2); o.toString" + ), + 0.0 + ), + 1.0 + ); + // Writing through the prototype itself changes nothing for anyone. + assert_close!(f.num(&format!("{set} put.call(Object.prototype, 2); ({{}}).toString === undefined ? 0 : 1"), 0.0), 1.0); + // An ordinary assignment finds the own property first and never reaches the setter at all. + assert_close!(f.num("var o = { toString: 1 }; o.toString = 2; o.toString", 0.0), 2.0); +} + +/// `for…of`, destructuring and spread all go through the iterator intrinsics, and those are the +/// `prototype` of no global function, so freezing what the globals reach never touched them (#2133). +#[test] +fn an_expression_cannot_poison_iteration_for_the_next_one() { + let f = fx(); + assert_close!(f.num("Object.getPrototypeOf([][Symbol.iterator]()).next = function () { return { done: true } };\n1", 0.0), 1.0); + assert_close!(f.num("var s = 0; for (var x of [1, 2, 3]) s += x; s", 0.0), 6.0); + assert_close!(f.num("var [a, b] = [4, 5]; a + b", 0.0), 9.0); + assert_close!(f.num("Math.max(...[1, 7, 3])", 0.0), 7.0); +} + +/// Shadowing an inherited name on an instance has to keep working for symbol keys too: making an +/// object iterable by assigning `Symbol.iterator` is ordinary JavaScript, and a frozen +/// `Array.prototype` drops that write in silence, so `for…of` walks the array's own elements and +/// quietly returns a different number rather than failing (#2133). +#[test] +fn an_expression_can_give_its_own_array_an_iterator() { + let f = fx(); + assert_close!( + f.num( + "var a = [1, 2]; + a[Symbol.iterator] = function () { var d = false; return { next: function () { if (d) return { done: true }; d = true; return { value: 9, done: false } } } }; + var s = 0; for (var x of a) s += x; s", + 0.0 + ), + 9.0 + ); + assert_close!(f.num("var a = [1, 2]; a[Symbol.iterator] = 1; a[Symbol.iterator] === 1 ? 1 : 0", 0.0), 1.0); + // `Object.prototype` carries no symbol keys, so a plain object was never affected. + assert_close!(f.num("var o = {}; o[Symbol.iterator] = 1; o[Symbol.iterator] === 1 ? 1 : 0", 0.0), 1.0); + // An inherited name that is already non-writable is left alone, as it is outside the prelude: + // `Date.prototype[Symbol.toPrimitive]` drops the write on `main` and in node as well. + assert_close!(f.num("var d = new Date(0); d[Symbol.toPrimitive] = 1; d[Symbol.toPrimitive] === 1 ? 1 : 0", 0.0), 0.0); + // Repairing the key must not reopen it: the shared prototype still cannot be written through. + assert_close!( + f.num( + "Array.prototype[Symbol.iterator] = function () { return { next: function () { return { done: true } } } }; +1", + 0.0 + ), + 1.0 + ); + assert_close!(f.num("var s = 0; for (var x of [1, 2, 3]) s += x; s", 0.0), 6.0); +} + #[test] fn comments_are_ignored() { let f = fx(); @@ -1672,3 +1981,47 @@ fn an_expression_cant_change_the_prelude_for_the_next() { } mod property_group; + +/// Timing harness for the prelude's cost on the evaluation path. Ignored: it is a measurement, not +/// an assertion. `cargo test -p effectcraft-expr --release --lib bench_evaluation -- --ignored +/// --nocapture`. +#[test] +#[ignore = "timing measurement, not a test"] +fn bench_evaluation() { + let f = fx(); + // Each case runs on a property whose dimension its answer fits: a scalar handed to `position` + // is rejected after the script has already run, so the case would be timing an error path. + // Ordinary expressions first, then three that read an inherited name in a loop: the repair + // turns each of those reads into an accessor call, and `Array.prototype.map` is the one an + // expression is most likely to reach for in quantity. + let cases: [(&str, &str, &str, u32); 10] = [ + ("array maths", POS, "value + [10, 0]", 2500), + ("scalar maths", ROT, "time * 100 + 5", 3000), + ("prelude call", ROT, "linear(time, 0, 1, 0, 100)", 2500), + ("cross-layer read", ROT, "thisComp.layer(\"B\").transform.position[0]", 200), + ("effect read", ROT, "effect(\"Slider Control\")(\"Slider\")", 150), + ("wiggle", POS, "wiggle(2, 10)", 800), + ("dictionary", ROT, "var c = {}; var k = 'constructor'; c[k] = 1; c[k] + value", 2500), + ("toString x100", ROT, "var o = {}; var n = 0; for (var i = 0; i < 100; i++) { if (o.toString) n++; } n", 250), + ("hasOwnProperty x100", ROT, "var o = {}; var n = 0; for (var i = 0; i < 100; i++) { if (o.hasOwnProperty) n++; } n", 250), + ("array method x100", ROT, "var a = [1]; var n = 0; for (var i = 0; i < 100; i++) { if (a.map) n++; } n", 250), + ]; + for (label, path, text, iters) in cases { + let mut best = u128::MAX; + // Three passes, keeping the fastest: a slower pass is scheduling noise, not the prelude. + for _ in 0..3 { + // The warm-up checks as it goes, at every time the measured loop uses: that loop drops + // its answer by design, so a case that had started failing would be timed on its error + // path and still print a number. + for i in 0..50 { + assert!(f.ev(f.a, path, text, f64::from(i % 10) * 0.1).is_ok(), "{label}"); + } + let start = std::time::Instant::now(); + for i in 0..iters { + let _ = f.ev(f.a, path, text, f64::from(i % 10) * 0.1); + } + best = best.min(start.elapsed().as_nanos() / u128::from(iters)); + } + println!("BENCH {best:>9} ns/eval {label} (min of 3 x {iters})"); + } +} From e730da14356abb84143d544786b295dff828a2f9 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 16:51:53 +0100 Subject: [PATCH 16/25] AEP import: read effect values recorded in the default state (tdsb 3) After Effects records some effect parameters as value records in the default state (tdsb 3) that still hold the value it uses. A survey of 839 corpus and sample files finds 25 such effect records, all in Lumetri Color (its hidden controls) and Cryptomatte (Selection Mode on); every one differs from its pard default and reads back in After Effects as recorded. No effect record in the default state holds a value that does not count. The general reader now takes them: EffectParam::recorded_value (the same name, signature and body as in the Lumetri import) reads a record whatever its state, value() and is_default() use it, and the importer's mapped parameters read it, so such a record imports like a written one instead of being reported as an unreadable value with the default used. stored_value keeps to written records. Layer, camera, light, shape and text properties in the default state all hold After Effects' defaults or values with no effect on that layer (the point light cone, a polygon's inner radius, a camera's opacity), so their readers are unchanged. docs/aep-format.md records both findings. Closes #3126 Co-Authored-By: Claude Opus 5.5 --- crates/aep/src/container/effects.rs | 27 ++++++-- crates/aep/src/import/effects.rs | 4 +- crates/aep/tests/it/import/synthetic.rs | 89 +++++++++++++++++++++++++ docs/aep-format.md | 8 +-- 4 files changed, 117 insertions(+), 11 deletions(-) diff --git a/crates/aep/src/container/effects.rs b/crates/aep/src/container/effects.rs index df1c2b9c1..29940a712 100644 --- a/crates/aep/src/container/effects.rs +++ b/crates/aep/src/container/effects.rs @@ -375,11 +375,24 @@ impl<'f, 'a> EffectParam<'f, 'a> { /// colours, popups and layer controls. A record counts when its `tdsb` is 1 and it has no /// keyframes; a layer control reads the `tdpi` chunk; a button has no value. pub fn stored_value(&self) -> Option { + let p = self.stored?; + if self.kind != ParamKind::Layer && p.flags() != Some(1) { + return None; + } + self.recorded_value() + } + + /// The value in the value record whatever the record's state (`tdsb`): what + /// [`EffectParam::stored_value`] reads, also from a record in the default state (`tdsb` 3). + /// Lumetri Color writes the values of its hidden parameters so (High Dynamic Range, HDR + /// Specular, Look: 1200 `lumetri-01-set-all`, `-02-sec-top`, `-06-sec-light`; After Effects + /// reads back the recorded values). `None` with keyframes or without a record. + pub fn recorded_value(&self) -> Option { let p = self.stored?; if self.kind == ParamKind::Layer { return be_u32(p.chunk().child(b"tdpi")?.bytes()?, 0).map(ParamValue::Layer); } - if p.flags() != Some(1) || p.is_animated() { + if p.is_animated() { return None; } let v = p.static_value()?; @@ -443,8 +456,9 @@ impl<'f, 'a> EffectParam<'f, 'a> { /// True when the parameter is at its default: it has no value record (a record exists exactly /// when the value differs from the `pard` default; setting a value back to the default removes - /// the record again), or the record equals the default. `None` when that cannot be decided - /// (unknown parameter type). + /// the record again), or the record equals the default. A record in the default state (`tdsb` + /// 3) counts with its value ([`EffectParam::recorded_value`]). `None` when that cannot be + /// decided (unknown parameter type, keyframes). pub fn is_default(&self) -> Option { match (self.stored, self.kind) { // A Layer Control always has its record, and a popup selection overwrites the default @@ -453,7 +467,7 @@ impl<'f, 'a> EffectParam<'f, 'a> { (None, _) => Some(true), (Some(_), ParamKind::Popup) => Some(false), (Some(_), _) => { - let stored = self.stored_value()?; + let stored = self.recorded_value()?; // A checkbox's default is in `pard` too (on for Repeat Edge Pixels, 412). Some(self.pard_default().is_some_and(|d| d == stored)) } @@ -541,10 +555,11 @@ impl<'f, 'a> EffectParam<'f, 'a> { self.items.as_ref()?.get(index).map(String::as_str) } - /// The effective value: the stored value when there is one, else the `pard` default (absent = + /// The effective value: the value in the record when there is one, also in a record in the + /// default state (`tdsb` 3, [`EffectParam::recorded_value`]), else the `pard` default (absent = /// default); a checkbox without a record is off. pub fn value(&self) -> Option { - self.stored_value().or_else(|| self.pard_default()).or(match self.kind { + self.recorded_value().or_else(|| self.pard_default()).or(match self.kind { ParamKind::Checkbox if self.stored.is_none() => Some(ParamValue::Bool(false)), _ => None, }) diff --git a/crates/aep/src/import/effects.rs b/crates/aep/src/import/effects.rs index 477a25df4..5ab0c1514 100644 --- a/crates/aep/src/import/effects.rs +++ b/crates/aep/src/import/effects.rs @@ -269,7 +269,9 @@ fn import_param(fx: &AepEffect<'_, '_>, map: &EffectMap, p: &EffectParam<'_, '_> } return; } - let value = match (p.stored(), p.stored_value()) { + // A record in the default state (`tdsb` 3) still holds the value After Effects uses (1200, + // 1400: every such record reads back as recorded). + let value = match (p.stored(), p.recorded_value()) { (Some(_), Some(v)) => Some(v), (Some(_), None) => { col.lose(LossCategory::DamagedData, format!("{here}: parameter {label} holds an unreadable value, default used")); diff --git a/crates/aep/tests/it/import/synthetic.rs b/crates/aep/tests/it/import/synthetic.rs index fba757df7..af066ab7c 100644 --- a/crates/aep/tests/it/import/synthetic.rs +++ b/crates/aep/tests/it/import/synthetic.rs @@ -295,3 +295,92 @@ fn unknown_time_display_and_frame_start_codes_are_reported() { assert!(losses(&r.unwrap(), LossCategory::ProjectSettings).is_empty(), "nhed {at}: {bytes:?}"); } } + +/// The project with the value records named `param` (`-NNNN`: the instance's and the one in +/// the project's copy of the effect, `EfdG`) set to state `tdsb` and holding `value` (the `cdat` +/// starts with the big-endian `f64`). +fn with_record(param: &str, tdsb: u32, value: f64) -> Vec { + let mut file: AepFile<'_> = parse(SYNTHETIC).unwrap(); + let mut patched = 0; + visit(&mut file.chunks, &mut |c| { + if !c.is_list(b"tdgp") { + return; + } + let kids = c.children_mut(); + let Some(at) = kids.iter().position(|k| k.is(b"tdmn") && k.bytes().is_some_and(|b| b.split(|x| *x == 0).next() == Some(param.as_bytes()))) else { + return; + }; + let Some(record) = kids.get_mut(at + 1).filter(|k| k.is_list(b"tdbs")) else { return }; + for k in record.children_mut() { + if k.is(b"tdsb") { + set_data(k, 0, &tdsb.to_be_bytes()); + } else if k.is(b"cdat") { + set_data(k, 0, &value.to_be_bytes()); + } + } + patched += 1; + }); + assert_eq!(patched, 2, "{param}: the instance and the EfdG copy"); + write(&file).unwrap() +} + +/// The values of the imported effect parameters, by property id, in file order. +fn effect_values(p: &Project) -> Vec<(String, effectcraft_keyframe::Value)> { + let mut out = Vec::new(); + for item in p.items.values() { + if let ItemKind::Comp(c) = &item.kind { + for layer in &c.layers { + if let Some(fx) = layer.effects() { + fx.walk("", &mut |_, prop| out.push((prop.match_id.clone(), prop.value.clone()))); + } + } + } + } + out +} + +#[test] +fn a_default_state_effect_record_holds_the_value() { + // After Effects writes the values of some parameters as records in the default state (`tdsb` + // 3): Lumetri Color's hidden controls (1200) and Cryptomatte's Selection Mode (1400), each + // read back as recorded. The reader reads them like a written record (`tdsb` 1). + let bytes = with_record("ADBE Glo2-0011", 3, 0.75); + let file = parse(&bytes).unwrap(); + let project = effectcraft_aep::Project::new(&file).unwrap(); + let defs = project.effect_definition_index(); + let mut seen = 0; + for item in project.all_items() { + let Some(comp) = item.comp() else { continue }; + for layer in comp.layers() { + for fx in layer.effects().iter().filter(|fx| fx.match_name() == "ADBE Glo2") { + let params = defs.params(fx); + let p = params.iter().find(|p| p.match_name == "ADBE Glo2-0011").unwrap(); + assert_eq!(p.stored().and_then(|s| s.flags()), Some(3)); + // `stored_value` keeps reading written records only. + assert_eq!(p.stored_value(), None); + assert_eq!(p.recorded_value(), Some(effectcraft_aep::ParamValue::Number(0.75))); + assert_eq!(p.value(), Some(effectcraft_aep::ParamValue::Number(0.75))); + assert_eq!(p.is_default(), Some(false)); + seen += 1; + } + } + } + assert_eq!(seen, 1); +} + +#[test] +fn default_state_effect_records_import_like_written_ones() { + // Before, such a record was reported as an unreadable value and the default used. + for (param, value) in [("ADBE Glo2-0011", 0.75), ("ADBE Easy Levels2-0008", 1.0)] { + let (written, r1) = run(&with_record(param, 1, value), true); + let (default_state, r3) = run(&with_record(param, 3, value), true); + let (r1, r3) = (r1.unwrap(), r3.unwrap()); + assert!(losses(&r3, LossCategory::DamagedData).is_empty(), "{param}: {:?}", r3.report.losses); + assert_eq!(r3.report.losses, r1.report.losses, "{param}"); + assert_eq!(effect_values(&default_state), effect_values(&written), "{param}"); + } + // Glow's A & B Midpoint 0.75 is 75 % (the default is 50). + let (p, _) = run(&with_record("ADBE Glo2-0011", 3, 0.75), true); + let mid: Vec<_> = effect_values(&p).into_iter().filter(|(id, _)| id == "abMidpoint").collect(); + assert_eq!(mid, vec![("abMidpoint".to_string(), effectcraft_keyframe::Value::Scalar(75.0))]); +} diff --git a/docs/aep-format.md b/docs/aep-format.md index fcc8f53be..5897453c6 100644 --- a/docs/aep-format.md +++ b/docs/aep-format.md @@ -328,7 +328,7 @@ Property record (`LIST tdbs`: `tdsb`, `tdsn`, `tdb4`, then `cdat` or a `LIST lis | keyed spatial-template flag | `tdb4` | 79 | u8 | 1 on a static colour or point effect value, 0 once it is keyed (359 / 364, 369 / 382); 0 on keyed and static sliders (362 / 363, 391 / 397); meaning **?** | 359, 362-364, 369, 382, 391, 397 | V (values) | | expression flags | `tdb4` | 76, 77, 120 | u8 | all three are 1 when an expression exists. 76 and 77 are also 1 on a camera's or light's computed (auto) Position, which has no expression | 060, 213, 251 | V | | **expression disabled** | `tdb4` | 119 | u8 | 1 = the expression is disabled; the text and bytes 76, 77, 120 stay. On the built-in view layers the byte flips by itself after the first save, so it counts only together with an expression | 060 (0), 213 (1) | V | -| state | `tdsb` | 0 | u32 | 1 = written (non-default value, keys or expression), 3 = default state | 040, 071, 073 (1); 3 on the unwritten properties of 000 | V (1), E (3) | +| state | `tdsb` | 0 | u32 | 1 = written (non-default value, keys or expression), 3 = default state. **A layer, camera, light, shape or text property in the default state holds After Effects' default** or a value with no effect on that layer (survey of 839 corpus and sample files: Material Options, cone 90 / 50 and falloff 500 / 500 of point lights, Position `[0, 0, -1333.33]` of ambient and environment lights, Opacity 1 and Scale `[1, 1, 1]` of cameras and lights, a polygon's Inner Radius 50, Miter Limit 4, the inert Time Remap 0), so the importer skips it. **An effect parameter's record in the default state holds the effective value** (3.7, stored value) | 040, 071, 073 (1); 3 on the unwritten properties of 000; 1200, 1400 (effects) | V (1), E (3), V (effects) | | static value | `cdat` | 0 | `components` x f64 | the first `components` doubles are the value | 040, 041, 043, 071 | V | | user name | `tdsn/Utf8` | | text | `-_0_/-` means "no custom name" | 091 (`Mask 1`), 238 (`Group 1`) | V | @@ -437,9 +437,9 @@ The expression engine is project-wide (`LIST ExEn/Utf8`, `javascript-1.0`). API: | **float slider valid minimum, slider range** | `pard` | 104; 112 / 116 | f32; f32 x 2 | the smallest valid value (Exposure -100, Offset -2, Gamma Correction 0.1 as f32) and the range the slider covers (Exposure -4..4, Offset -0.5..0.5, Gamma 0.1..10); the slider range is repeated as `tdum` / `tduM` (f64) in each value record (391-394). @120 holds the default again as f32 (Gamma 1.0, Blurriness 25), u16 @124 the display decimals (**H**: Exposure 2, Offset 4) | 389, 391-394 | V | | **definitions of a second instance** | top-level `LIST EfdG` | | | one `EfDf` per effect TYPE (`tdmn` = match name, then a copy of ONE instance's `sspc`, including that instance's value records); the `EfDC` count before them is a little-endian u32. When two layers carry the same effect type, only one instance keeps a full `parT`; **the other has an empty `LIST parT` (4 bytes)** but its own `tdgp` (input layer `-0000` with its own `tdpi`, the button record, and its value records). Its definitions come from the `EfdG` copy by match name, its values from its own records, never from the copy (383: the copy holds S100's Start [0.1, 0.2]; L's Ramp reads the 368 defaults). Which instance keeps the full `parT` is **?** (in 383 S100 is both first in the file and newest). The `EfDf` order follows the stack order (365 / 366); `Pefl/pjef` does not | 365, 366, 383 | V | | popup items | `parT/pdnm/Utf8` | | text | items separated by `\|` | 077 (`Item 1\|Item 2\|Item 3`), 214 | V | -| stored value | `sspc/tdgp/-NNNN` | | `tdbs` | present only for a parameter that differs from its default; `tdsb` 1, `cdat` as in 3.4. **Absent record = the `pard` default** | see below | V | +| stored value | `sspc/tdgp/-NNNN` | | `tdbs` | present only for a parameter that differs from its default; `tdsb` 1, `cdat` as in 3.4. **Absent record = the `pard` default.** Some parameters are recorded in the default state (`tdsb` 3) and the record still holds the value After Effects uses: every one of the 25 such records in the corpus and the samples differs from its `pard` default and reads back as recorded (Lumetri Color's hidden controls High Dynamic Range, HDR Specular, the second Look, Legacy Tone And Saturation, Face Detection, Auto Color, Auto Intensity and SemanticAutoTone Progress, 1200; Cryptomatte's Selection Mode on, 1400). No effect record in the default state holds a value that does not count. The reader takes the value whatever the state (`EffectParam::recorded_value`, `EffectParam::value`); `EffectParam::stored_value` keeps to written records. That other effects record values so is H (no sample shows one) | see below; 1200, 1400 | V (1200, 1400), H (other effects) | -Stored values by type (all `tdsb` 1): +Stored values by type (`tdsb` 1, or 3 as above): | Type | Stored | Evidence | Conf. | |---|---|---|---| @@ -823,7 +823,7 @@ another value). **Masks, shapes** - Masks: locked, colour, feather falloff, rotobezier, keyed expansion, a path whose last vertex has only handles, the counter after a mask is deleted, a mask on a scaled layer (whether the box stays layer-relative), an animated path with three or more keys. -- Shape layers: open shape paths (in shape groups), stroke dashes, taper and wave with values, merge paths, repeater, offset paths and the other path operators, item blend modes and composite order, shape direction, gradient colour stops other than the default (**?**: they appear only as an `ADBE Vector Grad Colors` entry), highlight length and angle other than 0, gradient scale and rotation; why a radial gradient writes its highlight records with `tdsb` 1 and a polygon its inner radius with `tdsb` 3 (**?**). +- Shape layers: open shape paths (in shape groups), stroke dashes, taper and wave with values, merge paths, repeater, offset paths and the other path operators, item blend modes and composite order, shape direction, gradient colour stops other than the default (**?**: they appear only as an `ADBE Vector Grad Colors` entry), highlight length and angle other than 0, gradient scale and rotation; why a radial gradient writes its highlight records with `tdsb` 1 and a polygon its inner radius with `tdsb` 3 (**?**; the polygon's records hold the defaults 50 and 0, as every non-effect record in the default state does: 3.4, state). **Text** From b9182972b2101c9ed86b63c620fc7c51b569bd87 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 17:02:26 +0100 Subject: [PATCH 17/25] HDR Highlight Compression: match After Effects' measured transfer curve (#3123) The effect used a soft knee on the brightest channel and left everything below the knee unchanged. After Effects compresses the whole range. Fitting our own 16-bit renders of an HDR ramp and coloured patches (Amount 0, 84, 90, 100) gives: Y = max(0.30 R + 0.59 G + 0.11 B, 0) comp = min(c / (1 + Y), 1) per channel out = ((1 - t) c^0.4 + t comp^0.4)^(1/0.4) t = Amount / 100 A grey value x maps to x / (1 + x) at Amount 100. The worst error against AE drops from 985 to about 2-4 of 65535 at Amounts 100, 90 and 84. Amount 0 stays identical. CPU and GPU change together. Tests: a reference table of measured values, GPU parity at Amount 90, and over-range input that exercises the clip at 1. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/utility.rs | 91 +++++++++++++++++++++------ crates/gpu/src/fx_pixel2.rs | 2 +- crates/gpu/src/shaders/fx_pixel2.wgsl | 20 +++--- crates/gpu/src/tests/fx_pixel2.rs | 17 +++++ 4 files changed, 101 insertions(+), 29 deletions(-) diff --git a/crates/effects/src/utility.rs b/crates/effects/src/utility.rs index e41b42e99..2448733b0 100644 --- a/crates/effects/src/utility.rs +++ b/crates/effects/src/utility.rs @@ -55,29 +55,39 @@ fn overbrights(ctx: &EffectCtx, mut b: Buf) -> Buf { b } -/// Soft-knee, hue-preserving compression of the brightest channel into [0, 1). -fn hdr_compress_value(m: f32, knee: f32) -> f32 { - if m <= knee { - return m; - } - let w = 1.0 - knee; - knee + w * (m - knee) / ((m - knee) + w) +/// Luma weights HDR Highlight Compression divides by (measured against After Effects 2026). +const HDR_LUMA: [f32; 3] = [0.30, 0.59, 0.11]; +/// Exponent of the domain the Amount blend happens in (out^0.4 is mixed linearly). +const HDR_BLEND_GAMMA: f32 = 0.4; + +#[inline] +fn hdr_spow(v: f32, e: f32) -> f32 { + v.signum() * v.abs().powf(e) +} + +/// One straight-colour pixel through HDR Highlight Compression at blend `t` (Amount / 100, > 0). +/// +/// Measured behaviour (16-bit renders of a 0–4 HDR ramp and coloured patches at Amount 100, +/// 90 and 84): every channel is divided by `1 + Y` (`Y` = 0.30 R + 0.59 G + 0.11 B, so a grey +/// value `x` lands on `x / (1 + x)`), clipped at 1, then mixed with the input as `x^0.4` values. +fn hdr_compress_pixel(c: [f32; 3], t: f32) -> [f32; 3] { + let y = (HDR_LUMA[0] * c[0] + HDR_LUMA[1] * c[1] + HDR_LUMA[2] * c[2]).max(0.0); + let s = 1.0 / (1.0 + y); + let g = HDR_BLEND_GAMMA; + c.map(|x| { + let comp = (x * s).min(1.0); + let mixed = (1.0 - t) * hdr_spow(x, g) + t * hdr_spow(comp, g); + let o = hdr_spow(mixed, 1.0 / g); + if o.is_finite() { o } else { x } + }) } fn hdr_compression(ctx: &EffectCtx, mut b: Buf) -> Buf { let t = (ctx.params.f("amount") / 100.0).clamp(0.0, 1.0) as f32; - if t <= 0.0 { + if t <= 0.0 || !t.is_finite() { return b; } - let knee = 1.0 - 0.5 * t; - b.img.map_straight(|c| { - let m = c[0].max(c[1]).max(c[2]); - if m <= knee { - return c; - } - let k = hdr_compress_value(m, knee) / m; - c.map(|v| v * k) - }); + b.img.map_straight(|c| hdr_compress_pixel(c, t)); b } @@ -575,10 +585,51 @@ mod tests { let mut img = Image::new(2, 1); img.set(0, 0, [0.2, 0.3, 0.4, 1.0]); img.set(1, 0, [4.0, 2.0, 1.0, 1.0]); + let out = run("ec.utility.hdrcompression", &[("amount", num(0.0))], img.clone()); + assert_eq!(out.img.data, img.data, "Amount 0 leaves the image alone"); let out = run("ec.utility.hdrcompression", &[], img); - assert_eq!(out.img.data[0], [0.2, 0.3, 0.4, 1.0]); - let o = out.img.data[1]; - assert!(o[0] <= 1.0 && o[0] > 0.5 && (o[0] / o[1] - 2.0).abs() < 1e-4, "{o:?}"); + for o in &out.img.data { + assert!(o.iter().all(|v| v.is_finite() && *v <= 1.0), "{o:?}"); + } + // Negative and huge input stays finite. + let mut img = Image::new(2, 1); + img.set(0, 0, [-0.5, 0.2, 3.0, 1.0]); + img.set(1, 0, [1e30, -1e30, 0.0, 1.0]); + let out = run("ec.utility.hdrcompression", &[("amount", num(50.0))], img); + assert!(out.img.data.iter().flatten().all(|v| v.is_finite()), "{:?}", out.img.data); + } + + /// Our own measurements of After Effects 2026 (32 bpc, no working space; 16-bit PNG renders + /// decoded back to linear): input pixel, then the output at Amount 100, 90 and 84. + #[test] + fn hdr_compression_matches_measured_reference() { + #[rustfmt::skip] + let table: [([f32; 3], [[f32; 3]; 3]); 9] = [ + ([0.18005371; 3], [[0.1524; 3], [0.1550; 3], [0.1566; 3]]), + ([0.49365234; 3], [[0.3302; 3], [0.3450; 3], [0.3540; 3]]), + ([1.0058594; 3], [[0.5014; 3], [0.5427; 3], [0.5683; 3]]), + ([2.0058594; 3], [[0.6673; 3], [0.7637; 3], [0.8256; 3]]), + ([2.9941406; 3], [[0.7496; 3], [0.8962; 3], [0.9922; 3]]), + ([3.9941406; 3], [[0.7998; 3], [0.9929; 3], [1.1213; 3]]), + ([0.7998047, 0.099975586, 0.099975586], [[0.6101, 0.0762, 0.0762], [0.6279, 0.0784, 0.0784], [0.6386, 0.0798, 0.0798]]), + ([0.099975586, 0.7001953, 0.19995117], [[0.0681, 0.4776, 0.1363], [0.0711, 0.4976, 0.1420], [0.0727, 0.5097, 0.1456]]), + ([0.30004883, 0.5, 2.5], [[0.1807, 0.3012, 0.9998], [0.1911, 0.3184, 1.1147], [0.1972, 0.3288, 1.1860]]), + ]; + let mut img = Image::new(table.len() as u32, 1); + for (x, (c, _)) in table.iter().enumerate() { + img.set(x as u32, 0, [c[0], c[1], c[2], 1.0]); + } + for (i, amount) in [100.0, 90.0, 84.0].into_iter().enumerate() { + let out = run("ec.utility.hdrcompression", &[("amount", num(amount))], img.clone()); + for (x, (input, want)) in table.iter().enumerate() { + let got = out.img.data[x]; + for c in 0..3 { + let w = want[i][c]; + let tol = 2e-3_f32.max(w * 2e-3); + assert!((got[c] - w).abs() <= tol, "amount {amount}, input {input:?}, channel {c}: {} vs AE {w}", got[c]); + } + } + } } #[test] diff --git a/crates/gpu/src/fx_pixel2.rs b/crates/gpu/src/fx_pixel2.rs index a62cbaff0..1fe1fec72 100644 --- a/crates/gpu/src/fx_pixel2.rs +++ b/crates/gpu/src/fx_pixel2.rs @@ -242,7 +242,7 @@ fn point(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { return Some(b); } p.u[0][0] = 2; - p.f[0][0] = 1.0 - 0.5 * t; + p.f[0][0] = t; } "ec.utility.ccoverbrights" => { p.u[0] = [3, pr.e("channel"), 0, 0]; diff --git a/crates/gpu/src/shaders/fx_pixel2.wgsl b/crates/gpu/src/shaders/fx_pixel2.wgsl index 583225299..2944f03b2 100644 --- a/crates/gpu/src/shaders/fx_pixel2.wgsl +++ b/crates/gpu/src/shaders/fx_pixel2.wgsl @@ -336,17 +336,21 @@ fn fp2_point(@builtin(global_invocation_id) gid: vec3) { o = vec4(r * px.w, px.w); } } - // f[0].x = knee. + // utility::hdr_compress_pixel; f[0].x = blend t (Amount / 100). case 2u: { if (px.w > 0.0) { let c = px.xyz / px.w; - let m = max(max(c.x, c.y), c.z); - let knee = P.f[0].x; - if (m > knee) { - let w = 1.0 - knee; - let k = (knee + w * (m - knee) / ((m - knee) + w)) / m; - o = vec4(c * k * px.w, px.w); - } + let t = P.f[0].x; + let y = max(0.30 * c.x + 0.59 * c.y + 0.11 * c.z, 0.0); + let comp = min(c * (1.0 / (1.0 + y)), vec3(1.0)); + let g = 0.4; + let mixed = vec3( + (1.0 - t) * fp2_spow(c.x, g) + t * fp2_spow(comp.x, g), + (1.0 - t) * fp2_spow(c.y, g) + t * fp2_spow(comp.y, g), + (1.0 - t) * fp2_spow(c.z, g) + t * fp2_spow(comp.z, g), + ); + let r = vec3(fp2_spow(mixed.x, 1.0 / g), fp2_spow(mixed.y, 1.0 / g), fp2_spow(mixed.z, 1.0 / g)); + o = vec4(r * px.w, px.w); } } // u[0].y = channel; f[0] = highlight colour. diff --git a/crates/gpu/src/tests/fx_pixel2.rs b/crates/gpu/src/tests/fx_pixel2.rs index 4a7886afe..9660592e7 100644 --- a/crates/gpu/src/tests/fx_pixel2.rs +++ b/crates/gpu/src/tests/fx_pixel2.rs @@ -184,6 +184,7 @@ fn composites_and_colour() { effect_case("ec.utility.hdrcompander", &[("mode", e(1)), ("gain", n(3.0)), ("gamma", n(1.5))]); effect_case("ec.utility.hdrcompression", &[]); effect_case("ec.utility.hdrcompression", &[("amount", n(40.0))]); + effect_case("ec.utility.hdrcompression", &[("amount", n(90.0))]); effect_case("ec.utility.growbounds", &[("pixels", n(9.0))]); effect_case("ec.utility.ccoverbrights", &[]); for ch in 1..=5 { @@ -191,6 +192,22 @@ fn composites_and_colour() { } } +/// HDR Highlight Compression on over-range input (HDR Compander expanding first), where the +/// compressed channel clips at 1 before the Amount blend. +#[test] +fn hdr_compression_over_range() { + for amount in [100.0, 60.0] { + let mut s = Scene::new(BitDepth::Bpc32); + let bg = s.solid([0.15, 0.1, 0.2], 97, 61); + s.push(bg); + let mut l = s.footage(70, 44); + s.effect(&mut l, "ec.utility.hdrcompander", &[("mode", e(1)), ("gain", n(4.0)), ("gamma", n(1.0))]); + s.effect(&mut l, "ec.utility.hdrcompression", &[("amount", n(amount))]); + s.push(l); + check(&format!("hdr compression over range {amount}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + } +} + #[test] fn stylize() { let _flips = allow_llvmpipe_boundary_flips(); From af7044d7db041fefeb913fc989776def6174ddb0 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 17:39:00 +0100 Subject: [PATCH 18/25] HDR Highlight Compression: check the curve at Amount 25, 50 and 75 (#3123) New After Effects 2026 renders of the same EXR at Amount 25, 50 and 75 (our own 1110-extra sample, 16-bit PNGs) confirm the transfer curve that was fitted at 84, 90 and 100. The worst channel error over all pixels is 4.0, 5.5 and 4.3 of 65535, against 997, 2415 and 3867 for the old soft knee. The code is unchanged. The measured-reference test now covers all six Amounts. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/utility.rs | 36 ++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/crates/effects/src/utility.rs b/crates/effects/src/utility.rs index 2448733b0..65de35cf6 100644 --- a/crates/effects/src/utility.rs +++ b/crates/effects/src/utility.rs @@ -68,7 +68,7 @@ fn hdr_spow(v: f32, e: f32) -> f32 { /// One straight-colour pixel through HDR Highlight Compression at blend `t` (Amount / 100, > 0). /// /// Measured behaviour (16-bit renders of a 0–4 HDR ramp and coloured patches at Amount 100, -/// 90 and 84): every channel is divided by `1 + Y` (`Y` = 0.30 R + 0.59 G + 0.11 B, so a grey +/// 90, 84, 75, 50 and 25): every channel is divided by `1 + Y` (`Y` = 0.30 R + 0.59 G + 0.11 B, so a grey /// value `x` lands on `x / (1 + x)`), clipped at 1, then mixed with the input as `x^0.4` values. fn hdr_compress_pixel(c: [f32; 3], t: f32) -> [f32; 3] { let y = (HDR_LUMA[0] * c[0] + HDR_LUMA[1] * c[1] + HDR_LUMA[2] * c[2]).max(0.0); @@ -600,26 +600,36 @@ mod tests { } /// Our own measurements of After Effects 2026 (32 bpc, no working space; 16-bit PNG renders - /// decoded back to linear): input pixel, then the output at Amount 100, 90 and 84. + /// decoded back to linear): input pixel, then the output at Amount 100, 90, 84, 75, 50 and 25. #[test] fn hdr_compression_matches_measured_reference() { + const AMOUNTS: [f64; 6] = [100.0, 90.0, 84.0, 75.0, 50.0, 25.0]; #[rustfmt::skip] - let table: [([f32; 3], [[f32; 3]; 3]); 9] = [ - ([0.18005371; 3], [[0.1524; 3], [0.1550; 3], [0.1566; 3]]), - ([0.49365234; 3], [[0.3302; 3], [0.3450; 3], [0.3540; 3]]), - ([1.0058594; 3], [[0.5014; 3], [0.5427; 3], [0.5683; 3]]), - ([2.0058594; 3], [[0.6673; 3], [0.7637; 3], [0.8256; 3]]), - ([2.9941406; 3], [[0.7496; 3], [0.8962; 3], [0.9922; 3]]), - ([3.9941406; 3], [[0.7998; 3], [0.9929; 3], [1.1213; 3]]), - ([0.7998047, 0.099975586, 0.099975586], [[0.6101, 0.0762, 0.0762], [0.6279, 0.0784, 0.0784], [0.6386, 0.0798, 0.0798]]), - ([0.099975586, 0.7001953, 0.19995117], [[0.0681, 0.4776, 0.1363], [0.0711, 0.4976, 0.1420], [0.0727, 0.5097, 0.1456]]), - ([0.30004883, 0.5, 2.5], [[0.1807, 0.3012, 0.9998], [0.1911, 0.3184, 1.1147], [0.1972, 0.3288, 1.1860]]), + let table: [([f32; 3], [[f32; 3]; 6]); 9] = [ + ([0.18005371; 3], [[0.1524; 3], [0.1550; 3], [0.1566; 3], [0.1590; 3], [0.1660; 3], [0.1729; 3]]), + ([0.49365234; 3], [[0.3302; 3], [0.3450; 3], [0.3540; 3], [0.3674; 3], [0.4073; 3], [0.4492; 3]]), + ([1.0058594; 3], [[0.5014; 3], [0.5427; 3], [0.5683; 3], [0.6085; 3], [0.7280; 3], [0.8602; 3]]), + ([2.0058594; 3], [[0.6673; 3], [0.7637; 3], [0.8256; 3], [0.9232; 3], [1.2292; 3], [1.5892; 3]]), + ([2.9941406; 3], [[0.7496; 3], [0.8962; 3], [0.9922; 3], [1.1465; 3], [1.6475; 3], [2.2604; 3]]), + ([3.9941406; 3], [[0.7998; 3], [0.9929; 3], [1.1213; 3], [1.3303; 3], [2.0312; 3], [2.9141; 3]]), + ([0.7998047, 0.099975586, 0.099975586], [ + [0.6101, 0.0762, 0.0762], [0.6279, 0.0784, 0.0784], [0.6386, 0.0798, 0.0798], + [0.6547, 0.0817, 0.0817], [0.7012, 0.0877, 0.0877], [0.7496, 0.0937, 0.0937], + ]), + ([0.099975586, 0.7001953, 0.19995117], [ + [0.0681, 0.4776, 0.1363], [0.0711, 0.4976, 0.1420], [0.0727, 0.5097, 0.1456], + [0.0754, 0.5287, 0.1508], [0.0832, 0.5829, 0.1662], [0.0912, 0.6397, 0.1825], + ]), + ([0.30004883, 0.5, 2.5], [ + [0.1807, 0.3012, 0.9998], [0.1911, 0.3184, 1.1147], [0.1972, 0.3288, 1.1860], + [0.2071, 0.3453, 1.2995], [0.2359, 0.3931, 1.6484], [0.2669, 0.4444, 2.0482], + ]), ]; let mut img = Image::new(table.len() as u32, 1); for (x, (c, _)) in table.iter().enumerate() { img.set(x as u32, 0, [c[0], c[1], c[2], 1.0]); } - for (i, amount) in [100.0, 90.0, 84.0].into_iter().enumerate() { + for (i, amount) in AMOUNTS.into_iter().enumerate() { let out = run("ec.utility.hdrcompression", &[("amount", num(amount))], img.clone()); for (x, (input, want)) in table.iter().enumerate() { let got = out.img.data[x]; From 3a9188e0fb6192e84c141799d25abb48c2ffb5b0 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 18:11:29 +0100 Subject: [PATCH 19/25] HDR Highlight Compression on the GPU: a NaN Amount keeps the layer, as on the CPU Doc comments on the touched helpers. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/utility.rs | 3 +++ crates/gpu/src/fx_pixel2.rs | 5 ++++- crates/gpu/src/tests/fx_pixel2.rs | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/effects/src/utility.rs b/crates/effects/src/utility.rs index 65de35cf6..667b90242 100644 --- a/crates/effects/src/utility.rs +++ b/crates/effects/src/utility.rs @@ -60,6 +60,7 @@ const HDR_LUMA: [f32; 3] = [0.30, 0.59, 0.11]; /// Exponent of the domain the Amount blend happens in (out^0.4 is mixed linearly). const HDR_BLEND_GAMMA: f32 = 0.4; +/// `v^e` with the sign of `v` kept (negative input stays negative, as on the GPU). #[inline] fn hdr_spow(v: f32, e: f32) -> f32 { v.signum() * v.abs().powf(e) @@ -82,6 +83,8 @@ fn hdr_compress_pixel(c: [f32; 3], t: f32) -> [f32; 3] { }) } +/// HDR Highlight Compression: every straight-colour pixel through [`hdr_compress_pixel`] at +/// Amount / 100; Amount 0 or NaN returns the layer unchanged. fn hdr_compression(ctx: &EffectCtx, mut b: Buf) -> Buf { let t = (ctx.params.f("amount") / 100.0).clamp(0.0, 1.0) as f32; if t <= 0.0 || !t.is_finite() { diff --git a/crates/gpu/src/fx_pixel2.rs b/crates/gpu/src/fx_pixel2.rs index 1fe1fec72..5b20976f4 100644 --- a/crates/gpu/src/fx_pixel2.rs +++ b/crates/gpu/src/fx_pixel2.rs @@ -215,6 +215,8 @@ fn composite_mode(m: u32) -> BlendMode { } } +/// The per-pixel effects of `fp2_point` (Cineon, HDR Compander and Highlight Compression, CC +/// Overbrights, CC Composite, Color Link): their parameters packed for the kernel, or `None`. fn point(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { let pr = ctx.params; let mut p = Params::default(); @@ -238,7 +240,8 @@ fn point(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { } "ec.utility.hdrcompression" => { let t = (pr.f("amount") / 100.0).clamp(0.0, 1.0) as f32; - if t <= 0.0 { + // A NaN Amount keeps the layer, as on the CPU. + if t <= 0.0 || !t.is_finite() { return Some(b); } p.u[0][0] = 2; diff --git a/crates/gpu/src/tests/fx_pixel2.rs b/crates/gpu/src/tests/fx_pixel2.rs index 9660592e7..16f2bd392 100644 --- a/crates/gpu/src/tests/fx_pixel2.rs +++ b/crates/gpu/src/tests/fx_pixel2.rs @@ -185,6 +185,8 @@ fn composites_and_colour() { effect_case("ec.utility.hdrcompression", &[]); effect_case("ec.utility.hdrcompression", &[("amount", n(40.0))]); effect_case("ec.utility.hdrcompression", &[("amount", n(90.0))]); + // A NaN Amount keeps the layer (the CPU returns it unchanged). + effect_case("ec.utility.hdrcompression", &[("amount", n(f64::NAN))]); effect_case("ec.utility.growbounds", &[("pixels", n(9.0))]); effect_case("ec.utility.ccoverbrights", &[]); for ch in 1..=5 { From ab6e76973a9aec31e0a180ed6ac5aff52f92f7ba Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 18:47:54 +0100 Subject: [PATCH 20/25] ProEXR test: a non-sRGB output curve (Rec. 709) decodes with its own curve Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/tests/frame_export.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/engine/src/tests/frame_export.rs b/crates/engine/src/tests/frame_export.rs index 2686e810e..8b69ac25d 100644 --- a/crates/engine/src/tests/frame_export.rs +++ b/crates/engine/src/tests/frame_export.rs @@ -294,6 +294,26 @@ fn proexr_samples(path: &str) -> Vec<(String, Vec)> { out } +/// ProEXR decodes with the output space's own curve. A Rec. 709 output has the sRGB working +/// space's primaries but another curve, so the composite and the isolated layer must hold the +/// solid's light (its sRGB working value decoded), not the sRGB decode of its Rec. 709 encoding. +#[test] +fn proexr_decodes_a_non_srgb_output_curve() { + let dir = out_dir("frame-exr-rec709"); + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Frame", "width": 32, "height": 32, "frameRate": 24, "duration": 1})).unwrap(); + s.execute("file.projectSettings", json!({"bitDepth": 32, "workingSpace": "srgb", "linearize": true, "outputSpace": "rec709"})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Warm", "color": [0.8, 0.5, 0.2]})).unwrap(); + let path = dir.join("rec709.exr").to_string_lossy().to_string(); + s.execute("comp.saveFrameAsExr", json!({"path": path})).unwrap(); + let got = proexr_samples(&path); + let at = |n: &str| got.iter().find(|c| c.0 == n).unwrap_or_else(|| panic!("no {n}")).1[0]; + let lin = effectcraft_color::srgb_to_linear; + for (n, want) in [("R", lin(0.8)), ("G", lin(0.5)), ("B", lin(0.2)), ("Warm.R", lin(0.8)), ("Warm.B", lin(0.2))] { + assert!((at(n) - want).abs() <= 1e-3 * want, "{n}: {} vs {want}", at(n)); + } +} + /// ProEXR samples in Clip mode (sRGB output) for each bit depth. #[test] fn proexr_clip_samples_are_unchanged() { From e25025545aeaf55fe3b17c7be31d7c1b82aed13c Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 18:02:10 +0100 Subject: [PATCH 21/25] AEP import: an effect record is read only in the written or the default state A record in another tdsb state is reported as damaged data again, as before the default-state change. The synthetic test checks Levels' recorded popup directly. Co-Authored-By: Claude Opus 5.5 --- crates/aep/src/container/effects.rs | 13 +++++++------ crates/aep/src/import/effects.rs | 2 ++ crates/aep/tests/it/import/synthetic.rs | 19 +++++++++++++++++++ docs/aep-format.md | 2 +- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/crates/aep/src/container/effects.rs b/crates/aep/src/container/effects.rs index 29940a712..8fe084f0f 100644 --- a/crates/aep/src/container/effects.rs +++ b/crates/aep/src/container/effects.rs @@ -382,17 +382,18 @@ impl<'f, 'a> EffectParam<'f, 'a> { self.recorded_value() } - /// The value in the value record whatever the record's state (`tdsb`): what - /// [`EffectParam::stored_value`] reads, also from a record in the default state (`tdsb` 3). - /// Lumetri Color writes the values of its hidden parameters so (High Dynamic Range, HDR - /// Specular, Look: 1200 `lumetri-01-set-all`, `-02-sec-top`, `-06-sec-light`; After Effects - /// reads back the recorded values). `None` with keyframes or without a record. + /// The value in the value record when the record is written (`tdsb` 1) or in the default + /// state (`tdsb` 3): what [`EffectParam::stored_value`] reads, also from a default-state + /// record. Lumetri Color writes the values of its hidden parameters so (High Dynamic Range, + /// HDR Specular, Look: 1200 `lumetri-01-set-all`, `-02-sec-top`, `-06-sec-light`; After + /// Effects reads back the recorded values). `None` with keyframes, without a record or for a + /// record in any other state (no sample has one; the importer reports it as damaged). pub fn recorded_value(&self) -> Option { let p = self.stored?; if self.kind == ParamKind::Layer { return be_u32(p.chunk().child(b"tdpi")?.bytes()?, 0).map(ParamValue::Layer); } - if p.is_animated() { + if !matches!(p.flags(), Some(1 | 3)) || p.is_animated() { return None; } let v = p.static_value()?; diff --git a/crates/aep/src/import/effects.rs b/crates/aep/src/import/effects.rs index 5ab0c1514..d2924f2bd 100644 --- a/crates/aep/src/import/effects.rs +++ b/crates/aep/src/import/effects.rs @@ -133,6 +133,8 @@ struct ParamOut<'a> { col: &'a mut Collector, } +/// Imports one parameter of `fx` into its EffectCraft property: its keyframes, the value in its +/// record or the `pard` default, with a loss for what cannot be read or mapped. fn import_param(fx: &AepEffect<'_, '_>, map: &EffectMap, p: &EffectParam<'_, '_>, cx: &LayerCtx<'_>, shown: &str, out: ParamOut<'_>) { let ParamOut { group, refs, col } = out; if carries_nothing(p) { diff --git a/crates/aep/tests/it/import/synthetic.rs b/crates/aep/tests/it/import/synthetic.rs index af066ab7c..18709f9e5 100644 --- a/crates/aep/tests/it/import/synthetic.rs +++ b/crates/aep/tests/it/import/synthetic.rs @@ -339,6 +339,7 @@ fn effect_values(p: &Project) -> Vec<(String, effectcraft_keyframe::Value)> { out } +/// A default-state record (`tdsb` 3) reads like a written one, except through `stored_value`. #[test] fn a_default_state_effect_record_holds_the_value() { // After Effects writes the values of some parameters as records in the default state (`tdsb` @@ -368,6 +369,7 @@ fn a_default_state_effect_record_holds_the_value() { assert_eq!(seen, 1); } +/// Default-state records import with their values and no loss, like written records. #[test] fn default_state_effect_records_import_like_written_ones() { // Before, such a record was reported as an unreadable value and the default used. @@ -383,4 +385,21 @@ fn default_state_effect_records_import_like_written_ones() { let (p, _) = run(&with_record("ADBE Glo2-0011", 3, 0.75), true); let mid: Vec<_> = effect_values(&p).into_iter().filter(|(id, _)| id == "abMidpoint").collect(); assert_eq!(mid, vec![("abMidpoint".to_string(), effectcraft_keyframe::Value::Scalar(75.0))]); + // Levels' Clip To Output Black (`-0008`) recorded as After Effects' item 1, On: EffectCraft's + // On is item 1 of Off, On, Off for 32 bpc Color (the default is item 2). + let (p, _) = run(&with_record("ADBE Easy Levels2-0008", 3, 1.0), true); + let clip: Vec<_> = effect_values(&p).into_iter().filter(|(id, _)| id == "clipToOutputBlack").collect(); + assert_eq!(clip, vec![("clipToOutputBlack".to_string(), effectcraft_keyframe::Value::Enum(1))]); +} + +/// A record in a state other than 1 or 3 is not read and is reported. +#[test] +fn an_effect_record_in_another_state_is_reported_as_damaged() { + // Only the written (1) and the default (3) state hold a value; any other state is not read. + for tdsb in [0, 2, 5] { + let (_, r) = run(&with_record("ADBE Glo2-0011", tdsb, 0.75), true); + let r = r.unwrap(); + let damaged = losses(&r, LossCategory::DamagedData); + assert!(damaged.iter().any(|l| l.contains("unreadable value")), "tdsb {tdsb}: {:?}", r.report.losses); + } } diff --git a/docs/aep-format.md b/docs/aep-format.md index 5897453c6..08737bc8e 100644 --- a/docs/aep-format.md +++ b/docs/aep-format.md @@ -437,7 +437,7 @@ The expression engine is project-wide (`LIST ExEn/Utf8`, `javascript-1.0`). API: | **float slider valid minimum, slider range** | `pard` | 104; 112 / 116 | f32; f32 x 2 | the smallest valid value (Exposure -100, Offset -2, Gamma Correction 0.1 as f32) and the range the slider covers (Exposure -4..4, Offset -0.5..0.5, Gamma 0.1..10); the slider range is repeated as `tdum` / `tduM` (f64) in each value record (391-394). @120 holds the default again as f32 (Gamma 1.0, Blurriness 25), u16 @124 the display decimals (**H**: Exposure 2, Offset 4) | 389, 391-394 | V | | **definitions of a second instance** | top-level `LIST EfdG` | | | one `EfDf` per effect TYPE (`tdmn` = match name, then a copy of ONE instance's `sspc`, including that instance's value records); the `EfDC` count before them is a little-endian u32. When two layers carry the same effect type, only one instance keeps a full `parT`; **the other has an empty `LIST parT` (4 bytes)** but its own `tdgp` (input layer `-0000` with its own `tdpi`, the button record, and its value records). Its definitions come from the `EfdG` copy by match name, its values from its own records, never from the copy (383: the copy holds S100's Start [0.1, 0.2]; L's Ramp reads the 368 defaults). Which instance keeps the full `parT` is **?** (in 383 S100 is both first in the file and newest). The `EfDf` order follows the stack order (365 / 366); `Pefl/pjef` does not | 365, 366, 383 | V | | popup items | `parT/pdnm/Utf8` | | text | items separated by `\|` | 077 (`Item 1\|Item 2\|Item 3`), 214 | V | -| stored value | `sspc/tdgp/-NNNN` | | `tdbs` | present only for a parameter that differs from its default; `tdsb` 1, `cdat` as in 3.4. **Absent record = the `pard` default.** Some parameters are recorded in the default state (`tdsb` 3) and the record still holds the value After Effects uses: every one of the 25 such records in the corpus and the samples differs from its `pard` default and reads back as recorded (Lumetri Color's hidden controls High Dynamic Range, HDR Specular, the second Look, Legacy Tone And Saturation, Face Detection, Auto Color, Auto Intensity and SemanticAutoTone Progress, 1200; Cryptomatte's Selection Mode on, 1400). No effect record in the default state holds a value that does not count. The reader takes the value whatever the state (`EffectParam::recorded_value`, `EffectParam::value`); `EffectParam::stored_value` keeps to written records. That other effects record values so is H (no sample shows one) | see below; 1200, 1400 | V (1200, 1400), H (other effects) | +| stored value | `sspc/tdgp/-NNNN` | | `tdbs` | present only for a parameter that differs from its default; `tdsb` 1, `cdat` as in 3.4. **Absent record = the `pard` default.** Some parameters are recorded in the default state (`tdsb` 3) and the record still holds the value After Effects uses: every one of the 25 such records in the corpus and the samples differs from its `pard` default and reads back as recorded (Lumetri Color's hidden controls High Dynamic Range, HDR Specular, the second Look, Legacy Tone And Saturation, Face Detection, Auto Color, Auto Intensity and SemanticAutoTone Progress, 1200; Cryptomatte's Selection Mode on, 1400). No effect record in the default state holds a value that does not count. The reader takes the value in either state (`EffectParam::recorded_value`, `EffectParam::value`); `EffectParam::stored_value` keeps to written records, and a record in any other state (no sample has one) is not read: the importer reports it as damaged data. That other effects record values so is H (no sample shows one) | see below; 1200, 1400 | V (1200, 1400), H (other effects) | Stored values by type (`tdsb` 1, or 3 as above): From 0a103c055c93c8a7316017cb043ca4a71cc0f845 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 19:31:17 +0100 Subject: [PATCH 22/25] HDR Highlight Compression test: 20 PNG codes up to 1, 0.1 % above The reference test now checks the bound the PR states in the code space of the renders (the sRGB curve x 6553.5) instead of a linear 2e-3, which allowed about 28 codes at the dark end. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/utility.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/effects/src/utility.rs b/crates/effects/src/utility.rs index 667b90242..e87c6d3ff 100644 --- a/crates/effects/src/utility.rs +++ b/crates/effects/src/utility.rs @@ -602,6 +602,11 @@ mod tests { assert!(out.img.data.iter().flatten().all(|v| v.is_finite()), "{:?}", out.img.data); } + /// A linear value up to 1 as a code of After Effects' 16-bit PNG renders: the sRGB curve × 6553.5. + fn png_code(x: f32) -> f32 { + (if x <= 0.003_130_8 { 12.92 * x } else { 1.055 * x.powf(1.0 / 2.4) - 0.055 }) * 6553.5 + } + /// Our own measurements of After Effects 2026 (32 bpc, no working space; 16-bit PNG renders /// decoded back to linear): input pixel, then the output at Amount 100, 90, 84, 75, 50 and 25. #[test] @@ -638,8 +643,10 @@ mod tests { let got = out.img.data[x]; for c in 0..3 { let w = want[i][c]; - let tol = 2e-3_f32.max(w * 2e-3); - assert!((got[c] - w).abs() <= tol, "amount {amount}, input {input:?}, channel {c}: {} vs AE {w}", got[c]); + // Up to 1: within 20 codes of the 16-bit PNG (the sRGB curve × 6553.5) the + // renders were read from. Above 1 the PNG's lookup table is not known: 0.1 %. + let ok = if w <= 1.0 { (png_code(got[c]) - png_code(w)).abs() <= 20.0 } else { (got[c] - w).abs() <= w * 1e-3 }; + assert!(ok, "amount {amount}, input {input:?}, channel {c}: {} vs AE {w}", got[c]); } } } From 7ce61c68fcab3b10cd1c3a4f249969870a54b196 Mon Sep 17 00:00:00 2001 From: MarekSpodBiedry <73122968+MarekSpodBiedry@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:52:10 +0100 Subject: [PATCH 23/25] Grow the buffer to the pinned quad so Corner Pin draws outside the layer rectangle (#1938) (#1967) * Grow the buffer to the pinned quad so Corner Pin draws outside the layer rectangle (#1938) * Corner Pin: grow through pad_sides, skip adjustment layers, clamp to the pixel budget (#1938) grow_sides returns the four sides for Buf::pad_sides on both paths, replacing grow_to/regrow and the inline GPU copy. Growth is skipped on adjustment layers, cut proportionally to the shared pixel budget (and 16384 per side) instead of giving up, and covered by tests for a far pin, an adjustment layer and a degenerate quad. * Corner Pin: give the grow_sides test its own doc comment (#1938) * effects: Corner Pin keeps a finite buffer for non-finite pins and grows without popping as a pin moves far away * effects: rustfmt the Corner Pin non-finite pin test --------- Co-authored-by: skylar --- crates/effects/src/distort.rs | 80 ++++++++++++++++++++++++++++ crates/effects/src/lib.rs | 85 ++++++++++++++++++++++++++++-- crates/gpu/src/fx_stylize.rs | 11 ++++ crates/gpu/src/tests/fx_stylize.rs | 1 + 4 files changed, 172 insertions(+), 5 deletions(-) diff --git a/crates/effects/src/distort.rs b/crates/effects/src/distort.rs index 916e162f6..610f73f4c 100644 --- a/crates/effects/src/distort.rs +++ b/crates/effects/src/distort.rs @@ -384,6 +384,18 @@ fn transform(ctx: &EffectCtx, mut b: Buf) -> Buf { fn corner_pin(ctx: &EffectCtx, mut b: Buf) -> Buf { let (w, h) = (b.layer_w(ctx), b.layer_h(ctx)); + // The warped layer is drawn wherever the corners are, so grow the buffer to hold the quad; + // an adjustment layer keeps the size of the comp below it. + if !ctx.adjustment { + let pts = ["ul", "ur", "lr", "ll"].map(|k| { + let p = b.to_px(ctx.params.v2(k)); + [p.0, p.1] + }); + let (lo, hi) = crate::quad_bounds(&pts); + if let Some(sides) = crate::grow_sides(lo, hi, b.img.width, b.img.height) { + b.pad_sides(sides); + } + } let q = ["ul", "ur", "lr", "ll"].map(|k| { let p = b.to_px(ctx.params.v2(k)); vec2(p.0, p.1) @@ -542,6 +554,74 @@ mod tests { crate::apply(s, &ctx, Buf { img: im, offset: [0.0, 0.0], scale: 1.0 }) } + #[test] + fn corner_pin_draws_outside_the_layer_rectangle() { + // A 20×10 layer pinned to a quad that lies entirely right of and below it. + let im = crate::util::gen_image(20, 10, |_, _| [1.0, 0.0, 0.0, 1.0]); + let set = [("ul", Value::Vec2([40.0, 30.0])), ("ur", Value::Vec2([60.0, 30.0])), ("lr", Value::Vec2([60.0, 40.0])), ("ll", Value::Vec2([40.0, 40.0]))]; + let o = run("ec.distort.cornerpin", &set, im); + assert!(o.img.width >= 60 && o.img.height >= 40, "{}x{}", o.img.width, o.img.height); + let (cx, cy) = (o.offset[0] as i64 + 50, o.offset[1] as i64 + 35); + assert!(o.img.get(cx, cy)[0] > 0.9); + } + + #[test] + fn corner_pin_keeps_the_size_on_an_adjustment_layer() { + let s = crate::find("ec.distort.cornerpin").unwrap(); + let im = crate::util::gen_image(20, 10, |_, _| [1.0, 0.0, 0.0, 1.0]); + let mut params = Params { values: s.params.iter().map(|p| (p.id.to_string(), crate::default_value(p, [20.0, 10.0]))).collect() }; + params.values.insert("lr".to_string(), Value::Vec2([400.0, 300.0])); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [20.0, 10.0], seed: 3, adjustment: true, env: Default::default() }; + let o = crate::apply(s, &ctx, Buf { img: im, offset: [0.0, 0.0], scale: 1.0 }); + assert_eq!((o.img.width, o.img.height, o.offset), (20, 10, [0.0, 0.0])); + } + + #[test] + fn corner_pin_with_a_far_pin_grows_to_it() { + let im = crate::util::gen_image(20, 10, |_, _| [1.0, 0.0, 0.0, 1.0]); + let o = run("ec.distort.cornerpin", &[("lr", Value::Vec2([1500.0, 1000.0]))], im); + assert!(o.img.width >= 1500 && o.img.height >= 1000, "{}x{}", o.img.width, o.img.height); + } + + #[test] + fn corner_pin_with_a_non_finite_pin_returns_a_buffer_without_panicking() { + for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + for corner in ["ul", "ur", "lr", "ll"] { + let im = crate::util::gen_image(20, 10, |_, _| [1.0, 0.0, 0.0, 1.0]); + let o = run("ec.distort.cornerpin", &[(corner, Value::Vec2([bad, 5.0]))], im); + assert!( + o.img.width >= 1 && o.img.height >= 1 && o.img.width <= 16384 && o.img.height <= 16384, + "{corner} {bad}: {}x{}", + o.img.width, + o.img.height + ); + assert!(o.offset.iter().all(|v| v.is_finite()), "{corner} {bad}: offset {:?}", o.offset); + } + } + } + + #[test] + fn corner_pin_growth_is_continuous_as_a_pin_moves_far_away() { + let mut last: Option = None; + for k in 0..60 { + let far = 100.0 * 1.4_f64.powi(k); + let im = crate::util::gen_image(20, 10, |_, _| [1.0, 0.0, 0.0, 1.0]); + let o = run("ec.distort.cornerpin", &[("lr", Value::Vec2([far, far]))], im); + let size = (o.img.width, o.img.height); + assert!(size.0 >= 20 && size.1 >= 10 && u64::from(size.0) * u64::from(size.1) <= 1 << 24, "pin {far}: {size:?}"); + let area = u64::from(size.0) * u64::from(size.1); + if let Some(prev) = last { + // The shared cut factor may trade width for height as the pin direction changes, + // but the buffer as a whole does not shrink (beyond whole-pixel flooring at the budget) + // as the pin moves further away. + assert!(area * 10_000 >= prev * 9_999, "pin {far}: the buffer shrank from {prev} to {area} pixels ({size:?})"); + } + let opaque = (0..o.img.height as i64).any(|y| (0..o.img.width as i64).any(|x| o.img.get(x, y)[3] > 0.0)); + assert!(opaque, "pin {far}: the pinned layer vanished"); + last = Some(area); + } + } + fn differs(a: Px4, b: Px4) -> bool { (0..4).any(|i| (a[i] - b[i]).abs() > 1e-4) } diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 0138cd9ca..da46836af 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -338,13 +338,19 @@ impl Buf { } } -/// `pad` on every side of a `w` × `h` effect buffer, cut so the padded buffer stays within -/// 16 Mpx (256 MiB of f32 RGBA) or four times its own area, whichever is larger: a small layer -/// far from an effect's centre can't ask for gigabytes (#2261). Shared with the GPU path. -pub fn pad_within_budget(w: u32, h: u32, pad: u32) -> u32 { +/// The most pixels a `w` × `h` effect buffer may grow to: 16 Mpx (256 MiB of f32 RGBA) or four +/// times its own area, whichever is larger. Shared by every effect that grows its buffer. +fn pixel_budget(w: f64, h: f64) -> f64 { const MIN_BUDGET: f64 = 16_777_216.0; + MIN_BUDGET.max(4.0 * w * h) +} + +/// `pad` on every side of a `w` × `h` effect buffer, cut so the padded buffer stays within +/// [`pixel_budget`]: a small layer far from an effect's centre can't ask for gigabytes (#2261). +/// Shared with the GPU path. +pub fn pad_within_budget(w: u32, h: u32, pad: u32) -> u32 { let (w, h) = (f64::from(w), f64::from(h)); - let budget = MIN_BUDGET.max(4.0 * w * h); + let budget = pixel_budget(w, h); if (w + 2.0 * f64::from(pad)) * (h + 2.0 * f64::from(pad)) <= budget { return pad; } @@ -353,6 +359,51 @@ pub fn pad_within_budget(w: u32, h: u32, pad: u32) -> u32 { (p.floor().max(0.0) as u32).min(pad) } +/// The most pixels on a side that [`grow_sides`] grows a buffer to. +const MAX_GROWN_SIDE: f64 = 16384.0; + +/// The transparent pixels to add (left, top, right, bottom; for [`Buf::pad_sides`]) so a +/// `width` × `height` buffer also spans the pixel box `lo`..`hi`, in the buffer's own coordinates. +/// When the full growth would pass [`MAX_GROWN_SIDE`] or the [`pixel_budget`], every side is cut +/// by the same factor instead of giving up, so the picture shrinks smoothly rather than popping. +/// `None` when the buffer already spans the box, or the box is not finite. +pub fn grow_sides(lo: [f64; 2], hi: [f64; 2], width: u32, height: u32) -> Option<[u32; 4]> { + if !lo.iter().chain(hi.iter()).all(|v| v.is_finite()) { + return None; + } + let (w, h) = (f64::from(width), f64::from(height)); + let want = |v: f64| v.clamp(0.0, MAX_GROWN_SIDE); + let [l, t] = [want(-lo[0].floor()), want(-lo[1].floor())]; + let [r, b] = [want(hi[0].ceil() - w), want(hi[1].ceil() - h)]; + let (sx, sy) = (l + r, t + b); + if sx == 0.0 && sy == 0.0 { + return None; + } + let mut f = 1.0_f64; + if sx > 0.0 { + f = f.min((MAX_GROWN_SIDE - w).max(0.0) / sx); + } + if sy > 0.0 { + f = f.min((MAX_GROWN_SIDE - h).max(0.0) / sy); + } + let budget = pixel_budget(w, h); + if (w + sx * f) * (h + sy * f) > budget { + // (w + sx·f)(h + sy·f) = budget, solved for f. + let (a, bq, c) = (sx * sy, w * sy + h * sx, w * h - budget); + let q = if a > 0.0 { (-bq + (bq * bq - 4.0 * a * c).max(0.0).sqrt()) / (2.0 * a) } else { -c / bq }; + f = f.min(q); + } + let f = if f.is_finite() { f.clamp(0.0, 1.0) } else { 0.0 }; + let sides = [l, t, r, b].map(|v| (v * f).floor() as u32); + sides.iter().any(|&v| v != 0).then_some(sides) +} + +/// The pixel box (min, max) enclosing the four points of a quad. +pub fn quad_bounds(q: &[[f64; 2]; 4]) -> ([f64; 2], [f64; 2]) { + let f = |i: usize, init: f64, g: fn(f64, f64) -> f64| q.iter().fold(init, |a, p| g(a, p[i])); + ([f(0, f64::INFINITY, f64::min), f(1, f64::INFINITY, f64::min)], [f(0, f64::NEG_INFINITY, f64::max), f(1, f64::NEG_INFINITY, f64::max)]) +} + /// Motion Tile's output frame for a buffer with layer origin `offset` (pixels) at `scale`: /// `(width, height, offset)`. Output Width / Height grow (or crop) the layer rectangle about its /// centre, on adjustment layers too, where the rectangle is the comp's and starts at the comp @@ -1416,6 +1467,30 @@ pub(crate) fn run_fx(id: &str, vals: &[(&str, Value)], img: Image, time: f64, en mod tests { use super::*; + /// #1938: Corner Pin's growth spans the pinned quad, and past the side cap or the pixel + /// budget it is cut by one factor instead of being dropped. + #[test] + fn grow_sides_spans_the_box_and_clamps_instead_of_giving_up() { + let area = |w: u32, h: u32, s: [u32; 4]| (u64::from(w) + u64::from(s[0]) + u64::from(s[2])) * (u64::from(h) + u64::from(s[1]) + u64::from(s[3])); + assert_eq!(grow_sides([-3.5, 2.0], [30.0, 40.2], 20, 10), Some([4, 0, 10, 31])); + assert_eq!(grow_sides([1.0, 1.0], [19.0, 9.0], 20, 10), None, "already inside"); + assert_eq!(grow_sides([f64::NAN, 0.0], [30.0, 40.0], 20, 10), None); + assert_eq!(grow_sides([0.0, 0.0], [f64::INFINITY, 40.0], 20, 10), None); + // A pin near (16000, 16000) on a small layer: cut to the budget, still growing, in the same ratio. + let s = grow_sides([0.0, 0.0], [16000.0, 16000.0], 20, 10).unwrap(); + assert!(s[2] > 1000 && s[3] > 1000 && area(20, 10, s) <= 1 << 24, "{s:?}"); + // An absurdly far pin is still bounded and still grows. + let s = grow_sides([-1e12, -1e12], [1e12, 1e12], 1920, 1080).unwrap(); + assert!(s.iter().all(|&v| v > 0) && area(1920, 1080, s) <= 1 << 24, "{s:?}"); + assert!(1920 + u64::from(s[0]) + u64::from(s[2]) <= 16384); + // A layer already past the side cap does not grow. + assert_eq!(grow_sides([0.0, 0.0], [20000.0, 100.0], 17000, 50), None); + // A degenerate quad (all four points equal) grows to its point only. + let q = [[50.0, 30.0]; 4]; + let (lo, hi) = quad_bounds(&q); + assert_eq!(grow_sides(lo, hi, 20, 10), Some([0, 0, 30, 20])); + } + /// #2261: a 20 × 20 layer with CC Radial Fast Blur's Center far away grows to at most 16 Mpx; /// big layers may still grow to four times their area, and small pads are untouched. #[test] diff --git a/crates/gpu/src/fx_stylize.rs b/crates/gpu/src/fx_stylize.rs index cc9045a41..34d434307 100644 --- a/crates/gpu/src/fx_stylize.rs +++ b/crates/gpu/src/fx_stylize.rs @@ -871,6 +871,17 @@ fn texturize(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { fn corner_pin(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { let (w, h) = (ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0)); + // Grow to hold the whole quad, like the CPU `corner_pin`. + if !ctx.adjustment { + let pts = ["ul", "ur", "lr", "ll"].map(|k| { + let p = b.to_px(ctx.params.v2(k)); + [p.0, p.1] + }); + let (lo, hi) = effectcraft_effects::quad_bounds(&pts); + if let Some(sides) = effectcraft_effects::grow_sides(lo, hi, b.img.width, b.img.height) { + b.pad_sides(e, sides)?; + } + } let q = ["ul", "ur", "lr", "ll"].map(|k| { let p = b.to_px(ctx.params.v2(k)); vec2(p.0, p.1) diff --git a/crates/gpu/src/tests/fx_stylize.rs b/crates/gpu/src/tests/fx_stylize.rs index d9ca95712..63d93f207 100644 --- a/crates/gpu/src/tests/fx_stylize.rs +++ b/crates/gpu/src/tests/fx_stylize.rs @@ -174,6 +174,7 @@ fn mirror_offset_polar_spherize_corner_pin() { effect_case("ec.distort.spherize", &[("radius", n(35.0)), ("center", pt(30.0, 25.0))]); effect_case("ec.distort.cornerpin", &[("ul", pt(5.0, 3.0)), ("ur", pt(60.0, 8.0)), ("lr", pt(66.0, 40.0)), ("ll", pt(2.0, 44.0))]); effect_case("ec.distort.cornerpin", &[("ul", pt(-8.0, 6.0)), ("ur", pt(75.0, -4.0)), ("lr", pt(50.0, 30.0)), ("ll", pt(12.0, 50.0))]); + effect_case("ec.distort.cornerpin", &[("ul", pt(70.0, 55.0)), ("ur", pt(120.0, 55.0)), ("lr", pt(120.0, 90.0)), ("ll", pt(70.0, 90.0))]); } #[test] From a408d990b23d0b60a17186d31ea715c8e0be9437 Mon Sep 17 00:00:00 2001 From: skylar taylor-barrick Date: Thu, 5 Nov 2026 22:45:23 +0100 Subject: [PATCH 24/25] Export: use conformant ProRes alpha run termination (#2225) * #293: use conformant ProRes alpha run termination Pin the direct ProRes encoder to FilmCraft e4be1577 while preserving all other codec pins. Add a full MOV export regression for final alpha runs across both 4444 profiles, alpha modes and blur settings. * #293: document ProRes alpha export regression * export: repin FilmCraft to the merged ProRes alpha fix and move the round-trip regression into roundtrip.rs All FilmCraft deps pinned to d5ead4ba (squash merge of filmcraft #2991). H.264 pinned movie bytes move 104341 -> 104342: only the encoder version string in the SEI (0.8.0 -> 0.12.0) and five size fields change. --- Cargo.lock | 144 ++++++++++++++------------ Cargo.toml | 36 +++---- ROADMAP.md | 6 +- crates/export/tests/it/movie_bytes.rs | 5 +- crates/export/tests/it/roundtrip.rs | 62 ++++++++++- docs/file-format-parity.md | 5 +- 6 files changed, 170 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ad887965..6ebde6757 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1132,6 +1132,14 @@ dependencies = [ "writeable", ] +[[package]] +name = "craft-media" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=f0ad6c1e1d297908915f4be315b0fcfc9be7b5d4#f0ad6c1e1d297908915f4be315b0fcfc9be7b5d4" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "craft-pdf" version = "0.1.0" @@ -2421,8 +2429,8 @@ dependencies = [ [[package]] name = "filmcraft-aac" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2430,16 +2438,16 @@ dependencies = [ [[package]] name = "filmcraft-ac3" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-apv" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2448,8 +2456,8 @@ dependencies = [ [[package]] name = "filmcraft-av1" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2458,23 +2466,26 @@ dependencies = [ [[package]] name = "filmcraft-avi" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" [[package]] name = "filmcraft-bitstream" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" +dependencies = [ + "craft-media", +] [[package]] name = "filmcraft-cfb" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" [[package]] name = "filmcraft-codecs" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-aac", "filmcraft-ac3", @@ -2510,16 +2521,16 @@ dependencies = [ [[package]] name = "filmcraft-color" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "serde", ] [[package]] name = "filmcraft-dng" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "rayon", "thiserror 2.0.21", @@ -2527,8 +2538,8 @@ dependencies = [ [[package]] name = "filmcraft-dnx" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2537,8 +2548,8 @@ dependencies = [ [[package]] name = "filmcraft-frame" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-color", "filmcraft-time", @@ -2548,16 +2559,16 @@ dependencies = [ [[package]] name = "filmcraft-geom" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "serde", ] [[package]] name = "filmcraft-h264" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2566,8 +2577,8 @@ dependencies = [ [[package]] name = "filmcraft-h264enc" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2576,8 +2587,8 @@ dependencies = [ [[package]] name = "filmcraft-hevc" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2586,8 +2597,8 @@ dependencies = [ [[package]] name = "filmcraft-interchange" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-cfb", "filmcraft-color", @@ -2603,22 +2614,24 @@ dependencies = [ [[package]] name = "filmcraft-isobmff" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ - "filmcraft-bitstream", - "thiserror 2.0.21", + "craft-media", ] [[package]] name = "filmcraft-matroska" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" +dependencies = [ + "craft-media", +] [[package]] name = "filmcraft-media" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-color", "filmcraft-dng", @@ -2632,8 +2645,8 @@ dependencies = [ [[package]] name = "filmcraft-mpa" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2641,8 +2654,8 @@ dependencies = [ [[package]] name = "filmcraft-mpeg2v" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "rayon", "thiserror 2.0.21", @@ -2650,31 +2663,34 @@ dependencies = [ [[package]] name = "filmcraft-mpegts" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-mxf" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" +dependencies = [ + "chrono", +] [[package]] name = "filmcraft-ogg" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" [[package]] name = "filmcraft-opus" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" [[package]] name = "filmcraft-project" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-color", "filmcraft-geom", @@ -2687,8 +2703,8 @@ dependencies = [ [[package]] name = "filmcraft-prores" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2697,24 +2713,24 @@ dependencies = [ [[package]] name = "filmcraft-time" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "serde", ] [[package]] name = "filmcraft-vp8" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-vp9" -version = "0.8.0" -source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +version = "0.12.0" +source = "git+https://github.com/storytold/filmcraft?rev=d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5#d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" dependencies = [ "filmcraft-bitstream", "rayon", diff --git a/Cargo.toml b/Cargo.toml index e512ecf15..8ffe7b988 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,29 +84,29 @@ tempfile = "3.27" # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). # Only `crates/media`, `crates/export` and `crates/interchange` may depend on these. -filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } # encoders + muxer (crates/export) -filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } # container headers read for AAC channel layouts (crates/media) -filmcraft-matroska = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-mpegts = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-matroska = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-mpegts = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) -filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } # decoders used as the round-trip oracles of effectcraft-hevcenc / effectcraft-av1enc (dev-dependencies only) -filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } # timeline interchange (EDL, FCP7 XML, FCPXML, OTIO) for Premiere Pro interop (crates/interchange only) -filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } +filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "d5ead4ba1ee2f1cd13cb86a4239aee2730a3d9f5" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } diff --git a/ROADMAP.md b/ROADMAP.md index dff795d85..e41cbe2f1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ **Stage: alpha** · next: beta, ~30 points of "ready for real work" and ~350–620 h away -> **Last reviewed:** 2026-10-22 · **Last updated:** 2026-10-30 · **Change:** minor (features of 30 October in the progress log: text Line Join, Timeline drag-reorder and multi-select of effects, viewer edge pan, ScriptUI drag capture and dock size; previously: localization recount after #1774) · **Target:** Adobe After Effects 2026 (26.5.0.89) +> **Last reviewed:** 2026-10-22 · **Last updated:** 2026-11-05 · **Change:** minor (ProRes alpha final-run conformance; previously: features of 30 October in the progress log: text Line Join, Timeline drag-reorder and multi-select of effects, viewer edge pan, ScriptUI drag capture and dock size; previously: localization recount after #1774) · **Target:** Adobe After Effects 2026 (26.5.0.89) EffectCraft aims to do what After Effects does, with the same panels, menus and behaviour, written from scratch in Rust. This page is the one-page summary; the evidence is in @@ -117,6 +117,10 @@ Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). ## Progress log +- 2026-11-05: ProRes 4444 and XQ exports write the required run after the final alpha + difference. The FilmCraft encoder pin fixes alpha movies rejected by native macOS + decoding; an export regression checks the stored bytes for both profiles, alpha modes, + and motion-blur settings (#293). Resolve itself still needs confirmation. - 2026-10-30: Text strokes take a Line Join (Miter, Round or Bevel) in the Character panel, Miter for new text as in After Effects (#2077). In the Timeline, effects, masks and shape items drag to a new place in their stack (#2210) and Shift-click adds them to the selection, so one diff --git a/crates/export/tests/it/movie_bytes.rs b/crates/export/tests/it/movie_bytes.rs index c72ec5427..76b89d1c7 100644 --- a/crates/export/tests/it/movie_bytes.rs +++ b/crates/export/tests/it/movie_bytes.rs @@ -95,8 +95,9 @@ fn export_bytes(format: OutputFormat, name: &str) -> Vec { fn h264_movie_bytes_are_pinned() { let data = export_bytes(OutputFormat::H264, "/out/pinned.mp4"); // FilmCraft 09aaf9b's encoder lists its settings in the first frame's user-data SEI (211 - // bytes where 0.1.1's name took 50): 161 bytes more than the FilmCraft 2c5ba7a pin. - assert_eq!((data.len(), fnv1a(&data)), (104_341, 17_655_680_090_264_218_914), "H.264 MP4"); + // bytes where 0.1.1's name took 50): 161 bytes more than the FilmCraft 2c5ba7a pin. The encoder's + // version string "0.8.0" -> "0.12.0" in that SEI adds one more byte (and one to the size fields). + assert_eq!((data.len(), fnv1a(&data)), (104_342, 17_327_612_315_319_115_693), "H.264 MP4"); } #[test] diff --git a/crates/export/tests/it/roundtrip.rs b/crates/export/tests/it/roundtrip.rs index 3ad174691..8531eed54 100644 --- a/crates/export/tests/it/roundtrip.rs +++ b/crates/export/tests/it/roundtrip.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use effectcraft_color::Label; -use effectcraft_export::render_queue::{AudioOutput, Channels, OutputFormat, OutputModule, ProResProfile, RenderSettings, TimeSpan, sequence_path}; +use effectcraft_export::render_queue::{AlphaMode, AudioOutput, Channels, OutputFormat, OutputModule, ProResProfile, RenderSettings, TimeSpan, sequence_path}; use effectcraft_export::{Job, Progress, export}; use effectcraft_media::MediaPool; use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, Project, Solid, build}; @@ -70,6 +70,7 @@ fn settings() -> RenderSettings { RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() } } +/// Exports the standard test comp and checks progress for all [`FRAMES`] frames. fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path) -> effectcraft_export::Report { run_frames(p, cid, footage, om, path, FRAMES) } @@ -251,6 +252,65 @@ fn prores_4444_with_alpha() { } } +/// Checks stored alpha run codewords in exported MOVs across both 4444 profiles, +/// alpha modes and motion-blur settings, without relying on permissive decoders. +#[test] +fn prores_alpha_final_run_is_present_in_exported_movies() { + let dir = out_dir("prores-alpha-final-run"); + let png = dir.join("alpha.png"); + // One macroblock with 255 transparent pixels followed by one opaque pixel. + // The last alpha difference is -1 (modulo 65536), requiring a singleton run. + let mut pixels = image::RgbaImage::new(16, 16); + pixels.put_pixel(15, 15, image::Rgba([255, 80, 20, 255])); + pixels.save(&png).expect("synthetic alpha PNG"); + + for profile in [ProResProfile::P4444, ProResProfile::P4444Xq] { + for alpha_mode in [AlphaMode::Straight, AlphaMode::Premultiplied] { + for motion_blur in [false, true] { + let mut project = Project::default(); + let rate = FrameRate::FPS_30; + let mut comp = Comp::new(16, 16, rate, rate.tick_of(1)); + comp.enable_motion_blur = motion_blur; + let footage = effectcraft_media::probe(&png).expect("probe synthetic PNG"); + let item = project.add_item("Alpha", Label::SeaFoam, None, ItemKind::Footage(footage)); + let mut layer = build::layer(&mut project, &comp, "Alpha", LayerSource::Footage { item }, (16, 16), None); + layer.switches.motion_blur = motion_blur; + comp.layers.push(layer); + let comp = project.add_item("Alpha", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let output = OutputModule { + channels: Channels::Rgba, + prores_profile: profile, + alpha_mode, + audio: AudioOutput::Off, + ..OutputModule::for_format(OutputFormat::ProRes) + }; + let path = dir.join(format!("{profile:?}-{alpha_mode:?}-blur-{motion_blur}.mov")); + run_frames(&project, comp, &MediaPool::new(), &output, &path, 1); + + let bytes = std::fs::read(&path).expect("exported MOV"); + let movie = filmcraft_isobmff::open(bytes.as_slice()).expect("demux exported MOV"); + let video = movie.tracks.iter().find(|track| track.kind == filmcraft_isobmff::TrackKind::Video).expect("video track"); + assert_eq!(video.samples.len(), 1); + let sample = &video.samples[0]; + let frame = &bytes[sample.offset as usize..][..sample.size as usize]; + let (header, start, size) = filmcraft_prores::FrameHeader::parse(frame).expect("ProRes header"); + assert_eq!((header.width, header.height), (16, 16)); + let picture = &frame[start..size]; + let header = filmcraft_prores::PictureHeader::parse(picture).expect("picture header"); + assert_eq!(header.num_slices, 1); + let slice = &picture[header.header_size + 2..]; + let color_bytes: usize = slice[2..8].as_chunks::<2>().0.iter().map(|b| u16::from_be_bytes(*b) as usize).sum(); + let alpha = &slice[(slice[0] >> 3) as usize + color_bytes..]; + // RDD 36:2022 Tables 12–14: +1, run 255, -1, run 1, alignment. + // The old encoder stopped after -1, giving [00 00 fe 01]. Its + // own decoder and FFmpeg accept that truncated payload, so a + // round-trip alone cannot enforce the native decoder's syntax. + assert_eq!(alpha, [0x00, 0x00, 0xfe, 0x01, 0x80], "{profile:?}, {alpha_mode:?}, blur={motion_blur}"); + } + } + } +} + fn check_still_dir(dir: &Path, ext: &str, tol: f32, alpha: bool) { for k in 0..FRAMES { let path = sequence_path(&dir.join(format!("f_[#####].{ext}")).to_string_lossy(), k as i64); diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 5ef21511d..1dc95e3a6 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File format parity -> **Last reviewed:** 2026-10-19 · **Last updated:** 2026-10-19 · **Change:** `.aep` / `.aepx` import (`crates/aep`: `container`, `import`) · **Target:** Adobe After Effects 2026 +> **Last reviewed:** 2026-10-19 · **Last updated:** 2026-11-05 · **Change:** ProRes alpha final-run conformance; previously `.aep` / `.aepx` import (`crates/aep`: `container`, `import`) · **Target:** Adobe After Effects 2026 Every format After Effects reads or writes, with EffectCraft's support. After Effects' list comes from its installed bundle (26.5.0.89: `Info.plist` `CFBundleDocumentTypes`, `Plug-ins/Format`, @@ -62,7 +62,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | MP4 / MOV with H.264 | R/W | yes | yes (own encoder, + AAC) | decoder bit-exact on conformance streams (FilmCraft) | | HEVC (H.265) | R/W | yes | yes (MP4 `hvc1`, Main / Main 10) | encoder bit-exact with ffmpeg's decode; no B-frames / SAO, larger files | | AV1 | R | yes | yes (MP4 `av01`, WebM) | beyond After Effects' export; no B-frames / CDEF | -| Apple ProRes (422 family, 4444) | R/W | yes | yes | good; 4444 colour tag wrong (#340); no ProRes RAW | +| Apple ProRes (422 family, 4444) | R/W | yes | yes | alpha final-run syntax checked in exported MOVs (#293); 4444 colour tag wrong (#340); no ProRes RAW | | WebM / Matroska (VP9, AV1, Opus) | R (limited) | yes, incl. VP9 alpha | yes | beyond After Effects | | MXF (OP1a / OP-Atom) | R/W | yes (FilmCraft's demuxer: AVC, MPEG-2 incl. XDCAM / D-10, VC-3 / DNxHD, ProRes; PCM audio) | none | partial: import (#767), checked against ffmpeg; no MXF output | | AVI | R | yes (FilmCraft's demuxer: AVI 1.0 / OpenDML; Motion JPEG, H.264, HEVC, uncompressed; PCM, MP3, MP2, AC-3 audio) | none | partial: import (#1701), checked against ffmpeg; H.264 needs its parameter sets in the chunks (global headers and copies with an `avcC` record don't decode until storytold/filmcraft#2303), and a damaged video index loses the video (storytold/filmcraft#2302); DV, MPEG-4 Part 2 (DivX / Xvid) and other legacy codecs import but show why they have no picture; no AVI output | @@ -146,6 +146,7 @@ relinking 6–12 h, `.mogrt`-compatible templates 15–30 h (partly parallel wit | Date | Change | Summary | |---|---|---| +| 2026-11-05 | minor | ProRes encoder pin and export regression for final alpha runs; native macOS decode validated, Resolve still unverified (#293) | | 2026-10-30 | minor | Render-queue source-protection behavior and limits consolidated here from the standalone page | | 2026-10-12 | minor | #324 and #514 marked closed; open `.aep` PRs #251 and #860 noted | | 2026-10-10 | major | First file format parity document, from the After Effects 26.5 bundle and this repository's import/export registries | From 4a89e68f9fb69ff444cc08e4064305d4760e5d2b Mon Sep 17 00:00:00 2001 From: gmrmad1 <86235830+gmrmad1@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:21:11 +0530 Subject: [PATCH 25/25] Content-Aware Fill: spill the range and stream the output (#2570 phases 2-4) (#3079) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Content-Aware Fill: bit-pack holes, spill frames to disk, stream outputs (#2570 phases 2-4) * Content-Aware Fill phases 2-4: bit-pack holes, spill frames, stream outputs (clippy clean) * Address CodeRabbit review: spill safety, streaming correctness, perf, memory guard (#2570) * Address the remaining CodeRabbit review findings on Content-Aware Fill (#2570) A non-streaming fill returned empty frames, progress could jump backwards between the sink and the fill, the memory guard ignored the membrane frames still resident for the flow pass (and wasm's in-RAM copies), and the GPU fill stages ran outside an error scope, so an OOM retired the shared context instead of falling back to the CPU for that frame. 🤖 Generated with Codebuff Co-Authored-By: Codebuff * Content-Aware Fill: sample spilled frames without decoding them, check spill errors (#2570) Tracing sampled spilled frames by decoding whole frames under the shared cache lock, so parallel traces across neighbouring frames evicted each other and mostly paid a full-frame read while every other thread waited. Sample only the 2x2 bilinear support with positioned reads on an open handle instead, and report a failed spill read in the no-hole branch rather than streaming black frames as a successful fill. 🤖 Generated with Codebuff Co-Authored-By: Codebuff * Content-Aware Fill: spill membrane/propagated frames, unquantized input, surface scratch errors (#2570) Flow and propagation now write their intermediate frames to scratch spills beside the output instead of holding the whole range as float RGBA, and the input spill uses 32-bit floats whatever the project depth so the fill sees the same values as the old in-memory path. Spill reads/writes and scratch creation report through a shared error slot that run_plan surfaces instead of masking disk failures as a cancellation. Sampling uses positioned reads; the guard models the packed masks plus per-thread scratch. * Content-Aware Fill: report a membrane-spill write failure instead of cancelling (#2570) * Content-Aware Fill: include the flow fields in the memory guard (#2570) * Content-Aware Fill: scale the memory guard with workers and check scratch space (#2570) The guard's scratch term was a fixed 8 frames, which under-counted a wide machine with Lighting Correction by several times. It now models the sequential synthesis stage plus a few frames per worker (from available parallelism) and takes Lighting Correction into account. Native fills write three f32 copies of the range to scratch beside the output, so a range that cannot fit on that volume is now refused up front with the free space in the message, before any frame is rendered. Tests cover the worker/lighting budget, the scratch formula, a shared error slot between the input and secondary spills, an uncreatable spill folder, and the engine surfacing a spill failure as an error instead of a cancellation. * Content-Aware Fill: resolve a relative output folder for the scratch check (#2570) A bare relative output name has an empty parent, so the walk-up found no existing ancestor and skipped the free-space check. Resolve it against the process directory first, and cover that in the test. * Do not reformat the GPU fill tests in the spill branch (#2570) The file was already not rustfmt-clean upstream, so the formatter only added noise to a file this PR no longer needs to touch: phase 1's call-site changes are already on the base branch. * Content-Aware Fill: measure peak RSS and correct the per-worker scratch model (#2570) The ignored measure_peak_rss_of_a_fill test fills a spilled 640x360 range and reports VmHWM. On this 18-worker machine 90 frames peak at 340-470 MB across runs (35 MB with RAYON_NUM_THREADS=1) and 600 frames at 582 MB, so the scratch slope is ~7 frames a worker, not the 2 the first guard revision assumed. Scaling the summary down showed a clean linear slope at 8 and 4 workers, and Lighting Correction stayed below the flow stage's peak. Budget 8 frames a worker (9 with Lighting Correction), just above the measured top of the range, and adjust the guard test to the real numbers. At 600 frames the measured peak is 582 MB against ~11.1 GB for the old five-copies model, a 19x reduction. * Contributors: credit Madhav Agarwal in the About window Adds the self-chosen real and display name for @gmrmad1. The generated contributors.json picks it up once the PR is merged. * Address CodeRabbit: size the guard from the rayon pool; migrate GPU tests (#2570) fill_workers used available_parallelism, but the parallel fill stages run on the rayon pool, which RAYON_NUM_THREADS can size larger. Use rayon::current_num_threads so the scratch budget follows the threads that actually run; wasm keeps rayon's single-thread fallback. The GPU inpaint tests still used the pre-spill FillInput API. The earlier 'do not reformat' revert dropped their API migration along with the formatting, leaving cargo test -p effectcraft-gpu uncompilable. Restore the Frames::Memory / HoleMask / sink migration and verify the 15 inpaint tests. * Fix gate: flatten the wasm prop lookup and keep the budget wrapper test-only (#2570) * Address CodeRabbit review: spill error check, global-pool preflight, docs (#2570) * Include RAM-backed spill in the preflight and guard streaming reads (#2570) * Resolve relative spill paths, prefer stacked mounts, guard the Edge Blend sink (#2570) * Make the fill spill anonymous and test the streaming claims (#2570) The spill was a named file in the output folder that only Drop removed, so an out-of-memory kill or a force-quit left a multi-gigabyte effectcraft-fill-* file in the user's project folder, and a failed set_len after keep() leaked it too. The spill is now an anonymous temporary file (tempfile_in), which the system reclaims when its last handle closes; every read and write is positioned on that one handle (pwrite/seek_write join the existing pread/seek_read), and the folder is kept only so the propagated frames spill beside the input. Also: - Edge Blend checks for a failed read before handing a frame to the sink, so a black fallback frame never streams. - The propagation workers stop as soon as the spill records an error. - HoleMask::get/set use get()/get_mut(), so a mask whose public words are shorter than len cannot panic. New tests cover the PR's headline claims: a spilled range fills bit-identically to the same frames in RAM (returned and streamed), the sink sees frames strictly in order, a truncated spill fails with a recorded error and streams nothing, a failure mid-stream stops before the next frame, propagation stops after a failure, the spill has no name in its folder, and HoleMask round-trips at word boundaries. Co-Authored-By: Claude Opus 5.5 * Content-Aware Fill: clean up a stopped fill and count two scratch copies (#2570) - A fill that fails or is cancelled after frames started streaming deletes the PNGs it wrote, and the output folder when the fill made it and it is left empty (OutputFiles, dropped after the spill so Windows can remove the folder). - After a successful fill, run_plan checks that every frame has a file, and apply_plan/add_fill_layer return an error on an empty range or file list (a job worker's reply) instead of indexing [0]. - scratch_need charges two f32 copies (32 B/px): flow_pairs drops the membrane spill before the propagated one is created, so the third copy never coexists. - The spill-creation error carries a stable "spill: cannot make the scratch file" marker; the panel test asserted on the OS wording, which on Windows is "Cannot create a file when that file already exists". Co-Authored-By: Claude Opus 5.5 * Content-Aware Fill: a stopped fill leaves files it didn't create (#2570) A fill into a given outputDir that already holds an earlier sequence overwrites its frames as it goes (as before). The cleanup of a failed or stopped fill deleted every path it wrote, including those earlier frames, so a layer still using that sequence lost them. It now deletes only the files the fill created (CodeRabbit on #3079). Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Codebuff Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 1 + crates/engine/src/cmd/content_fill.rs | 383 +++++++- crates/engine/src/system/sysinfo.rs | 107 +++ crates/engine/src/tests/panels.rs | 52 ++ crates/gpu/src/tests/inpaint.rs | 34 +- crates/raster/Cargo.toml | 1 + crates/raster/src/inpaint.rs | 1233 +++++++++++++++++++++++-- 7 files changed, 1671 insertions(+), 140 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ebde6757..7d58b050d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1855,6 +1855,7 @@ dependencies = [ "effectcraft-color", "effectcraft-geom", "rayon", + "tempfile", ] [[package]] diff --git a/crates/engine/src/cmd/content_fill.rs b/crates/engine/src/cmd/content_fill.rs index 04779db8b..720e780b9 100644 --- a/crates/engine/src/cmd/content_fill.rs +++ b/crates/engine/src/cmd/content_fill.rs @@ -13,7 +13,9 @@ use std::sync::Arc; use effectcraft_color::Label; use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, LayerId, LayerSource, build}; use effectcraft_raster::Image; -use effectcraft_raster::inpaint::{FillInput, FillMethod, FillOpts, dilate}; +#[cfg(not(target_arch = "wasm32"))] +use effectcraft_raster::inpaint::SpillHandle; +use effectcraft_raster::inpaint::{FillInput, FillMethod, FillOpts, Frames, HoleMask, dilate}; use effectcraft_render::{Accelerator, EvalCtx, ExprHost, FootageSource, LayerCache, Renderer}; use effectcraft_time::Tick; use serde::{Deserialize, Serialize}; @@ -170,11 +172,76 @@ pub struct FillPlan { pub dir: String, } -/// The fill holds every frame as float RGBA (16 bytes a pixel), its hole mask (1) and the filled -/// copy (16) until it finishes (#1789): when even that is more than `available`, the error to -/// report up front, instead of running out of memory partway through. +/// Transient full frames one parallel stage holds per worker: the membrane reads a frame and +/// returns a copy, the flow stage holds the neighbouring pair plus the luma pyramids, and +/// Lighting Correction adds the mismatch ring (`diff`, its mask) on top. +/// +/// Measured with `measure_peak_rss_of_a_fill` (90 frames, 640×360, Surface): peak RSS grows by +/// ~7 frames a rayon worker at the top of its run-to-run spread, and Lighting Correction stays +/// below the flow stage's peak, so 8 (9 with lighting) is a conservative constant (#2570). +fn per_worker_frames(lighting: bool) -> u64 { + if lighting { 9 } else { 8 } +} + +/// The sequential synthesis stage: the stored frame, the PatchMatch pyramid and its +/// coarse-to-fine copies, the warped previous frame, and the sink's prepared copy. +const SYNTH_FRAMES: u64 = 6; + +/// Workers a range of `frames` can keep busy (rayon never runs more tasks than frames). The +/// scratch estimate must follow the pool that actually runs the parallel stages: a synchronous +/// fill stays in the caller's pool, while an asynchronous one runs on `spawn_task`'s own OS +/// thread and so uses the global pool, whatever pool the command was called from (#2570). +fn fill_workers(frames: usize, wait: bool) -> u64 { + let pool = if wait { rayon::current_num_threads() as u64 } else { global_pool_threads() }; + pool.min(frames.max(1) as u64) +} + +/// The global Rayon pool's thread count, read from a thread outside every pool: +/// `rayon::current_num_threads()` answers for the caller's pool, not the one the background +/// task uses. Falls back to the current pool when the probe thread cannot start. +fn global_pool_threads() -> u64 { + let (tx, rx) = std::sync::mpsc::channel(); + match std::thread::Builder::new().name("fill-pool-probe".into()).spawn(move || { + let _ = tx.send(rayon::current_num_threads()); + }) { + Ok(handle) => { + let n = rx.recv().map(|n| n as u64).unwrap_or_else(|_| rayon::current_num_threads() as u64); + let _ = handle.join(); + n + } + Err(_) => rayon::current_num_threads() as u64, + } +} + +/// Native builds spill the input, the membrane-filled neighbours and the propagated copies, and +/// pack the holes to one bit per pixel, so only the two packed masks and the two flow fields +/// (~1/2 byte a pixel a frame) plus the workers' scratch stay resident (#2570). wasm has no +/// spill and keeps the input and the working copies in RAM (~32 bytes a pixel a frame). When +/// even that peak is more than `available`, report it up front instead of running out of memory +/// partway through (#1789, #2570 review). +#[cfg(test)] fn memory_shortfall(frames: usize, w: u32, h: u32, available: u64) -> Option { - let need = (frames as u64).saturating_mul(u64::from(w)).saturating_mul(u64::from(h)).saturating_mul(33); + memory_shortfall_with(frames, w, h, available, fill_workers(frames, true), false, 0) +} + +/// `workers` and `lighting` are parameters so the budget is testable: the scratch term used to +/// be a fixed 8 frames, which under-counted a wide machine by several times (#2570 review). +/// `resident` adds storage that would otherwise be spill: when the spill volume is RAM-backed, +/// its files use the same memory as the working set (#2570 review). +fn memory_shortfall_with(frames: usize, w: u32, h: u32, available: u64, workers: u64, lighting: bool, resident: u64) -> Option { + let px = u64::from(w).saturating_mul(u64::from(h)); + let per_frame = (frames as u64).saturating_mul(px); + // `holes` and the leftover propagation masks, packed to a bit a pixel each. + let masks = per_frame / 4; + // The forward and backward flow fields, ~0.125 byte a pixel each. + let flows = per_frame / 4; + // wasm keeps the input and one working copy in RAM at every stage; a RAM-backed spill + // volume counts there too. + let resident = resident.saturating_add(if cfg!(target_arch = "wasm32") { per_frame.saturating_mul(32) } else { 0 }); + // The synthesis stage is sequential; every parallel stage holds a few frames a worker. + let scratch_frames = SYNTH_FRAMES.saturating_add(per_worker_frames(lighting).saturating_mul(workers)); + let scratch = px.saturating_mul(16).saturating_mul(scratch_frames); + let need = masks.saturating_add(flows).saturating_add(resident).saturating_add(scratch); (need > available).then(|| { let gb = |b: u64| b as f64 / 1e9; format!( @@ -185,6 +252,40 @@ fn memory_shortfall(frames: usize, w: u32, h: u32, available: u64) -> Option u64 { + let bytes = (frames as u64).saturating_mul(u64::from(w)).saturating_mul(u64::from(h)).saturating_mul(32); + bytes.saturating_add(bytes / 10) +} + +/// Refuse up front when the output volume cannot hold the fill's scratch. The folder may not +/// exist yet (the spill creates it), so query the nearest existing ancestor. `None` when the +/// volume cannot be read; the write itself still reports a real error then (#2570 review). +#[cfg(not(target_arch = "wasm32"))] +fn scratch_shortfall(frames: usize, w: u32, h: u32, dir: &std::path::Path) -> Option { + let need = scratch_need(frames, w, h); + // Resolve a relative output folder against the process directory; `parent()` of a bare + // name is empty, which would otherwise skip the check. + let base = if dir.is_absolute() { dir.to_path_buf() } else { std::env::current_dir().ok()?.join(dir) }; + let mut probe = base.as_path(); + while !probe.exists() { + probe = probe.parent()?; + } + let (_, free) = crate::sysinfo::disk_space(probe)?; + (need > free).then(|| { + let gb = |b: u64| b as f64 / 1e9; + format!( + "Content-Aware Fill of {frames} frames at {w}×{h} needs up to {:.1} GB of scratch space beside the output, and {:.1} GB is free there: fill a shorter range (set the work area and choose Work Area) or choose an output folder on a larger disk", + gb(need), + gb(free) + ) + }) +} + /// The status line shown while the frames are filled. Edge Blend calls only the membrane, which /// the device backend leaves to the CPU reference, so it is not a device fill even when a `fill` /// backend is present; Object and Surface run the flow and PatchMatch on the device. @@ -192,6 +293,43 @@ fn filling_status(on_device: bool, method: FillMethod) -> &'static str { if on_device && method != FillMethod::EdgeBlend { "Filling (GPU)" } else { "Filling" } } +/// The PNGs a fill has written so far. Unless the fill finishes ([`OutputFiles::keep`]), dropping +/// it deletes the ones it created, and the output folder too when the fill made it and nothing +/// else is in it, so a failed or stopped fill leaves no partial sequence behind (#2570 review). A +/// file that was already there (an `outputDir` holding an earlier sequence) is never deleted. +struct OutputFiles { + dir: std::path::PathBuf, + /// The folder was missing before the fill (the spill creates it). + made_dir: bool, + paths: Vec, + /// Paths among `paths` that existed before the fill wrote them: not the fill's to delete. + spare: Vec, + kept: bool, +} + +impl OutputFiles { + /// The fill finished: keep its files. + fn keep(mut self) -> Vec { + self.kept = true; + std::mem::take(&mut self.paths) + } +} + +impl Drop for OutputFiles { + fn drop(&mut self) { + if self.kept { + return; + } + for path in self.paths.iter().filter(|p| !self.spare.contains(p)) { + let _ = std::fs::remove_file(path); + } + // `remove_dir` leaves a folder that still holds anything else. + if self.made_dir { + let _ = std::fs::remove_dir(&self.dir); + } + } +} + /// Run a fill: the layer's frames (rendered in passes with a job worker's GPU), the fill, the /// PNG sequence written into `plan.dir` (`Services::store_file`). Returns the files. /// @@ -224,17 +362,34 @@ pub(crate) async fn run_plan( let ctx0 = EvalCtx { expr, ..EvalCtx::new(project, cid, comp, Tick::ZERO) }; let times = &plan.times; let n = times.len() as u64; - let mut frames = Vec::with_capacity(times.len()); let mut holes = Vec::with_capacity(times.len()); + #[cfg(target_arch = "wasm32")] + let mut frames = Vec::with_capacity(times.len()); + // Declared before the spill, so on an early return the spill is closed first and the folder + // it was made in can be removed. + let dir = std::path::Path::new(&plan.dir); + let mut written = OutputFiles { dir: dir.to_path_buf(), made_dir: !dir.exists(), paths: Vec::with_capacity(times.len()), spare: Vec::new(), kept: false }; + // Spill 32-bit floats whatever the project depth: the old in-memory path kept f32 all the + // way through, so quantising here would change the fill (#2570, "no quality loss"). Scratch + // lives beside the output when one is set; the default temp folder may be RAM-backed, which + // the memory preflight accounts for when it chooses the spill folder. + #[cfg(not(target_arch = "wasm32"))] + let spill = SpillHandle::create_in(Some(dir), w as usize, h as usize, times.len(), 32) + .map_err(|e| format!("spill: cannot make the scratch file in {}: {e}", plan.dir))?; + #[cfg(not(target_arch = "wasm32"))] + let spill_probe = spill.clone(); for (k, t) in times.iter().enumerate() { let (img, hole) = crate::offload::in_passes(accel, |a| masked_frame(&r.with_accel(a), &ctx0.at(*t), layer, w, h)).await; - holes.push(dilate(&hole, w as usize, h as usize, plan.expand)); + holes.push(HoleMask::from_vec(&dilate(&hole, w as usize, h as usize, plan.expand))); + #[cfg(target_arch = "wasm32")] frames.push(img); + #[cfg(not(target_arch = "wasm32"))] + spill.write_frame(k, &img)?; if !progress(k as u64 + 1, n * 3) { return Err(crate::render_queue::CANCELLED.into()); } } - if !holes.iter().any(|h| h.iter().any(|b| *b)) { + if !holes.iter().any(|h| h.any()) { return Err("the layer has no transparent area to fill: draw a mask (Subtract) around what to remove".into()); } let mut references = vec![]; @@ -249,31 +404,94 @@ pub(crate) async fn run_plan( } let opts = FillOpts { method: FillMethod::from_name(&plan.method).unwrap_or_default(), lighting: plan.lighting, ..Default::default() }; message(filling_status(fill.is_some(), opts.method)); - let input = FillInput { frames, holes, references }; + #[cfg(target_arch = "wasm32")] + let input_frames = Frames::Memory(frames); + #[cfg(not(target_arch = "wasm32"))] + let input_frames = Frames::Spilled(spill); + let input = FillInput { frames: input_frames, holes, references }; let cpu = effectcraft_raster::inpaint::CpuFill; let backend: &dyn effectcraft_raster::inpaint::FillBackend = fill.unwrap_or(&cpu); - let out = effectcraft_raster::inpaint::fill_sequence_with(input, &opts, backend, &mut |d, t| progress(n + (d as u64 * n / t.max(1) as u64), n * 3)) - .ok_or(crate::render_queue::CANCELLED)?; - message("Writing the fill layer"); - let dir = &plan.dir; - let mut paths = vec![]; - for (k, img) in out.iter().enumerate() { - let path = std::path::Path::new(dir).join(format!("fill_{k:05}.png")).to_string_lossy().to_string(); - services.store_file(&path, &png(img)?).map_err(|e| format!("{path}: {e}"))?; - paths.push(path); - if !progress(2 * n + k as u64 + 1, n * 3) { - return Err(crate::render_queue::CANCELLED.into()); + let progress = std::rc::Rc::new(std::cell::RefCell::new(progress)); + // The fill and the sink both advance the bar: a streamed frame reports ahead of the fill's own + // progress, so both report a shared high-water mark and the bar never moves backwards. + let high = std::cell::Cell::new(0u64); + let report = |done: u64| -> bool { + let done = done.max(high.get()); + high.set(done); + (progress.borrow_mut())(done, n * 3) + }; + let paths: Vec = (0..n as usize).map(|k| dir.join(format!("fill_{k:05}.png")).to_string_lossy().to_string()).collect(); + let mut sink_error: Option = None; + let mut sink = |idx: usize, img: &Image| -> std::result::Result<(), String> { + let r = (|| -> std::result::Result<(), String> { + // A failed spilled read returns a black fallback frame; never publish it (#2570 + // review). + #[cfg(not(target_arch = "wasm32"))] + if spill_probe.has_error() { + return Err(spill_probe.take_error().unwrap_or_else(|| "spill: read failed".to_string())); + } + let existed = services.exists(&paths[idx]); + services.store_file(&paths[idx], &png(img)?).map_err(|e| format!("{}: {e}", paths[idx]))?; + written.paths.push(paths[idx].clone()); + if existed { + written.spare.push(paths[idx].clone()); + } + if !report(2 * n + idx as u64 + 1) { + return Err(crate::render_queue::CANCELLED.to_string()); + } + Ok(()) + })(); + if let Err(e) = &r + && sink_error.is_none() + { + sink_error = Some(e.clone()); + } + r + }; + let produced = + effectcraft_raster::inpaint::fill_sequence_with(input, &opts, backend, &mut |d, t| report(n + (d as u64 * n / t.max(1) as u64)), Some(&mut sink)); + #[cfg(not(target_arch = "wasm32"))] + if let Some(e) = spill_probe.take_error() { + return Err(e); + } + if produced.is_none() { + if let Some(e) = sink_error.take() { + return Err(e); + } + return Err(crate::render_queue::CANCELLED.into()); + } + let produced = produced.unwrap_or_default(); + // Non-streaming fallback (tests): write any retained images. + if !produced.is_empty() { + for (k, img) in produced.iter().enumerate() { + let path = paths[k].clone(); + services.store_file(&path, &png(img)?).map_err(|e| format!("{path}: {e}"))?; + if written.paths.len() <= k { + written.paths.push(path); + } + if !report(2 * n + k as u64 + 1) { + return Err(crate::render_queue::CANCELLED.to_string()); + } } } - Ok(paths) + // Every frame must have a file: the fill layer is built on the first one, and a short + // sequence would play the wrong frames. + if written.paths.len() != times.len() { + return Err(format!("Content-Aware Fill wrote {} of {} frames", written.paths.len(), times.len())); + } + Ok(written.keep()) } /// Add the fill layer of a finished plan (one undo step). +/// The plan comes back from a job worker, so an empty range is an error, not a crash. pub(crate) fn apply_plan(s: &mut Session, plan: &FillPlan, paths: Vec) -> Result { let importer = s.importer.clone(); - add_fill_layer(s, plan.comp, plan.layer, &plan.name, paths, (plan.width, plan.height), plan.times[0], plan.times.len(), importer) + let start = plan.times.first().copied().ok_or_else(|| bad("contentFill.generate", "the fill has no frames"))?; + add_fill_layer(s, plan.comp, plan.layer, &plan.name, paths, (plan.width, plan.height), start, plan.times.len(), importer) } +/// `contentFill.generate`: validate the layer and range, refuse a fill that cannot fit in +/// memory or scratch space, then run it (in the background unless `wait` is set). fn generate(s: &mut Session, p: &Value) -> Result { let c = "contentFill.generate"; let (cid, lid) = layer_p(s, p, c)?; @@ -307,17 +525,35 @@ fn generate(s: &mut Session, p: &Value) -> Result { } let (fa, fb) = (fr.frame_at(fr.snap_nearest(a)), fr.frame_at(b - Tick(1))); let times: Vec = (fa..=fb).map(|f| fr.tick_of(f)).collect(); - if let Some(e) = crate::sysinfo::memory().and_then(|m| memory_shortfall(times.len(), w, h, m.available)) { - return Err(bad(c, e)); - } let opts = FillOpts { method: FillMethod::from_name(&st.method).unwrap_or_default(), lighting: lighting(&st.lighting_correction).unwrap_or(0.0), ..Default::default() }; + let dir = str_p(p, "outputDir").map(str::to_string).unwrap_or_else(|| default_dir(s, &layer.name)); + // The spill lives beside the output. On a RAM-backed volume (`tmpfs`/`ramfs`, where the + // default temp dir may live) its files use the same memory as the working set, so the + // memory preflight counts them; disk-backed volumes get the space check below instead. + #[cfg(not(target_arch = "wasm32"))] + let ram_spill = if crate::sysinfo::is_memory_backed(std::path::Path::new(&dir)) { scratch_need(times.len(), w, h) } else { 0 }; + #[cfg(target_arch = "wasm32")] + let ram_spill = 0; + let wait = b_p(p, "wait").unwrap_or(false); + // `spawn_task` runs a fill inline on wasm, and `runs_inline` selects the pool it will use. + let runs_inline = wait || cfg!(target_arch = "wasm32"); + if let Some(e) = crate::sysinfo::memory() + .and_then(|m| memory_shortfall_with(times.len(), w, h, m.available, fill_workers(times.len(), runs_inline), opts.lighting > 0.0, ram_spill)) + { + return Err(bad(c, e)); + } let expand = st.alpha_expansion.round().max(0.0) as usize; let reference = st.reference_layer.filter(|r| comp.layer(*r).is_some()).map(|r| (r, Tick::from_seconds_f64(st.reference_time.unwrap_or(0.0)))); - let dir = str_p(p, "outputDir").map(str::to_string).unwrap_or_else(|| default_dir(s, &layer.name)); + // Native fills keep up to two f32 copies of the range in scratch beside the output; a volume + // that cannot hold them is refused before any frame is rendered (#2570 review). + #[cfg(not(target_arch = "wasm32"))] + if let Some(e) = scratch_shortfall(times.len(), w, h, std::path::Path::new(&dir)) { + return Err(bad(c, e)); + } let plan = FillPlan { comp: cid, layer: lid, @@ -331,7 +567,6 @@ fn generate(s: &mut Session, p: &Value) -> Result { reference, dir, }; - let wait = b_p(p, "wait").unwrap_or(false); // The browser: a job worker (its own GPU), the page keeps running. if !wait && s.offloads() { s.offload_analysis(crate::offload::WorkerJob::ContentFill { plan: Box::new(plan) }).map_err(EngineError::Other)?; @@ -377,12 +612,13 @@ fn add_fill_layer( count: usize, importer: Option>, ) -> Result { + let first = paths.first().cloned().ok_or_else(|| bad("contentFill.generate", "the fill wrote no frames"))?; let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); let src = comp.layer(lid).ok_or_else(|| bad("contentFill.generate", "the source layer is gone"))?.clone(); let fr = comp.frame_rate; let dur = fr.tick_of(count as i64); - let mut footage = importer.and_then(|i| i.probe(&paths[0]).ok()).filter(|f| f.has_video).unwrap_or_else(|| Footage { - path: paths[0].clone(), + let mut footage = importer.and_then(|i| i.probe(&first).ok()).filter(|f| f.has_video).unwrap_or_else(|| Footage { + path: first.clone(), kind: FootageKind::Sequence, width: size.0, height: size.1, @@ -468,7 +704,11 @@ pub fn specs() -> Vec { #[cfg(test)] mod tests { - use super::{filling_status, memory_shortfall}; + #[cfg(not(target_arch = "wasm32"))] + use super::scratch_need; + #[cfg(not(target_arch = "wasm32"))] + use super::scratch_shortfall; + use super::{filling_status, memory_shortfall, memory_shortfall_with}; /// Edge Blend fills with the membrane, which the device backend leaves to the CPU reference: /// it must not claim a GPU fill just because a backend is available. @@ -481,13 +721,88 @@ mod tests { assert_eq!(filling_status(false, FillMethod::Object), "Filling"); } - /// #1789: 10 s of 1080p30 (300 frames) needs at least 20 GB, which 4 GB free can't hold. + /// #1789 / #2570: a range that cannot fit is refused up front instead of OOM-ing. The + /// scratch term scales with the workers and Lighting Correction, not a fixed frame count. #[test] fn a_fill_that_cannot_fit_in_memory_is_refused_up_front() { - let e = memory_shortfall(300, 1920, 1080, 4_000_000_000).expect("refused"); + let e = memory_shortfall_with(300, 1920, 1080, 1_000_000, 8, false, 0).expect("refused"); assert!(e.contains("300 frames") && e.contains("Work Area"), "{e}"); - assert_eq!(memory_shortfall(30, 1920, 1080, 4_000_000_000), None, "a second of it fits"); + // Four workers, no lighting: the packed masks and flows plus ~38 scratch frames fit. + assert_eq!(memory_shortfall_with(30, 1920, 1080, 4_000_000_000, 4, false, 0), None); + assert_eq!(memory_shortfall_with(300, 1920, 1080, 2_000_000_000, 4, false, 0), None); + // A wide machine with Lighting Correction holds nine frames a worker: 300 frames no + // longer fit in 5 GB, which the old fixed 8-frame scratch term missed (#2570 review). + assert!(memory_shortfall_with(300, 1920, 1080, 1_000_000_000, 16, true, 0).is_some()); + assert!(memory_shortfall_with(300, 1920, 1080, 5_000_000_000, 16, true, 0).is_some()); + assert_eq!(memory_shortfall_with(300, 1920, 1080, 6_000_000_000, 16, true, 0), None); + // The two packed masks still grow with the range: a very long 1080p range cannot fit. + assert!(memory_shortfall_with(1000, 1920, 1080, 500_000_000, 8, false, 0).is_some()); + // A RAM-backed spill volume counts its bytes as resident, so the same range that fit on + // a disk-backed volume is refused (#2570 review). + assert_eq!(memory_shortfall_with(30, 1920, 1080, 2_500_000_000, 4, false, 0), None); + assert!(memory_shortfall_with(30, 1920, 1080, 2_500_000_000, 4, false, 2_000_000_000).is_some()); // Hostile sizes saturate instead of overflowing. - assert!(memory_shortfall(usize::MAX, u32::MAX, u32::MAX, u64::MAX - 1).is_some()); + assert!(memory_shortfall(usize::MAX, u32::MAX, u32::MAX, 0).is_some()); + } + + /// The native spill holds at most two f32 copies of the range beside the output at once (the + /// membrane spill is gone before the propagated one is made), plus the PNGs. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn scratch_space_is_checked_before_a_fill() { + let copies = 300u64 * 1920 * 1080 * 32; + assert_eq!(scratch_need(300, 1920, 1080), copies + copies / 10); + let tmp = std::env::temp_dir(); + // A range that cannot possibly fit on the volume is refused up front. + assert!(scratch_shortfall(usize::MAX, u32::MAX, u32::MAX, &tmp).is_some()); + // A relative folder that does not exist is resolved against the process directory. + assert!(scratch_shortfall(usize::MAX, u32::MAX, u32::MAX, std::path::Path::new("effectcraft-missing-fill-folder")).is_some()); + // A tiny fill fits. + assert_eq!(scratch_shortfall(1, 16, 16, &tmp), None); + } + + /// A fill that fails or is stopped after some frames were written deletes them, and the + /// output folder it made; a finished fill keeps its files (#2570 review). + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_stopped_fill_removes_its_partial_output() { + use super::OutputFiles; + let root = tempfile::tempdir().unwrap(); + let write = |dir: &std::path::Path, made_dir: bool| { + std::fs::create_dir_all(dir).unwrap(); + let paths: Vec = (0..2).map(|k| dir.join(format!("fill_{k:05}.png")).to_string_lossy().to_string()).collect(); + for p in &paths { + std::fs::write(p, b"png").unwrap(); + } + OutputFiles { dir: dir.to_path_buf(), made_dir, paths, spare: Vec::new(), kept: false } + }; + // The fill made the folder: the files and the folder go. + let made = root.path().join("made"); + drop(write(&made, true)); + assert!(!made.exists(), "the partial sequence and its folder are removed"); + // The folder was already there: only the fill's own files go. + let existing = root.path().join("existing"); + drop(write(&existing, false)); + assert!(existing.exists()); + assert_eq!(std::fs::read_dir(&existing).unwrap().count(), 0); + // Something else in a folder the fill made keeps the folder. + let shared = root.path().join("shared"); + let files = write(&shared, true); + std::fs::write(shared.join("notes.txt"), b"keep").unwrap(); + drop(files); + assert_eq!(std::fs::read_dir(&shared).unwrap().count(), 1); + // A file that was there before the fill wrote over it (an earlier sequence in a given + // `outputDir`) is not the fill's to delete. + let earlier = root.path().join("earlier"); + let mut files = write(&earlier, false); + files.spare.push(files.paths[0].clone()); + drop(files); + assert!(earlier.join("fill_00000.png").exists(), "an earlier fill's frame stays"); + assert!(!earlier.join("fill_00001.png").exists(), "the fill's own frame goes"); + // A finished fill keeps its files. + let done = root.path().join("done"); + let kept = write(&done, true).keep(); + assert_eq!(kept.len(), 2); + assert!(kept.iter().all(|p| std::path::Path::new(p).exists())); } } diff --git a/crates/engine/src/system/sysinfo.rs b/crates/engine/src/system/sysinfo.rs index bfb471e92..80b51b462 100644 --- a/crates/engine/src/system/sysinfo.rs +++ b/crates/engine/src/system/sysinfo.rs @@ -207,6 +207,75 @@ pub fn disk_space(path: &std::path::Path) -> Option<(u64, u64)> { } } +/// Whether `path` is on a RAM-backed filesystem (`tmpfs`/`ramfs`), where bytes written count +/// against memory as well as space. Linux reads `/proc/self/mountinfo`; other systems report +/// `false` (their temporary folders are disk-backed by default). `path` may not exist yet: its +/// nearest existing ancestor decides, as when the fill creates its scratch folder. +pub fn is_memory_backed(path: &std::path::Path) -> bool { + #[cfg(target_os = "linux")] + { + let Some(canonical) = canonical_ancestor(path) else { return false }; + return std::fs::read_to_string("/proc/self/mountinfo").is_ok_and(|mountinfo| mountinfo_is_memory_backed(&mountinfo, &canonical)); + } + #[allow(unreachable_code)] + { + let _ = path; + false + } +} + +/// The canonical path of `path`'s nearest existing ancestor (or itself). Relative paths are +/// resolved against the current directory first, so a new relative output folder still names +/// the volume it will be created on. +#[cfg(target_os = "linux")] +fn canonical_ancestor(path: &std::path::Path) -> Option { + let absolute = if path.is_absolute() { path.to_path_buf() } else { std::env::current_dir().ok()?.join(path) }; + let mut p = absolute.as_path(); + loop { + if let Ok(canonical) = std::fs::canonicalize(p) { + return Some(canonical); + } + p = p.parent()?; + } +} + +/// Whether a `/proc/self/mountinfo` listing mounts `path` on `tmpfs` or `ramfs`; the deepest +/// matching mount point wins, and a later entry at the same mount point is the visible one. +#[cfg(target_os = "linux")] +pub(crate) fn mountinfo_is_memory_backed(mountinfo: &str, path: &std::path::Path) -> bool { + let mut best: Option<(usize, &str)> = None; + for line in mountinfo.lines() { + let Some(mount) = line.split_whitespace().nth(4) else { continue }; + let Some(fstype) = line.split(" - ").nth(1).and_then(|rest| rest.split_whitespace().next()) else { continue }; + let mount = unescape_mount(mount); + if path.starts_with(&mount) && best.is_none_or(|(longest, _)| mount.len() >= longest) { + best = Some((mount.len(), fstype)); + } + } + matches!(best, Some((_, "tmpfs" | "ramfs"))) +} + +/// Undo mountinfo's octal escapes (`\040` space, `\011` tab, `\012` newline, `\134` backslash). +#[cfg(target_os = "linux")] +fn unescape_mount(field: &str) -> String { + let mut out = String::with_capacity(field.len()); + let mut chars = field.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\\' + && chars.clone().take(3).count() == 3 + && let Ok(code) = u8::from_str_radix(&chars.clone().take(3).collect::(), 8) + { + for _ in 0..3 { + chars.next(); + } + out.push(code as char); + } else { + out.push(c); + } + } + out +} + /// CPU model name. pub fn cpu_name() -> Option { #[cfg(target_os = "linux")] @@ -339,4 +408,42 @@ mod tests { assert_eq!(m.available, 4100 * 16384); assert!(m.is_low()); } + + /// The default temp folder may be a `tmpfs`; a nested disk mount under `/tmp` or a nested + /// `tmpfs` must be resolved to the deepest mount point. + #[cfg(target_os = "linux")] + #[test] + fn mountinfo_finds_memory_backed_storage() { + let mountinfo = "\ +25 1 0:22 / / rw,relatime - ext4 /dev/nvme0n1p2 rw +26 25 0:23 / /tmp rw,nosuid - tmpfs tmpfs rw +27 25 0:24 / /tmp/disk rw - ext4 /dev/sda1 rw +28 25 0:25 / /mnt/ram rw - ramfs none rw +29 25 0:26 / /media/My\\040Disk rw - ext4 /dev/sdb1 rw +"; + assert!(mountinfo_is_memory_backed(mountinfo, std::path::Path::new("/tmp/effectcraft-fill"))); + assert!(!mountinfo_is_memory_backed(mountinfo, std::path::Path::new("/tmp/disk/fill"))); + assert!(mountinfo_is_memory_backed(mountinfo, std::path::Path::new("/mnt/ram/a"))); + assert!(!mountinfo_is_memory_backed(mountinfo, std::path::Path::new("/home/fill"))); + assert!(!mountinfo_is_memory_backed(mountinfo, std::path::Path::new("/"))); + // A path that merely shares a prefix is not on that mount. + assert!(!mountinfo_is_memory_backed(mountinfo, std::path::Path::new("/tmpdisk/fill"))); + // Escaped spaces in a mount point decode before matching. + assert!(!mountinfo_is_memory_backed(mountinfo, std::path::Path::new("/media/My Disk/fill"))); + assert_eq!(unescape_mount("/media/My\\040Disk"), "/media/My Disk"); + // The later entry at the same mount point is the visible mount: a disk mount stacked on + // a tmpfs is disk-backed, and the reverse order is memory-backed (#2570 review). + let stacked_disk = "\ +30 25 0:30 / /tmp/stack rw - tmpfs tmpfs rw +31 25 0:31 / /tmp/stack rw - ext4 /dev/sdb2 rw +"; + let stacked_ram = "\ +30 25 0:30 / /tmp/stack rw - ext4 /dev/sdb2 rw +31 25 0:31 / /tmp/stack rw - tmpfs tmpfs rw +"; + assert!(!mountinfo_is_memory_backed(stacked_disk, std::path::Path::new("/tmp/stack/fill"))); + assert!(mountinfo_is_memory_backed(stacked_ram, std::path::Path::new("/tmp/stack/fill"))); + // A new relative folder resolves against the current directory before walking up. + assert!(canonical_ancestor(std::path::Path::new("no-such-relative-folder/fill")).is_some()); + } } diff --git a/crates/engine/src/tests/panels.rs b/crates/engine/src/tests/panels.rs index 622088745..547b7f100 100644 --- a/crates/engine/src/tests/panels.rs +++ b/crates/engine/src/tests/panels.rs @@ -321,6 +321,58 @@ fn content_aware_fill_removes_an_object() { assert!(s2.execute("contentFill.generate", json!({"wait": true, "outputDir": tmp("caf2").to_string_lossy()})).is_err()); } +/// A scratch folder that cannot be created surfaces as an error, not a cancelled fill or a +/// black frame (#2570 review). +#[test] +fn content_aware_fill_reports_a_spill_failure() { + let (mut s, _, _, _) = setup(48, 32, 0.2, |f| { + let mut img = Image::new(48, 32); + for y in 0..32 { + for x in 0..48 { + img.set(x, y, bg(x as f32 + f as f32, y as f32)); + } + } + img + }); + s.execute("layer.addMask", json!({"rect": [20.0, 10.0, 8.0, 8.0], "mode": "subtract"})).unwrap(); + let dir = tmp("cafspill"); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("not-a-folder"); + std::fs::write(&file, b"x").unwrap(); + // The fill wants to create its spill here; the parent is a file. + let out = file.join("fills"); + let e = s.execute("contentFill.generate", json!({"method": "object", "outputDir": out.to_string_lossy(), "wait": true})).unwrap_err(); + // The OS wording differs ("Not a directory" on Unix, "Cannot create a file when that file + // already exists" on Windows), so look for the fill's own marker. + let msg = e.to_string(); + assert!(msg.contains("spill: cannot make the scratch file"), "not a spill error: {msg}"); + std::fs::remove_dir_all(&dir).unwrap(); +} + +/// A fill plan that comes back without files or frames (a job worker that wrote none) is an +/// error, not a crash on the first file (#2570 review). +#[test] +fn content_aware_fill_without_files_is_an_error() { + let (mut s, cid, lid, _) = setup(48, 32, 0.2, |_| Image::filled(48, 32, [0.5, 0.5, 0.5, 1.0])); + let plan = crate::commands::content_fill::FillPlan { + comp: cid, + layer: lid, + name: "Clip".into(), + width: 48, + height: 32, + times: vec![Tick::ZERO], + method: "object".into(), + lighting: 0.0, + expand: 0, + reference: None, + dir: "Clip_Fill_1".into(), + }; + assert!(crate::commands::content_fill::apply_plan(&mut s, &plan, vec![]).is_err()); + let empty = crate::commands::content_fill::FillPlan { times: vec![], ..plan }; + assert!(crate::commands::content_fill::apply_plan(&mut s, &empty, vec!["Clip_Fill_1/fill_00000.png".into()]).is_err()); + assert_eq!(s.project.comp(cid).unwrap().layers.len(), 1, "no fill layer was added"); +} + #[test] fn content_aware_fill_runs_in_the_background() { let (mut s, _, _, _) = setup(48, 32, 0.2, |f| { diff --git a/crates/gpu/src/tests/inpaint.rs b/crates/gpu/src/tests/inpaint.rs index c6901bb45..dacda05e2 100644 --- a/crates/gpu/src/tests/inpaint.rs +++ b/crates/gpu/src/tests/inpaint.rs @@ -147,16 +147,20 @@ fn object_fill_recovers_background_and_is_stable() { holes.push(hole); gts.push(gt); } - let input = FillInput { frames, holes, references: vec![] }; - let out = fill_sequence_with(input.clone(), &FillOpts::default(), g, &mut |_, _| true).expect("fill"); + let input = FillInput { + frames: effectcraft_raster::inpaint::Frames::Memory(frames), + holes: holes.into_iter().map(|h| effectcraft_raster::inpaint::HoleMask::from_vec(&h)).collect(), + references: vec![], + }; + let out = fill_sequence_with(input.clone(), &FillOpts::default(), g, &mut |_, _| true, None).expect("fill"); for t in 0..out.len() { - let db = psnr(&out[t], >s[t], Some(&input.holes[t])); + let db = psnr(&out[t], >s[t], Some(&input.holes[t].to_vec())); assert!(db > 26.0, "frame {t} pan {pan}: {db:.1} dB"); assert_eq!(out[t].data[0][3], 1.0); } if pan == 0.0 { for t in 1..out.len() { - let both: Vec = input.holes[t].iter().zip(&input.holes[t - 1]).map(|(a, b)| *a || *b).collect(); + let both: Vec = input.holes[t].to_vec().iter().zip(input.holes[t - 1].to_vec().iter()).map(|(a, b)| *a || *b).collect(); let db = psnr(&out[t], &out[t - 1], Some(&both)); assert!(db > 30.0, "frames {}–{t} differ: {db:.1} dB", t - 1); } @@ -172,13 +176,17 @@ fn edge_blend_takes_the_cpu_membrane() { let (w, h) = (80u32, 60u32); let gt = background(w, h, 0.0); let hole = rect_hole(w, h, 30, 20, 14); - let input = FillInput { frames: vec![cut(>, &hole); 4], holes: vec![hole.clone(); 4], references: vec![] }; + let input = FillInput { + frames: effectcraft_raster::inpaint::Frames::Memory(vec![cut(>, &hole); 4]), + holes: vec![effectcraft_raster::inpaint::HoleMask::from_vec(&hole); 4], + references: vec![], + }; let opts = FillOpts { method: effectcraft_raster::inpaint::FillMethod::EdgeBlend, ..Default::default() }; - let edge = fill_sequence_with(input.clone(), &opts, g, &mut |_, _| true).expect("fill"); + let edge = fill_sequence_with(input.clone(), &opts, g, &mut |_, _| true, None).expect("fill"); for t in 1..4 { assert!(psnr(&edge[t], &edge[0], Some(&hole)) > 40.0, "frame {t}"); } - assert!(psnr(&edge[0], &membrane_fill(&input.frames[0], &hole), Some(&hole)) > 60.0); + assert!(psnr(&edge[0], &membrane_fill(&input.frames.get(0), &hole), Some(&hole)) > 60.0); } /// The dilation the level plans are built from (kept honest: the flags' target bit is `near`). @@ -299,11 +307,15 @@ fn sequence_fill_with_the_device_flow_matches_the_cpu() { frames.push(cut(>, &hole)); holes.push(hole); } - let input = FillInput { frames, holes, references: vec![] }; - let dev = fill_sequence_with(input.clone(), &FillOpts::default(), g, &mut |_, _| true).expect("gpu fill"); - let ref_cpu = fill_sequence_with(input.clone(), &FillOpts::default(), &cpu, &mut |_, _| true).expect("cpu fill"); + let input = FillInput { + frames: effectcraft_raster::inpaint::Frames::Memory(frames), + holes: holes.into_iter().map(|h| effectcraft_raster::inpaint::HoleMask::from_vec(&h)).collect(), + references: vec![], + }; + let dev = fill_sequence_with(input.clone(), &FillOpts::default(), g, &mut |_, _| true, None).expect("gpu fill"); + let ref_cpu = fill_sequence_with(input.clone(), &FillOpts::default(), &cpu, &mut |_, _| true, None).expect("cpu fill"); for t in 0..dev.len() { - let db = psnr(&dev[t], &ref_cpu[t], Some(&input.holes[t])); + let db = psnr(&dev[t], &ref_cpu[t], Some(&input.holes[t].to_vec())); assert!(db > 40.0, "frame {t} pan {pan}: device vs CPU {db:.1} dB"); } } diff --git a/crates/raster/Cargo.toml b/crates/raster/Cargo.toml index e27d55317..7cd4fbe5a 100644 --- a/crates/raster/Cargo.toml +++ b/crates/raster/Cargo.toml @@ -13,3 +13,4 @@ workspace = true effectcraft-geom = { workspace = true } effectcraft-color = { workspace = true } rayon = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/raster/src/inpaint.rs b/crates/raster/src/inpaint.rs index 5625b242c..8b7127b4d 100644 --- a/crates/raster/src/inpaint.rs +++ b/crates/raster/src/inpaint.rs @@ -34,6 +34,7 @@ use rayon::prelude::*; use crate::Image; +use crate::Px; use crate::flow::{Flow, block_flow}; /// Fill method. @@ -605,13 +606,469 @@ pub fn complete_flow(mut f: Flow, hole_a: &[bool], hole_b: &[bool], w: usize, h: f } +/// A hole mask packed into 64-bit words (1 bit per pixel) to keep the per-frame cost +/// down across a whole range (#2570). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HoleMask { + pub words: Vec, + pub len: usize, +} + +impl HoleMask { + /// Pack one `bool` a pixel into 64-bit words. + pub fn from_vec(hole: &[bool]) -> Self { + let mut words = vec![0u64; hole.len().div_ceil(64)]; + for (i, &h) in hole.iter().enumerate() { + if h { + words[i / 64] |= 1u64 << (i % 64); + } + } + HoleMask { words, len: hole.len() } + } + /// An all-known mask of `n` pixels. + pub fn zeros(n: usize) -> Self { + HoleMask { words: vec![0u64; n.div_ceil(64)], len: n } + } + /// Whether pixel `i` is in the hole. `words` and `len` are public, so a mask whose words are + /// shorter than `len` reads the missing pixels as known instead of panicking. + pub fn get(&self, i: usize) -> bool { + i < self.len && self.words.get(i / 64).is_some_and(|w| (w >> (i % 64)) & 1 == 1) + } + /// Mark pixel `i` as missing (`true`) or known (`false`). A pixel outside the mask is ignored. + pub fn set(&mut self, i: usize, v: bool) { + if i >= self.len { + return; + } + let Some(w) = self.words.get_mut(i / 64) else { + return; + }; + if v { + *w |= 1u64 << (i % 64); + } else { + *w &= !(1u64 << (i % 64)); + } + } + /// Whether any pixel is in the hole. + pub fn any(&self) -> bool { + self.words.iter().any(|&w| w != 0) + } + /// Unpack to one `bool` a pixel. + pub fn to_vec(&self) -> Vec { + (0..self.len).map(|i| self.get(i)).collect() + } +} + +/// Where a fill's input frames live. `Memory` keeps them in RAM (tests, small ranges); +/// `Spilled` stores them on disk and reads them on demand, so a whole range costs ~0 RAM. +#[derive(Clone)] +pub enum Frames { + Memory(Vec), + Spilled(SpillHandle), +} + +impl Frames { + /// The number of frames in the range. + pub fn len(&self) -> usize { + match self { + Frames::Memory(v) => v.len(), + Frames::Spilled(h) => h.n, + } + } + /// Whether the range has no frames. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Frame `k`: a clone from RAM, or a decode from the spill. + pub fn get(&self, k: usize) -> Image { + match self { + Frames::Memory(v) => v[k].clone(), + Frames::Spilled(h) => h.get(k), + } + } + /// Frame size without decoding a whole image. + pub fn size(&self) -> Option<(usize, usize)> { + match self { + Frames::Memory(v) => v.first().map(|i| (i.width as usize, i.height as usize)), + Frames::Spilled(h) => Some((h.w, h.h)), + } + } + /// Bilinear sample of one pixel. A spilled frame reads only the 2x2 support from disk, so + /// the hot trace path never decodes or clones a whole frame (#2570). + pub fn sample(&self, k: usize, x: f64, y: f64) -> Px { + match self { + Frames::Memory(v) => v[k].sample_bilinear_clamped(x, y), + Frames::Spilled(h) => h.sample(k, x, y), + } + } + /// Directory of the spill file, so a secondary spill can live beside it. + pub fn scratch_dir(&self) -> Option { + match self { + Frames::Memory(_) => None, + Frames::Spilled(h) => h.dir().map(std::path::Path::to_path_buf), + } + } + /// Whether a read or write of a spilled frame failed. + pub fn has_error(&self) -> bool { + match self { + Frames::Memory(_) => false, + Frames::Spilled(h) => h.has_error(), + } + } + /// Take the first spill failure, clearing it. + pub fn take_error(&self) -> Option { + match self { + Frames::Memory(_) => None, + Frames::Spilled(h) => h.take_error(), + } + } + /// The error slot shared by every spill in this fill, so a caller holding a clone of the + /// input handle also sees failures in the propagated-frame spill. + pub fn error_cell(&self) -> Option>>> { + match self { + Frames::Memory(_) => None, + Frames::Spilled(h) => Some(h.error_cell()), + } + } +} + +/// A spill file holding `n` frames of `w x h` pixels at `bpc` bits per channel. +/// Frames are read back on demand. The file is shared (reference-counted), so it lives until the +/// last clone is dropped (#2570 review). +#[derive(Clone)] +pub struct SpillHandle { + inner: std::sync::Arc, + pub w: usize, + pub h: usize, + pub n: usize, + pub bpc: u8, +} + +struct SpillInner { + /// The folder the spill was created in (`None`: the system temporary folder), so the fill's + /// other spills go beside it. + dir: Option, + /// An anonymous temporary file: it has no name in `dir`, and the system reclaims it when the + /// last handle closes, even when the app is killed or runs out of memory partway through a + /// fill, so a multi-gigabyte scratch file is never left in the user's folder (#2570 review). + /// Every read and write is positioned on this one handle: several trace threads read the same + /// file at once, and a shared file offset would need a lock around every read (#2570). + file: std::fs::File, + /// Shared with any secondary spill so one probe reports every scratch failure. + error: std::sync::Arc>>, +} + +impl SpillHandle { + /// The folder the spill lives in (`None`: the system temporary folder). + pub fn dir(&self) -> Option<&std::path::Path> { + self.inner.dir.as_deref() + } + + /// Bytes a pixel in the spill: RGBA `u8`, or four `f32` above 8 bits a channel. + fn bytes_per_px(&self) -> usize { + if self.bpc <= 8 { 4 } else { 16 } + } + + /// Record a scratch failure in the shared slot, keeping the first. + pub fn record_error(&self, message: String) { + if let Ok(mut e) = self.inner.error.lock() + && e.is_none() + { + *e = Some(message); + } + } + + /// The error slot shared with any secondary spill. + pub fn error_cell(&self) -> std::sync::Arc>> { + std::sync::Arc::clone(&self.inner.error) + } + + /// Whether a read or write failed (`true` when the slot is poisoned). + pub fn has_error(&self) -> bool { + self.inner.error.lock().map(|e| e.is_some()).unwrap_or(true) + } + + /// Take the first failure, clearing it. + pub fn take_error(&self) -> Option { + self.inner.error.lock().ok().and_then(|mut e| e.take()) + } + + /// Read one whole frame from the spill, reporting a short or failed read. + fn load_frame(&self, k: usize) -> Image { + let mut img = Image::new(self.w as u32, self.h as u32); + let Some(px) = self.w.checked_mul(self.h) else { + self.record_error("spill: pixel count overflow".into()); + return img; + }; + let bytes_per_px = self.bytes_per_px(); + let Some(off) = k.checked_mul(px).and_then(|v| v.checked_mul(bytes_per_px)) else { + self.record_error(format!("spill: offset overflow for frame {k}")); + return img; + }; + let Some(len) = px.checked_mul(bytes_per_px) else { + self.record_error("spill: frame size overflow".into()); + return img; + }; + let mut buf = vec![0u8; len]; + if let Err(e) = self.read_at(off as u64, &mut buf) { + self.record_error(format!("spill: {e}")); + return img; + } + if self.bpc <= 8 { + for (i, p) in img.data.iter_mut().enumerate() { + let j = i * 4; + p[0] = buf[j] as f32 / 255.0; + p[1] = buf[j + 1] as f32 / 255.0; + p[2] = buf[j + 2] as f32 / 255.0; + p[3] = buf[j + 3] as f32 / 255.0; + } + } else { + for (i, p) in img.data.iter_mut().enumerate() { + let base = i * 16; + for c in 0..4 { + let mut b = [0u8; 4]; + b.copy_from_slice(&buf[base + c * 4..base + c * 4 + 4]); + p[c] = f32::from_bits(u32::from_le_bytes(b)); + } + } + } + img + } + + /// Read a whole frame, decoding it. + pub fn get(&self, k: usize) -> Image { + self.load_frame(k) + } + + /// A positioned read: `pread`/`seek_read` use the shared handle without touching a shared + /// file offset, so tracing threads do not serialise on the file. + fn read_at(&self, off: u64, buf: &mut [u8]) -> std::io::Result<()> { + let file = &self.inner.file; + #[cfg(unix)] + { + use std::os::unix::fs::FileExt; + file.read_exact_at(buf, off) + } + #[cfg(windows)] + { + use std::os::windows::fs::FileExt; + let mut done = 0usize; + while done < buf.len() { + match file.seek_read(&mut buf[done..], off.saturating_add(done as u64)) { + Ok(0) => return Err(std::io::ErrorKind::UnexpectedEof.into()), + Ok(n) => done += n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) + } + // Elsewhere, seek the shared handle. Spills are only made natively: wasm keeps the + // propagated frames in RAM and has no file system to spill the input to. + #[cfg(not(any(unix, windows)))] + { + use std::io::{Read, Seek, SeekFrom}; + let mut f = file; + f.seek(SeekFrom::Start(off))?; + f.read_exact(buf) + } + } + + /// A positioned write on the same handle as `read_at`, so the propagation workers can store + /// their frames at once and the file never needs a name to be reopened by. + fn write_at(&self, off: u64, buf: &[u8]) -> std::io::Result<()> { + let file = &self.inner.file; + #[cfg(unix)] + { + use std::os::unix::fs::FileExt; + file.write_all_at(buf, off) + } + #[cfg(windows)] + { + use std::os::windows::fs::FileExt; + let mut done = 0usize; + while done < buf.len() { + match file.seek_write(&buf[done..], off.saturating_add(done as u64)) { + Ok(0) => return Err(std::io::ErrorKind::WriteZero.into()), + Ok(n) => done += n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) + } + #[cfg(not(any(unix, windows)))] + { + use std::io::{Seek, SeekFrom, Write}; + let mut f = file; + f.seek(SeekFrom::Start(off))?; + f.write_all(buf) + } + } + + /// One pixel decoded from `buf` at index `i` (transparent if `buf` is short). + fn px_at(&self, buf: &[u8], i: usize) -> Px { + if self.bpc <= 8 { + let j = i * 4; + [ + buf.get(j).copied().unwrap_or(0) as f32 / 255.0, + buf.get(j + 1).copied().unwrap_or(0) as f32 / 255.0, + buf.get(j + 2).copied().unwrap_or(0) as f32 / 255.0, + buf.get(j + 3).copied().unwrap_or(0) as f32 / 255.0, + ] + } else { + let base = i * 16; + let mut p = [0.0f32; 4]; + for (c, v) in p.iter_mut().enumerate() { + let at = base + c * 4; + let Some(b) = buf.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()) else { + return [0.0; 4]; + }; + *v = f32::from_bits(u32::from_le_bytes(b)); + } + p + } + } + + /// The two horizontally adjacent pixels (`x`, `x + 1`, both clamped) of row `y` (clamped), + /// read straight from the spill file. + fn edge(&self, k: usize, y: i64, x: i64) -> Option<(Px, Px)> { + let (w, h) = (self.w, self.h); + if w == 0 || h == 0 || k >= self.n { + return None; + } + let (iw, ih) = (i64::try_from(w).ok()?, i64::try_from(h).ok()?); + let y = y.clamp(0, ih - 1) as usize; + let xa = x.clamp(0, iw - 1) as usize; + let xb = x.saturating_add(1).clamp(0, iw - 1) as usize; + let bpp = self.bytes_per_px(); + let off = k + .checked_mul(w) + .and_then(|v| v.checked_mul(h)) + .and_then(|v| v.checked_add(y.checked_mul(w)?)) + .and_then(|v| v.checked_add(xa)) + .and_then(|v| v.checked_mul(bpp)); + let Some(off) = off.and_then(|v| u64::try_from(v).ok()) else { + self.record_error("spill: sample offset overflow".into()); + return None; + }; + // One read covers both pixels unless the right edge clamps them together. A stack buffer + // keeps the per-sample allocation out of the trace loop. + let len = (xb - xa + 1) * bpp; + let mut held = [0u8; 32]; + let buf = held.get_mut(..len)?; + if let Err(e) = self.read_at(off, buf) { + self.record_error(format!("spill: {e}")); + return None; + } + let a = self.px_at(buf, 0); + let b = if xb > xa { self.px_at(buf, 1) } else { a }; + Some((a, b)) + } + + /// Bilinear sample of one pixel, reading only its 2x2 support. Matches + /// `Image::sample_bilinear_clamped` exactly, with the same edge clamping. + pub fn sample(&self, k: usize, x: f64, y: f64) -> Px { + let fx = x - 0.5; + let fy = y - 0.5; + let (x0, y0) = (fx.floor(), fy.floor()); + let (tx, ty) = ((fx - x0) as f32, (fy - y0) as f32); + // `as i64` saturates NaN and out-of-range values, as in `sample_bilinear_clamped`. + let (x0, y0) = (x0 as i64, y0 as i64); + let (Some((a, b)), Some((c, d))) = (self.edge(k, y0, x0), self.edge(k, y0.saturating_add(1), x0)) else { + return [0.0; 4]; + }; + let mut o = [0.0; 4]; + for i in 0..4 { + let top = a[i] + (b[i] - a[i]) * tx; + let bot = c[i] + (d[i] - c[i]) * tx; + o[i] = top + (bot - top) * ty; + } + o + } + + /// Write frame `k` into the spill, reporting a disk error. + pub fn write_frame(&self, k: usize, img: &Image) -> std::result::Result<(), String> { + if k >= self.n { + return Err(format!("spill: frame {k} out of range (n = {})", self.n)); + } + if img.width as usize != self.w || img.height as usize != self.h { + return Err(format!("spill: frame {k} is {}x{}, expected {}x{}", img.width, img.height, self.w, self.h)); + } + let bytes_per_px = self.bytes_per_px(); + let px = self.w.checked_mul(self.h).ok_or("overflow")?; + let off = k.checked_mul(px).and_then(|v| v.checked_mul(bytes_per_px)).ok_or("overflow")?; + let buf = if self.bpc <= 8 { + let mut buf = vec![0u8; px * 4]; + for (i, p) in img.data.iter().enumerate() { + buf[i * 4] = (p[0] * 255.0).round().clamp(0.0, 255.0) as u8; + buf[i * 4 + 1] = (p[1] * 255.0).round().clamp(0.0, 255.0) as u8; + buf[i * 4 + 2] = (p[2] * 255.0).round().clamp(0.0, 255.0) as u8; + buf[i * 4 + 3] = (p[3] * 255.0).round().clamp(0.0, 255.0) as u8; + } + buf + } else { + let mut buf = vec![0u8; px * 16]; + for (i, p) in img.data.iter().enumerate() { + let base = i * 16; + for c in 0..4 { + buf[base + c * 4..base + c * 4 + 4].copy_from_slice(&p[c].to_le_bytes()); + } + } + buf + }; + self.write_at(off as u64, &buf).map_err(|e| format!("spill: {e}")) + } + + /// Create a spill file. `dir` lets the caller keep scratch next to the output (so a + /// RAM-backed `/tmp` does not defeat the point of spilling); `None` uses the temp dir. + pub fn create_in(dir: Option<&std::path::Path>, w: usize, h: usize, n: usize, bpc: u8) -> std::io::Result { + Self::create_in_with_error(dir, w, h, n, bpc, None) + } + + /// `error`: share an existing error slot, so a secondary spill reports through the same + /// handle the caller already watches (#2570 review). + pub fn create_in_with_error( + dir: Option<&std::path::Path>, + w: usize, + h: usize, + n: usize, + bpc: u8, + error: Option>>>, + ) -> std::io::Result { + let bytes_per_px: usize = if bpc <= 8 { 4 } else { 16 }; + let total = n + .checked_mul(w) + .and_then(|v| v.checked_mul(h)) + .and_then(|v| v.checked_mul(bytes_per_px)) + .ok_or(std::io::Error::new(std::io::ErrorKind::InvalidInput, "overflow"))?; + // Anonymous, so nothing is left behind if `set_len` fails below or the app never gets + // to clean up. + let file = match dir { + Some(d) => { + std::fs::create_dir_all(d)?; + tempfile::tempfile_in(d)? + } + None => tempfile::tempfile()?, + }; + file.set_len(total as u64)?; + let error = error.unwrap_or_else(|| std::sync::Arc::new(std::sync::Mutex::new(None))); + let inner = std::sync::Arc::new(SpillInner { dir: dir.map(std::path::Path::to_path_buf), file, error }); + Ok(SpillHandle { inner, w, h, n, bpc }) + } + + /// Create a spill in the system temporary directory. + pub fn create(w: usize, h: usize, n: usize, bpc: u8) -> std::io::Result { + Self::create_in(None, w, h, n, bpc) + } +} + /// The input of a sequence fill. #[derive(Clone)] pub struct FillInput { - /// Frames (premultiplied RGBA, all the same size). - pub frames: Vec, - /// Per frame, `true` where the pixel is missing. - pub holes: Vec>, + /// Frames (premultiplied RGBA, all the same size), in RAM or spilled to disk. + pub frames: Frames, + /// Per frame, packed hole mask (`true` where the pixel is missing). + pub holes: Vec, /// Complete images for some frames: (frame index, image). pub references: Vec<(usize, Image)>, } @@ -621,7 +1078,10 @@ struct TraceContext<'a> { fwd: &'a [Flow], bwd: &'a [Flow], known: &'a (dyn Fn(usize, i64, i64) -> bool + Sync), - frames: &'a [&'a Image], + frames: &'a Frames, + ref_images: &'a [Option], + w: usize, + h: usize, max: usize, } @@ -636,7 +1096,7 @@ struct TraceStart { /// Trace a point to the nearest visible frame: (sampled colour, frames travelled). fn trace(context: &TraceContext<'_>, start: TraceStart) -> Option<([f32; 4], usize)> { let n = context.frames.len(); - let (w, h) = (context.frames[0].width as f64, context.frames[0].height as f64); + let (w, h) = (context.w as f64, context.h as f64); let (mut px, mut py) = (start.x, start.y); let mut k = start.frame as i64; for d in 1..=context.max { @@ -663,7 +1123,13 @@ fn trace(context: &TraceContext<'_>, start: TraceStart) -> Option<([f32; 4], usi .iter() .all(|&(xx, yy)| (context.known)(k as usize, xx.clamp(0, w as i64 - 1), yy.clamp(0, h as i64 - 1))); if ok { - return Some((context.frames[k as usize].sample_bilinear_clamped(px, py), d)); + let img = if let Some(r) = &context.ref_images[k as usize] { + r.sample_bilinear_clamped(px, py) + } else { + // Samples a pixel; never clones a whole spilled frame (the trace hot path). + context.frames.sample(k as usize, px, py) + }; + return Some((img, d)); } } None @@ -709,144 +1175,365 @@ pub struct CpuFill; impl FillBackend for CpuFill {} +/// Holds the propagated frames between the parallel propagation pass and the sequential +/// synthesis pass. Native builds spill them beside the input, so a whole range is never +/// resident at once (#2570); wasm keeps them in memory (it has no shared file system). +enum PropStore { + #[cfg(not(target_arch = "wasm32"))] + Spilled(SpillHandle), + #[cfg(target_arch = "wasm32")] + Memory(std::sync::Mutex>>), +} + +impl PropStore { + /// Open the propagated-frame store: spilled natively, in RAM on wasm. + fn new( + dir: Option<&std::path::Path>, + w: usize, + h: usize, + n: usize, + error: Option>>>, + ) -> std::io::Result { + #[cfg(not(target_arch = "wasm32"))] + { + // 32-bit floats: propagating quantized input would lose quality (#2570). + Ok(PropStore::Spilled(SpillHandle::create_in_with_error(dir, w, h, n, 32, error)?)) + } + #[cfg(target_arch = "wasm32")] + { + let _ = (dir, w, h, error); + Ok(PropStore::Memory(std::sync::Mutex::new((0..n).map(|_| None).collect()))) + } + } + + /// Store frame `k`, reporting a spill write failure. + fn put(&self, k: usize, img: Image) -> std::result::Result<(), String> { + match self { + #[cfg(not(target_arch = "wasm32"))] + PropStore::Spilled(h) => h.write_frame(k, &img), + #[cfg(target_arch = "wasm32")] + PropStore::Memory(v) => { + let mut v = v.lock().map_err(|_| "propagation store poisoned".to_string())?; + if let Some(slot) = v.get_mut(k) { + *slot = Some(img); + } + Ok(()) + } + } + } + + /// Read a stored frame without consuming it (a frame can be shared by two flows). + fn get(&self, k: usize) -> Image { + match self { + #[cfg(not(target_arch = "wasm32"))] + PropStore::Spilled(h) => h.get(k), + #[cfg(target_arch = "wasm32")] + PropStore::Memory(v) => v.lock().ok().and_then(|v| v.get(k).cloned()).flatten().unwrap_or_default(), + } + } + + /// Read a propagated frame once, freeing it where the backing store allows it. + fn take(&self, k: usize) -> Image { + match self { + #[cfg(not(target_arch = "wasm32"))] + PropStore::Spilled(h) => h.get(k), + #[cfg(target_arch = "wasm32")] + PropStore::Memory(v) => v.lock().ok().and_then(|mut v| v.get_mut(k).and_then(Option::take)).unwrap_or_default(), + } + } + + /// Record a failure in the spill, so the caller reports it instead of a bare cancellation. + fn record_error(&self, message: String) { + match self { + #[cfg(not(target_arch = "wasm32"))] + PropStore::Spilled(h) => h.record_error(message), + #[cfg(target_arch = "wasm32")] + PropStore::Memory(_) => { + let _ = message; + } + } + } + + /// Whether the spill failed (always `false` on wasm, which keeps frames in RAM). + fn has_error(&self) -> bool { + match self { + #[cfg(not(target_arch = "wasm32"))] + PropStore::Spilled(h) => h.has_error(), + #[cfg(target_arch = "wasm32")] + PropStore::Memory(_) => false, + } + } +} + +/// Publish a scratch failure through the input spill's shared error slot, so the caller reports +/// the disk error instead of a bare cancellation. +fn record_spill_error(cell: &Option>>>, message: String) { + if let Some(cell) = cell + && let Ok(mut e) = cell.lock() + && e.is_none() + { + *e = Some(message); + } +} + +/// Membrane-fill every frame and return the forward/backward flow for each neighbouring pair. +/// The filled frames are written to a scratch spill, so the flow pass never holds the range in +/// RAM (#2570). +#[allow(clippy::too_many_arguments)] +fn flow_pairs( + n: usize, + w: usize, + h: usize, + holes: &[HoleMask], + is_ref: &[bool], + ref_images: &[Option], + frames: &Frames, + prop_dir: Option<&std::path::Path>, + error_cell: Option>>>, + backend: &dyn FillBackend, +) -> Option> { + let store = match PropStore::new(prop_dir, w, h, n, error_cell.clone()) { + Ok(s) => s, + Err(e) => { + record_spill_error(&error_cell, format!("spill: {e}")); + return None; + } + }; + let pre = (0..n).into_par_iter().try_for_each(|t| { + let base = if is_ref[t] { ref_images[t].clone().unwrap_or_default() } else { frames.get(t) }; + let img = backend.membrane(&base, &holes[t].to_vec()); + store.put(t, img) + }); + if let Err(e) = pre { + record_spill_error(&error_cell, e); + return None; + } + if frames.has_error() { + return None; + } + let pairs: Vec<(Flow, Flow)> = (0..n.saturating_sub(1)) + .into_par_iter() + .map(|i| { + let a = store.get(i); + let b = store.get(i + 1); + let ha = holes[i].to_vec(); + let hb = holes[i + 1].to_vec(); + (backend.flow(&a, &b, &ha, &hb), backend.flow(&b, &a, &hb, &ha)) + }) + .collect(); + let store_failed = store.has_error(); + drop(store); + if store_failed || frames.has_error() { None } else { Some(pairs) } +} + /// Fill a range of frames on the CPU. `progress(done, total)` returns `false` to cancel /// (→ `None`). pub fn fill_sequence(input: &FillInput, o: &FillOpts, progress: &mut dyn FnMut(usize, usize) -> bool) -> Option> { - fill_sequence_with(input.clone(), o, &CpuFill, progress) + fill_sequence_with(input.clone(), o, &CpuFill, progress, None) } /// [`fill_sequence`] with a hardware backend for the stages it can take (see [`FillBackend`]). -pub fn fill_sequence_with(mut input: FillInput, o: &FillOpts, backend: &dyn FillBackend, progress: &mut dyn FnMut(usize, usize) -> bool) -> Option> { +pub fn fill_sequence_with( + mut input: FillInput, + o: &FillOpts, + backend: &dyn FillBackend, + progress: &mut dyn FnMut(usize, usize) -> bool, + mut sink: Option<&mut dyn FnMut(usize, &Image) -> Result<(), String>>, +) -> Option> { let n = input.frames.len(); if n == 0 { return Some(vec![]); } - let (w, h) = (input.frames[0].width as usize, input.frames[0].height as usize); + let (w, h) = input.frames.size().unwrap_or((0, 0)); + if w == 0 || h == 0 { + return Some(vec![]); + } // Steps: flows (1) + propagation (1) + per-frame synthesis (n). let total = n + 2; + let streaming = sink.is_some(); if o.method == FillMethod::EdgeBlend { let mut out = Vec::with_capacity(n); - for (i, (f, hl)) in input.frames.iter().zip(&input.holes).enumerate() { - out.push(opaque_fill(backend.membrane(f, hl), hl)); + for i in 0..n { + let f = input.frames.get(i); + // A failed read returns a black frame, which must never reach the sink. + if input.frames.has_error() { + return None; + } + let hl = input.holes[i].to_vec(); + let img = opaque_fill(backend.membrane(&f, &hl), &hl); + if let Some(cb) = sink.as_mut() + && cb(i, &img).is_err() + { + return None; + } + if !streaming { + out.push(img); + } if !progress(i + 1, n) { return None; } } + if input.frames.has_error() { + return None; + } + if streaming { + return Some(Vec::new()); + } return Some(out); } // Reference frames make their frame fully known. - let mut frames: Vec = std::mem::take(&mut input.frames); - let mut holes: Vec> = std::mem::take(&mut input.holes); + let mut holes: Vec = std::mem::take(&mut input.holes); let references = std::mem::take(&mut input.references); let mut is_ref = vec![false; n]; - for (k, img) in &references { - if *k < n && img.width as usize == w && img.height as usize == h { - frames[*k] = img.clone(); - holes[*k] = vec![false; w * h]; - is_ref[*k] = true; + let mut ref_images: Vec> = vec![None; n]; + for (k, img) in references { + if k < n && img.width as usize == w && img.height as usize == h { + ref_images[k] = Some(img); + holes[k] = HoleMask::zeros(w * h); + is_ref[k] = true; } } - // 1. Flows between membrane-filled neighbours, completed inside the holes. - let pre: Vec = frames.par_iter().zip(&holes).map(|(f, hl)| backend.membrane(f, hl)).collect(); - let pairs: Vec<(Flow, Flow)> = (0..n.saturating_sub(1)) - .into_par_iter() - .map(|i| (backend.flow(&pre[i], &pre[i + 1], &holes[i], &holes[i + 1]), backend.flow(&pre[i + 1], &pre[i], &holes[i + 1], &holes[i]))) - .collect(); + let prop_dir = input.frames.scratch_dir(); + let error_cell = input.frames.error_cell(); + // 1. Flows between membrane-filled neighbours, completed inside the holes. The + // membrane-filled frames are spilled too, so the flow pass does not hold the range. + let pairs = flow_pairs(n, w, h, &holes, &is_ref, &ref_images, &input.frames, prop_dir.as_deref(), error_cell.clone(), backend)?; let (fwd, bwd): (Vec, Vec) = pairs.into_iter().unzip(); - drop(pre); if !progress(1, total) { return None; } - // 2. Propagation (and the lighting-correction border ring). - let refs: Vec<&Image> = frames.iter().collect(); - let known = |k: usize, x: i64, y: i64| !holes[k][y as usize * w + x as usize]; - let trace_context = TraceContext { fwd: &fwd, bwd: &bwd, known: &known, frames: &refs, max: n }; + // 2. Propagation (and the lighting-correction border ring). The filled frames are + // spilled as they are produced, so only the leftover masks stay in RAM. + let store = match PropStore::new(prop_dir.as_deref(), w, h, n, error_cell.clone()) { + Ok(s) => s, + Err(e) => { + record_spill_error(&error_cell, format!("spill: {e}")); + return None; + } + }; + let known = |k: usize, x: i64, y: i64| !holes[k].get(y as usize * w + x as usize); + let trace_context = TraceContext { fwd: &fwd, bwd: &bwd, known: &known, frames: &input.frames, ref_images: &ref_images, w, h, max: n }; let ring_r = 2usize; - let mut propagated: Vec<(Image, Vec)> = (0..n) + let propagated_left: Vec = (0..n) .into_par_iter() .map(|t| { - let mut img = frames[t].clone(); - let mut left = holes[t].clone(); - if is_ref[t] || !holes[t].iter().any(|b| *b) { - return (img, left); + // Once a frame failed to spill or read the fill is lost: stop tracing the rest (the + // masks are dropped below). + if store.has_error() || input.frames.has_error() { + return HoleMask::default(); } - let sample = |i: usize| -> Option<[f32; 4]> { - let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); - let f = trace(&trace_context, TraceStart { frame: t, x, y, step: 1 }); - let b = trace(&trace_context, TraceStart { frame: t, x, y, step: -1 }); - match (f, b) { - (Some((a, da)), Some((c, dc))) => { - let (wa, wc) = (1.0 / da as f32, 1.0 / dc as f32); - Some(std::array::from_fn(|k| (a[k] * wa + c[k] * wc) / (wa + wc))) + let mut img = if is_ref[t] { ref_images[t].clone().unwrap_or_default() } else { input.frames.get(t) }; + let mut left = holes[t].to_vec(); + if !is_ref[t] && holes[t].any() { + let sample = |i: usize| -> Option<[f32; 4]> { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let f = trace(&trace_context, TraceStart { frame: t, x, y, step: 1 }); + let b = trace(&trace_context, TraceStart { frame: t, x, y, step: -1 }); + match (f, b) { + (Some((a, da)), Some((c, dc))) => { + let (wa, wc) = (1.0 / da as f32, 1.0 / dc as f32); + Some(std::array::from_fn(|k| (a[k] * wa + c[k] * wc) / (wa + wc))) + } + (Some((a, _)), None) | (None, Some((a, _))) => Some(a), + _ => None, } - (Some((a, _)), None) | (None, Some((a, _))) => Some(a), - _ => None, + }; + let filled: Vec<(usize, [f32; 4])> = (0..w * h).filter(|&i| holes[t].get(i)).filter_map(|i| sample(i).map(|v| (i, v))).collect(); + for (i, v) in &filled { + img.data[*i] = *v; + left[*i] = false; } - }; - let filled: Vec<(usize, [f32; 4])> = (0..w * h).filter(|&i| holes[t][i]).filter_map(|i| sample(i).map(|v| (i, v))).collect(); - for (i, v) in &filled { - img.data[*i] = *v; - left[*i] = false; - } - // Lighting correction: membrane of the border mismatch, added to propagated pixels. - if o.lighting > 0.0 && !filled.is_empty() { - let grown = dilate(&holes[t], w, h, ring_r); - let mut diff = vec![[0.0f32; 4]; w * h]; - let mut dk = vec![true; w * h]; - for i in 0..w * h { - if holes[t][i] { - dk[i] = false; - } else if grown[i] - && let Some(v) = sample(i) - { - let a = frames[t].data[i]; - diff[i] = std::array::from_fn(|c| if c < 3 { a[c] - v[c] } else { 0.0 }); + // Lighting correction: membrane of the border mismatch, added to propagated pixels. + // Non-hole pixels are untouched above, so `img` is still the original there. + if o.lighting > 0.0 && !filled.is_empty() { + let grown = dilate(&holes[t].to_vec(), w, h, ring_r); + let mut diff = vec![[0.0f32; 4]; w * h]; + let mut dk = vec![true; w * h]; + for i in 0..w * h { + if holes[t].get(i) { + dk[i] = false; + } else if grown[i] + && let Some(v) = sample(i) + { + let a = img.data[i]; + diff[i] = std::array::from_fn(|c| if c < 3 { a[c] - v[c] } else { 0.0 }); + } } - } - backend.membrane_known(&mut diff, &dk, w, h); - for (i, _) in &filled { - for c in 0..3 { - img.data[*i][c] += o.lighting * diff[*i][c]; + backend.membrane_known(&mut diff, &dk, w, h); + for (i, _) in &filled { + for c in 0..3 { + img.data[*i][c] += o.lighting * diff[*i][c]; + } } } } - (img, left) + if let Err(e) = store.put(t, img) { + store.record_error(e); + } + HoleMask::from_vec(&left) }) .collect(); - drop(refs); - drop(frames); + if input.frames.has_error() || store.has_error() { + return None; + } + drop(input); if !progress(2, total) { return None; } // 3. Synthesis of what no frame shows. let mut out: Vec> = vec![None; n]; - let first = (0..n).find(|&t| propagated[t].1.iter().any(|b| *b)); + let first = (0..n).find(|&t| propagated_left[t].any()); let Some(t0) = first else { - return Some(propagated.into_iter().zip(&holes).map(|((img, _), hl)| opaque_fill(img, hl)).collect()); + if streaming { + for t in 0..n { + let img = opaque_fill(take_checked(&store, t)?, &holes[t].to_vec()); + if !stream_frame(&mut sink, t, Some(&img), &holes) { + return None; + } + } + return Some(Vec::new()); + } + let out: Vec = (0..n).map(|t| opaque_fill(store.take(t), &holes[t].to_vec())).collect(); + if store.has_error() { + return None; + } + return Some(out); }; // Frames before t0 are complete already. - for (t, slot) in out.iter_mut().enumerate().take(t0) { - *slot = Some(std::mem::take(&mut propagated[t].0)); + for t in 0..t0 { + let img = take_checked(&store, t)?; + if streaming { + if !stream_frame(&mut sink, t, Some(&img), &holes) { + return None; + } + } else { + out[t] = Some(img); + } } let mut done = 2; - let (img0, left0) = std::mem::take(&mut propagated[t0]); - out[t0] = Some(backend.patch(&img0, &left0, None, o.patch, o.seed)); - drop((img0, left0)); + let left0 = propagated_left[t0].to_vec(); + out[t0] = Some(backend.patch(&take_checked(&store, t0)?, &left0, None, o.patch, o.seed)); + if !stream_frame(&mut sink, t0, out[t0].as_ref(), &holes) { + return None; + } done += 1; if !progress(done, total) { return None; } for t in t0 + 1..n { - // Taken, not cloned: each propagated frame is freed once its output exists. - let (img, left) = std::mem::take(&mut propagated[t]); - let (img, left) = (&img, &left); - if !left.iter().any(|b| *b) { - out[t] = Some(img.clone()); + let complete = !propagated_left[t].any(); + if complete { + out[t] = Some(take_checked(&store, t)?); } else { + let img = take_checked(&store, t)?; + let left = propagated_left[t].to_vec(); // Warp the previous result here along the backward flow (t → t − 1). // Frames after `t0` are filled in order, so the previous one is set. let Some(prev) = out[t - 1].as_ref() else { - out[t] = Some(backend.patch(img, left, None, o.patch, o.seed)); + out[t] = Some(backend.patch(&img, &left, None, o.patch, o.seed)); + if !stream_frame(&mut sink, t, out[t].as_ref(), &holes) { + return None; + } continue; }; let mut warped = img.clone(); @@ -859,15 +1546,47 @@ pub fn fill_sequence_with(mut input: FillInput, o: &FillOpts, backend: &dyn Fill } out[t] = Some(match o.method { FillMethod::Surface => warped, - _ => backend.patch(img, left, Some(&warped), o.patch, o.seed), + _ => backend.patch(&img, &left, Some(&warped), o.patch, o.seed), }); } + // Only the streaming sink has consumed the frame; a non-streaming result returns it. + if streaming && t >= 2 { + out[t - 2] = None; + } + if !stream_frame(&mut sink, t, out[t].as_ref(), &holes) { + return None; + } done += 1; if !progress(done, total) { return None; } } - Some(out.into_iter().zip(&holes).map(|(img, hl)| opaque_fill(img.unwrap_or_default(), hl)).collect()) + if store.has_error() { + return None; + } + if streaming { + for slot in out.iter_mut() { + *slot = None; + } + return Some(Vec::new()); + } + Some(out.into_iter().enumerate().map(|(i, img)| opaque_fill(img.unwrap_or_default(), &holes[i].to_vec())).collect()) +} + +/// Take frame `k` from the store, or `None` when the spill read failed. A failed read yields a +/// black frame, which must never be emitted (#2570 review). +fn take_checked(store: &PropStore, k: usize) -> Option { + let img = store.take(k); + (!store.has_error()).then_some(img) +} + +/// Hand frame `t` to the streaming sink with its filled pixels made opaque. `true` when there is +/// no sink or it accepted the frame. +fn stream_frame(sink: &mut Option<&mut dyn FnMut(usize, &Image) -> Result<(), String>>, t: usize, img: Option<&Image>, holes: &[HoleMask]) -> bool { + let (Some(cb), Some(img)) = (sink.as_mut(), img) else { + return true; + }; + cb(t, &opaque_fill(img.clone(), &holes[t].to_vec())).is_ok() } /// Filled pixels are opaque. @@ -914,6 +1633,47 @@ mod tests { (0..w * h).map(|i| (x0..x0 + s).contains(&(i % w)) && (y0..y0 + s).contains(&(i / w))).collect() } + /// Packing and unpacking a hole mask round-trips every pixel. + #[test] + fn holemask_roundtrip() { + let v: Vec = (0..1000).map(|i| i % 3 == 0).collect(); + let m = HoleMask::from_vec(&v); + assert_eq!(m.to_vec(), v); + for i in 0..v.len() { + assert_eq!(m.get(i), v[i]); + } + } + + /// Packing round-trips where the last word is full, holds one bit or is one bit short, and + /// `set` builds the same mask; a mask whose public `words` are shorter than `len` reads and + /// writes without panicking (#2570 review). + #[test] + fn holemask_roundtrips_at_word_boundaries() { + for len in [1usize, 63, 64, 65, 127, 128, 129, 191, 192] { + let v: Vec = (0..len).map(|i| i % 3 == 0 || i == len - 1).collect(); + let m = HoleMask::from_vec(&v); + assert_eq!(m.words.len(), len.div_ceil(64), "len {len}"); + assert_eq!(m.to_vec(), v, "len {len}"); + let mut s = HoleMask::zeros(len); + for (i, &h) in v.iter().enumerate() { + s.set(i, h); + } + assert_eq!(s, m, "len {len}: set builds the packed mask"); + for i in 0..len { + s.set(i, false); + } + assert!(!s.any(), "len {len}: every pixel cleared"); + // A pixel past the end is ignored. + s.set(len, true); + assert!(!s.get(len), "len {len}"); + assert!(!s.any(), "len {len}"); + } + let mut short = HoleMask { words: vec![], len: 100 }; + assert!(!short.get(70)); + short.set(70, true); + assert!(!short.any()); + } + #[test] fn membrane_matches_a_linear_ramp() { let (w, h) = (40u32, 30u32); @@ -957,10 +1717,10 @@ mod tests { let gt = background(w, h, t as f32 * pan); let hole = rect_hole(w, h, 10 + 8 * t as u32, 22, 18); frames.push(cut(>, &hole)); - holes.push(hole); + holes.push(HoleMask::from_vec(&hole)); gts.push(gt); } - (FillInput { frames, holes, references: vec![] }, gts) + (FillInput { frames: Frames::Memory(frames), holes, references: vec![] }, gts) } #[test] @@ -969,7 +1729,7 @@ mod tests { let (input, gts) = sequence(8, pan); let out = fill_sequence(&input, &FillOpts::default(), &mut |_, _| true).expect("fill"); for t in 0..out.len() { - let db = psnr(&out[t], >s[t], Some(&input.holes[t])); + let db = psnr(&out[t], >s[t], Some(&input.holes[t].to_vec())); assert!(db > 26.0, "frame {t} pan {pan}: {db:.1} dB"); assert_eq!(out[t].data[0][3], 1.0); } @@ -977,7 +1737,7 @@ mod tests { // Temporal consistency: on a static background the filled pixels agree between // frames. for t in 1..out.len() { - let both: Vec = input.holes[t].iter().zip(&input.holes[t - 1]).map(|(a, b)| *a || *b).collect(); + let both: Vec = input.holes[t].to_vec().iter().zip(input.holes[t - 1].to_vec().iter()).map(|(a, b)| *a || *b).collect(); let db = psnr(&out[t], &out[t - 1], Some(&both)); assert!(db > 30.0, "frames {}–{t} differ: {db:.1} dB", t - 1); } @@ -991,14 +1751,14 @@ mod tests { let (w, h) = (80u32, 60u32); let gt = background(w, h, 0.0); let hole = rect_hole(w, h, 30, 20, 14); - let input = FillInput { frames: vec![cut(>, &hole); 4], holes: vec![hole.clone(); 4], references: vec![] }; + let input = FillInput { frames: Frames::Memory(vec![cut(>, &hole); 4]), holes: vec![HoleMask::from_vec(&hole); 4], references: vec![] }; let surf = fill_sequence(&input, &FillOpts { method: FillMethod::Surface, ..Default::default() }, &mut |_, _| true).expect("fill"); for t in 1..4 { // Surface carries one fill through: identical frames. assert!(psnr(&surf[t], &surf[0], Some(&hole)) > 40.0); } let edge = fill_sequence(&input, &FillOpts { method: FillMethod::EdgeBlend, ..Default::default() }, &mut |_, _| true).expect("fill"); - assert!(psnr(&edge[0], &membrane_fill(&input.frames[0], &hole), Some(&hole)) > 60.0); + assert!(psnr(&edge[0], &membrane_fill(&input.frames.get(0), &hole), Some(&hole)) > 60.0); // A clean reference frame recovers the background exactly. let mut with_ref = FillInput { frames: input.frames.clone(), holes: input.holes.clone(), references: vec![(2, gt.clone())] }; let out = fill_sequence(&with_ref, &FillOpts::default(), &mut |_, _| true).expect("fill"); @@ -1014,9 +1774,13 @@ mod tests { fn lighting_correction_absorbs_a_brightness_change() { // The background brightens over time; propagated pixels come from darker frames. let (mut input, mut gts) = sequence(6, 0.0); + let mut frames = match std::mem::replace(&mut input.frames, Frames::Memory(vec![])) { + Frames::Memory(v) => v, + _ => vec![], + }; for t in 0..6 { let k = 1.0 + 0.06 * t as f32; - for img in [&mut input.frames[t], &mut gts[t]] { + for img in [&mut frames[t], &mut gts[t]] { for p in &mut img.data { for c in 0..3 { p[c] *= k; @@ -1024,9 +1788,288 @@ mod tests { } } } + input.frames = Frames::Memory(frames); let plain = fill_sequence(&input, &FillOpts::default(), &mut |_, _| true).expect("fill"); let lit = fill_sequence(&input, &FillOpts { lighting: 1.0, ..Default::default() }, &mut |_, _| true).expect("fill"); - let score = |o: &[Image]| (0..6).map(|t| psnr(&o[t], >s[t], Some(&input.holes[t]))).sum::(); + let score = |o: &[Image]| (0..6).map(|t| psnr(&o[t], >s[t], Some(&input.holes[t].to_vec()))).sum::(); assert!(score(&lit) > score(&plain), "lit {} vs plain {}", score(&lit), score(&plain)); } + + /// `SpillHandle::sample` reads only the 2x2 support; it must still agree with sampling the + /// decoded frame, including the edges and coordinates outside the image (#2570 review). + #[test] + fn spilled_samples_match_the_decoded_frame() { + let (w, h, n) = (7u32, 5u32, 3usize); + for bpc in [8u8, 32] { + let mut frames = Vec::new(); + for k in 0..n { + let mut img = Image::new(w, h); + for (i, p) in img.data.iter_mut().enumerate() { + *p = [(i % 7) as f32 / 7.0, (k * 3 + i % 5) as f32 / 5.0, 0.25, 1.0]; + } + frames.push(img); + } + let spill = SpillHandle::create(w as usize, h as usize, n, bpc).expect("spill"); + for (k, img) in frames.iter().enumerate() { + spill.write_frame(k, img).expect("write"); + } + let spilled = Frames::Spilled(spill.clone()); + assert!(!spilled.has_error()); + for k in 0..n { + let decoded = spill.get(k); + for &(x, y) in &[(0.0, 0.0), (0.5, 0.5), (3.4, 2.6), (6.5, 4.5), (7.9, 5.9), (-1.0, -1.0), (2.25, 3.75)] { + let a = spilled.sample(k, x, y); + let b = decoded.sample_bilinear_clamped(x, y); + for c in 0..4 { + assert!((a[c] - b[c]).abs() <= 1e-6, "bpc {bpc} frame {k} at ({x}, {y}) channel {c}: {} vs {}", a[c], b[c]); + } + } + } + assert!(!spilled.has_error(), "sampling a valid spill records no error"); + } + } + + /// The input's error slot is shared with the membrane and propagation spills, so one probe + /// reports any scratch failure (#2570 review). + #[test] + fn a_secondary_spill_reports_through_the_input_error_slot() { + let input = SpillHandle::create(4, 4, 2, 32).expect("input spill"); + let prop = SpillHandle::create_in_with_error(None, 4, 4, 2, 32, Some(input.error_cell())).expect("prop spill"); + prop.record_error("spill: No space left on device".into()); + assert!(input.has_error()); + assert_eq!(input.take_error().as_deref(), Some("spill: No space left on device")); + assert!(!prop.has_error(), "take clears the shared slot"); + } + + /// A spill whose scratch folder cannot be created is an error, not a silent fallback. + #[test] + fn a_spill_that_cannot_create_its_folder_is_an_error() { + let dir = tempfile::tempdir().expect("temp"); + let file = dir.path().join("not-a-folder"); + std::fs::write(&file, b"x").expect("write"); + assert!(SpillHandle::create_in(Some(&file), 4, 4, 1, 32).is_err()); + } + + /// The scratch file has no name in its folder, so a fill that is killed partway through + /// leaves nothing behind, and every read and write still reaches the frames (#2570 review). + #[test] + fn a_spill_leaves_no_file_in_its_folder() { + let dir = tempfile::tempdir().expect("temp"); + let spill = SpillHandle::create_in(Some(dir.path()), 4, 3, 2, 32).expect("spill"); + let mut img = Image::new(4, 3); + for (i, p) in img.data.iter_mut().enumerate() { + *p = [i as f32 / 12.0, 0.5, 0.25, 1.0]; + } + spill.write_frame(1, &img).expect("write"); + assert_eq!(spill.get(1).data, img.data); + assert!(!spill.has_error()); + assert_eq!(spill.dir(), Some(dir.path())); + assert_eq!(Frames::Spilled(spill.clone()).scratch_dir().as_deref(), Some(dir.path())); + // Elsewhere the file is unlinked as soon as it is made; Windows keeps a delete-on-close + // file listed until its last handle closes. + #[cfg(not(windows))] + assert_eq!(std::fs::read_dir(dir.path()).expect("list").count(), 0, "the open spill has a name in the folder"); + drop(spill); + assert_eq!(std::fs::read_dir(dir.path()).expect("list").count(), 0, "scratch left in the folder"); + } + + /// The same frames spilled to 32 bpc scratch. + fn spill_frames(frames: &Frames) -> SpillHandle { + let (w, h) = frames.size().expect("size"); + let spill = SpillHandle::create(w, h, frames.len(), 32).expect("spill"); + for k in 0..frames.len() { + spill.write_frame(k, &frames.get(k)).expect("write"); + } + spill + } + + /// Every channel of every pixel, as bits. + fn bits(img: &Image) -> Vec { + img.data.iter().flatten().map(|v| v.to_bits()).collect() + } + + /// A spilled range fills bit-identically to the same frames in RAM, streamed or returned: the + /// spill keeps 32-bit floats and samples the same 2x2 support (#2570, "no quality loss"). + #[test] + fn a_spilled_range_fills_exactly_like_one_in_memory() { + let (input, _) = sequence(6, 1.0); + for method in [FillMethod::Object, FillMethod::Surface, FillMethod::EdgeBlend] { + let opts = FillOpts { method, lighting: 0.5, ..Default::default() }; + let memory = fill_sequence(&input, &opts, &mut |_, _| true).expect("memory fill"); + let spilled = FillInput { frames: Frames::Spilled(spill_frames(&input.frames)), holes: input.holes.clone(), references: vec![] }; + let returned = fill_sequence(&spilled, &opts, &mut |_, _| true).expect("spilled fill"); + let mut streamed = Vec::new(); + let mut sink = |_: usize, img: &Image| -> Result<(), String> { + streamed.push(img.clone()); + Ok(()) + }; + let rest = fill_sequence_with(spilled, &opts, &CpuFill, &mut |_, _| true, Some(&mut sink)); + assert!(rest.is_some_and(|v| v.is_empty()), "{method:?}: the sink took every frame"); + assert_eq!((memory.len(), returned.len(), streamed.len()), (6, 6, 6), "{method:?}"); + for t in 0..6 { + assert!(bits(&returned[t]) == bits(&memory[t]), "{method:?} frame {t}: spilled fill differs from RAM"); + assert!(bits(&streamed[t]) == bits(&memory[t]), "{method:?} frame {t}: streamed fill differs from RAM"); + } + } + } + + /// The sink sees every frame once and strictly in order, whichever stage finishes it: the + /// complete frames before the first synthesised one, the synthesised run, and Edge Blend. + #[test] + fn the_sink_sees_frames_in_order() { + let (input, gts) = sequence(8, 1.0); + let mut frames: Vec = (0..8).map(|k| input.frames.get(k)).collect(); + let mut holes = input.holes.clone(); + // The first two frames are clean, so they stream ahead of the first synthesised frame. + for k in 0..2 { + frames[k] = gts[k].clone(); + holes[k] = HoleMask::zeros(frames[k].data.len()); + } + let input = FillInput { frames: Frames::Spilled(spill_frames(&Frames::Memory(frames))), holes, references: vec![] }; + for method in [FillMethod::Object, FillMethod::Surface, FillMethod::EdgeBlend] { + let mut seen = Vec::new(); + let mut sink = |t: usize, _: &Image| -> Result<(), String> { + seen.push(t); + Ok(()) + }; + let out = fill_sequence_with(input.clone(), &FillOpts { method, ..Default::default() }, &CpuFill, &mut |_, _| true, Some(&mut sink)); + assert!(out.is_some(), "{method:?}"); + assert_eq!(seen, (0..8).collect::>(), "{method:?}"); + } + } + + /// A spill that cannot be read fails the fill with the disk error in the slot the caller + /// probes, and the black fallback frame never reaches the sink (#2570 review). + #[test] + fn a_spill_read_failure_is_an_error_and_never_streamed() { + let (input, _) = sequence(4, 1.0); + let spill = spill_frames(&input.frames); + // Truncate the scratch under the fill: every read comes back short. + spill.inner.file.set_len(0).expect("truncate"); + for method in [FillMethod::Object, FillMethod::Surface, FillMethod::EdgeBlend] { + let mut seen = Vec::new(); + let mut sink = |t: usize, _: &Image| -> Result<(), String> { + seen.push(t); + Ok(()) + }; + let failed = FillInput { frames: Frames::Spilled(spill.clone()), holes: input.holes.clone(), references: vec![] }; + let out = fill_sequence_with(failed, &FillOpts { method, ..Default::default() }, &CpuFill, &mut |_, _| true, Some(&mut sink)); + assert!(out.is_none(), "{method:?}"); + assert!(seen.is_empty(), "{method:?}: streamed frames {seen:?} from an unreadable spill"); + let e = spill.take_error().expect("the read failure is recorded"); + assert!(e.starts_with("spill:"), "{method:?}: {e}"); + } + } + + /// A scratch failure while the output streams stops the fill before the next frame reaches + /// the sink, and leaves the failure for the caller to report. + #[test] + fn a_spill_failure_mid_stream_stops_before_the_next_frame() { + let (input, _) = sequence(6, 1.0); + let spill = spill_frames(&input.frames); + let cell = spill.error_cell(); + let mut seen = Vec::new(); + let mut sink = |t: usize, _: &Image| -> Result<(), String> { + seen.push(t); + if t == 2 { + *cell.lock().unwrap() = Some("spill: injected".into()); + } + Ok(()) + }; + let streamed = FillInput { frames: Frames::Spilled(spill.clone()), holes: input.holes.clone(), references: vec![] }; + let out = fill_sequence_with(streamed, &FillOpts::default(), &CpuFill, &mut |_, _| true, Some(&mut sink)); + assert!(out.is_none()); + assert_eq!(seen, vec![0, 1, 2]); + assert_eq!(spill.take_error().as_deref(), Some("spill: injected")); + } + + /// Once a propagated frame fails to spill, the other propagation workers stop instead of + /// tracing the rest of a range whose fill is already lost (#2570 review). + #[test] + fn propagation_stops_after_a_spill_failure() { + /// Fails the spill from inside the propagation stage (Lighting Correction's membrane). + struct FailInPropagation { + cell: std::sync::Arc>>, + calls: std::sync::atomic::AtomicUsize, + } + impl FillBackend for FailInPropagation { + fn membrane_known(&self, data: &mut [[f32; 4]], known: &[bool], w: usize, h: usize) { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + *self.cell.lock().unwrap() = Some("spill: injected".into()); + membrane_core(data, known, w, h); + } + } + let (input, _) = sequence(6, 1.0); + let spill = spill_frames(&input.frames); + let backend = FailInPropagation { cell: spill.error_cell(), calls: std::sync::atomic::AtomicUsize::new(0) }; + let input = FillInput { frames: Frames::Spilled(spill.clone()), holes: input.holes, references: vec![] }; + let opts = FillOpts { lighting: 1.0, ..Default::default() }; + // One worker, so the frames run one after another and the count is exact. + let pool = rayon::ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let out = pool.install(|| fill_sequence_with(input, &opts, &backend, &mut |_, _| true, None)); + assert!(out.is_none()); + assert_eq!(backend.calls.load(std::sync::atomic::Ordering::SeqCst), 1, "propagation kept going after the failure"); + assert_eq!(spill.take_error().as_deref(), Some("spill: injected")); + } + + /// Peak RSS of one synthetic fill, quoted in the PR. Ignored so CI does not run it. Run it + /// under `/usr/bin/time -v` and read `VmHWM` (`Maximum resident set size`): + /// + /// ```text + /// cargo test -p effectcraft-raster --lib measure_peak_rss --no-run + /// target/debug/deps/effectcraft_raster-* inpaint::tests::measure_peak_rss_of_a_fill \ + /// --ignored --exact --nocapture --test-threads=1 + /// ``` + /// + /// `RAYON_NUM_THREADS`, `MEASURE_LIGHTING=1` and `MEASURE_FRAMES` select the configuration. + /// Measured on 18 workers (2026-11-04): 90 frames of 640×360 peak at 340–470 MB across runs + /// (35 MB with one worker) and 600 frames at 582 MB, i.e. ~7 frames of scratch a worker, so + /// the guard budgets 8 (9 with Lighting Correction) (#2570). + #[cfg(not(target_arch = "wasm32"))] + #[test] + #[ignore = "memory measurement; run explicitly"] + fn measure_peak_rss_of_a_fill() { + let n: usize = std::env::var("MEASURE_FRAMES").ok().and_then(|v| v.parse().ok()).unwrap_or(90); + let (w, h) = (640u32, 360u32); + let spill = SpillHandle::create_in(None, w as usize, h as usize, n, 32).expect("spill"); + let hole = rect_hole(w, h, 240, 120, 160); + let mut holes = Vec::with_capacity(n); + for k in 0..n { + let img = background(w, h, 2.0 * k as f32); + spill.write_frame(k, &img).expect("write"); + holes.push(HoleMask::from_vec(&hole)); + } + let input = FillInput { frames: Frames::Spilled(spill), holes, references: vec![] }; + // MEASURE_LIGHTING=1 exercises the propagation ring too (one process per reading: VmHWM + // never goes down). + let lighting = std::env::var("MEASURE_LIGHTING").is_ok_and(|v| v == "1"); + let opts = FillOpts { method: FillMethod::Surface, lighting: if lighting { 1.0 } else { 0.0 }, ..Default::default() }; + let before = peak_rss_kb(); + let started = std::time::Instant::now(); + let mut sink = |_: usize, _: &Image| -> Result<(), String> { Ok(()) }; + let out = fill_sequence_with(input, &opts, &CpuFill, &mut |_, _| true, Some(&mut sink)); + eprintln!( + "measure_peak_rss: {n} frames of {w}x{h}, surface, lighting={}, {:.2}s, peak RSS {} KB -> {} KB", + if lighting { "on" } else { "off" }, + started.elapsed().as_secs_f64(), + before.unwrap_or(0), + peak_rss_kb().unwrap_or(0) + ); + assert!(out.is_some_and(|v| v.is_empty()), "the sink consumed every frame"); + } + + /// Peak resident set size of this process (`VmHWM`), in kilobytes. + #[cfg(target_os = "linux")] + fn peak_rss_kb() -> Option { + std::fs::read_to_string("/proc/self/status").ok()?.lines().find_map(|l| { + let v = l.strip_prefix("VmHWM:")?; + v.split_whitespace().next()?.parse().ok() + }) + } + + /// Peak resident set size is only measured on Linux. + #[cfg(not(target_os = "linux"))] + fn peak_rss_kb() -> Option { + None + } }