From e7c208434b6a966f67731aa98abe625651f52778 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 00:27:45 +0100 Subject: [PATCH 1/5] #474: pin OpenEXR pictures and aux channels to exact hashes Synthetic single-layer, multi-layer, multi-part and windowed files in half, float and uint, with RLE, ZIP, PIZ and DWAA, decoded through the media pool picture first and channels first. The hashes are taken on main before the decode is reworked. Co-Authored-By: Claude Opus 5.5 --- crates/media/tests/it/exr_golden.rs | 338 ++++++++++++++++++++++++++++ crates/media/tests/it/main.rs | 1 + 2 files changed, 339 insertions(+) create mode 100644 crates/media/tests/it/exr_golden.rs diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs new file mode 100644 index 000000000..498944882 --- /dev/null +++ b/crates/media/tests/it/exr_golden.rs @@ -0,0 +1,338 @@ +//! OpenEXR pictures and aux channels pinned to exact hashes (#474). +//! +//! Synthetic files (single-layer, multi-layer with and without an unnamed RGB layer, multi-part, +//! data windows smaller and larger than the display window, half / float / uint channels, RLE, +//! ZIP, PIZ and DWAA) are decoded through the media pool as the renderer asks for them: the +//! picture in each alpha interpretation, and the channels EXtractoR and Cryptomatte read. The +//! hashes were taken before the picture and the channels started sharing one decode, so a change +//! to how either is decoded must leave every bit as it was. Only bit-exact float steps are +//! pinned (linear light, no `powf`), so the hashes hold on every platform; DWAA (whose decoder +//! uses `powf`) and the sRGB / gamma 2.4 curves are checked for agreement between the two ways a +//! frame can be decoded instead. Re-pin with `EXR_GOLDEN_PRINT=1` after an intended change. + +use std::sync::Arc; + +use effectcraft_media::{MediaPool, probe_bytes}; +use effectcraft_project::{AlphaMode, Footage, ItemId}; +use effectcraft_render::{FloatCurve, FootageSource}; +use effectcraft_time::Tick; +use exr::prelude::*; + +const W: usize = 37; +const H: usize = 23; + +/// A deterministic value for pixel (x, y) of channel `c`: negative, in range and above 1. +fn value(x: usize, y: usize, c: usize) -> f32 { + ((x * 7 + y * 13 + c * 29) % 97) as f32 / 40.0 - 0.3 +} + +/// A deterministic alpha in 0..=1 (0 included, for the premultiplied divide). +fn alpha(x: usize, y: usize) -> f32 { + ((x * 3 + y * 5) % 11) as f32 / 10.0 +} + +#[derive(Clone, Copy)] +enum Kind { + Half, + Float, + Uint, +} + +/// A channel `name` of `size` filled from [`value`] (or [`alpha`] for an `A` / `a` channel). +fn channel(name: &str, kind: Kind, size: (usize, usize), c: usize) -> AnyChannel { + let is_alpha = name.rsplit('.').next().is_some_and(|n| n.eq_ignore_ascii_case("a")); + let px = |i: usize| { + let (x, y) = (i % size.0, i / size.0); + if is_alpha { alpha(x, y) } else { value(x, y, c) } + }; + let n = size.0 * size.1; + let samples = match kind { + Kind::Half => FlatSamples::F16((0..n).map(|i| f16::from_f32(px(i))).collect()), + Kind::Float => FlatSamples::F32((0..n).map(px).collect()), + Kind::Uint => FlatSamples::U32((0..n).map(|i| (i as u32).wrapping_mul(2_654_435_761) >> 8).collect()), + }; + AnyChannel::new(name, samples) +} + +fn channels(list: &[(&str, Kind)], size: (usize, usize)) -> AnyChannels { + AnyChannels::sort(list.iter().enumerate().map(|(c, (n, k))| channel(n, *k, size, c)).collect::>().into()) +} + +fn encoding(compression: Compression) -> Encoding { + Encoding { compression, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing } +} + +fn with_manifest(mut attrs: LayerAttributes) -> LayerAttributes { + attrs.other.insert(Text::from("cryptomatte/abc1234/name"), AttributeValue::Text(Text::from("CryptoObject"))); + attrs.other.insert(Text::from("cryptomatte/abc1234/manifest"), AttributeValue::Text(Text::from(r#"{"bunny":"13851a76","floor":"0a1b2c3d"}"#))); + attrs +} + +/// One part: `list` channels over a data window of `size` at `pos`. +fn part(name: Option<&str>, list: &[(&str, Kind)], size: (usize, usize), pos: (i32, i32), compression: Compression) -> Layer> { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, with_manifest(attrs), encoding(compression), channels(list, size)) +} + +/// Encode `layers` with the display window `display` (position, size). +fn write(layers: Vec>>, display: ((i32, i32), (usize, usize))) -> Vec { + let mut attrs = ImageAttributes::new(IntegerBounds::new(display.0, display.1)); + attrs.pixel_aspect = 1.0; + let image = Image::from_layers(attrs, layers); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().non_parallel().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() +} + +const RGBA_HALF: [(&str, Kind); 4] = [("R", Kind::Half), ("G", Kind::Half), ("B", Kind::Half), ("A", Kind::Half)]; +const CRYPTO: [(&str, Kind); 4] = + [("CryptoObject00.r", Kind::Float), ("CryptoObject00.g", Kind::Float), ("CryptoObject00.b", Kind::Float), ("CryptoObject00.a", Kind::Float)]; + +/// The synthetic files: (name, bytes, whether the hashes are pinned). +fn files() -> Vec<(&'static str, Vec, bool)> { + let full = (W, H); + let origin = ((0, 0), full); + let cat = |a: &[(&'static str, Kind)], b: &[(&'static str, Kind)]| -> Vec<(&'static str, Kind)> { a.iter().chain(b).copied().collect() }; + let layered: Vec<(&str, Kind)> = cat( + &[("Image.R", Kind::Half), ("Image.G", Kind::Half), ("Image.B", Kind::Half), ("Image.A", Kind::Half), ("depth.Z", Kind::Float), ("ObjectID", Kind::Uint)], + &CRYPTO, + ); + let unnamed_plus: Vec<(&str, Kind)> = + cat(&cat(&RGBA_HALF, &[("Diffuse.R", Kind::Half), ("Diffuse.G", Kind::Half), ("Diffuse.B", Kind::Half), ("Z", Kind::Float)]), &CRYPTO); + let rgb_float = [("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("depth.Z", Kind::Float)]; + vec![ + ("rgba_half_zip16", write(vec![part(None, &RGBA_HALF, full, (0, 0), Compression::ZIP16)], origin), true), + ("rgb_float_piz", write(vec![part(None, &[("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("Z", Kind::Float)], full, (0, 0), Compression::PIZ)], origin), true), + ("layered_rle", write(vec![part(None, &layered, full, (0, 0), Compression::RLE)], origin), true), + ("unnamed_rgb_plus_layers_zip1", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::ZIP1)], origin), true), + ( + "multipart_zip16", + write( + vec![ + part(Some("Image"), &RGBA_HALF, full, (0, 0), Compression::ZIP16), + part(Some("depth"), &[("Z", Kind::Float)], full, (0, 0), Compression::ZIP16), + part(Some("CryptoObject00"), &[("r", Kind::Float), ("g", Kind::Float), ("b", Kind::Float), ("a", Kind::Float)], full, (0, 0), Compression::ZIP16), + ], + origin, + ), + true, + ), + ( + // The first part with R, G and B is not the beauty pass. + "multipart_diffuse_then_beauty_zip1", + write( + vec![ + part(Some("diffuse"), &[("R", Kind::Half), ("G", Kind::Half), ("B", Kind::Half)], full, (0, 0), Compression::ZIP1), + part(Some("beauty"), &RGBA_HALF, (20, 10), (5, 7), Compression::ZIP1), + part(Some("depth"), &[("Z", Kind::Float)], full, (0, 0), Compression::ZIP1), + ], + origin, + ), + true, + ), + ( + "multipart_blender5_rle", + write( + vec![ + part(Some("ViewLayer.Depth"), &[("ViewLayer.Depth.Z", Kind::Float)], full, (0, 0), Compression::RLE), + part( + Some("ViewLayer.Combined"), + &[("ViewLayer.Combined.R", Kind::Half), ("ViewLayer.Combined.G", Kind::Half), ("ViewLayer.Combined.B", Kind::Half), ("ViewLayer.Combined.A", Kind::Half)], + full, + (0, 0), + Compression::RLE, + ), + ], + origin, + ), + true, + ), + ("unnamed_rgb_window_smaller_zip16", write(vec![part(None, &unnamed_plus, (20, 10), (5, 7), Compression::ZIP16)], origin), true), + ("rgb_float_window_larger_piz", write(vec![part(None, &rgb_float, (45, 30), (-1, -5), Compression::PIZ)], ((3, -2), full)), true), + ("layered_window_smaller_zip16", write(vec![part(None, &layered, (20, 10), (5, 7), Compression::ZIP16)], origin), true), + ("layered_window_larger_piz", write(vec![part(None, &layered, (45, 30), (-4, -3), Compression::PIZ)], origin), true), + ("unnamed_rgb_plus_layers_dwaa", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::DWAA(None))], origin), false), + ("layered_dwaa", write(vec![part(None, &layered, full, (0, 0), Compression::DWAA(None))], origin), false), + ] +} + +/// FNV-1a over 32-bit words. +fn fnv(words: impl IntoIterator) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for w in words { + for b in w.to_le_bytes() { + h ^= u64::from(b); + h = h.wrapping_mul(0x0100_0000_01b3); + } + } + h +} + +fn picture_hash(pool: &MediaPool, f: &Footage, float: FloatCurve, size: Option<(u32, u32)>) -> u64 { + let img = pool.frame_with(ItemId(1), f, Tick::ZERO, size, float).expect("picture"); + fnv([img.width, img.height].into_iter().chain(img.data.iter().flat_map(|p| p.map(f32::to_bits)))) +} + +fn aux_hash(pool: &MediaPool, f: &Footage) -> u64 { + let Some(aux) = pool.aux(ItemId(1), f, Tick::ZERO) else { return 0 }; + let mut words = vec![aux.width, aux.height, aux.scale.to_bits() as u32]; + for (name, plane) in &aux.channels { + words.extend(name.bytes().map(u32::from)); + words.extend(plane.iter().map(|v| v.to_bits())); + } + for (name, objects) in &aux.manifests { + words.extend(name.bytes().map(u32::from)); + for (o, h) in objects { + words.extend(o.bytes().map(u32::from)); + words.push(*h); + } + } + fnv(words) +} + +/// The interpretations a picture is decoded in: (label, footage, curve, size, pinned). +fn variants(f: &Footage) -> Vec<(String, Footage, FloatCurve, Option<(u32, u32)>, bool)> { + let mut out = vec![]; + for (mode, label) in [(None, "probed"), (Some(AlphaMode::Straight), "straight"), (Some(AlphaMode::Premultiplied), "premul"), (Some(AlphaMode::Ignore), "ignore")] { + let mut g = f.clone(); + if let Some(m) = mode { + g.alpha = m; + } + g.premul_color = [0.2, 0.4, 0.6]; + out.push((format!("{label}/linear"), g.clone(), FloatCurve::Linear, None, true)); + out.push((format!("{label}/srgb"), g.clone(), FloatCurve::Srgb, None, false)); + out.push((format!("{label}/gamma24"), g.clone(), FloatCurve::Gamma24, None, false)); + } + let preserve = Footage { preserve_rgb: true, ..f.clone() }; + out.push(("preserve_rgb".into(), preserve, FloatCurve::Srgb, None, true)); + out.push(("half_size/srgb".into(), f.clone(), FloatCurve::Srgb, Some((f.width.div_ceil(2), f.height.div_ceil(2))), false)); + out +} + +/// Every hash of `file`: the size and alpha import finds, each picture and the channels, decoded +/// picture first (the footage alone) or channels first (an effect reads them). +fn hashes(name: &str, bytes: &[u8], aux_first: bool) -> Vec<(String, u64, bool)> { + let order = if aux_first { "aux-first" } else { "picture-first" }; + let path = format!("/exr-golden-474/{order}/{name}.exr"); + let b: Arc<[u8]> = bytes.into(); + let pool = MediaPool::new(); + pool.add_bytes(&path, b.clone()); + let f = probe_bytes(&path, b).expect("probe"); + let mut out = vec![("probe".to_string(), fnv([f.width, f.height, f.alpha as u32]), true)]; + let aux = if aux_first { Some(aux_hash(&pool, &f)) } else { None }; + for (label, g, float, size, pinned) in variants(&f) { + out.push((label, picture_hash(&pool, &g, float, size), pinned)); + } + out.push(("aux".into(), aux.unwrap_or_else(|| aux_hash(&pool, &f)), true)); + out +} + +/// Hashes taken on main before #474's shared decode (`EXR_GOLDEN_PRINT=1`). +const PINNED: &[(&str, &str, u64)] = &[ + ("rgba_half_zip16", "probe", 0x0a32e074a28c1f66), + ("rgba_half_zip16", "probed/linear", 0xa84600283dce98f6), + ("rgba_half_zip16", "straight/linear", 0xf028e1367a2fbb30), + ("rgba_half_zip16", "premul/linear", 0xa84600283dce98f6), + ("rgba_half_zip16", "ignore/linear", 0xa90a57f9a67f7468), + ("rgba_half_zip16", "preserve_rgb", 0xc8e0050f5efe75b7), + ("rgba_half_zip16", "aux", 0xdd555e6cf67d75f3), + ("rgb_float_piz", "probe", 0x6a38346c4c5662d5), + ("rgb_float_piz", "probed/linear", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "straight/linear", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "premul/linear", 0x515f476c176229c9), + ("rgb_float_piz", "ignore/linear", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "preserve_rgb", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "aux", 0x554623d237c2e5ef), + ("layered_rle", "probe", 0x0a32e074a28c1f66), + ("layered_rle", "probed/linear", 0xa84600283dce98f6), + ("layered_rle", "straight/linear", 0xf028e1367a2fbb30), + ("layered_rle", "premul/linear", 0xa84600283dce98f6), + ("layered_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_rle", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_zip1", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_zip1", "probed/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_zip1", "straight/linear", 0xf028e1367a2fbb30), + ("unnamed_rgb_plus_layers_zip1", "premul/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_zip1", "ignore/linear", 0xa90a57f9a67f7468), + ("unnamed_rgb_plus_layers_zip1", "preserve_rgb", 0xc8e0050f5efe75b7), + ("unnamed_rgb_plus_layers_zip1", "aux", 0x27cc9b53d0862db1), + ("multipart_zip16", "probe", 0x0a32e074a28c1f66), + ("multipart_zip16", "probed/linear", 0xa84600283dce98f6), + ("multipart_zip16", "straight/linear", 0xf028e1367a2fbb30), + ("multipart_zip16", "premul/linear", 0xa84600283dce98f6), + ("multipart_zip16", "ignore/linear", 0xa90a57f9a67f7468), + ("multipart_zip16", "preserve_rgb", 0xc8e0050f5efe75b7), + ("multipart_zip16", "aux", 0x567ad99a6690d03d), + ("multipart_diffuse_then_beauty_zip1", "probe", 0x6a38346c4c5662d5), + ("multipart_diffuse_then_beauty_zip1", "probed/linear", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "straight/linear", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "premul/linear", 0x7ab18932e25c5cc8), + ("multipart_diffuse_then_beauty_zip1", "ignore/linear", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "preserve_rgb", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "aux", 0x6724d6dfd56a04cb), + ("multipart_blender5_rle", "probe", 0x0a32e074a28c1f66), + ("multipart_blender5_rle", "probed/linear", 0xa84600283dce98f6), + ("multipart_blender5_rle", "straight/linear", 0xf028e1367a2fbb30), + ("multipart_blender5_rle", "premul/linear", 0xa84600283dce98f6), + ("multipart_blender5_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("multipart_blender5_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("multipart_blender5_rle", "aux", 0x522ae4819dd92e89), + ("unnamed_rgb_window_smaller_zip16", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_window_smaller_zip16", "probed/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_window_smaller_zip16", "straight/linear", 0x2c73f9acaa936420), + ("unnamed_rgb_window_smaller_zip16", "premul/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_window_smaller_zip16", "ignore/linear", 0x3b86057d18a16bf0), + ("unnamed_rgb_window_smaller_zip16", "preserve_rgb", 0x5b999014e4a64bb2), + ("unnamed_rgb_window_smaller_zip16", "aux", 0x8f78d3233450acf3), + ("rgb_float_window_larger_piz", "probe", 0x6a38346c4c5662d5), + ("rgb_float_window_larger_piz", "probed/linear", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "straight/linear", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "premul/linear", 0x0998eefc456dcdc9), + ("rgb_float_window_larger_piz", "ignore/linear", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "preserve_rgb", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "aux", 0x7ccc9f86e6ff6140), + ("layered_window_smaller_zip16", "probe", 0x0a32e074a28c1f66), + ("layered_window_smaller_zip16", "probed/linear", 0x19eb3b1a88d108a7), + ("layered_window_smaller_zip16", "straight/linear", 0x2c73f9acaa936420), + ("layered_window_smaller_zip16", "premul/linear", 0x19eb3b1a88d108a7), + ("layered_window_smaller_zip16", "ignore/linear", 0x3b86057d18a16bf0), + ("layered_window_smaller_zip16", "preserve_rgb", 0x5b999014e4a64bb2), + ("layered_window_smaller_zip16", "aux", 0x443480a1d789e63c), + ("layered_window_larger_piz", "probe", 0x0a32e074a28c1f66), + ("layered_window_larger_piz", "probed/linear", 0x443afe4029748836), + ("layered_window_larger_piz", "straight/linear", 0xb32b44880f746826), + ("layered_window_larger_piz", "premul/linear", 0x443afe4029748836), + ("layered_window_larger_piz", "ignore/linear", 0xaa01706accce2cc4), + ("layered_window_larger_piz", "preserve_rgb", 0xb2cbb40b10bc56b4), + ("layered_window_larger_piz", "aux", 0xb7d91c8eea56a772), +]; + +#[test] +fn exr_pictures_and_channels_are_bit_identical() { + let print = std::env::var_os("EXR_GOLDEN_PRINT").is_some(); + let mut failures = vec![]; + for (name, bytes, pinned_file) in files() { + let first = hashes(name, &bytes, false); + let second = hashes(name, &bytes, true); + for ((label, h, pinned), (_, h2, _)) in first.iter().zip(&second) { + if h != h2 { + failures.push(format!("{name} {label}: picture first {h:#018x}, channels first {h2:#018x}")); + } + if print { + println!(" (\"{name}\", \"{label}\", {h:#018x}),{}", if *pinned && pinned_file { "" } else { " // not pinned" }); + continue; + } + if !(*pinned && pinned_file) { + continue; + } + match PINNED.iter().find(|(n, l, _)| *n == name && l == label) { + Some((_, _, want)) if want == h => {} + Some((_, _, want)) => failures.push(format!("{name} {label}: {h:#018x}, pinned {want:#018x}")), + None => failures.push(format!("{name} {label}: {h:#018x} is not pinned")), + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rs index 69bed4a9f..08c5ec389 100644 --- a/crates/media/tests/it/main.rs +++ b/crates/media/tests/it/main.rs @@ -3,6 +3,7 @@ mod aac_layout; mod downmix; mod exif_orientation; +mod exr_golden; mod footage; mod open_in_place; mod tga; From 8add8ac7b401634fc2af0c6d2cde9ce96d1d3f08 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 00:51:59 +0100 Subject: [PATCH 2/5] #474: decode a multi-layer EXR once per frame, and only its picture when nothing else reads it - The channel cache decodes a file once however many callers ask for it at the same time (the others wait for that decode), and is bounded by the bytes of its planes (1 GiB, least recently used out first, a file counting as used when its decode finishes, the newest file always kept) instead of four files of any size. - Once an effect has read a footage item's channels, its pictures are made of the cached planes, also for files with an unnamed RGB layer (bit for bit what the image crate shows, which still decodes the files the planes can't stand in for); footage alone decodes only its picture's channels (a multi-layer file's were all decoded and cached for every frame). - Channels are decoded straight into their display-window planes, and the picture is converted from the planes without two interleaved copies. Measured on a 30-frame 2560x1440 sequence of 32 channels (DWAA), one frame at a time: footage alone 0.185 -> 0.073 s per frame and 4.4 -> 1.8 GB peak; with an unnamed RGB layer plus Cryptomatte 0.244 -> 0.159 s; multi-part plus EXtractoR 0.194 -> 0.127 s. Co-Authored-By: Claude Opus 5.5 --- crates/media/src/convert.rs | 52 +- crates/media/src/exr_channels.rs | 816 +++++++++++++++++++++++++------ crates/media/src/pool.rs | 30 +- docs/effects-parity.md | 2 +- docs/file-format-parity.md | 2 +- docs/footage.md | 5 + docs/gaps.md | 2 +- 7 files changed, 742 insertions(+), 167 deletions(-) diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index 9e88ab4dd..a2b24b3ae 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -376,30 +376,20 @@ fn convert_yuv(img: &mut Image, source: YuvSource<'_, T>, k: &Yuv, op pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image { use image::DynamicImage as D; let (w, h) = (img.width(), img.height()); + if let D::ImageRgb32F(_) | D::ImageRgba32F(_) = img { + let px = img.to_rgba32f(); + let raw = px.as_raw(); + return float_to_image(w, h, op, |i| { + let s = &raw[i * 4..i * 4 + 4]; + [s[0], s[1], s[2], s[3]] + }); + } let mut out = Image::new(w, h); let wu = w as usize; if wu == 0 || h == 0 { return out; } match img { - D::ImageRgb32F(_) | D::ImageRgba32F(_) => { - let px = img.to_rgba32f(); - let raw = px.as_raw(); - let premul = op.mode == AlphaMode::Premultiplied; - out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { - for (x, o) in row.iter_mut().enumerate() { - let s = &raw[(y * wu + x) * 4..(y * wu + x) * 4 + 4]; - let a = s[3]; - let c = if premul { - let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; - [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] - } else { - [op.float(s[0]), op.float(s[1]), op.float(s[2])] - }; - *o = op.apply(c, a.clamp(0.0, 1.0)); - } - }); - } D::ImageLuma16(_) | D::ImageLumaA16(_) | D::ImageRgb16(_) | D::ImageRgba16(_) => { let px = img.to_rgba16(); let raw = px.as_raw(); @@ -425,6 +415,32 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image out } +/// Float RGBA (linear light, alpha straight or premultiplied as the footage says), `px(i)` for +/// pixel `i` in rows, to an [`Image`]: how [`dynamic_to_image`] converts float stills, also used +/// for OpenEXR pictures made from decoded channel planes without an interleaved copy (#474). +pub(crate) fn float_to_image(w: u32, h: u32, op: AlphaOp, px: impl Fn(usize) -> [f32; 4] + Sync) -> Image { + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + let premul = op.mode == AlphaMode::Premultiplied; + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = px(y * wu + x); + let a = s[3]; + let c = if premul { + let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; + [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] + } else { + [op.float(s[0]), op.float(s[1]), op.float(s[2])] + }; + *o = op.apply(c, a.clamp(0.0, 1.0)); + } + }); + out +} + /// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`]. pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image { let mut out = Image::new(w, h); diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index d372ec849..747882337 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -6,12 +6,25 @@ //! 0, or the background depth for depth channels). Cryptomatte manifests come from the //! `cryptomatte//name` and `cryptomatte//manifest` header attributes (the published //! Cryptomatte metadata convention: the manifest is a JSON object of name → hex hash). +//! +//! Decoded files are kept in a small cache ([`CACHE_BUDGET`]) that the picture of an EXR +//! footage item shares with EXtractoR / Cryptomatte once an effect has read its channels +//! (#474); a file is decoded once however many callers ask for it at the same time. use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError}; -use effectcraft_raster::AuxChannels; use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, split_channel}; +use effectcraft_raster::{AuxChannels, Image}; +use exr::block::UncompressedBlock; +use exr::block::chunk::TileCoordinates; +use exr::image::read::image::ReadImage; +use exr::image::read::layers::{ChannelsReader, ReadAllLayers, ReadChannels}; +use exr::meta::attribute::SampleType; +use exr::meta::header::Header; + +use crate::convert::{AlphaOp, float_to_image}; /// The largest display window decoded (16384²; vector footage and Continuously Rasterize /// stop at the same size). @@ -33,6 +46,11 @@ fn channel_name(part: Option<&str>, channel: &str) -> String { } } +/// The part name channels are prefixed with (`None`: unnamed). +fn part_name(header: &Header) -> Option { + header.own_attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()) +} + /// Parse a Cryptomatte manifest (`{"name": "hexhash", …}`). pub fn parse_manifest(json: &str) -> Vec<(String, u32)> { let Ok(serde_json::Value::Object(m)) = serde_json::from_str::(json) else { return vec![] }; @@ -53,23 +71,209 @@ pub fn unsupported_compression(bytes: &[u8]) -> Option<&'static str> { .then_some("OpenEXR HTJ2K compression isn't supported yet: re-save the file with ZIP, PIZ or DWAA compression") } -/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. -pub fn read_exr_channels(bytes: &[u8]) -> Option { - use exr::prelude::*; - let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_buffered(std::io::Cursor::new(bytes)).ok()?; - let dw = img.attributes.display_window; - let (w, h) = (dw.size.0, dw.size.1); - if w == 0 || h == 0 { - return None; +/// Which channels a decode keeps, by plane name, and the display window they are placed in +/// (the first part's, as the `exr` crate reports the image's). +struct Planes<'k> { + keep: &'k dyn Fn(&str) -> bool, + display: exr::meta::attribute::IntegerBounds, +} + +/// One part's kept channels while it is decoded: each line of samples goes straight into its +/// channel's display-window plane (no copy of the part's samples in their own type first). +struct PlanesReader { + /// Per channel of the part: its plane name, sample type and plane (`None`: not kept). + planes: Vec)>>, + /// The data window's position in the display window, and the data window's and display + /// window's sizes. + offset: (i64, i64), + layer: (usize, usize), + display: (usize, usize), + f16s: Vec, + u32s: Vec, + f32s: Vec, + /// The part's channel indices of exact `R`, `G`, `B` and `A`, when it has R, G and B. + rgb: Option<([usize; 3], Option)>, + /// The part has a channel that doesn't sample every pixel. + subsampled: bool, + /// The part's display window differs from the first part's. + own_display: bool, +} + +/// A decoded part: its kept planes (in channel order) and what [`PlanesReader`] found. +struct PartPlanes { + planes: Vec<(String, Vec)>, + /// Positions in `planes` of the part's R, G, B (and A) channels, when it has them and keeps + /// them all. + rgb: Option<([usize; 3], Option)>, + has_rgb: bool, + exact: bool, +} + +impl<'s> ReadChannels<'s> for Planes<'_> { + type Reader = PlanesReader; + + fn create_channels_reader(&'s self, header: &Header) -> exr::error::Result { + if header.deep { + return Err(exr::error::Error::NotSupported("deep data".into())); + } + let dw = self.display; + let display = (dw.size.0, dw.size.1); + // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a + // multi-gigabyte allocation per channel (#2141). + let n = display.0.checked_mul(display.1).filter(|n| *n > 0 && *n <= MAX_DISPLAY_PIXELS).ok_or(exr::error::Error::Invalid("display window".into()))?; + let prefix = part_name(header); + let pos = header.own_attributes.layer_position; + let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); + let planes = header + .channels + .list + .iter() + .map(|ch| { + let name = channel_name(prefix.as_deref(), &ch.name.to_string()); + let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; + (self.keep)(&name).then(|| (name, ch.sample_type, vec![fill; n])) + }) + .collect(); + let find = |c: &str| header.channels.find_index_of_channel(&exr::meta::attribute::Text::from(c)); + let rgb = match (find("R"), find("G"), find("B")) { + (Some(r), Some(g), Some(b)) => Some(([r, g, b], find("A"))), + _ => None, + }; + Ok(PlanesReader { + planes, + offset, + layer: (header.layer_size.0, header.layer_size.1), + display, + f16s: vec![], + u32s: vec![], + f32s: vec![], + rgb, + subsampled: header.channels.list.iter().any(|c| c.sampling != exr::math::Vec2(1, 1)), + own_display: header.shared_attributes.display_window != dw, + }) } - let mut aux = AuxChannels::new(w as u32, h as u32, 1.0); +} + +impl ChannelsReader for PlanesReader { + type Channels = PartPlanes; + + fn filter_block(&self, tile: TileCoordinates) -> bool { + tile.is_largest_resolution_level() && self.planes.iter().any(Option::is_some) + } + + fn read_block(&mut self, header: &Header, block: UncompressedBlock) -> exr::error::UnitResult { + let ((lw, lh), (w, h)) = (self.layer, self.display); + for line in block.lines(&header.channels) { + let Some(Some((_, kind, plane))) = self.planes.get_mut(line.location.channel) else { continue }; + let (x0, y, count) = (line.location.position.0, line.location.position.1, line.location.sample_count); + // The samples of data-window row `y`, columns `x0..x0 + count`, at display row `ty` + // from display column `tx0`; clipped to the display window. + if y >= lh || x0.checked_add(count).is_none_or(|end| end > lw) { + continue; + } + let Some(ty) = usize::try_from(y as i64 + self.offset.1).ok().filter(|ty| *ty < h) else { continue }; + let tx0 = x0 as i64 + self.offset.0; + let skip = usize::try_from(-tx0).unwrap_or(0).min(count); + let tx = usize::try_from(tx0.max(0)).unwrap_or(usize::MAX); + let take = (count - skip).min(w.saturating_sub(tx)); + let start = ty.checked_mul(w).and_then(|row| row.checked_add(tx)); + let Some(dst) = start.filter(|_| take > 0).and_then(|s| plane.get_mut(s..s.checked_add(take)?)) else { continue }; + match kind { + SampleType::F16 => { + self.f16s.resize(count, exr::prelude::f16::ZERO); + line.read_samples_into_slice(&mut self.f16s)?; + for (d, s) in dst.iter_mut().zip(self.f16s.iter().skip(skip)) { + *d = s.to_f32(); + } + } + SampleType::F32 => { + self.f32s.resize(count, 0.0); + line.read_samples_into_slice(&mut self.f32s)?; + for (d, s) in dst.iter_mut().zip(self.f32s.iter().skip(skip)) { + *d = *s; + } + } + SampleType::U32 => { + self.u32s.resize(count, 0); + line.read_samples_into_slice(&mut self.u32s)?; + for (d, s) in dst.iter_mut().zip(self.u32s.iter().skip(skip)) { + *d = *s as f32; + } + } + } + } + Ok(()) + } + + fn into_channels(self) -> PartPlanes { + // Positions of the kept channels among the kept planes. + let mut at = vec![None; self.planes.len()]; + let mut planes = Vec::with_capacity(self.planes.len()); + for (i, p) in self.planes.into_iter().enumerate() { + if let Some((name, _, data)) = p { + if let Some(slot) = at.get_mut(i) { + *slot = Some(planes.len()); + } + planes.push((name, data)); + } + } + let kept = |i: usize| at.get(i).copied().flatten(); + let has_rgb = self.rgb.is_some(); + let rgb = self.rgb.and_then(|([r, g, b], a)| { + let a = match a { + Some(a) => Some(kept(a)?), + None => None, + }; + Some(([kept(r)?, kept(g)?, kept(b)?], a)) + }); + PartPlanes { planes, rgb, has_rgb, exact: !self.subsampled && !self.own_display } + } +} + +/// How the `image` crate's OpenEXR decoder would show a file: the first part with exact `R`, `G` +/// and `B` channels (alpha `A` when it has one, else opaque), placed in the display window with +/// 0 outside the data window. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Rgb { + /// No part has R, G and B: the decoder refuses the file (a multi-layer file, shown with + /// [`layered_image`]'s rule). + None, + /// The channels (indices in [`AuxChannels::channels`]) hold exactly what it would show. + Planes([usize; 3], Option), + /// The planes can't stand in for it (subsampled channels, a depth-filled plane, another + /// display window): decode with the `image` crate. + Unsupported, +} + +/// A decoded file: its channels and how its picture is made of them. +pub(crate) struct Decoded { + pub(crate) aux: Arc, + rgb: Rgb, +} + +impl Decoded { + /// Bytes held (the planes; names and manifests are small). + fn bytes(&self) -> usize { + self.aux.channels.iter().map(|(n, p)| n.len() + std::mem::size_of_val(p.as_slice())).sum() + } +} + +/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). `None` when it +/// is not a readable EXR. +fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let display = meta.headers.first()?.shared_attributes.display_window; + let read = ReadImage::new(ReadAllLayers { read_channels: Planes { keep, display } }, |_| {}); + let img = read.from_buffered(std::io::Cursor::new(bytes)).ok()?; + let dw = img.attributes.display_window; + let mut aux = AuxChannels::new(dw.size.0 as u32, dw.size.1 as u32, 1.0); let mut crypto: HashMap, Option)> = HashMap::new(); - let mut collect_meta = |other: &HashMap| { + let mut collect_meta = |other: &HashMap| { for (k, v) in other { let key = k.to_string(); let Some(rest) = key.strip_prefix("cryptomatte/") else { continue }; let Some((id, field)) = rest.split_once('/') else { continue }; - let AttributeValue::Text(t) = v else { continue }; + let exr::meta::attribute::AttributeValue::Text(t) = v else { continue }; let e = crypto.entry(id.to_string()).or_default(); match field { "name" => e.0 = Some(t.to_string()), @@ -79,24 +283,22 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } }; collect_meta(&img.attributes.other); - // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a - // multi-gigabyte allocation per channel (#2141). - let n = w.checked_mul(h).filter(|n| *n <= MAX_DISPLAY_PIXELS)?; - // The decoded image is consumed channel by channel, so each channel's samples are freed (or - // moved) as its plane is made instead of the whole file being held twice (#474). + let mut rgb = Rgb::None; for layer in img.layer_data { collect_meta(&layer.attributes.other); - let prefix = layer.attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()); - let size = (layer.size.0, layer.size.1); - let pos = layer.attributes.layer_position; - let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); - for ch in layer.channel_data.list { - let name = channel_name(prefix.as_deref(), &ch.name.to_string()); - let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; - if let Some(plane) = place(ch.sample_data, size, offset, (w, h), n, fill) { - aux.channels.push((name, plane)); - } + let part = layer.channel_data; + if rgb == Rgb::None && part.has_rgb { + let base = aux.channels.len(); + rgb = match part.rgb { + Some(([r, g, b], a)) if part.exact => { + let picked = [Some(r), Some(g), Some(b), a]; + let depth = picked.iter().flatten().any(|i| part.planes.get(*i).is_some_and(|(n, _)| is_depth(n))); + if depth { Rgb::Unsupported } else { Rgb::Planes([base + r, base + g, base + b], a.map(|a| base + a)) } + } + _ => Rgb::Unsupported, + }; } + aux.channels.extend(part.planes); } for (_, (name, manifest)) in crypto { if let (Some(n), Some(m)) = (name, manifest) { @@ -104,46 +306,13 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } } aux.manifests.sort(); - Some(aux) + Some(Decoded { aux: Arc::new(aux), rgb }) } -/// One channel's samples (`size`, at `offset` in the `display` window of `n` pixels) as a -/// display-window plane, `fill` outside its data window. A channel covering exactly the display -/// window is converted in one pass, and an f32 one is moved without a copy. `None` when the -/// channel has fewer samples than its size. -fn place(samples: exr::image::FlatSamples, size: (usize, usize), offset: (i64, i64), display: (usize, usize), n: usize, fill: f32) -> Option> { - use exr::image::FlatSamples; - let (lw, lh) = size; - if samples.len() < lw.checked_mul(lh)? { - return None; - } - if size == display && offset == (0, 0) { - return Some(match samples { - FlatSamples::F32(mut v) => { - v.truncate(n); - v - } - FlatSamples::F16(v) => v.iter().take(n).map(|s| s.to_f32()).collect(), - FlatSamples::U32(v) => v.iter().take(n).map(|s| *s as f32).collect(), - }); - } - let value = |i: usize| match &samples { - FlatSamples::F32(v) => v.get(i).copied(), - FlatSamples::F16(v) => v.get(i).map(|s| s.to_f32()), - FlatSamples::U32(v) => v.get(i).map(|s| *s as f32), - }; - let (w, h) = display; - let mut plane = vec![fill; n]; - for y in 0..lh { - let Some(ty) = usize::try_from(y as i64 + offset.1).ok().filter(|ty| *ty < h) else { continue }; - for x in 0..lw { - let Some(tx) = usize::try_from(x as i64 + offset.0).ok().filter(|tx| *tx < w) else { continue }; - if let (Some(v), Some(p)) = (value(y * lw + x), plane.get_mut(ty * w + tx)) { - *p = v; - } - } - } - Some(plane) +/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. +pub fn read_exr_channels(bytes: &[u8]) -> Option { + let aux = decode(bytes, &|_| true)?.aux; + Some(Arc::try_unwrap(aux).unwrap_or_else(|a| (*a).clone())) } /// Names of the beauty pass, most preferred first, matched against the last part of a layer's @@ -186,6 +355,36 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { .copied() } +/// The planes a multi-layer file's picture is made of (see [`layered_image`]): red, green, blue +/// (one channel thrice for grey) and alpha, by name. Only the names of `aux` are read. +fn picture_names(aux: &AuxChannels) -> Option<([String; 3], Option)> { + let layers = aux.layers(); + let colour = picture_layer(aux, &layers)?; + let names = aux.layer_rgba(colour); + // A layer without R, G and B channels shows its first channel as grey. + let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); + let pick = |k: usize| names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first).to_string(); + let a = names.get(3).filter(|n| !n.is_empty() && aux.get(n).is_some()).cloned(); + Some(([pick(0), pick(1), pick(2)], a)) +} + +/// The channels of the picture of a multi-layer file and nothing else (names first from the +/// headers, then only those channels decoded; in a multi-part file the other parts' pixels are +/// not even decompressed), with the names of its planes. +fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let mut names = AuxChannels::new(0, 0, 1.0); + for header in meta.headers.iter() { + let prefix = part_name(header); + names.channels.extend(header.channels.list.iter().map(|ch| (channel_name(prefix.as_deref(), &ch.name.to_string()), vec![]))); + } + let (rgb, a) = picture_names(&names)?; + // Planes are looked up by name ignoring case, so every channel that could answer is kept. + let keep = |n: &str| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(n)); + let decoded = decode(bytes, &keep)?; + Some((decoded, rgb, a)) +} + /// The picture of a multi-layer OpenEXR file without an unnamed RGB layer (Blender's /// `ViewLayer.Combined.R`, the compositor's `Image.R`, Nuke's `beauty.R`, …), which the image /// decoder rejects. After Effects shows a file's main RGBA channels and leaves the other layers @@ -193,26 +392,13 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { /// that is not a data pass (never Cryptomatte, depth or normals, #412), else the first channel /// as grey. Linear, straight RGBA as the file stores it. `None` when it is not a readable EXR. pub fn layered_image(bytes: &[u8]) -> Option { - picture_of(&read_exr_channels(bytes)?) -} - -/// [`layered_image`] of the EXR at `path` from the channel cache, so the picture and EXtractoR / -/// Cryptomatte share one decode of the file (#474). -pub(crate) fn cached_layered_image(path: &str, bytes: Arc<[u8]>) -> Option { - let aux = cached(path, || Some(bytes))?; - picture_of(&aux) -} - -/// The picture of decoded channels (see [`layered_image`]). -fn picture_of(aux: &AuxChannels) -> Option { - let layers = aux.layers(); - let colour = picture_layer(aux, &layers)?; - let names = aux.layer_rgba(colour); - // A layer without R, G and B channels shows its first channel as grey. - let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); - let plane = |k: usize| -> Option<&[f32]> { aux.get(names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first)) }; - let (r, g, b) = (plane(0)?, plane(1)?, plane(2)?); - let a = names.get(3).filter(|n| !n.is_empty()).and_then(|n| aux.get(n)); + let (d, rgb, a) = picture_only(bytes)?; + let aux = &d.aux; + let [r, g, b] = [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?]; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; let n = (aux.width as usize).checked_mul(aux.height as usize)?; let mut data = Vec::with_capacity(n.checked_mul(4)?); for i in 0..n { @@ -227,40 +413,214 @@ fn picture_of(aux: &AuxChannels) -> Option { } } -type Cache = Mutex>)>>; +/// The picture made of planes `rgb` and `a` (opaque without one) of `aux`, as +/// [`crate::convert::dynamic_to_image`] makes it of the same values. `None` when a plane is +/// missing or short. +fn planes_picture(aux: &AuxChannels, rgb: [&[f32]; 3], a: Option<&[f32]>, op: AlphaOp) -> Option { + let n = (aux.width as usize).checked_mul(aux.height as usize)?; + if rgb.iter().chain(&a).any(|p| p.len() < n) { + return None; + } + let [r, g, b] = rgb; + let at = |p: &[f32], i: usize| p.get(i).copied().unwrap_or(0.0); + Some(float_to_image(aux.width, aux.height, op, |i| [at(r, i), at(g, i), at(b, i), a.map_or(1.0, |a| at(a, i))])) +} -fn cache() -> &'static Cache { - static CACHE: OnceLock = OnceLock::new(); - CACHE.get_or_init(|| Mutex::new(Vec::new())) +/// The picture of a decoded file by [`layered_image`]'s rule. +fn layered_picture_of(aux: &AuxChannels, op: AlphaOp) -> Option { + let (rgb, a) = picture_names(aux)?; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the multi-layer EXR at `path` ([`layered_image`]'s rule) as an [`Image`]: +/// from the channel cache when `shared` (an effect reads the footage's channels, so they are +/// decoded anyway), else of the picture's channels alone. +pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Option { + if shared { + let d = cache().get(path, || Some(bytes))?; + return layered_picture_of(&d.aux, op); + } + let (d, rgb, a) = picture_only(&bytes)?; + let aux = &d.aux; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the EXR at `path` from the channel cache (#474: EXtractoR / Cryptomatte read +/// this footage's channels, so the picture shares their decode): what the `image` crate's +/// decoder shows for a file it reads, [`layered_image`]'s rule for one it refuses. `None` when +/// the planes can't stand in for the `image` crate's picture (decode with it), or the file is +/// not a readable EXR. +pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Option { + let d = cache().get(path, || Some(bytes))?; + let aux = &d.aux; + match d.rgb { + Rgb::None => layered_picture_of(aux, op), + Rgb::Planes([r, g, b], a) => { + let plane = |i: usize| aux.channels.get(i).map(|(_, p)| p.as_slice()); + let a = match a { + Some(a) => Some(plane(a)?), + None => None, + }; + planes_picture(aux, [plane(r)?, plane(g)?, plane(b)?], a, op) + } + Rgb::Unsupported => None, + } +} + +/// How many bytes of decoded channels the cache keeps, like the frame cache's default +/// ([`crate::DEFAULT_BUDGET`]): two 2560×1440 files of 32 channels (470 MB each, as every plane +/// is f32). The newest file is kept whatever its size, so a frame's picture and its effects +/// share its decode. (Measured on such a sequence decoded one frame at a time: 1 GiB held peak +/// memory about 1 GB lower than 2 GiB, at the same speed.) +#[cfg(not(target_arch = "wasm32"))] +pub const CACHE_BUDGET: usize = 1 << 30; +#[cfg(target_arch = "wasm32")] +pub const CACHE_BUDGET: usize = 512 << 20; + +/// At most this many files are cached (unreadable ones hold no bytes). +const CACHE_FILES: usize = 64; + +/// A file's decode: done once, by the first caller; the others wait for it. +type Slot = Arc>>>; + +struct Entry { + path: String, + slot: Slot, + /// Bytes of the finished decode (0 while decoding). + bytes: usize, + /// When it was last asked for (a counter). + used: u64, +} + +/// Decoded EXR files by path, bounded by bytes, least recently used out first. +pub(crate) struct ChannelCache { + entries: Mutex<(Vec, u64)>, + budget: usize, + /// Files decoded (for tests). + decodes: AtomicU64, +} + +impl ChannelCache { + pub(crate) const fn new(budget: usize) -> Self { + Self { entries: Mutex::new((Vec::new(), 0)), budget, decodes: AtomicU64::new(0) } + } + + /// The decoded file at `path`, read with `read` by the first caller; callers asking for the + /// same file meanwhile wait for that decode instead of decoding it again. + pub(crate) fn get(&self, path: &str, read: impl FnOnce() -> Option>) -> Option> { + let slot = { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + match entries.iter_mut().find(|e| e.path == path) { + Some(e) => { + e.used = *clock; + e.slot.clone() + } + None => { + let slot = Slot::default(); + entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: *clock }); + slot + } + } + }; + let mut decoded_here = false; + let v = slot + .get_or_init(|| { + decoded_here = true; + self.decodes.fetch_add(1, Ordering::Relaxed); + read().and_then(|b| decode(&b, &|_| true)).map(Arc::new) + }) + .clone(); + if decoded_here { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + // (gone when the file was forgotten meanwhile) Used as of now: a file asked for + // before another but decoded after it is not the older one, or concurrent frames + // would evict each other's files before their effects read them. + if let Some(e) = entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, &slot)) { + e.bytes = v.as_ref().map_or(0, |d| d.bytes()); + e.used = *clock; + } + Self::evict(entries, self.budget, &slot); + } + v + } + + /// Drop least recently used finished files until the rest fit the budget, keeping `newest`. + fn evict(entries: &mut Vec, budget: usize, newest: &Slot) { + while entries.iter().map(|e| e.bytes).sum::() > budget || entries.len() > CACHE_FILES { + let oldest = entries + .iter() + .enumerate() + .filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && e.slot.get().is_some()) + .min_by_key(|(_, e)| e.used) + .map(|(i, _)| i); + let Some(i) = oldest else { break }; + entries.remove(i); + } + } + + pub(crate) fn forget(&self, path: &str) { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.retain(|e| e.path != path); + } + + #[cfg(test)] + fn decodes(&self) -> u64 { + self.decodes.load(Ordering::Relaxed) + } + + #[cfg(test)] + fn cached(&self) -> Vec { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.iter().map(|e| e.path.clone()).collect() + } +} + +fn cache() -> &'static ChannelCache { + static CACHE: ChannelCache = ChannelCache::new(CACHE_BUDGET); + &CACHE } /// Drop the cached channels of `path` (the file changed on disk). pub(crate) fn forget(path: &str) { - if let Ok(mut c) = cache().lock() { - c.retain(|(p, _)| p != path); - } + cache().forget(path); } -/// Read (and cache, a few files) the channels of the EXR at `path` via `read`. +/// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { - let cache = cache(); - if let Some(v) = cache.lock().ok().and_then(|c| c.iter().find(|(p, _)| p == path).map(|(_, v)| v.clone())) { - return v; - } - let v = read().and_then(|b| read_exr_channels(&b)).map(Arc::new); - if let Ok(mut c) = cache.lock() { - if c.len() >= 4 { - c.remove(0); - } - c.push((path.to_string(), v.clone())); - } - v + cache().get(path, read).map(|d| d.aux.clone()) } #[cfg(test)] mod tests { use super::*; + /// An EXR file of `layers` (with their display window `display`). + fn file(layers: Vec>>, display: exr::prelude::IntegerBounds) -> Vec { + use exr::prelude::*; + let image = Image::from_layers(ImageAttributes::new(display), layers); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() + } + + /// A single-part file of float channels named `names`, `w`×`h`, every sample `v`. + fn flat(names: &[&str], (w, h): (usize, usize), v: f32) -> Vec { + use exr::prelude::*; + let list: Vec> = names.iter().map(|n| AnyChannel::new(*n, FlatSamples::F32(vec![v; w * h]))).collect(); + let layer = Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())); + file(vec![layer], IntegerBounds::from_dimensions((w, h))) + } + #[test] fn reads_layered_channels_and_cryptomatte_metadata() { use exr::prelude::*; @@ -294,53 +654,229 @@ mod tests { assert!(read_exr_channels(b"not an exr").is_none()); } - /// #474: a full-window f32 channel (Cryptomatte ranks, …) becomes its plane without a copy, - /// so a decode no longer holds the file's samples twice; other channels are placed in the - /// display window as before. + /// #474: each line of samples goes straight into its display-window plane: a data window + /// inside the display window is placed with the fill around it (the background depth for + /// depth), one partly outside is clipped, never out of bounds; half and uint samples are + /// converted as the file stores them. #[test] - fn channels_become_planes_without_an_extra_copy() { - use exr::image::FlatSamples; - let v = vec![0.25f32; 6]; - let ptr = v.as_ptr(); - let plane = place(FlatSamples::F32(v), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(plane.as_ptr(), ptr, "moved, not copied"); - assert_eq!(plane, [0.25; 6]); - let half = |x: f32| exr::prelude::f16::from_f32(x); - let full = place(FlatSamples::F16(vec![half(0.5); 6]), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(full, [0.5; 6]); - // A 2×1 data window at (1, 1) of a 3×2 display window; the rest is the fill. - let part = place(FlatSamples::F16(vec![half(1.0), half(2.0)]), (2, 1), (1, 1), (3, 2), 6, 9.0).unwrap(); - assert_eq!(part, [9.0, 9.0, 9.0, 9.0, 1.0, 2.0]); - // Partly outside the display window: clipped, never out of bounds. - let off = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, -1), (3, 2), 6, 0.0).unwrap(); - assert_eq!(off, [0.0; 6]); - let edge = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(edge, [0.0, 0.0, 7.0, 0.0, 0.0, 0.0]); - assert!(place(FlatSamples::F32(vec![1.0; 5]), (3, 2), (0, 0), (3, 2), 6, 0.0).is_none(), "too few samples"); + fn channels_are_placed_in_the_display_window() { + use exr::prelude::*; + let half = |x: f32| f16::from_f32(x); + let part = |pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = LayerAttributes::default(); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::from_dimensions((3, 2)); + // A 2×1 data window at (1, 1). + let inside = file( + vec![part((1, 1), (2, 1), vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))])], + display, + ); + let aux = read_exr_channels(&inside).unwrap(); + assert_eq!(aux.get("G").unwrap(), [0.0, 0.0, 0.0, 0.0, 1.0, 2.0]); + let bg = BACKGROUND_DEPTH; + assert_eq!(aux.get("Z").unwrap(), [bg, bg, bg, bg, 7.0, 8.0]); + // A 2×2 data window at (2, -1): only its bottom-left sample lands at (2, 0). + let off = file(vec![part((2, -1), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0, 2.0, 3.0, 4.0]))])], display); + assert_eq!(read_exr_channels(&off).unwrap().get("G").unwrap(), [0.0, 0.0, 3.0, 0.0, 0.0, 0.0]); + // Entirely outside: all fill. + let away = file(vec![part((-5, 0), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0; 4]))])], display); + assert_eq!(read_exr_channels(&away).unwrap().get("G").unwrap(), [0.0; 6]); + } + + /// #474: concurrent requests for a file that isn't cached decode it once; the others wait + /// for that decode and share its planes. + #[test] + fn concurrent_requests_decode_a_file_once() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "Z"], (64, 64), 0.5).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let reads = AtomicU64::new(0); + let barrier = std::sync::Barrier::new(8); + let got: Vec> = std::thread::scope(|s| { + let jobs: Vec<_> = (0..8) + .map(|_| { + s.spawn(|| { + barrier.wait(); + cache + .get("/474-concurrent.exr", || { + reads.fetch_add(1, Ordering::Relaxed); + // Long enough for the others to arrive while this one decodes. + std::thread::sleep(std::time::Duration::from_millis(50)); + Some(bytes.clone()) + }) + .unwrap() + }) + }) + .collect(); + jobs.into_iter().map(|j| j.join().unwrap()).collect() + }); + assert_eq!(cache.decodes(), 1, "one decode"); + assert_eq!(reads.load(Ordering::Relaxed), 1, "the file is read once"); + assert!(got.iter().all(|d| Arc::ptr_eq(d, &got[0])), "every caller has the same planes"); + assert_eq!(got[0].aux.get("Z").and_then(|z| z.first()).copied(), Some(0.5)); + } + + /// #474: the cache is bounded by the bytes of its planes, least recently used out first; the + /// newest file stays even when it alone is over the budget, and unreadable files are cached + /// (as such) without counting bytes. + #[test] + fn cache_evicts_least_recently_used_by_bytes() { + // 4 float planes of 32×32: 16 KiB of samples (plus a few bytes of names) per file. + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + let get = |p: &str| cache.get(p, || Some(bytes.clone())).unwrap(); + assert_eq!(get("/a.exr").bytes(), one); + get("/b.exr"); + get("/a.exr"); // a is now more recent than b + get("/c.exr"); + assert_eq!(cache.cached(), ["/a.exr", "/c.exr"], "b was least recently used"); + assert_eq!(cache.decodes(), 3); + get("/a.exr"); + assert_eq!(cache.decodes(), 3, "a hit"); + // A file over the budget on its own is still kept while it is the newest. + let small = ChannelCache::new(one / 2); + assert!(small.get("/big.exr", || Some(bytes.clone())).is_some()); + assert_eq!(small.cached(), ["/big.exr"]); + small.get("/big2.exr", || Some(bytes.clone())); + assert_eq!(small.cached(), ["/big2.exr"]); + // An unreadable file is remembered (not read again) without bytes, and forgotten on request. + assert!(cache.get("/broken.exr", || Some(Arc::from(&b"not an exr"[..]))).is_none()); + assert!(cache.get("/broken.exr", || panic!("not read again")).is_none()); + cache.forget("/broken.exr"); + assert!(!cache.cached().contains(&"/broken.exr".to_string())); + // At most CACHE_FILES files, however small. + for i in 0..CACHE_FILES + 10 { + cache.get(&format!("/none{i}.exr"), || None); + } + assert!(cache.cached().len() <= CACHE_FILES); + } + + /// #474: a file counts as used when its decode finishes: one asked for first but decoded + /// after another is the newer of the two, so concurrent frames don't evict each other's + /// files before their effects read them. + #[test] + fn a_file_is_used_as_of_its_decode() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + cache.get("/first.exr", || { + // Another frame's file is asked for and decoded while this one decodes. + cache.get("/second.exr", || Some(bytes.clone())); + Some(bytes.clone()) + }); + cache.get("/third.exr", || Some(bytes.clone())); + assert_eq!(cache.cached(), ["/first.exr", "/third.exr"], "the file decoded first went first"); + } + + /// #474: a file forgotten (changed on disk) while it decodes isn't cached under its path when + /// that decode finishes. + #[test] + fn a_file_forgotten_while_it_decodes_is_not_cached() { + let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let d = cache.get("/changed.exr", || { + cache.forget("/changed.exr"); + Some(bytes.clone()) + }); + assert!(d.is_some()); + assert!(cache.cached().is_empty()); } /// #295: a multi-layer EXR whose colour is only in named layers (`diffuse.R`, as Blender and /// Nuke write them) failed to import ("does not contain non-deep rgb channels"); it now /// shows its colour layer and keeps every channel for EXtractoR. - /// #474: the picture and EXtractoR / Cryptomatte share one decode of a multi-layer file: - /// after the frame is decoded, the aux channels come from the cache without reading the file. + /// #474: once an effect has read a footage item's channels, the picture of each later frame + /// shares one decode with them: the frame's decode is cached and the channels come from it + /// without reading the file again. #[test] fn picture_and_aux_channels_share_one_decode() { use exr::prelude::*; let (w, h) = (2usize, 2usize); - let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); - let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", 0.25)].into()); - let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); - let mut bytes = std::io::Cursor::new(Vec::new()); - image.write().to_buffered(&mut bytes).unwrap(); - let b: Arc<[u8]> = bytes.into_inner().into(); - let path = "/474-shared-decode.exr"; + let frame = |v: f32| -> Arc<[u8]> { + let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); + let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", v)].into()); + let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner().into() + }; + let paths = ["/474-shared-decode_0001.exr", "/474-shared-decode_0002.exr"]; let pool = crate::MediaPool::new(); - pool.add_bytes(path, b.clone()); - let f = crate::probe_bytes(path, b).unwrap(); + pool.add_bytes(paths[0], frame(0.25)); + pool.add_bytes(paths[1], frame(0.75)); + let mut f = crate::probe_bytes(paths[0], frame(0.25)).unwrap(); + f.kind = effectcraft_project::FootageKind::Sequence; + f.sequence = paths.iter().map(|p| p.to_string()).collect(); + let t1 = f.frame_rate.tick_of(1); + use effectcraft_render::FootageSource; + // The first frame: the picture, then an effect reads the channels. pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap(); - let aux = cached(path, || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + let aux = pool.aux(effectcraft_project::ItemId(1), &f, effectcraft_time::Tick::ZERO).unwrap(); assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); + // The next frame's picture decodes the file into the cache, so its channels are not read. + let px = pool.frame_at(&f, t1).unwrap().get(0, 0); + let aux = cached(paths[1], || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.75)); + assert_eq!(px, pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap().get(0, 0), "the same picture"); + } + + /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the + /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 + /// around it), in every alpha interpretation and float curve; a file it can't stand in for + /// (a subsampled channel) is left to the `image` crate. + #[test] + fn picture_from_planes_is_the_image_crates() { + use effectcraft_project::{AlphaMode, Footage}; + use effectcraft_render::FloatCurve; + use exr::prelude::*; + let half = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F16((0..len).map(|i| f16::from_f32(v + i as f32 * 0.37 - 1.0)).collect())); + let float = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F32((0..len).map(|i| ((i * 7) % 11) as f32 * v).collect())); + let part = |name: Option<&str>, pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::SMALL_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::new((1, -1), (5, 4)); + let n = 20; + let files = [ + file(vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], display), + file(vec![part(None, (0, 0), (3, 2), vec![float("R", 0.4, 6), float("G", 0.5, 6), float("B", 0.6, 6), half("Image.R", 0.1, 6)])], display), + file( + vec![ + part(Some("diffuse"), (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n)]), + part(Some("beauty"), (2, 0), (2, 2), vec![half("R", 0.4, 4), half("G", 0.5, 4), half("B", 0.6, 4), half("A", 0.5, 4)]), + ], + display, + ), + ]; + let bits = |i: &effectcraft_raster::Image| i.data.iter().flat_map(|p| p.map(f32::to_bits)).collect::>(); + for (k, bytes) in files.iter().enumerate() { + let path = format!("/474-planes-{k}.exr"); + let d = cache().get(&path, || Some(bytes.as_slice().into())).unwrap(); + assert!(matches!(d.rgb, Rgb::Planes(..)), "file {k}: {:?}", d.rgb); + let img = crate::probe::load_oriented(bytes, Some(image::ImageFormat::OpenExr)).unwrap(); + for alpha in [AlphaMode::Straight, AlphaMode::Premultiplied, AlphaMode::Ignore] { + for curve in [FloatCurve::Srgb, FloatCurve::Linear, FloatCurve::Gamma24] { + let op = AlphaOp::of(&Footage { alpha, premul_color: [0.2, 0.4, 0.6], ..Default::default() }, curve); + let want = crate::convert::dynamic_to_image(&img, op); + let got = cached_picture(&path, bytes.as_slice().into(), op).unwrap(); + assert_eq!((got.width, got.height), (want.width, want.height)); + assert_eq!(bits(&got), bits(&want), "file {k} {alpha:?} {curve:?}"); + } + } + } + // A subsampled channel in the picture's part: the `image` crate decodes it. + let mut sub = part(None, (0, 0), (4, 2), vec![half("R", 0.1, 8), half("G", 0.2, 8), half("B", 0.3, 8), half("C", 0.3, 2)]); + if let Some(c) = sub.channel_data.list.iter_mut().find(|c| c.name == *"C") { + c.sampling = Vec2(2, 2); + } + let mut bytes = std::io::Cursor::new(Vec::new()); + if Image::from_layer(sub).write().to_buffered(&mut bytes).is_ok() { + let d = cache().get("/474-subsampled.exr", || Some(bytes.into_inner().into())).map(|d| d.rgb); + assert!(d.is_none_or(|r| r == Rgb::Unsupported), "{d:?}"); + } } /// #2141: one pixel of data in a 60000² display window is refused, not allocated per channel. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 655ca4ed8..9f1f2aa23 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -245,6 +245,9 @@ struct Inner { /// Movies that failed to open or decode (path to a readable reason; see /// [`FootageSource::problem`]). The first failure of a file is logged, later ones are not. problems: Mutex, String>>, + /// OpenEXR footage (by path) whose channels an effect has read (EXtractoR, Cryptomatte, the 3D + /// Channel effects): its pictures come from the same cached decode (#474). + aux_footage: Mutex>, } /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus @@ -315,6 +318,7 @@ impl MediaPool { conforming: Mutex::default(), conform_failed: Mutex::default(), problems: Mutex::default(), + aux_footage: Mutex::default(), }), } } @@ -856,13 +860,19 @@ impl Inner { { return Err(MediaError::Decode(format!("{path}: {why}"))); } - let loaded = crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)); - let img = match loaded { - Ok(img) => img, + // An effect reads this footage's channels: the picture is made of the same decode (#474). + let shared = lock(&self.aux_footage).contains(&footage.path); + if shared + && path.to_ascii_lowercase().ends_with(".exr") + && let Some(img) = crate::exr_channels::cached_picture(path, bytes.clone(), op) + { + return Ok(img); + } + match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { + Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. - Err(e) => crate::exr_channels::cached_layered_image(path, bytes).ok_or_else(|| MediaError::Decode(format!("{path}: {e}")))?, - }; - Ok(dynamic_to_image(&img, op)) + Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared).ok_or_else(|| MediaError::Decode(format!("{path}: {e}"))), + } } fn decode_media(&self, loc: &Loc, footage: &Footage) -> Result { @@ -962,6 +972,14 @@ impl FootageSource for MediaPool { return None; } let path = loc.key.path.clone(); + { + let mut wanted = lock(&self.inner.aux_footage); + // (a bound: names of footage items, never cleared otherwise) + if wanted.len() >= 4096 { + wanted.clear(); + } + wanted.insert(footage.path.clone()); + } crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) } diff --git a/docs/effects-parity.md b/docs/effects-parity.md index bd81a09df..ce4debd01 100644 --- a/docs/effects-parity.md +++ b/docs/effects-parity.md @@ -26,7 +26,7 @@ corpus compares renders, each effect is assumed to differ somewhere, and users k | Category | Effects | On the GPU | Known differences or bugs | |---|---:|---:|---| -| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR decode shared (#474 open: f16 storage and cache sizing remain) | +| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache bounded at 1 GB (#474 open: f16 storage remains) | | Audio | 13 | 0 | CPU by design (applied in the mixdown) | | Blur & Sharpen | 16 | 15 | Radial Blur keeps the layer edges as After Effects 2026 (713 now pixel-identical); Gaussian Blur interaction and output to be measured first ([fidelity/ae-2026-acceptance.md](fidelity/ae-2026-acceptance.md)) | | Channel | 13 | 13 | — | diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 6eeef1719..741582d9e 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); EXRs without an unnamed RGB layer decode twice (#474) | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, footage alone only its picture's channels (#474; decoded channels are held as f32) | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index f7f1df21b..0ebbeeecc 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -120,6 +120,11 @@ Nuke's `beauty`) show the beauty pass: the layer named Combined, Beauty, Image, Color, else the first colour layer that isn't a data pass. Cryptomatte, depth, mist, normals, vectors, positions and IDs are never the picture. EXtractoR reads every other layer and channel. +Footage on its own decodes only the channels of its picture. Once an effect reads an item's +channels (EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for +the picture and the effects together, and kept for a while (up to 1 GB of decoded channels); +several frames asking for the same file at once wait for one decode. + ## Replacing and reloading Importing a layered file as a composition refreshes the decoded sources used by its layers, diff --git a/docs/gaps.md b/docs/gaps.md index 016aeff22..46a96983a 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -65,7 +65,7 @@ are grouped by area; the issue tracker has the rest. | 6 | **Linux and Windows interaction bugs** | Open: #488 rename arrows (hardening landed, reporter asked for details), #492 anchor drag, #268 file drops on Wayland (winit), #613 Direct3D 12 compositor. Now closed: #491, #493, #467, #320, #302, #596 | Many reports come from Linux; some Windows GPUs fail to start the compositor | 25–45 h | G2 · [ui-parity.md](ui-parity.md) | | 7 | **Animation presets: 76 against 621** | After Effects ships 621 `.ffx` presets (Behaviors, Backgrounds, Shapes, Synthetics, Text, Transitions…); we ship 76 original presets (8 text-animator; 23 Transitions: wipes, clock and fan reveals, blinds, dissolves, card flips, CC wipes, fades, #757; 23 Backgrounds: drifting gradients, fog, cloud and smoke, plasma and magma, cells, hexagons, scrolling grids and checks, pulse rings, bokeh, light sweep, #1096, driven by expressions so they keep moving, not seamless loops; 22 Behaviors: wiggle, shake, float, orbit, bounce, drift, pendulum, spin, pulse, heartbeat, blink, strobe, flicker on transform and opacity, #1510) and read only our `.ecpreset`. Missing: Shapes, Synthetics, Image and more Text | Presets are how many designers start; tutorials assume them | 15–35 h (original presets; `.ffx` reading is out of clean-room scope) | G8 · [effects-parity.md](effects-parity.md) | | 8 | **Mask and shape bugs** | Open: #1004 mask vertex multi-select, #999 auto-pan, #1042 / #1033 vertex nodes under the bounding box, #959 / #960 shape group toggles and operators. Now closed: #510, #513, #527, #509, #303, #553, #932 (Convert To Bezier Path on a shape layer's parametric path) | Masks are core compositing | 10–20 h | G2 · [target-app-parity.md](target-app-parity.md) | -| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (decode shared, faffad3a; f16 storage and cache sizing open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | +| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | | 10 | **Colour pipeline differences** | #340 ProRes 4444 tagged bt709 but sRGB-encoded (open); ACES 2.0 / OCIO 2.5 configs (26.5) missing; now closed: #494 32-bpc blend modes, #495 and #750 Working Gamma 2.4, #546 / #547 viewer colour | Wrong colour in delivery is a client-visible failure | 15–30 h | G1 · [file-format-parity.md](file-format-parity.md) | | 11 | **Text rendering and layout** | #453 CJK text in comps: the Han fallback now reaches the installed Chinese faces; open: #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | From 2939b3769cb98274af5c7c83a2e302656fba820e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 02:18:30 +0100 Subject: [PATCH 3/5] #474: format the OpenEXR golden test Formatting only; no hash changes. Co-Authored-By: Claude Opus 5.5 --- crates/media/tests/it/exr_golden.rs | 34 ++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs index 498944882..f94b3cac0 100644 --- a/crates/media/tests/it/exr_golden.rs +++ b/crates/media/tests/it/exr_golden.rs @@ -95,7 +95,14 @@ fn files() -> Vec<(&'static str, Vec, bool)> { let origin = ((0, 0), full); let cat = |a: &[(&'static str, Kind)], b: &[(&'static str, Kind)]| -> Vec<(&'static str, Kind)> { a.iter().chain(b).copied().collect() }; let layered: Vec<(&str, Kind)> = cat( - &[("Image.R", Kind::Half), ("Image.G", Kind::Half), ("Image.B", Kind::Half), ("Image.A", Kind::Half), ("depth.Z", Kind::Float), ("ObjectID", Kind::Uint)], + &[ + ("Image.R", Kind::Half), + ("Image.G", Kind::Half), + ("Image.B", Kind::Half), + ("Image.A", Kind::Half), + ("depth.Z", Kind::Float), + ("ObjectID", Kind::Uint), + ], &CRYPTO, ); let unnamed_plus: Vec<(&str, Kind)> = @@ -103,7 +110,11 @@ fn files() -> Vec<(&'static str, Vec, bool)> { let rgb_float = [("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("depth.Z", Kind::Float)]; vec![ ("rgba_half_zip16", write(vec![part(None, &RGBA_HALF, full, (0, 0), Compression::ZIP16)], origin), true), - ("rgb_float_piz", write(vec![part(None, &[("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("Z", Kind::Float)], full, (0, 0), Compression::PIZ)], origin), true), + ( + "rgb_float_piz", + write(vec![part(None, &[("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("Z", Kind::Float)], full, (0, 0), Compression::PIZ)], origin), + true, + ), ("layered_rle", write(vec![part(None, &layered, full, (0, 0), Compression::RLE)], origin), true), ("unnamed_rgb_plus_layers_zip1", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::ZIP1)], origin), true), ( @@ -112,7 +123,13 @@ fn files() -> Vec<(&'static str, Vec, bool)> { vec![ part(Some("Image"), &RGBA_HALF, full, (0, 0), Compression::ZIP16), part(Some("depth"), &[("Z", Kind::Float)], full, (0, 0), Compression::ZIP16), - part(Some("CryptoObject00"), &[("r", Kind::Float), ("g", Kind::Float), ("b", Kind::Float), ("a", Kind::Float)], full, (0, 0), Compression::ZIP16), + part( + Some("CryptoObject00"), + &[("r", Kind::Float), ("g", Kind::Float), ("b", Kind::Float), ("a", Kind::Float)], + full, + (0, 0), + Compression::ZIP16, + ), ], origin, ), @@ -138,7 +155,12 @@ fn files() -> Vec<(&'static str, Vec, bool)> { part(Some("ViewLayer.Depth"), &[("ViewLayer.Depth.Z", Kind::Float)], full, (0, 0), Compression::RLE), part( Some("ViewLayer.Combined"), - &[("ViewLayer.Combined.R", Kind::Half), ("ViewLayer.Combined.G", Kind::Half), ("ViewLayer.Combined.B", Kind::Half), ("ViewLayer.Combined.A", Kind::Half)], + &[ + ("ViewLayer.Combined.R", Kind::Half), + ("ViewLayer.Combined.G", Kind::Half), + ("ViewLayer.Combined.B", Kind::Half), + ("ViewLayer.Combined.A", Kind::Half), + ], full, (0, 0), Compression::RLE, @@ -194,7 +216,9 @@ fn aux_hash(pool: &MediaPool, f: &Footage) -> u64 { /// The interpretations a picture is decoded in: (label, footage, curve, size, pinned). fn variants(f: &Footage) -> Vec<(String, Footage, FloatCurve, Option<(u32, u32)>, bool)> { let mut out = vec![]; - for (mode, label) in [(None, "probed"), (Some(AlphaMode::Straight), "straight"), (Some(AlphaMode::Premultiplied), "premul"), (Some(AlphaMode::Ignore), "ignore")] { + for (mode, label) in + [(None, "probed"), (Some(AlphaMode::Straight), "straight"), (Some(AlphaMode::Premultiplied), "premul"), (Some(AlphaMode::Ignore), "ignore")] + { let mut g = f.clone(); if let Some(m) = mode { g.alpha = m; From d4816719474e4b1d5fa3ae6272f781333b421b57 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 02:20:15 +0100 Subject: [PATCH 4/5] #474: golden OpenEXR files for tiles, mip levels, line order, uncompressed and uint RGB Adds tiled (ZIP16, RLE), tiled mip-mapped (ZIP16, PIZ), decreasing line order, uncompressed, a uint R/G/B picture and a multi-part file whose first R/G/B part is depth-named (the planes can't stand in for the image crate's picture there). The new hashes were taken with main's crates/media/src at a646bdcc (extracted with git archive) and the same test, before the decode rework. Co-Authored-By: Claude Opus 5.5 --- crates/media/tests/it/exr_golden.rs | 156 +++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs index f94b3cac0..d99de635a 100644 --- a/crates/media/tests/it/exr_golden.rs +++ b/crates/media/tests/it/exr_golden.rs @@ -70,9 +70,36 @@ fn with_manifest(mut attrs: LayerAttributes) -> LayerAttributes { /// One part: `list` channels over a data window of `size` at `pos`. fn part(name: Option<&str>, list: &[(&str, Kind)], size: (usize, usize), pos: (i32, i32), compression: Compression) -> Layer> { + part_with(name, list, size, pos, encoding(compression)) +} + +/// [`part`] with its own block layout (tiles) and line order. +fn part_with(name: Option<&str>, list: &[(&str, Kind)], size: (usize, usize), pos: (i32, i32), encoding: Encoding) -> Layer> { let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); attrs.layer_position = Vec2(pos.0, pos.1); - Layer::new(size, with_manifest(attrs), encoding(compression), channels(list, size)) + Layer::new(size, with_manifest(attrs), encoding, channels(list, size)) +} + +/// A tiled single-part file of `list` with mip levels down to 1×1 (each level its own values); +/// the picture and the channels are its largest level. +fn mipmapped(list: &[(&str, Kind)], compression: Compression) -> Vec { + let levels = |name: &str, kind: Kind, c: usize| { + let (mut data, mut size) = (vec![], (W, H)); + loop { + data.push(channel(name, kind, size, c + data.len()).sample_data); + if size == (1, 1) { + break; + } + size = ((size.0 / 2).max(1), (size.1 / 2).max(1)); + } + AnyChannel::new(name, Levels::Mip { rounding_mode: exr::math::RoundingMode::Down, level_data: data.into() }) + }; + let list: Vec>> = list.iter().enumerate().map(|(c, (n, k))| levels(n, *k, c)).collect(); + let encoding = Encoding { compression, blocks: Blocks::Tiles(Vec2(16, 16)), line_order: LineOrder::Increasing }; + let layer = Layer::new((W, H), with_manifest(LayerAttributes::default()), encoding, AnyChannels::sort(list.into())); + let mut bytes = std::io::Cursor::new(Vec::new()); + Image::from_layer(layer).write().non_parallel().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() } /// Encode `layers` with the display window `display` (position, size). @@ -176,6 +203,63 @@ fn files() -> Vec<(&'static str, Vec, bool)> { ("layered_window_larger_piz", write(vec![part(None, &layered, (45, 30), (-4, -3), Compression::PIZ)], origin), true), ("unnamed_rgb_plus_layers_dwaa", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::DWAA(None))], origin), false), ("layered_dwaa", write(vec![part(None, &layered, full, (0, 0), Compression::DWAA(None))], origin), false), + ( + "unnamed_rgb_plus_layers_tiled_zip16", + write( + vec![part_with(None, &unnamed_plus, (30, 20), (3, 2), Encoding { blocks: Blocks::Tiles(Vec2(16, 8)), ..encoding(Compression::ZIP16) })], + origin, + ), + true, + ), + ( + "layered_tiled_rle", + write(vec![part_with(None, &layered, full, (0, 0), Encoding { blocks: Blocks::Tiles(Vec2(8, 8)), ..encoding(Compression::RLE) })], origin), + true, + ), + ("unnamed_rgb_plus_layers_mipmapped_zip16", mipmapped(&unnamed_plus, Compression::ZIP16), true), + ("layered_mipmapped_piz", mipmapped(&layered, Compression::PIZ), true), + ( + "unnamed_rgb_plus_layers_decreasing_zip1", + write( + vec![part_with(None, &unnamed_plus, (20, 10), (5, 7), Encoding { line_order: LineOrder::Decreasing, ..encoding(Compression::ZIP1) })], + origin, + ), + true, + ), + ( + "layered_decreasing_rle", + write(vec![part_with(None, &layered, full, (0, 0), Encoding { line_order: LineOrder::Decreasing, ..encoding(Compression::RLE) })], origin), + true, + ), + ("unnamed_rgb_plus_layers_uncompressed", write(vec![part(None, &unnamed_plus, (20, 10), (-3, 4), Compression::Uncompressed)], origin), true), + ("layered_uncompressed", write(vec![part(None, &layered, full, (0, 0), Compression::Uncompressed)], origin), true), + ( + "uint_rgb_zip16", + write( + vec![part( + None, + &[("R", Kind::Uint), ("G", Kind::Uint), ("B", Kind::Uint), ("A", Kind::Half), ("depth.Z", Kind::Float)], + full, + (0, 0), + Compression::ZIP16, + )], + origin, + ), + true, + ), + ( + // The image crate shows the part "depth" (the first with R, G and B), whose planes are + // filled with the background depth outside its data window: decoded by the image crate. + "multipart_depth_named_rgb_zip16", + write( + vec![ + part(Some("depth"), &[("R", Kind::Half), ("G", Kind::Half), ("B", Kind::Half)], (20, 10), (5, 7), Compression::ZIP16), + part(Some("beauty"), &RGBA_HALF, full, (0, 0), Compression::ZIP16), + ], + origin, + ), + true, + ), ] } @@ -331,6 +415,76 @@ const PINNED: &[(&str, &str, u64)] = &[ ("layered_window_larger_piz", "ignore/linear", 0xaa01706accce2cc4), ("layered_window_larger_piz", "preserve_rgb", 0xb2cbb40b10bc56b4), ("layered_window_larger_piz", "aux", 0xb7d91c8eea56a772), + ("unnamed_rgb_plus_layers_tiled_zip16", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_tiled_zip16", "probed/linear", 0x7612f20634fe957e), + ("unnamed_rgb_plus_layers_tiled_zip16", "straight/linear", 0xaa5f42cb6a31c78a), + ("unnamed_rgb_plus_layers_tiled_zip16", "premul/linear", 0x7612f20634fe957e), + ("unnamed_rgb_plus_layers_tiled_zip16", "ignore/linear", 0xbbf236635dcaac90), + ("unnamed_rgb_plus_layers_tiled_zip16", "preserve_rgb", 0x15b555a243d79613), + ("unnamed_rgb_plus_layers_tiled_zip16", "aux", 0x3d1439eed2202e41), + ("layered_tiled_rle", "probe", 0x0a32e074a28c1f66), + ("layered_tiled_rle", "probed/linear", 0xa84600283dce98f6), + ("layered_tiled_rle", "straight/linear", 0xf028e1367a2fbb30), + ("layered_tiled_rle", "premul/linear", 0xa84600283dce98f6), + ("layered_tiled_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_tiled_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_tiled_rle", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "probed/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "straight/linear", 0xf028e1367a2fbb30), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "premul/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "ignore/linear", 0xa90a57f9a67f7468), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "preserve_rgb", 0xc8e0050f5efe75b7), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "aux", 0x27cc9b53d0862db1), + ("layered_mipmapped_piz", "probe", 0x0a32e074a28c1f66), + ("layered_mipmapped_piz", "probed/linear", 0xa84600283dce98f6), + ("layered_mipmapped_piz", "straight/linear", 0xf028e1367a2fbb30), + ("layered_mipmapped_piz", "premul/linear", 0xa84600283dce98f6), + ("layered_mipmapped_piz", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_mipmapped_piz", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_mipmapped_piz", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_decreasing_zip1", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_decreasing_zip1", "probed/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_plus_layers_decreasing_zip1", "straight/linear", 0x2c73f9acaa936420), + ("unnamed_rgb_plus_layers_decreasing_zip1", "premul/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_plus_layers_decreasing_zip1", "ignore/linear", 0x3b86057d18a16bf0), + ("unnamed_rgb_plus_layers_decreasing_zip1", "preserve_rgb", 0x5b999014e4a64bb2), + ("unnamed_rgb_plus_layers_decreasing_zip1", "aux", 0x8f78d3233450acf3), + ("layered_decreasing_rle", "probe", 0x0a32e074a28c1f66), + ("layered_decreasing_rle", "probed/linear", 0xa84600283dce98f6), + ("layered_decreasing_rle", "straight/linear", 0xf028e1367a2fbb30), + ("layered_decreasing_rle", "premul/linear", 0xa84600283dce98f6), + ("layered_decreasing_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_decreasing_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_decreasing_rle", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_uncompressed", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_uncompressed", "probed/linear", 0xa27668bf31d9426b), + ("unnamed_rgb_plus_layers_uncompressed", "straight/linear", 0xfc73fe89457ce029), + ("unnamed_rgb_plus_layers_uncompressed", "premul/linear", 0xa27668bf31d9426b), + ("unnamed_rgb_plus_layers_uncompressed", "ignore/linear", 0xb3276cc5126776d4), + ("unnamed_rgb_plus_layers_uncompressed", "preserve_rgb", 0xdee1105fa6e6822a), + ("unnamed_rgb_plus_layers_uncompressed", "aux", 0x854a5e84e7aaa5c2), + ("layered_uncompressed", "probe", 0x0a32e074a28c1f66), + ("layered_uncompressed", "probed/linear", 0xa84600283dce98f6), + ("layered_uncompressed", "straight/linear", 0xf028e1367a2fbb30), + ("layered_uncompressed", "premul/linear", 0xa84600283dce98f6), + ("layered_uncompressed", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_uncompressed", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_uncompressed", "aux", 0xa6b9bd9f02c192ad), + ("uint_rgb_zip16", "probe", 0x0a32e074a28c1f66), + ("uint_rgb_zip16", "probed/linear", 0xed89310ef0cdb9fd), + ("uint_rgb_zip16", "straight/linear", 0xf9fe11ab8ed97cc8), + ("uint_rgb_zip16", "premul/linear", 0xed89310ef0cdb9fd), + ("uint_rgb_zip16", "ignore/linear", 0xf78cd263c0d74f12), + ("uint_rgb_zip16", "preserve_rgb", 0x2a19d67b07cf4359), + ("uint_rgb_zip16", "aux", 0x03e42411f8521856), + ("multipart_depth_named_rgb_zip16", "probe", 0x6a38346c4c5662d5), + ("multipart_depth_named_rgb_zip16", "probed/linear", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "straight/linear", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "premul/linear", 0x869374e9b0384cea), + ("multipart_depth_named_rgb_zip16", "ignore/linear", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "preserve_rgb", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "aux", 0x0fa56884e83b4417), ]; #[test] From eedd64906c5859b9118bc3b0264646c3719bb699 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 02:25:07 +0100 Subject: [PATCH 5/5] #474: keep files in use in the channel cache, cap plane memory, fall back on a missing cache entry Review follow-ups: - Files in use are never dropped from the channel cache: planes an effect holds, the files of the last 4 pictures made of their planes until their effects ask for the channels, and decodes in progress, which now count the planes they will allocate (from the header) against the budget. With 4 frames at once over a budget of one file, each file decodes once (test). - A decode that panics removes its entry instead of leaving it decoding. - One decode allocates at most 8 GiB of planes (512 MiB on 32-bit and wasm); a header asking for more is refused with the reason. - A shared picture whose file has no cached channels (a failed read, a broken part it doesn't use) is decoded from its own channels. - Comment why waiting on a decode may block a rayon worker; drop the subsampling check and test (the exr crate refuses subsampled files). - Docs: the cache is a soft bound; a single-part file still decompresses every channel; the first frame and the image crate's route decode twice. Co-Authored-By: Claude Opus 5.5 --- crates/media/src/exr_channels.rs | 389 +++++++++++++++++++++------- crates/media/src/pool.rs | 3 +- crates/media/tests/it/exr_golden.rs | 2 +- docs/effects-parity.md | 2 +- docs/file-format-parity.md | 2 +- docs/footage.md | 10 +- docs/gaps.md | 2 +- 7 files changed, 309 insertions(+), 101 deletions(-) diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index 747882337..8a8f5edcc 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -93,8 +93,6 @@ struct PlanesReader { f32s: Vec, /// The part's channel indices of exact `R`, `G`, `B` and `A`, when it has R, G and B. rgb: Option<([usize; 3], Option)>, - /// The part has a channel that doesn't sample every pixel. - subsampled: bool, /// The part's display window differs from the first part's. own_display: bool, } @@ -148,7 +146,6 @@ impl<'s> ReadChannels<'s> for Planes<'_> { u32s: vec![], f32s: vec![], rgb, - subsampled: header.channels.list.iter().any(|c| c.sampling != exr::math::Vec2(1, 1)), own_display: header.shared_attributes.display_window != dw, }) } @@ -226,7 +223,7 @@ impl ChannelsReader for PlanesReader { }; Some(([kept(r)?, kept(g)?, kept(b)?], a)) }); - PartPlanes { planes, rgb, has_rgb, exact: !self.subsampled && !self.own_display } + PartPlanes { planes, rgb, has_rgb, exact: !self.own_display } } } @@ -240,8 +237,8 @@ enum Rgb { None, /// The channels (indices in [`AuxChannels::channels`]) hold exactly what it would show. Planes([usize; 3], Option), - /// The planes can't stand in for it (subsampled channels, a depth-filled plane, another - /// display window): decode with the `image` crate. + /// The planes can't stand in for it (a depth-named channel, whose plane is filled with the + /// background depth, or another display window): decode with the `image` crate. Unsupported, } @@ -258,13 +255,52 @@ impl Decoded { } } -/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). `None` when it -/// is not a readable EXR. -fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { - let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; - let display = meta.headers.first()?.shared_attributes.display_window; +/// The most bytes of planes one decode allocates (every kept channel is a display-window plane +/// of f32, allocated before any pixels are read): a header can ask for far more than the file +/// holds. +#[cfg(target_pointer_width = "64")] +const MAX_DECODE_BYTES: usize = 8 << 30; +#[cfg(not(target_pointer_width = "64"))] +const MAX_DECODE_BYTES: usize = 512 << 20; + +/// Why a file's channels weren't decoded. +#[derive(Debug, PartialEq)] +enum Refused { + /// Not a readable OpenEXR file. + Unreadable, + /// Its kept channels need more memory than [`MAX_DECODE_BYTES`] (the reason, to show). + TooLarge(String), +} + +/// Bytes the planes of the channels `keep` names would take (`None`: not a readable header). +fn planes_bytes(meta: &exr::meta::MetaData, keep: &dyn Fn(&str) -> bool) -> Option { + let dw = meta.headers.first()?.shared_attributes.display_window; + let n = dw.size.0.checked_mul(dw.size.1)?; + let mut kept = 0usize; + for header in meta.headers.iter() { + let prefix = part_name(header); + kept = kept.saturating_add(header.channels.list.iter().filter(|ch| keep(&channel_name(prefix.as_deref(), &ch.name.to_string()))).count()); + } + Some(n.saturating_mul(kept).saturating_mul(std::mem::size_of::())) +} + +/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). +fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Result { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).map_err(|_| Refused::Unreadable)?; + let display = meta.headers.first().ok_or(Refused::Unreadable)?.shared_attributes.display_window; + let need = planes_bytes(&meta, keep).ok_or(Refused::Unreadable)?; + if need > MAX_DECODE_BYTES { + let gb = |b: usize| b as f64 / f64::from(1u32 << 30); + return Err(Refused::TooLarge(format!( + "its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", + display.size.0, + display.size.1, + gb(need), + gb(MAX_DECODE_BYTES) + ))); + } let read = ReadImage::new(ReadAllLayers { read_channels: Planes { keep, display } }, |_| {}); - let img = read.from_buffered(std::io::Cursor::new(bytes)).ok()?; + let img = read.from_buffered(std::io::Cursor::new(bytes)).map_err(|_| Refused::Unreadable)?; let dw = img.attributes.display_window; let mut aux = AuxChannels::new(dw.size.0 as u32, dw.size.1 as u32, 1.0); let mut crypto: HashMap, Option)> = HashMap::new(); @@ -306,12 +342,12 @@ fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { } } aux.manifests.sort(); - Some(Decoded { aux: Arc::new(aux), rgb }) + Ok(Decoded { aux: Arc::new(aux), rgb }) } /// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. pub fn read_exr_channels(bytes: &[u8]) -> Option { - let aux = decode(bytes, &|_| true)?.aux; + let aux = decode(bytes, &|_| true).ok()?.aux; Some(Arc::try_unwrap(aux).unwrap_or_else(|a| (*a).clone())) } @@ -371,18 +407,18 @@ fn picture_names(aux: &AuxChannels) -> Option<([String; 3], Option)> { /// The channels of the picture of a multi-layer file and nothing else (names first from the /// headers, then only those channels decoded; in a multi-part file the other parts' pixels are /// not even decompressed), with the names of its planes. -fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> { - let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; +fn picture_only(bytes: &[u8]) -> Result<(Decoded, [String; 3], Option), Refused> { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).map_err(|_| Refused::Unreadable)?; let mut names = AuxChannels::new(0, 0, 1.0); for header in meta.headers.iter() { let prefix = part_name(header); names.channels.extend(header.channels.list.iter().map(|ch| (channel_name(prefix.as_deref(), &ch.name.to_string()), vec![]))); } - let (rgb, a) = picture_names(&names)?; + let (rgb, a) = picture_names(&names).ok_or(Refused::Unreadable)?; // Planes are looked up by name ignoring case, so every channel that could answer is kept. let keep = |n: &str| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(n)); let decoded = decode(bytes, &keep)?; - Some((decoded, rgb, a)) + Ok((decoded, rgb, a)) } /// The picture of a multi-layer OpenEXR file without an unnamed RGB layer (Blender's @@ -392,7 +428,7 @@ fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> /// that is not a data pass (never Cryptomatte, depth or normals, #412), else the first channel /// as grey. Linear, straight RGBA as the file stores it. `None` when it is not a readable EXR. pub fn layered_image(bytes: &[u8]) -> Option { - let (d, rgb, a) = picture_only(bytes)?; + let (d, rgb, a) = picture_only(bytes).ok()?; let aux = &d.aux; let [r, g, b] = [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?]; let a = match &a { @@ -438,19 +474,24 @@ fn layered_picture_of(aux: &AuxChannels, op: AlphaOp) -> Option { /// The picture of the multi-layer EXR at `path` ([`layered_image`]'s rule) as an [`Image`]: /// from the channel cache when `shared` (an effect reads the footage's channels, so they are -/// decoded anyway), else of the picture's channels alone. -pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Option { - if shared { - let d = cache().get(path, || Some(bytes))?; - return layered_picture_of(&d.aux, op); +/// decoded anyway), else of the picture's channels alone (also when the cache has no channels +/// for the file: a read that failed, or a broken part the picture doesn't use). `Err(Some(why))` +/// when the file is too large to decode, `Err(None)` when it isn't a readable multi-layer EXR. +pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Result> { + if shared && let Some(d) = cache().get(path, Use::Picture, || Some(bytes.clone())) { + return layered_picture_of(&d.aux, op).ok_or(None); } - let (d, rgb, a) = picture_only(&bytes)?; + let (d, rgb, a) = picture_only(&bytes).map_err(|r| match r { + Refused::TooLarge(why) => Some(why), + Refused::Unreadable => None, + })?; let aux = &d.aux; + let plane = |n: &str| aux.get(n).ok_or(None); let a = match &a { - Some(n) => Some(aux.get(n)?), + Some(n) => Some(plane(n)?), None => None, }; - planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) + planes_picture(aux, [plane(&rgb[0])?, plane(&rgb[1])?, plane(&rgb[2])?], a, op).ok_or(None) } /// The picture of the EXR at `path` from the channel cache (#474: EXtractoR / Cryptomatte read @@ -459,7 +500,7 @@ pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: /// the planes can't stand in for the `image` crate's picture (decode with it), or the file is /// not a readable EXR. pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Option { - let d = cache().get(path, || Some(bytes))?; + let d = cache().get(path, Use::Picture, || Some(bytes))?; let aux = &d.aux; match d.rgb { Rgb::None => layered_picture_of(aux, op), @@ -477,9 +518,11 @@ pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Optio /// How many bytes of decoded channels the cache keeps, like the frame cache's default /// ([`crate::DEFAULT_BUDGET`]): two 2560×1440 files of 32 channels (470 MB each, as every plane -/// is f32). The newest file is kept whatever its size, so a frame's picture and its effects -/// share its decode. (Measured on such a sequence decoded one frame at a time: 1 GiB held peak -/// memory about 1 GB lower than 2 GiB, at the same speed.) +/// is f32). A soft bound: the newest file is kept whatever its size (so a frame's picture and its +/// effects share its decode), and files still in use are never dropped (planes an effect holds, +/// [`HELD_FILES`] pictures waiting for their effects, decodes in progress), so memory can go +/// above it while they are. (Measured on such a sequence decoded one frame at a time: 1 GiB held +/// peak memory about 1 GB lower than 2 GiB, at the same speed.) #[cfg(not(target_arch = "wasm32"))] pub const CACHE_BUDGET: usize = 1 << 30; #[cfg(target_arch = "wasm32")] @@ -488,90 +531,178 @@ pub const CACHE_BUDGET: usize = 512 << 20; /// At most this many files are cached (unreadable ones hold no bytes). const CACHE_FILES: usize = 64; +/// At most this many files whose picture was made of their planes are kept for their effects +/// (the renderer asks for a frame's picture, then its channels): about as many frames as are +/// rendered at once. The oldest stops being kept when another comes. +const HELD_FILES: usize = 4; + /// A file's decode: done once, by the first caller; the others wait for it. type Slot = Arc>>>; struct Entry { path: String, slot: Slot, - /// Bytes of the finished decode (0 while decoding). + /// Bytes of the finished decode, or the planes it will allocate while it decodes. bytes: usize, - /// When it was last asked for (a counter). + /// When it was last asked for or finished decoding (a counter). used: u64, } +impl Entry { + /// Dropping it would free nothing or lose a decode in progress: an effect (or a picture + /// waiting for its effects) still holds its planes, or it is decoding. + fn in_use(&self) -> bool { + match self.slot.get() { + None => true, + Some(None) => false, + Some(Some(d)) => Arc::strong_count(d) > 1 || Arc::strong_count(&d.aux) > 1, + } + } +} + +struct State { + entries: Vec, + clock: u64, + /// Files whose picture was made of their planes and whose channels haven't been asked for + /// since, oldest first (at most [`HELD_FILES`]). + held: std::collections::VecDeque<(String, Arc)>, +} + +/// Who asks for a file's channels. +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum Use { + /// The picture of a footage item whose effects read its channels next: kept for them. + Picture, + /// The channels themselves (EXtractoR, Cryptomatte, the 3D Channel effects). + Channels, +} + /// Decoded EXR files by path, bounded by bytes, least recently used out first. pub(crate) struct ChannelCache { - entries: Mutex<(Vec, u64)>, + state: Mutex, budget: usize, /// Files decoded (for tests). decodes: AtomicU64, } +/// Removes a decode's entry if the decode panics, so the file doesn't stay "decoding" (never +/// dropped, its estimated bytes counted) until someone asks for it again. +struct Abandon<'a> { + cache: &'a ChannelCache, + slot: &'a Slot, +} + +impl Drop for Abandon<'_> { + fn drop(&mut self) { + if std::thread::panicking() { + self.cache.lock().entries.retain(|e| !Arc::ptr_eq(&e.slot, self.slot)); + } + } +} + impl ChannelCache { pub(crate) const fn new(budget: usize) -> Self { - Self { entries: Mutex::new((Vec::new(), 0)), budget, decodes: AtomicU64::new(0) } + Self { state: Mutex::new(State { entries: Vec::new(), clock: 0, held: std::collections::VecDeque::new() }), budget, decodes: AtomicU64::new(0) } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Set the bytes of the entry of `slot` (if it's still cached), mark it used now and drop + /// least recently used files until the rest fit the budget. + fn account(&self, slot: &Slot, bytes: usize) { + let mut state = self.lock(); + state.clock += 1; + let clock = state.clock; + // (gone when the file was forgotten meanwhile) Used as of now: a file asked for before + // another but decoded after it is not the older one. + if let Some(e) = state.entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, slot)) { + e.bytes = bytes; + e.used = clock; + } + Self::evict(&mut state.entries, self.budget, slot); } /// The decoded file at `path`, read with `read` by the first caller; callers asking for the /// same file meanwhile wait for that decode instead of decoding it again. - pub(crate) fn get(&self, path: &str, read: impl FnOnce() -> Option>) -> Option> { + pub(crate) fn get(&self, path: &str, by: Use, read: impl FnOnce() -> Option>) -> Option> { let slot = { - let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); - let (entries, clock) = &mut *guard; - *clock += 1; - match entries.iter_mut().find(|e| e.path == path) { + let mut state = self.lock(); + state.clock += 1; + let clock = state.clock; + match state.entries.iter_mut().find(|e| e.path == path) { Some(e) => { - e.used = *clock; + e.used = clock; e.slot.clone() } None => { let slot = Slot::default(); - entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: *clock }); + state.entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: clock }); slot } } }; let mut decoded_here = false; + // Waiting here can block a rayon worker (which `Inner::frame_at` avoids for frames): that + // is safe because the decode never waits on rayon's global pool (the exr crate + // decompresses on a pool of its own and this thread only receives the blocks), so the + // decoding thread can't be waiting for this one. let v = slot .get_or_init(|| { decoded_here = true; self.decodes.fetch_add(1, Ordering::Relaxed); - read().and_then(|b| decode(&b, &|_| true)).map(Arc::new) + let _abandon = Abandon { cache: self, slot: &slot }; + let bytes = read()?; + // The planes it will allocate count against the budget while it decodes. + if let Ok(meta) = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(&bytes[..]), false) + && let Some(need) = planes_bytes(&meta, &|_| true) + { + self.account(&slot, need.min(MAX_DECODE_BYTES)); + } + match decode(&bytes, &|_| true) { + Ok(d) => Some(Arc::new(d)), + Err(Refused::TooLarge(why)) => { + log::warn!("media: {path}: {why}"); + None + } + Err(Refused::Unreadable) => None, + } }) .clone(); if decoded_here { - let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); - let (entries, clock) = &mut *guard; - *clock += 1; - // (gone when the file was forgotten meanwhile) Used as of now: a file asked for - // before another but decoded after it is not the older one, or concurrent frames - // would evict each other's files before their effects read them. - if let Some(e) = entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, &slot)) { - e.bytes = v.as_ref().map_or(0, |d| d.bytes()); - e.used = *clock; - } - Self::evict(entries, self.budget, &slot); + self.account(&slot, v.as_ref().map_or(0, |d| d.bytes())); } + let mut state = self.lock(); + match (by, &v) { + (Use::Picture, Some(d)) => { + if !state.held.iter().any(|(p, _)| p == path) { + state.held.push_back((path.to_string(), d.clone())); + if state.held.len() > HELD_FILES { + state.held.pop_front(); + } + } + } + (Use::Channels, _) => state.held.retain(|(p, _)| p != path), + (Use::Picture, None) => {} + } + drop(state); v } - /// Drop least recently used finished files until the rest fit the budget, keeping `newest`. + /// Drop least recently used files not in use until the rest fit the budget, keeping `newest`. fn evict(entries: &mut Vec, budget: usize, newest: &Slot) { while entries.iter().map(|e| e.bytes).sum::() > budget || entries.len() > CACHE_FILES { - let oldest = entries - .iter() - .enumerate() - .filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && e.slot.get().is_some()) - .min_by_key(|(_, e)| e.used) - .map(|(i, _)| i); + let oldest = entries.iter().enumerate().filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && !e.in_use()).min_by_key(|(_, e)| e.used).map(|(i, _)| i); let Some(i) = oldest else { break }; entries.remove(i); } } pub(crate) fn forget(&self, path: &str) { - self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.retain(|e| e.path != path); + let mut state = self.lock(); + state.entries.retain(|e| e.path != path); + state.held.retain(|(p, _)| p != path); } #[cfg(test)] @@ -581,7 +712,7 @@ impl ChannelCache { #[cfg(test)] fn cached(&self) -> Vec { - self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.iter().map(|e| e.path.clone()).collect() + self.lock().entries.iter().map(|e| e.path.clone()).collect() } } @@ -597,7 +728,7 @@ pub(crate) fn forget(path: &str) { /// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { - cache().get(path, read).map(|d| d.aux.clone()) + cache().get(path, Use::Channels, read).map(|d| d.aux.clone()) } #[cfg(test)] @@ -663,14 +794,17 @@ mod tests { use exr::prelude::*; let half = |x: f32| f16::from_f32(x); let part = |pos: (i32, i32), size: (usize, usize), list: Vec>| { - let mut attrs = LayerAttributes::default(); - attrs.layer_position = Vec2(pos.0, pos.1); + let attrs = LayerAttributes { layer_position: Vec2(pos.0, pos.1), ..Default::default() }; Layer::new(size, attrs, Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())) }; let display = IntegerBounds::from_dimensions((3, 2)); // A 2×1 data window at (1, 1). let inside = file( - vec![part((1, 1), (2, 1), vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))])], + vec![part( + (1, 1), + (2, 1), + vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))], + )], display, ); let aux = read_exr_channels(&inside).unwrap(); @@ -699,7 +833,7 @@ mod tests { s.spawn(|| { barrier.wait(); cache - .get("/474-concurrent.exr", || { + .get("/474-concurrent.exr", Use::Channels, || { reads.fetch_add(1, Ordering::Relaxed); // Long enough for the others to arrive while this one decodes. std::thread::sleep(std::time::Duration::from_millis(50)); @@ -726,7 +860,7 @@ mod tests { let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); let one = 4 * 32 * 32 * 4 + 4; let cache = ChannelCache::new(2 * one); - let get = |p: &str| cache.get(p, || Some(bytes.clone())).unwrap(); + let get = |p: &str| cache.get(p, Use::Channels, || Some(bytes.clone())).unwrap(); assert_eq!(get("/a.exr").bytes(), one); get("/b.exr"); get("/a.exr"); // a is now more recent than b @@ -737,18 +871,18 @@ mod tests { assert_eq!(cache.decodes(), 3, "a hit"); // A file over the budget on its own is still kept while it is the newest. let small = ChannelCache::new(one / 2); - assert!(small.get("/big.exr", || Some(bytes.clone())).is_some()); + assert!(small.get("/big.exr", Use::Channels, || Some(bytes.clone())).is_some()); assert_eq!(small.cached(), ["/big.exr"]); - small.get("/big2.exr", || Some(bytes.clone())); + small.get("/big2.exr", Use::Channels, || Some(bytes.clone())); assert_eq!(small.cached(), ["/big2.exr"]); // An unreadable file is remembered (not read again) without bytes, and forgotten on request. - assert!(cache.get("/broken.exr", || Some(Arc::from(&b"not an exr"[..]))).is_none()); - assert!(cache.get("/broken.exr", || panic!("not read again")).is_none()); + assert!(cache.get("/broken.exr", Use::Channels, || Some(Arc::from(&b"not an exr"[..]))).is_none()); + assert!(cache.get("/broken.exr", Use::Channels, || panic!("not read again")).is_none()); cache.forget("/broken.exr"); assert!(!cache.cached().contains(&"/broken.exr".to_string())); // At most CACHE_FILES files, however small. for i in 0..CACHE_FILES + 10 { - cache.get(&format!("/none{i}.exr"), || None); + cache.get(&format!("/none{i}.exr"), Use::Channels, || None); } assert!(cache.cached().len() <= CACHE_FILES); } @@ -761,22 +895,101 @@ mod tests { let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); let one = 4 * 32 * 32 * 4 + 4; let cache = ChannelCache::new(2 * one); - cache.get("/first.exr", || { + cache.get("/first.exr", Use::Channels, || { // Another frame's file is asked for and decoded while this one decodes. - cache.get("/second.exr", || Some(bytes.clone())); + cache.get("/second.exr", Use::Channels, || Some(bytes.clone())); Some(bytes.clone()) }); - cache.get("/third.exr", || Some(bytes.clone())); + cache.get("/third.exr", Use::Channels, || Some(bytes.clone())); assert_eq!(cache.cached(), ["/first.exr", "/third.exr"], "the file decoded first went first"); } + /// #474: frames rendered at once (each asks for its picture, then its effects for the + /// channels while other frames decode) decode each file once, though the budget holds only + /// one: a picture's file is kept for its effects, and files in use are never dropped. + #[test] + fn concurrent_frames_decode_each_file_once() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "Z"], (64, 64), 0.5).into(); + let cache = ChannelCache::new(4 * 64 * 64 * 4 + 4); + let (next, reads, frames) = (std::sync::atomic::AtomicUsize::new(0), AtomicU64::new(0), 24); + let read = || { + reads.fetch_add(1, Ordering::Relaxed); + Some(bytes.clone()) + }; + std::thread::scope(|s| { + for _ in 0..4 { + s.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= frames { + break; + } + let path = format!("/474-frames/{i:04}.exr"); + drop(cache.get(&path, Use::Picture, read).unwrap()); + std::thread::sleep(std::time::Duration::from_millis(3)); + let aux = cache.get(&path, Use::Channels, read).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(2)); + assert_eq!(aux.aux.get("Z").and_then(|z| z.first()).copied(), Some(0.5)); + } + }); + } + }); + assert_eq!(cache.decodes(), frames as u64, "one decode per frame"); + assert_eq!(reads.load(Ordering::Relaxed), frames as u64); + } + + /// #474: a decode that panics doesn't leave its file "decoding" in the cache (never dropped, + /// its planes counted): the entry goes, and the next request decodes the file. + #[test] + fn a_decode_that_panics_is_not_left_in_the_cache() { + let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| cache.get("/474-panics.exr", Use::Channels, || panic!("the read failed")))); + assert!(r.is_err()); + assert!(cache.cached().is_empty()); + assert!(cache.get("/474-panics.exr", Use::Channels, || Some(bytes.clone())).is_some()); + assert_eq!(cache.cached(), ["/474-panics.exr"]); + } + + /// #474: the picture of a multi-layer file whose effects read its channels comes from the + /// cache, and from the picture's own channels when the cache has none for the file (a read + /// that failed, or a broken part the picture doesn't use). + #[test] + fn a_shared_picture_without_cached_channels_is_still_decoded() { + use effectcraft_project::Footage; + use effectcraft_render::FloatCurve; + let bytes: Arc<[u8]> = flat(&["Image.R", "Image.G", "Image.B", "depth.Z"], (4, 2), 0.5).into(); + let path = "/474-shared-fallback.exr"; + assert!(cache().get(path, Use::Channels, || None).is_none(), "a read that failed"); + let op = AlphaOp::of(&Footage::default(), FloatCurve::Linear); + let img = layered_picture(path, bytes, op, true).unwrap(); + assert_eq!(img.get(1, 1), [0.5, 0.5, 0.5, 1.0]); + } + + /// #474: a header asking for more plane memory than one decode may allocate (16384² display + /// window, 40 channels: 43 GB) is refused with a reason before anything is allocated. + #[test] + fn a_header_asking_for_too_much_memory_is_refused() { + use exr::prelude::*; + let list: Vec> = (0..40).map(|i| AnyChannel::new(format!("pass{i:02}.Y").as_str(), FlatSamples::F32(vec![0.5]))).collect(); + let layer = Layer::new((1, 1), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())); + let bytes = file(vec![layer], IntegerBounds::new((0, 0), (16384, 16384))); + match decode(&bytes, &|_| true) { + Err(Refused::TooLarge(why)) => assert!(why.contains("16384×16384") && why.contains("GB"), "{why}"), + Err(Refused::Unreadable) => panic!("readable"), + Ok(_) => panic!("refused"), + } + assert!(read_exr_channels(&bytes).is_none()); + assert!(cached("/474-too-large.exr", || Some(bytes.as_slice().into())).is_none()); + } + /// #474: a file forgotten (changed on disk) while it decodes isn't cached under its path when /// that decode finishes. #[test] fn a_file_forgotten_while_it_decodes_is_not_cached() { let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); let cache = ChannelCache::new(CACHE_BUDGET); - let d = cache.get("/changed.exr", || { + let d = cache.get("/changed.exr", Use::Channels, || { cache.forget("/changed.exr"); Some(bytes.clone()) }); @@ -824,8 +1037,7 @@ mod tests { /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 - /// around it), in every alpha interpretation and float curve; a file it can't stand in for - /// (a subsampled channel) is left to the `image` crate. + /// around it), in every alpha interpretation and float curve. #[test] fn picture_from_planes_is_the_image_crates() { use effectcraft_project::{AlphaMode, Footage}; @@ -841,7 +1053,10 @@ mod tests { let display = IntegerBounds::new((1, -1), (5, 4)); let n = 20; let files = [ - file(vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], display), + file( + vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], + display, + ), file(vec![part(None, (0, 0), (3, 2), vec![float("R", 0.4, 6), float("G", 0.5, 6), float("B", 0.6, 6), half("Image.R", 0.1, 6)])], display), file( vec![ @@ -854,7 +1069,7 @@ mod tests { let bits = |i: &effectcraft_raster::Image| i.data.iter().flat_map(|p| p.map(f32::to_bits)).collect::>(); for (k, bytes) in files.iter().enumerate() { let path = format!("/474-planes-{k}.exr"); - let d = cache().get(&path, || Some(bytes.as_slice().into())).unwrap(); + let d = cache().get(&path, Use::Channels, || Some(bytes.as_slice().into())).unwrap(); assert!(matches!(d.rgb, Rgb::Planes(..)), "file {k}: {:?}", d.rgb); let img = crate::probe::load_oriented(bytes, Some(image::ImageFormat::OpenExr)).unwrap(); for alpha in [AlphaMode::Straight, AlphaMode::Premultiplied, AlphaMode::Ignore] { @@ -867,16 +1082,6 @@ mod tests { } } } - // A subsampled channel in the picture's part: the `image` crate decodes it. - let mut sub = part(None, (0, 0), (4, 2), vec![half("R", 0.1, 8), half("G", 0.2, 8), half("B", 0.3, 8), half("C", 0.3, 2)]); - if let Some(c) = sub.channel_data.list.iter_mut().find(|c| c.name == *"C") { - c.sampling = Vec2(2, 2); - } - let mut bytes = std::io::Cursor::new(Vec::new()); - if Image::from_layer(sub).write().to_buffered(&mut bytes).is_ok() { - let d = cache().get("/474-subsampled.exr", || Some(bytes.into_inner().into())).map(|d| d.rgb); - assert!(d.is_none_or(|r| r == Rgb::Unsupported), "{d:?}"); - } } /// #2141: one pixel of data in a 60000² display window is refused, not allocated per channel. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 9f1f2aa23..910f8238f 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -871,7 +871,8 @@ impl Inner { match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. - Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared).ok_or_else(|| MediaError::Decode(format!("{path}: {e}"))), + Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared) + .map_err(|why| MediaError::Decode(format!("{path}: {}", why.unwrap_or_else(|| e.to_string())))), } } diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs index d99de635a..f760ed7c2 100644 --- a/crates/media/tests/it/exr_golden.rs +++ b/crates/media/tests/it/exr_golden.rs @@ -92,7 +92,7 @@ fn mipmapped(list: &[(&str, Kind)], compression: Compression) -> Vec { } size = ((size.0 / 2).max(1), (size.1 / 2).max(1)); } - AnyChannel::new(name, Levels::Mip { rounding_mode: exr::math::RoundingMode::Down, level_data: data.into() }) + AnyChannel::new(name, Levels::Mip { rounding_mode: exr::math::RoundingMode::Down, level_data: data }) }; let list: Vec>> = list.iter().enumerate().map(|(c, (n, k))| levels(n, *k, c)).collect(); let encoding = Encoding { compression, blocks: Blocks::Tiles(Vec2(16, 16)), line_order: LineOrder::Increasing }; diff --git a/docs/effects-parity.md b/docs/effects-parity.md index ce4debd01..d80a874bd 100644 --- a/docs/effects-parity.md +++ b/docs/effects-parity.md @@ -26,7 +26,7 @@ corpus compares renders, each effect is assumed to differ somewhere, and users k | Category | Effects | On the GPU | Known differences or bugs | |---|---:|---:|---| -| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache bounded at 1 GB (#474 open: f16 storage remains) | +| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache softly bounded at 1 GB (#474; holding channels as f16 is still to do) | | Audio | 13 | 0 | CPU by design (applied in the mixdown) | | Blur & Sharpen | 16 | 15 | Radial Blur keeps the layer edges as After Effects 2026 (713 now pixel-identical); Gaussian Blur interaction and output to be measured first ([fidelity/ae-2026-acceptance.md](fidelity/ae-2026-acceptance.md)) | | Channel | 13 | 13 | — | diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 741582d9e..648eadcee 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, footage alone only its picture's channels (#474; decoded channels are held as f32) | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, except the first frame an effect reads and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index 0ebbeeecc..50bd4e7d1 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -120,10 +120,12 @@ Nuke's `beauty`) show the beauty pass: the layer named Combined, Beauty, Image, Color, else the first colour layer that isn't a data pass. Cryptomatte, depth, mist, normals, vectors, positions and IDs are never the picture. EXtractoR reads every other layer and channel. -Footage on its own decodes only the channels of its picture. Once an effect reads an item's -channels (EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for -the picture and the effects together, and kept for a while (up to 1 GB of decoded channels); -several frames asking for the same file at once wait for one decode. +Footage on its own keeps only the channels of its picture (a single-part file still +decompresses all of its pixels to read them). Once an effect reads an item's channels +(EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for the +picture and the effects together, and kept for a while: about 1 GB of decoded channels (512 MB +in the browser), a soft bound, as the newest file and files still in use stay whatever their +size. Several frames asking for the same file at once wait for one decode. ## Replacing and reloading diff --git a/docs/gaps.md b/docs/gaps.md index 46a96983a..75e49a1d6 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -65,7 +65,7 @@ are grouped by area; the issue tracker has the rest. | 6 | **Linux and Windows interaction bugs** | Open: #488 rename arrows (hardening landed, reporter asked for details), #492 anchor drag, #268 file drops on Wayland (winit), #613 Direct3D 12 compositor. Now closed: #491, #493, #467, #320, #302, #596 | Many reports come from Linux; some Windows GPUs fail to start the compositor | 25–45 h | G2 · [ui-parity.md](ui-parity.md) | | 7 | **Animation presets: 76 against 621** | After Effects ships 621 `.ffx` presets (Behaviors, Backgrounds, Shapes, Synthetics, Text, Transitions…); we ship 76 original presets (8 text-animator; 23 Transitions: wipes, clock and fan reveals, blinds, dissolves, card flips, CC wipes, fades, #757; 23 Backgrounds: drifting gradients, fog, cloud and smoke, plasma and magma, cells, hexagons, scrolling grids and checks, pulse rings, bokeh, light sweep, #1096, driven by expressions so they keep moving, not seamless loops; 22 Behaviors: wiggle, shake, float, orbit, bounce, drift, pendulum, spin, pulse, heartbeat, blink, strobe, flicker on transform and opacity, #1510) and read only our `.ecpreset`. Missing: Shapes, Synthetics, Image and more Text | Presets are how many designers start; tutorials assume them | 15–35 h (original presets; `.ffx` reading is out of clean-room scope) | G8 · [effects-parity.md](effects-parity.md) | | 8 | **Mask and shape bugs** | Open: #1004 mask vertex multi-select, #999 auto-pan, #1042 / #1033 vertex nodes under the bounding box, #959 / #960 shape group toggles and operators. Now closed: #510, #513, #527, #509, #303, #553, #932 (Convert To Bezier Path on a shape layer's parametric path) | Masks are core compositing | 10–20 h | G2 · [target-app-parity.md](target-app-parity.md) | -| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | +| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage still to do). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | | 10 | **Colour pipeline differences** | #340 ProRes 4444 tagged bt709 but sRGB-encoded (open); ACES 2.0 / OCIO 2.5 configs (26.5) missing; now closed: #494 32-bpc blend modes, #495 and #750 Working Gamma 2.4, #546 / #547 viewer colour | Wrong colour in delivery is a client-visible failure | 15–30 h | G1 · [file-format-parity.md](file-format-parity.md) | | 11 | **Text rendering and layout** | #453 CJK text in comps: the Han fallback now reaches the installed Chinese faces; open: #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) |