From 8add8ac7b401634fc2af0c6d2cde9ce96d1d3f08 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 00:51:59 +0100 Subject: [PATCH] #474: decode a multi-layer EXR once per frame, and only its picture when nothing else reads it - The channel cache decodes a file once however many callers ask for it at the same time (the others wait for that decode), and is bounded by the bytes of its planes (1 GiB, least recently used out first, a file counting as used when its decode finishes, the newest file always kept) instead of four files of any size. - Once an effect has read a footage item's channels, its pictures are made of the cached planes, also for files with an unnamed RGB layer (bit for bit what the image crate shows, which still decodes the files the planes can't stand in for); footage alone decodes only its picture's channels (a multi-layer file's were all decoded and cached for every frame). - Channels are decoded straight into their display-window planes, and the picture is converted from the planes without two interleaved copies. Measured on a 30-frame 2560x1440 sequence of 32 channels (DWAA), one frame at a time: footage alone 0.185 -> 0.073 s per frame and 4.4 -> 1.8 GB peak; with an unnamed RGB layer plus Cryptomatte 0.244 -> 0.159 s; multi-part plus EXtractoR 0.194 -> 0.127 s. Co-Authored-By: Claude Opus 5.5 --- crates/media/src/convert.rs | 52 +- crates/media/src/exr_channels.rs | 816 +++++++++++++++++++++++++------ crates/media/src/pool.rs | 30 +- docs/effects-parity.md | 2 +- docs/file-format-parity.md | 2 +- docs/footage.md | 5 + docs/gaps.md | 2 +- 7 files changed, 742 insertions(+), 167 deletions(-) diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index 9e88ab4dd..a2b24b3ae 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -376,30 +376,20 @@ fn convert_yuv(img: &mut Image, source: YuvSource<'_, T>, k: &Yuv, op pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image { use image::DynamicImage as D; let (w, h) = (img.width(), img.height()); + if let D::ImageRgb32F(_) | D::ImageRgba32F(_) = img { + let px = img.to_rgba32f(); + let raw = px.as_raw(); + return float_to_image(w, h, op, |i| { + let s = &raw[i * 4..i * 4 + 4]; + [s[0], s[1], s[2], s[3]] + }); + } let mut out = Image::new(w, h); let wu = w as usize; if wu == 0 || h == 0 { return out; } match img { - D::ImageRgb32F(_) | D::ImageRgba32F(_) => { - let px = img.to_rgba32f(); - let raw = px.as_raw(); - let premul = op.mode == AlphaMode::Premultiplied; - out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { - for (x, o) in row.iter_mut().enumerate() { - let s = &raw[(y * wu + x) * 4..(y * wu + x) * 4 + 4]; - let a = s[3]; - let c = if premul { - let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; - [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] - } else { - [op.float(s[0]), op.float(s[1]), op.float(s[2])] - }; - *o = op.apply(c, a.clamp(0.0, 1.0)); - } - }); - } D::ImageLuma16(_) | D::ImageLumaA16(_) | D::ImageRgb16(_) | D::ImageRgba16(_) => { let px = img.to_rgba16(); let raw = px.as_raw(); @@ -425,6 +415,32 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image out } +/// Float RGBA (linear light, alpha straight or premultiplied as the footage says), `px(i)` for +/// pixel `i` in rows, to an [`Image`]: how [`dynamic_to_image`] converts float stills, also used +/// for OpenEXR pictures made from decoded channel planes without an interleaved copy (#474). +pub(crate) fn float_to_image(w: u32, h: u32, op: AlphaOp, px: impl Fn(usize) -> [f32; 4] + Sync) -> Image { + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + let premul = op.mode == AlphaMode::Premultiplied; + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = px(y * wu + x); + let a = s[3]; + let c = if premul { + let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; + [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] + } else { + [op.float(s[0]), op.float(s[1]), op.float(s[2])] + }; + *o = op.apply(c, a.clamp(0.0, 1.0)); + } + }); + out +} + /// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`]. pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image { let mut out = Image::new(w, h); diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index d372ec849..747882337 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -6,12 +6,25 @@ //! 0, or the background depth for depth channels). Cryptomatte manifests come from the //! `cryptomatte//name` and `cryptomatte//manifest` header attributes (the published //! Cryptomatte metadata convention: the manifest is a JSON object of name → hex hash). +//! +//! Decoded files are kept in a small cache ([`CACHE_BUDGET`]) that the picture of an EXR +//! footage item shares with EXtractoR / Cryptomatte once an effect has read its channels +//! (#474); a file is decoded once however many callers ask for it at the same time. use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError}; -use effectcraft_raster::AuxChannels; use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, split_channel}; +use effectcraft_raster::{AuxChannels, Image}; +use exr::block::UncompressedBlock; +use exr::block::chunk::TileCoordinates; +use exr::image::read::image::ReadImage; +use exr::image::read::layers::{ChannelsReader, ReadAllLayers, ReadChannels}; +use exr::meta::attribute::SampleType; +use exr::meta::header::Header; + +use crate::convert::{AlphaOp, float_to_image}; /// The largest display window decoded (16384²; vector footage and Continuously Rasterize /// stop at the same size). @@ -33,6 +46,11 @@ fn channel_name(part: Option<&str>, channel: &str) -> String { } } +/// The part name channels are prefixed with (`None`: unnamed). +fn part_name(header: &Header) -> Option { + header.own_attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()) +} + /// Parse a Cryptomatte manifest (`{"name": "hexhash", …}`). pub fn parse_manifest(json: &str) -> Vec<(String, u32)> { let Ok(serde_json::Value::Object(m)) = serde_json::from_str::(json) else { return vec![] }; @@ -53,23 +71,209 @@ pub fn unsupported_compression(bytes: &[u8]) -> Option<&'static str> { .then_some("OpenEXR HTJ2K compression isn't supported yet: re-save the file with ZIP, PIZ or DWAA compression") } -/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. -pub fn read_exr_channels(bytes: &[u8]) -> Option { - use exr::prelude::*; - let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_buffered(std::io::Cursor::new(bytes)).ok()?; - let dw = img.attributes.display_window; - let (w, h) = (dw.size.0, dw.size.1); - if w == 0 || h == 0 { - return None; +/// Which channels a decode keeps, by plane name, and the display window they are placed in +/// (the first part's, as the `exr` crate reports the image's). +struct Planes<'k> { + keep: &'k dyn Fn(&str) -> bool, + display: exr::meta::attribute::IntegerBounds, +} + +/// One part's kept channels while it is decoded: each line of samples goes straight into its +/// channel's display-window plane (no copy of the part's samples in their own type first). +struct PlanesReader { + /// Per channel of the part: its plane name, sample type and plane (`None`: not kept). + planes: Vec)>>, + /// The data window's position in the display window, and the data window's and display + /// window's sizes. + offset: (i64, i64), + layer: (usize, usize), + display: (usize, usize), + f16s: Vec, + u32s: Vec, + f32s: Vec, + /// The part's channel indices of exact `R`, `G`, `B` and `A`, when it has R, G and B. + rgb: Option<([usize; 3], Option)>, + /// The part has a channel that doesn't sample every pixel. + subsampled: bool, + /// The part's display window differs from the first part's. + own_display: bool, +} + +/// A decoded part: its kept planes (in channel order) and what [`PlanesReader`] found. +struct PartPlanes { + planes: Vec<(String, Vec)>, + /// Positions in `planes` of the part's R, G, B (and A) channels, when it has them and keeps + /// them all. + rgb: Option<([usize; 3], Option)>, + has_rgb: bool, + exact: bool, +} + +impl<'s> ReadChannels<'s> for Planes<'_> { + type Reader = PlanesReader; + + fn create_channels_reader(&'s self, header: &Header) -> exr::error::Result { + if header.deep { + return Err(exr::error::Error::NotSupported("deep data".into())); + } + let dw = self.display; + let display = (dw.size.0, dw.size.1); + // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a + // multi-gigabyte allocation per channel (#2141). + let n = display.0.checked_mul(display.1).filter(|n| *n > 0 && *n <= MAX_DISPLAY_PIXELS).ok_or(exr::error::Error::Invalid("display window".into()))?; + let prefix = part_name(header); + let pos = header.own_attributes.layer_position; + let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); + let planes = header + .channels + .list + .iter() + .map(|ch| { + let name = channel_name(prefix.as_deref(), &ch.name.to_string()); + let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; + (self.keep)(&name).then(|| (name, ch.sample_type, vec![fill; n])) + }) + .collect(); + let find = |c: &str| header.channels.find_index_of_channel(&exr::meta::attribute::Text::from(c)); + let rgb = match (find("R"), find("G"), find("B")) { + (Some(r), Some(g), Some(b)) => Some(([r, g, b], find("A"))), + _ => None, + }; + Ok(PlanesReader { + planes, + offset, + layer: (header.layer_size.0, header.layer_size.1), + display, + f16s: vec![], + u32s: vec![], + f32s: vec![], + rgb, + subsampled: header.channels.list.iter().any(|c| c.sampling != exr::math::Vec2(1, 1)), + own_display: header.shared_attributes.display_window != dw, + }) } - let mut aux = AuxChannels::new(w as u32, h as u32, 1.0); +} + +impl ChannelsReader for PlanesReader { + type Channels = PartPlanes; + + fn filter_block(&self, tile: TileCoordinates) -> bool { + tile.is_largest_resolution_level() && self.planes.iter().any(Option::is_some) + } + + fn read_block(&mut self, header: &Header, block: UncompressedBlock) -> exr::error::UnitResult { + let ((lw, lh), (w, h)) = (self.layer, self.display); + for line in block.lines(&header.channels) { + let Some(Some((_, kind, plane))) = self.planes.get_mut(line.location.channel) else { continue }; + let (x0, y, count) = (line.location.position.0, line.location.position.1, line.location.sample_count); + // The samples of data-window row `y`, columns `x0..x0 + count`, at display row `ty` + // from display column `tx0`; clipped to the display window. + if y >= lh || x0.checked_add(count).is_none_or(|end| end > lw) { + continue; + } + let Some(ty) = usize::try_from(y as i64 + self.offset.1).ok().filter(|ty| *ty < h) else { continue }; + let tx0 = x0 as i64 + self.offset.0; + let skip = usize::try_from(-tx0).unwrap_or(0).min(count); + let tx = usize::try_from(tx0.max(0)).unwrap_or(usize::MAX); + let take = (count - skip).min(w.saturating_sub(tx)); + let start = ty.checked_mul(w).and_then(|row| row.checked_add(tx)); + let Some(dst) = start.filter(|_| take > 0).and_then(|s| plane.get_mut(s..s.checked_add(take)?)) else { continue }; + match kind { + SampleType::F16 => { + self.f16s.resize(count, exr::prelude::f16::ZERO); + line.read_samples_into_slice(&mut self.f16s)?; + for (d, s) in dst.iter_mut().zip(self.f16s.iter().skip(skip)) { + *d = s.to_f32(); + } + } + SampleType::F32 => { + self.f32s.resize(count, 0.0); + line.read_samples_into_slice(&mut self.f32s)?; + for (d, s) in dst.iter_mut().zip(self.f32s.iter().skip(skip)) { + *d = *s; + } + } + SampleType::U32 => { + self.u32s.resize(count, 0); + line.read_samples_into_slice(&mut self.u32s)?; + for (d, s) in dst.iter_mut().zip(self.u32s.iter().skip(skip)) { + *d = *s as f32; + } + } + } + } + Ok(()) + } + + fn into_channels(self) -> PartPlanes { + // Positions of the kept channels among the kept planes. + let mut at = vec![None; self.planes.len()]; + let mut planes = Vec::with_capacity(self.planes.len()); + for (i, p) in self.planes.into_iter().enumerate() { + if let Some((name, _, data)) = p { + if let Some(slot) = at.get_mut(i) { + *slot = Some(planes.len()); + } + planes.push((name, data)); + } + } + let kept = |i: usize| at.get(i).copied().flatten(); + let has_rgb = self.rgb.is_some(); + let rgb = self.rgb.and_then(|([r, g, b], a)| { + let a = match a { + Some(a) => Some(kept(a)?), + None => None, + }; + Some(([kept(r)?, kept(g)?, kept(b)?], a)) + }); + PartPlanes { planes, rgb, has_rgb, exact: !self.subsampled && !self.own_display } + } +} + +/// How the `image` crate's OpenEXR decoder would show a file: the first part with exact `R`, `G` +/// and `B` channels (alpha `A` when it has one, else opaque), placed in the display window with +/// 0 outside the data window. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Rgb { + /// No part has R, G and B: the decoder refuses the file (a multi-layer file, shown with + /// [`layered_image`]'s rule). + None, + /// The channels (indices in [`AuxChannels::channels`]) hold exactly what it would show. + Planes([usize; 3], Option), + /// The planes can't stand in for it (subsampled channels, a depth-filled plane, another + /// display window): decode with the `image` crate. + Unsupported, +} + +/// A decoded file: its channels and how its picture is made of them. +pub(crate) struct Decoded { + pub(crate) aux: Arc, + rgb: Rgb, +} + +impl Decoded { + /// Bytes held (the planes; names and manifests are small). + fn bytes(&self) -> usize { + self.aux.channels.iter().map(|(n, p)| n.len() + std::mem::size_of_val(p.as_slice())).sum() + } +} + +/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). `None` when it +/// is not a readable EXR. +fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let display = meta.headers.first()?.shared_attributes.display_window; + let read = ReadImage::new(ReadAllLayers { read_channels: Planes { keep, display } }, |_| {}); + let img = read.from_buffered(std::io::Cursor::new(bytes)).ok()?; + let dw = img.attributes.display_window; + let mut aux = AuxChannels::new(dw.size.0 as u32, dw.size.1 as u32, 1.0); let mut crypto: HashMap, Option)> = HashMap::new(); - let mut collect_meta = |other: &HashMap| { + let mut collect_meta = |other: &HashMap| { for (k, v) in other { let key = k.to_string(); let Some(rest) = key.strip_prefix("cryptomatte/") else { continue }; let Some((id, field)) = rest.split_once('/') else { continue }; - let AttributeValue::Text(t) = v else { continue }; + let exr::meta::attribute::AttributeValue::Text(t) = v else { continue }; let e = crypto.entry(id.to_string()).or_default(); match field { "name" => e.0 = Some(t.to_string()), @@ -79,24 +283,22 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } }; collect_meta(&img.attributes.other); - // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a - // multi-gigabyte allocation per channel (#2141). - let n = w.checked_mul(h).filter(|n| *n <= MAX_DISPLAY_PIXELS)?; - // The decoded image is consumed channel by channel, so each channel's samples are freed (or - // moved) as its plane is made instead of the whole file being held twice (#474). + let mut rgb = Rgb::None; for layer in img.layer_data { collect_meta(&layer.attributes.other); - let prefix = layer.attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()); - let size = (layer.size.0, layer.size.1); - let pos = layer.attributes.layer_position; - let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); - for ch in layer.channel_data.list { - let name = channel_name(prefix.as_deref(), &ch.name.to_string()); - let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; - if let Some(plane) = place(ch.sample_data, size, offset, (w, h), n, fill) { - aux.channels.push((name, plane)); - } + let part = layer.channel_data; + if rgb == Rgb::None && part.has_rgb { + let base = aux.channels.len(); + rgb = match part.rgb { + Some(([r, g, b], a)) if part.exact => { + let picked = [Some(r), Some(g), Some(b), a]; + let depth = picked.iter().flatten().any(|i| part.planes.get(*i).is_some_and(|(n, _)| is_depth(n))); + if depth { Rgb::Unsupported } else { Rgb::Planes([base + r, base + g, base + b], a.map(|a| base + a)) } + } + _ => Rgb::Unsupported, + }; } + aux.channels.extend(part.planes); } for (_, (name, manifest)) in crypto { if let (Some(n), Some(m)) = (name, manifest) { @@ -104,46 +306,13 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } } aux.manifests.sort(); - Some(aux) + Some(Decoded { aux: Arc::new(aux), rgb }) } -/// One channel's samples (`size`, at `offset` in the `display` window of `n` pixels) as a -/// display-window plane, `fill` outside its data window. A channel covering exactly the display -/// window is converted in one pass, and an f32 one is moved without a copy. `None` when the -/// channel has fewer samples than its size. -fn place(samples: exr::image::FlatSamples, size: (usize, usize), offset: (i64, i64), display: (usize, usize), n: usize, fill: f32) -> Option> { - use exr::image::FlatSamples; - let (lw, lh) = size; - if samples.len() < lw.checked_mul(lh)? { - return None; - } - if size == display && offset == (0, 0) { - return Some(match samples { - FlatSamples::F32(mut v) => { - v.truncate(n); - v - } - FlatSamples::F16(v) => v.iter().take(n).map(|s| s.to_f32()).collect(), - FlatSamples::U32(v) => v.iter().take(n).map(|s| *s as f32).collect(), - }); - } - let value = |i: usize| match &samples { - FlatSamples::F32(v) => v.get(i).copied(), - FlatSamples::F16(v) => v.get(i).map(|s| s.to_f32()), - FlatSamples::U32(v) => v.get(i).map(|s| *s as f32), - }; - let (w, h) = display; - let mut plane = vec![fill; n]; - for y in 0..lh { - let Some(ty) = usize::try_from(y as i64 + offset.1).ok().filter(|ty| *ty < h) else { continue }; - for x in 0..lw { - let Some(tx) = usize::try_from(x as i64 + offset.0).ok().filter(|tx| *tx < w) else { continue }; - if let (Some(v), Some(p)) = (value(y * lw + x), plane.get_mut(ty * w + tx)) { - *p = v; - } - } - } - Some(plane) +/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. +pub fn read_exr_channels(bytes: &[u8]) -> Option { + let aux = decode(bytes, &|_| true)?.aux; + Some(Arc::try_unwrap(aux).unwrap_or_else(|a| (*a).clone())) } /// Names of the beauty pass, most preferred first, matched against the last part of a layer's @@ -186,6 +355,36 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { .copied() } +/// The planes a multi-layer file's picture is made of (see [`layered_image`]): red, green, blue +/// (one channel thrice for grey) and alpha, by name. Only the names of `aux` are read. +fn picture_names(aux: &AuxChannels) -> Option<([String; 3], Option)> { + let layers = aux.layers(); + let colour = picture_layer(aux, &layers)?; + let names = aux.layer_rgba(colour); + // A layer without R, G and B channels shows its first channel as grey. + let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); + let pick = |k: usize| names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first).to_string(); + let a = names.get(3).filter(|n| !n.is_empty() && aux.get(n).is_some()).cloned(); + Some(([pick(0), pick(1), pick(2)], a)) +} + +/// The channels of the picture of a multi-layer file and nothing else (names first from the +/// headers, then only those channels decoded; in a multi-part file the other parts' pixels are +/// not even decompressed), with the names of its planes. +fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let mut names = AuxChannels::new(0, 0, 1.0); + for header in meta.headers.iter() { + let prefix = part_name(header); + names.channels.extend(header.channels.list.iter().map(|ch| (channel_name(prefix.as_deref(), &ch.name.to_string()), vec![]))); + } + let (rgb, a) = picture_names(&names)?; + // Planes are looked up by name ignoring case, so every channel that could answer is kept. + let keep = |n: &str| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(n)); + let decoded = decode(bytes, &keep)?; + Some((decoded, rgb, a)) +} + /// The picture of a multi-layer OpenEXR file without an unnamed RGB layer (Blender's /// `ViewLayer.Combined.R`, the compositor's `Image.R`, Nuke's `beauty.R`, …), which the image /// decoder rejects. After Effects shows a file's main RGBA channels and leaves the other layers @@ -193,26 +392,13 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { /// that is not a data pass (never Cryptomatte, depth or normals, #412), else the first channel /// as grey. Linear, straight RGBA as the file stores it. `None` when it is not a readable EXR. pub fn layered_image(bytes: &[u8]) -> Option { - picture_of(&read_exr_channels(bytes)?) -} - -/// [`layered_image`] of the EXR at `path` from the channel cache, so the picture and EXtractoR / -/// Cryptomatte share one decode of the file (#474). -pub(crate) fn cached_layered_image(path: &str, bytes: Arc<[u8]>) -> Option { - let aux = cached(path, || Some(bytes))?; - picture_of(&aux) -} - -/// The picture of decoded channels (see [`layered_image`]). -fn picture_of(aux: &AuxChannels) -> Option { - let layers = aux.layers(); - let colour = picture_layer(aux, &layers)?; - let names = aux.layer_rgba(colour); - // A layer without R, G and B channels shows its first channel as grey. - let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); - let plane = |k: usize| -> Option<&[f32]> { aux.get(names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first)) }; - let (r, g, b) = (plane(0)?, plane(1)?, plane(2)?); - let a = names.get(3).filter(|n| !n.is_empty()).and_then(|n| aux.get(n)); + let (d, rgb, a) = picture_only(bytes)?; + let aux = &d.aux; + let [r, g, b] = [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?]; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; let n = (aux.width as usize).checked_mul(aux.height as usize)?; let mut data = Vec::with_capacity(n.checked_mul(4)?); for i in 0..n { @@ -227,40 +413,214 @@ fn picture_of(aux: &AuxChannels) -> Option { } } -type Cache = Mutex>)>>; +/// The picture made of planes `rgb` and `a` (opaque without one) of `aux`, as +/// [`crate::convert::dynamic_to_image`] makes it of the same values. `None` when a plane is +/// missing or short. +fn planes_picture(aux: &AuxChannels, rgb: [&[f32]; 3], a: Option<&[f32]>, op: AlphaOp) -> Option { + let n = (aux.width as usize).checked_mul(aux.height as usize)?; + if rgb.iter().chain(&a).any(|p| p.len() < n) { + return None; + } + let [r, g, b] = rgb; + let at = |p: &[f32], i: usize| p.get(i).copied().unwrap_or(0.0); + Some(float_to_image(aux.width, aux.height, op, |i| [at(r, i), at(g, i), at(b, i), a.map_or(1.0, |a| at(a, i))])) +} -fn cache() -> &'static Cache { - static CACHE: OnceLock = OnceLock::new(); - CACHE.get_or_init(|| Mutex::new(Vec::new())) +/// The picture of a decoded file by [`layered_image`]'s rule. +fn layered_picture_of(aux: &AuxChannels, op: AlphaOp) -> Option { + let (rgb, a) = picture_names(aux)?; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the multi-layer EXR at `path` ([`layered_image`]'s rule) as an [`Image`]: +/// from the channel cache when `shared` (an effect reads the footage's channels, so they are +/// decoded anyway), else of the picture's channels alone. +pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Option { + if shared { + let d = cache().get(path, || Some(bytes))?; + return layered_picture_of(&d.aux, op); + } + let (d, rgb, a) = picture_only(&bytes)?; + let aux = &d.aux; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the EXR at `path` from the channel cache (#474: EXtractoR / Cryptomatte read +/// this footage's channels, so the picture shares their decode): what the `image` crate's +/// decoder shows for a file it reads, [`layered_image`]'s rule for one it refuses. `None` when +/// the planes can't stand in for the `image` crate's picture (decode with it), or the file is +/// not a readable EXR. +pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Option { + let d = cache().get(path, || Some(bytes))?; + let aux = &d.aux; + match d.rgb { + Rgb::None => layered_picture_of(aux, op), + Rgb::Planes([r, g, b], a) => { + let plane = |i: usize| aux.channels.get(i).map(|(_, p)| p.as_slice()); + let a = match a { + Some(a) => Some(plane(a)?), + None => None, + }; + planes_picture(aux, [plane(r)?, plane(g)?, plane(b)?], a, op) + } + Rgb::Unsupported => None, + } +} + +/// How many bytes of decoded channels the cache keeps, like the frame cache's default +/// ([`crate::DEFAULT_BUDGET`]): two 2560×1440 files of 32 channels (470 MB each, as every plane +/// is f32). The newest file is kept whatever its size, so a frame's picture and its effects +/// share its decode. (Measured on such a sequence decoded one frame at a time: 1 GiB held peak +/// memory about 1 GB lower than 2 GiB, at the same speed.) +#[cfg(not(target_arch = "wasm32"))] +pub const CACHE_BUDGET: usize = 1 << 30; +#[cfg(target_arch = "wasm32")] +pub const CACHE_BUDGET: usize = 512 << 20; + +/// At most this many files are cached (unreadable ones hold no bytes). +const CACHE_FILES: usize = 64; + +/// A file's decode: done once, by the first caller; the others wait for it. +type Slot = Arc>>>; + +struct Entry { + path: String, + slot: Slot, + /// Bytes of the finished decode (0 while decoding). + bytes: usize, + /// When it was last asked for (a counter). + used: u64, +} + +/// Decoded EXR files by path, bounded by bytes, least recently used out first. +pub(crate) struct ChannelCache { + entries: Mutex<(Vec, u64)>, + budget: usize, + /// Files decoded (for tests). + decodes: AtomicU64, +} + +impl ChannelCache { + pub(crate) const fn new(budget: usize) -> Self { + Self { entries: Mutex::new((Vec::new(), 0)), budget, decodes: AtomicU64::new(0) } + } + + /// The decoded file at `path`, read with `read` by the first caller; callers asking for the + /// same file meanwhile wait for that decode instead of decoding it again. + pub(crate) fn get(&self, path: &str, read: impl FnOnce() -> Option>) -> Option> { + let slot = { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + match entries.iter_mut().find(|e| e.path == path) { + Some(e) => { + e.used = *clock; + e.slot.clone() + } + None => { + let slot = Slot::default(); + entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: *clock }); + slot + } + } + }; + let mut decoded_here = false; + let v = slot + .get_or_init(|| { + decoded_here = true; + self.decodes.fetch_add(1, Ordering::Relaxed); + read().and_then(|b| decode(&b, &|_| true)).map(Arc::new) + }) + .clone(); + if decoded_here { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + // (gone when the file was forgotten meanwhile) Used as of now: a file asked for + // before another but decoded after it is not the older one, or concurrent frames + // would evict each other's files before their effects read them. + if let Some(e) = entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, &slot)) { + e.bytes = v.as_ref().map_or(0, |d| d.bytes()); + e.used = *clock; + } + Self::evict(entries, self.budget, &slot); + } + v + } + + /// Drop least recently used finished files until the rest fit the budget, keeping `newest`. + fn evict(entries: &mut Vec, budget: usize, newest: &Slot) { + while entries.iter().map(|e| e.bytes).sum::() > budget || entries.len() > CACHE_FILES { + let oldest = entries + .iter() + .enumerate() + .filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && e.slot.get().is_some()) + .min_by_key(|(_, e)| e.used) + .map(|(i, _)| i); + let Some(i) = oldest else { break }; + entries.remove(i); + } + } + + pub(crate) fn forget(&self, path: &str) { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.retain(|e| e.path != path); + } + + #[cfg(test)] + fn decodes(&self) -> u64 { + self.decodes.load(Ordering::Relaxed) + } + + #[cfg(test)] + fn cached(&self) -> Vec { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.iter().map(|e| e.path.clone()).collect() + } +} + +fn cache() -> &'static ChannelCache { + static CACHE: ChannelCache = ChannelCache::new(CACHE_BUDGET); + &CACHE } /// Drop the cached channels of `path` (the file changed on disk). pub(crate) fn forget(path: &str) { - if let Ok(mut c) = cache().lock() { - c.retain(|(p, _)| p != path); - } + cache().forget(path); } -/// Read (and cache, a few files) the channels of the EXR at `path` via `read`. +/// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { - let cache = cache(); - if let Some(v) = cache.lock().ok().and_then(|c| c.iter().find(|(p, _)| p == path).map(|(_, v)| v.clone())) { - return v; - } - let v = read().and_then(|b| read_exr_channels(&b)).map(Arc::new); - if let Ok(mut c) = cache.lock() { - if c.len() >= 4 { - c.remove(0); - } - c.push((path.to_string(), v.clone())); - } - v + cache().get(path, read).map(|d| d.aux.clone()) } #[cfg(test)] mod tests { use super::*; + /// An EXR file of `layers` (with their display window `display`). + fn file(layers: Vec>>, display: exr::prelude::IntegerBounds) -> Vec { + use exr::prelude::*; + let image = Image::from_layers(ImageAttributes::new(display), layers); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() + } + + /// A single-part file of float channels named `names`, `w`×`h`, every sample `v`. + fn flat(names: &[&str], (w, h): (usize, usize), v: f32) -> Vec { + use exr::prelude::*; + let list: Vec> = names.iter().map(|n| AnyChannel::new(*n, FlatSamples::F32(vec![v; w * h]))).collect(); + let layer = Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())); + file(vec![layer], IntegerBounds::from_dimensions((w, h))) + } + #[test] fn reads_layered_channels_and_cryptomatte_metadata() { use exr::prelude::*; @@ -294,53 +654,229 @@ mod tests { assert!(read_exr_channels(b"not an exr").is_none()); } - /// #474: a full-window f32 channel (Cryptomatte ranks, …) becomes its plane without a copy, - /// so a decode no longer holds the file's samples twice; other channels are placed in the - /// display window as before. + /// #474: each line of samples goes straight into its display-window plane: a data window + /// inside the display window is placed with the fill around it (the background depth for + /// depth), one partly outside is clipped, never out of bounds; half and uint samples are + /// converted as the file stores them. #[test] - fn channels_become_planes_without_an_extra_copy() { - use exr::image::FlatSamples; - let v = vec![0.25f32; 6]; - let ptr = v.as_ptr(); - let plane = place(FlatSamples::F32(v), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(plane.as_ptr(), ptr, "moved, not copied"); - assert_eq!(plane, [0.25; 6]); - let half = |x: f32| exr::prelude::f16::from_f32(x); - let full = place(FlatSamples::F16(vec![half(0.5); 6]), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(full, [0.5; 6]); - // A 2×1 data window at (1, 1) of a 3×2 display window; the rest is the fill. - let part = place(FlatSamples::F16(vec![half(1.0), half(2.0)]), (2, 1), (1, 1), (3, 2), 6, 9.0).unwrap(); - assert_eq!(part, [9.0, 9.0, 9.0, 9.0, 1.0, 2.0]); - // Partly outside the display window: clipped, never out of bounds. - let off = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, -1), (3, 2), 6, 0.0).unwrap(); - assert_eq!(off, [0.0; 6]); - let edge = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(edge, [0.0, 0.0, 7.0, 0.0, 0.0, 0.0]); - assert!(place(FlatSamples::F32(vec![1.0; 5]), (3, 2), (0, 0), (3, 2), 6, 0.0).is_none(), "too few samples"); + fn channels_are_placed_in_the_display_window() { + use exr::prelude::*; + let half = |x: f32| f16::from_f32(x); + let part = |pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = LayerAttributes::default(); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::from_dimensions((3, 2)); + // A 2×1 data window at (1, 1). + let inside = file( + vec![part((1, 1), (2, 1), vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))])], + display, + ); + let aux = read_exr_channels(&inside).unwrap(); + assert_eq!(aux.get("G").unwrap(), [0.0, 0.0, 0.0, 0.0, 1.0, 2.0]); + let bg = BACKGROUND_DEPTH; + assert_eq!(aux.get("Z").unwrap(), [bg, bg, bg, bg, 7.0, 8.0]); + // A 2×2 data window at (2, -1): only its bottom-left sample lands at (2, 0). + let off = file(vec![part((2, -1), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0, 2.0, 3.0, 4.0]))])], display); + assert_eq!(read_exr_channels(&off).unwrap().get("G").unwrap(), [0.0, 0.0, 3.0, 0.0, 0.0, 0.0]); + // Entirely outside: all fill. + let away = file(vec![part((-5, 0), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0; 4]))])], display); + assert_eq!(read_exr_channels(&away).unwrap().get("G").unwrap(), [0.0; 6]); + } + + /// #474: concurrent requests for a file that isn't cached decode it once; the others wait + /// for that decode and share its planes. + #[test] + fn concurrent_requests_decode_a_file_once() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "Z"], (64, 64), 0.5).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let reads = AtomicU64::new(0); + let barrier = std::sync::Barrier::new(8); + let got: Vec> = std::thread::scope(|s| { + let jobs: Vec<_> = (0..8) + .map(|_| { + s.spawn(|| { + barrier.wait(); + cache + .get("/474-concurrent.exr", || { + reads.fetch_add(1, Ordering::Relaxed); + // Long enough for the others to arrive while this one decodes. + std::thread::sleep(std::time::Duration::from_millis(50)); + Some(bytes.clone()) + }) + .unwrap() + }) + }) + .collect(); + jobs.into_iter().map(|j| j.join().unwrap()).collect() + }); + assert_eq!(cache.decodes(), 1, "one decode"); + assert_eq!(reads.load(Ordering::Relaxed), 1, "the file is read once"); + assert!(got.iter().all(|d| Arc::ptr_eq(d, &got[0])), "every caller has the same planes"); + assert_eq!(got[0].aux.get("Z").and_then(|z| z.first()).copied(), Some(0.5)); + } + + /// #474: the cache is bounded by the bytes of its planes, least recently used out first; the + /// newest file stays even when it alone is over the budget, and unreadable files are cached + /// (as such) without counting bytes. + #[test] + fn cache_evicts_least_recently_used_by_bytes() { + // 4 float planes of 32×32: 16 KiB of samples (plus a few bytes of names) per file. + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + let get = |p: &str| cache.get(p, || Some(bytes.clone())).unwrap(); + assert_eq!(get("/a.exr").bytes(), one); + get("/b.exr"); + get("/a.exr"); // a is now more recent than b + get("/c.exr"); + assert_eq!(cache.cached(), ["/a.exr", "/c.exr"], "b was least recently used"); + assert_eq!(cache.decodes(), 3); + get("/a.exr"); + assert_eq!(cache.decodes(), 3, "a hit"); + // A file over the budget on its own is still kept while it is the newest. + let small = ChannelCache::new(one / 2); + assert!(small.get("/big.exr", || Some(bytes.clone())).is_some()); + assert_eq!(small.cached(), ["/big.exr"]); + small.get("/big2.exr", || Some(bytes.clone())); + assert_eq!(small.cached(), ["/big2.exr"]); + // An unreadable file is remembered (not read again) without bytes, and forgotten on request. + assert!(cache.get("/broken.exr", || Some(Arc::from(&b"not an exr"[..]))).is_none()); + assert!(cache.get("/broken.exr", || panic!("not read again")).is_none()); + cache.forget("/broken.exr"); + assert!(!cache.cached().contains(&"/broken.exr".to_string())); + // At most CACHE_FILES files, however small. + for i in 0..CACHE_FILES + 10 { + cache.get(&format!("/none{i}.exr"), || None); + } + assert!(cache.cached().len() <= CACHE_FILES); + } + + /// #474: a file counts as used when its decode finishes: one asked for first but decoded + /// after another is the newer of the two, so concurrent frames don't evict each other's + /// files before their effects read them. + #[test] + fn a_file_is_used_as_of_its_decode() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + cache.get("/first.exr", || { + // Another frame's file is asked for and decoded while this one decodes. + cache.get("/second.exr", || Some(bytes.clone())); + Some(bytes.clone()) + }); + cache.get("/third.exr", || Some(bytes.clone())); + assert_eq!(cache.cached(), ["/first.exr", "/third.exr"], "the file decoded first went first"); + } + + /// #474: a file forgotten (changed on disk) while it decodes isn't cached under its path when + /// that decode finishes. + #[test] + fn a_file_forgotten_while_it_decodes_is_not_cached() { + let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let d = cache.get("/changed.exr", || { + cache.forget("/changed.exr"); + Some(bytes.clone()) + }); + assert!(d.is_some()); + assert!(cache.cached().is_empty()); } /// #295: a multi-layer EXR whose colour is only in named layers (`diffuse.R`, as Blender and /// Nuke write them) failed to import ("does not contain non-deep rgb channels"); it now /// shows its colour layer and keeps every channel for EXtractoR. - /// #474: the picture and EXtractoR / Cryptomatte share one decode of a multi-layer file: - /// after the frame is decoded, the aux channels come from the cache without reading the file. + /// #474: once an effect has read a footage item's channels, the picture of each later frame + /// shares one decode with them: the frame's decode is cached and the channels come from it + /// without reading the file again. #[test] fn picture_and_aux_channels_share_one_decode() { use exr::prelude::*; let (w, h) = (2usize, 2usize); - let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); - let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", 0.25)].into()); - let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); - let mut bytes = std::io::Cursor::new(Vec::new()); - image.write().to_buffered(&mut bytes).unwrap(); - let b: Arc<[u8]> = bytes.into_inner().into(); - let path = "/474-shared-decode.exr"; + let frame = |v: f32| -> Arc<[u8]> { + let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); + let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", v)].into()); + let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner().into() + }; + let paths = ["/474-shared-decode_0001.exr", "/474-shared-decode_0002.exr"]; let pool = crate::MediaPool::new(); - pool.add_bytes(path, b.clone()); - let f = crate::probe_bytes(path, b).unwrap(); + pool.add_bytes(paths[0], frame(0.25)); + pool.add_bytes(paths[1], frame(0.75)); + let mut f = crate::probe_bytes(paths[0], frame(0.25)).unwrap(); + f.kind = effectcraft_project::FootageKind::Sequence; + f.sequence = paths.iter().map(|p| p.to_string()).collect(); + let t1 = f.frame_rate.tick_of(1); + use effectcraft_render::FootageSource; + // The first frame: the picture, then an effect reads the channels. pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap(); - let aux = cached(path, || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + let aux = pool.aux(effectcraft_project::ItemId(1), &f, effectcraft_time::Tick::ZERO).unwrap(); assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); + // The next frame's picture decodes the file into the cache, so its channels are not read. + let px = pool.frame_at(&f, t1).unwrap().get(0, 0); + let aux = cached(paths[1], || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.75)); + assert_eq!(px, pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap().get(0, 0), "the same picture"); + } + + /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the + /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 + /// around it), in every alpha interpretation and float curve; a file it can't stand in for + /// (a subsampled channel) is left to the `image` crate. + #[test] + fn picture_from_planes_is_the_image_crates() { + use effectcraft_project::{AlphaMode, Footage}; + use effectcraft_render::FloatCurve; + use exr::prelude::*; + let half = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F16((0..len).map(|i| f16::from_f32(v + i as f32 * 0.37 - 1.0)).collect())); + let float = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F32((0..len).map(|i| ((i * 7) % 11) as f32 * v).collect())); + let part = |name: Option<&str>, pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::SMALL_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::new((1, -1), (5, 4)); + let n = 20; + let files = [ + file(vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], display), + file(vec![part(None, (0, 0), (3, 2), vec![float("R", 0.4, 6), float("G", 0.5, 6), float("B", 0.6, 6), half("Image.R", 0.1, 6)])], display), + file( + vec![ + part(Some("diffuse"), (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n)]), + part(Some("beauty"), (2, 0), (2, 2), vec![half("R", 0.4, 4), half("G", 0.5, 4), half("B", 0.6, 4), half("A", 0.5, 4)]), + ], + display, + ), + ]; + let bits = |i: &effectcraft_raster::Image| i.data.iter().flat_map(|p| p.map(f32::to_bits)).collect::>(); + for (k, bytes) in files.iter().enumerate() { + let path = format!("/474-planes-{k}.exr"); + let d = cache().get(&path, || Some(bytes.as_slice().into())).unwrap(); + assert!(matches!(d.rgb, Rgb::Planes(..)), "file {k}: {:?}", d.rgb); + let img = crate::probe::load_oriented(bytes, Some(image::ImageFormat::OpenExr)).unwrap(); + for alpha in [AlphaMode::Straight, AlphaMode::Premultiplied, AlphaMode::Ignore] { + for curve in [FloatCurve::Srgb, FloatCurve::Linear, FloatCurve::Gamma24] { + let op = AlphaOp::of(&Footage { alpha, premul_color: [0.2, 0.4, 0.6], ..Default::default() }, curve); + let want = crate::convert::dynamic_to_image(&img, op); + let got = cached_picture(&path, bytes.as_slice().into(), op).unwrap(); + assert_eq!((got.width, got.height), (want.width, want.height)); + assert_eq!(bits(&got), bits(&want), "file {k} {alpha:?} {curve:?}"); + } + } + } + // A subsampled channel in the picture's part: the `image` crate decodes it. + let mut sub = part(None, (0, 0), (4, 2), vec![half("R", 0.1, 8), half("G", 0.2, 8), half("B", 0.3, 8), half("C", 0.3, 2)]); + if let Some(c) = sub.channel_data.list.iter_mut().find(|c| c.name == *"C") { + c.sampling = Vec2(2, 2); + } + let mut bytes = std::io::Cursor::new(Vec::new()); + if Image::from_layer(sub).write().to_buffered(&mut bytes).is_ok() { + let d = cache().get("/474-subsampled.exr", || Some(bytes.into_inner().into())).map(|d| d.rgb); + assert!(d.is_none_or(|r| r == Rgb::Unsupported), "{d:?}"); + } } /// #2141: one pixel of data in a 60000² display window is refused, not allocated per channel. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 655ca4ed8..9f1f2aa23 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -245,6 +245,9 @@ struct Inner { /// Movies that failed to open or decode (path to a readable reason; see /// [`FootageSource::problem`]). The first failure of a file is logged, later ones are not. problems: Mutex, String>>, + /// OpenEXR footage (by path) whose channels an effect has read (EXtractoR, Cryptomatte, the 3D + /// Channel effects): its pictures come from the same cached decode (#474). + aux_footage: Mutex>, } /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus @@ -315,6 +318,7 @@ impl MediaPool { conforming: Mutex::default(), conform_failed: Mutex::default(), problems: Mutex::default(), + aux_footage: Mutex::default(), }), } } @@ -856,13 +860,19 @@ impl Inner { { return Err(MediaError::Decode(format!("{path}: {why}"))); } - let loaded = crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)); - let img = match loaded { - Ok(img) => img, + // An effect reads this footage's channels: the picture is made of the same decode (#474). + let shared = lock(&self.aux_footage).contains(&footage.path); + if shared + && path.to_ascii_lowercase().ends_with(".exr") + && let Some(img) = crate::exr_channels::cached_picture(path, bytes.clone(), op) + { + return Ok(img); + } + match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { + Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. - Err(e) => crate::exr_channels::cached_layered_image(path, bytes).ok_or_else(|| MediaError::Decode(format!("{path}: {e}")))?, - }; - Ok(dynamic_to_image(&img, op)) + Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared).ok_or_else(|| MediaError::Decode(format!("{path}: {e}"))), + } } fn decode_media(&self, loc: &Loc, footage: &Footage) -> Result { @@ -962,6 +972,14 @@ impl FootageSource for MediaPool { return None; } let path = loc.key.path.clone(); + { + let mut wanted = lock(&self.inner.aux_footage); + // (a bound: names of footage items, never cleared otherwise) + if wanted.len() >= 4096 { + wanted.clear(); + } + wanted.insert(footage.path.clone()); + } crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) } diff --git a/docs/effects-parity.md b/docs/effects-parity.md index bd81a09df..ce4debd01 100644 --- a/docs/effects-parity.md +++ b/docs/effects-parity.md @@ -26,7 +26,7 @@ corpus compares renders, each effect is assumed to differ somewhere, and users k | Category | Effects | On the GPU | Known differences or bugs | |---|---:|---:|---| -| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR decode shared (#474 open: f16 storage and cache sizing remain) | +| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache bounded at 1 GB (#474 open: f16 storage remains) | | Audio | 13 | 0 | CPU by design (applied in the mixdown) | | Blur & Sharpen | 16 | 15 | Radial Blur keeps the layer edges as After Effects 2026 (713 now pixel-identical); Gaussian Blur interaction and output to be measured first ([fidelity/ae-2026-acceptance.md](fidelity/ae-2026-acceptance.md)) | | Channel | 13 | 13 | — | diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 6eeef1719..741582d9e 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); EXRs without an unnamed RGB layer decode twice (#474) | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, footage alone only its picture's channels (#474; decoded channels are held as f32) | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index f7f1df21b..0ebbeeecc 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -120,6 +120,11 @@ Nuke's `beauty`) show the beauty pass: the layer named Combined, Beauty, Image, Color, else the first colour layer that isn't a data pass. Cryptomatte, depth, mist, normals, vectors, positions and IDs are never the picture. EXtractoR reads every other layer and channel. +Footage on its own decodes only the channels of its picture. Once an effect reads an item's +channels (EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for +the picture and the effects together, and kept for a while (up to 1 GB of decoded channels); +several frames asking for the same file at once wait for one decode. + ## Replacing and reloading Importing a layered file as a composition refreshes the decoded sources used by its layers, diff --git a/docs/gaps.md b/docs/gaps.md index 016aeff22..46a96983a 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -65,7 +65,7 @@ are grouped by area; the issue tracker has the rest. | 6 | **Linux and Windows interaction bugs** | Open: #488 rename arrows (hardening landed, reporter asked for details), #492 anchor drag, #268 file drops on Wayland (winit), #613 Direct3D 12 compositor. Now closed: #491, #493, #467, #320, #302, #596 | Many reports come from Linux; some Windows GPUs fail to start the compositor | 25–45 h | G2 · [ui-parity.md](ui-parity.md) | | 7 | **Animation presets: 76 against 621** | After Effects ships 621 `.ffx` presets (Behaviors, Backgrounds, Shapes, Synthetics, Text, Transitions…); we ship 76 original presets (8 text-animator; 23 Transitions: wipes, clock and fan reveals, blinds, dissolves, card flips, CC wipes, fades, #757; 23 Backgrounds: drifting gradients, fog, cloud and smoke, plasma and magma, cells, hexagons, scrolling grids and checks, pulse rings, bokeh, light sweep, #1096, driven by expressions so they keep moving, not seamless loops; 22 Behaviors: wiggle, shake, float, orbit, bounce, drift, pendulum, spin, pulse, heartbeat, blink, strobe, flicker on transform and opacity, #1510) and read only our `.ecpreset`. Missing: Shapes, Synthetics, Image and more Text | Presets are how many designers start; tutorials assume them | 15–35 h (original presets; `.ffx` reading is out of clean-room scope) | G8 · [effects-parity.md](effects-parity.md) | | 8 | **Mask and shape bugs** | Open: #1004 mask vertex multi-select, #999 auto-pan, #1042 / #1033 vertex nodes under the bounding box, #959 / #960 shape group toggles and operators. Now closed: #510, #513, #527, #509, #303, #553, #932 (Convert To Bezier Path on a shape layer's parametric path) | Masks are core compositing | 10–20 h | G2 · [target-app-parity.md](target-app-parity.md) | -| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (decode shared, faffad3a; f16 storage and cache sizing open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | +| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | | 10 | **Colour pipeline differences** | #340 ProRes 4444 tagged bt709 but sRGB-encoded (open); ACES 2.0 / OCIO 2.5 configs (26.5) missing; now closed: #494 32-bpc blend modes, #495 and #750 Working Gamma 2.4, #546 / #547 viewer colour | Wrong colour in delivery is a client-visible failure | 15–30 h | G1 · [file-format-parity.md](file-format-parity.md) | | 11 | **Text rendering and layout** | #453 CJK text in comps: the Han fallback now reaches the installed Chinese faces; open: #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) |