diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index 9e88ab4dd..a2b24b3ae 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -376,30 +376,20 @@ fn convert_yuv(img: &mut Image, source: YuvSource<'_, T>, k: &Yuv, op pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image { use image::DynamicImage as D; let (w, h) = (img.width(), img.height()); + if let D::ImageRgb32F(_) | D::ImageRgba32F(_) = img { + let px = img.to_rgba32f(); + let raw = px.as_raw(); + return float_to_image(w, h, op, |i| { + let s = &raw[i * 4..i * 4 + 4]; + [s[0], s[1], s[2], s[3]] + }); + } let mut out = Image::new(w, h); let wu = w as usize; if wu == 0 || h == 0 { return out; } match img { - D::ImageRgb32F(_) | D::ImageRgba32F(_) => { - let px = img.to_rgba32f(); - let raw = px.as_raw(); - let premul = op.mode == AlphaMode::Premultiplied; - out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { - for (x, o) in row.iter_mut().enumerate() { - let s = &raw[(y * wu + x) * 4..(y * wu + x) * 4 + 4]; - let a = s[3]; - let c = if premul { - let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; - [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] - } else { - [op.float(s[0]), op.float(s[1]), op.float(s[2])] - }; - *o = op.apply(c, a.clamp(0.0, 1.0)); - } - }); - } D::ImageLuma16(_) | D::ImageLumaA16(_) | D::ImageRgb16(_) | D::ImageRgba16(_) => { let px = img.to_rgba16(); let raw = px.as_raw(); @@ -425,6 +415,32 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image out } +/// Float RGBA (linear light, alpha straight or premultiplied as the footage says), `px(i)` for +/// pixel `i` in rows, to an [`Image`]: how [`dynamic_to_image`] converts float stills, also used +/// for OpenEXR pictures made from decoded channel planes without an interleaved copy (#474). +pub(crate) fn float_to_image(w: u32, h: u32, op: AlphaOp, px: impl Fn(usize) -> [f32; 4] + Sync) -> Image { + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + let premul = op.mode == AlphaMode::Premultiplied; + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = px(y * wu + x); + let a = s[3]; + let c = if premul { + let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; + [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] + } else { + [op.float(s[0]), op.float(s[1]), op.float(s[2])] + }; + *o = op.apply(c, a.clamp(0.0, 1.0)); + } + }); + out +} + /// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`]. pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image { let mut out = Image::new(w, h); diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index d372ec849..747882337 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -6,12 +6,25 @@ //! 0, or the background depth for depth channels). Cryptomatte manifests come from the //! `cryptomatte//name` and `cryptomatte//manifest` header attributes (the published //! Cryptomatte metadata convention: the manifest is a JSON object of name → hex hash). +//! +//! Decoded files are kept in a small cache ([`CACHE_BUDGET`]) that the picture of an EXR +//! footage item shares with EXtractoR / Cryptomatte once an effect has read its channels +//! (#474); a file is decoded once however many callers ask for it at the same time. use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError}; -use effectcraft_raster::AuxChannels; use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, split_channel}; +use effectcraft_raster::{AuxChannels, Image}; +use exr::block::UncompressedBlock; +use exr::block::chunk::TileCoordinates; +use exr::image::read::image::ReadImage; +use exr::image::read::layers::{ChannelsReader, ReadAllLayers, ReadChannels}; +use exr::meta::attribute::SampleType; +use exr::meta::header::Header; + +use crate::convert::{AlphaOp, float_to_image}; /// The largest display window decoded (16384²; vector footage and Continuously Rasterize /// stop at the same size). @@ -33,6 +46,11 @@ fn channel_name(part: Option<&str>, channel: &str) -> String { } } +/// The part name channels are prefixed with (`None`: unnamed). +fn part_name(header: &Header) -> Option { + header.own_attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()) +} + /// Parse a Cryptomatte manifest (`{"name": "hexhash", …}`). pub fn parse_manifest(json: &str) -> Vec<(String, u32)> { let Ok(serde_json::Value::Object(m)) = serde_json::from_str::(json) else { return vec![] }; @@ -53,23 +71,209 @@ pub fn unsupported_compression(bytes: &[u8]) -> Option<&'static str> { .then_some("OpenEXR HTJ2K compression isn't supported yet: re-save the file with ZIP, PIZ or DWAA compression") } -/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. -pub fn read_exr_channels(bytes: &[u8]) -> Option { - use exr::prelude::*; - let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_buffered(std::io::Cursor::new(bytes)).ok()?; - let dw = img.attributes.display_window; - let (w, h) = (dw.size.0, dw.size.1); - if w == 0 || h == 0 { - return None; +/// Which channels a decode keeps, by plane name, and the display window they are placed in +/// (the first part's, as the `exr` crate reports the image's). +struct Planes<'k> { + keep: &'k dyn Fn(&str) -> bool, + display: exr::meta::attribute::IntegerBounds, +} + +/// One part's kept channels while it is decoded: each line of samples goes straight into its +/// channel's display-window plane (no copy of the part's samples in their own type first). +struct PlanesReader { + /// Per channel of the part: its plane name, sample type and plane (`None`: not kept). + planes: Vec)>>, + /// The data window's position in the display window, and the data window's and display + /// window's sizes. + offset: (i64, i64), + layer: (usize, usize), + display: (usize, usize), + f16s: Vec, + u32s: Vec, + f32s: Vec, + /// The part's channel indices of exact `R`, `G`, `B` and `A`, when it has R, G and B. + rgb: Option<([usize; 3], Option)>, + /// The part has a channel that doesn't sample every pixel. + subsampled: bool, + /// The part's display window differs from the first part's. + own_display: bool, +} + +/// A decoded part: its kept planes (in channel order) and what [`PlanesReader`] found. +struct PartPlanes { + planes: Vec<(String, Vec)>, + /// Positions in `planes` of the part's R, G, B (and A) channels, when it has them and keeps + /// them all. + rgb: Option<([usize; 3], Option)>, + has_rgb: bool, + exact: bool, +} + +impl<'s> ReadChannels<'s> for Planes<'_> { + type Reader = PlanesReader; + + fn create_channels_reader(&'s self, header: &Header) -> exr::error::Result { + if header.deep { + return Err(exr::error::Error::NotSupported("deep data".into())); + } + let dw = self.display; + let display = (dw.size.0, dw.size.1); + // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a + // multi-gigabyte allocation per channel (#2141). + let n = display.0.checked_mul(display.1).filter(|n| *n > 0 && *n <= MAX_DISPLAY_PIXELS).ok_or(exr::error::Error::Invalid("display window".into()))?; + let prefix = part_name(header); + let pos = header.own_attributes.layer_position; + let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); + let planes = header + .channels + .list + .iter() + .map(|ch| { + let name = channel_name(prefix.as_deref(), &ch.name.to_string()); + let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; + (self.keep)(&name).then(|| (name, ch.sample_type, vec![fill; n])) + }) + .collect(); + let find = |c: &str| header.channels.find_index_of_channel(&exr::meta::attribute::Text::from(c)); + let rgb = match (find("R"), find("G"), find("B")) { + (Some(r), Some(g), Some(b)) => Some(([r, g, b], find("A"))), + _ => None, + }; + Ok(PlanesReader { + planes, + offset, + layer: (header.layer_size.0, header.layer_size.1), + display, + f16s: vec![], + u32s: vec![], + f32s: vec![], + rgb, + subsampled: header.channels.list.iter().any(|c| c.sampling != exr::math::Vec2(1, 1)), + own_display: header.shared_attributes.display_window != dw, + }) } - let mut aux = AuxChannels::new(w as u32, h as u32, 1.0); +} + +impl ChannelsReader for PlanesReader { + type Channels = PartPlanes; + + fn filter_block(&self, tile: TileCoordinates) -> bool { + tile.is_largest_resolution_level() && self.planes.iter().any(Option::is_some) + } + + fn read_block(&mut self, header: &Header, block: UncompressedBlock) -> exr::error::UnitResult { + let ((lw, lh), (w, h)) = (self.layer, self.display); + for line in block.lines(&header.channels) { + let Some(Some((_, kind, plane))) = self.planes.get_mut(line.location.channel) else { continue }; + let (x0, y, count) = (line.location.position.0, line.location.position.1, line.location.sample_count); + // The samples of data-window row `y`, columns `x0..x0 + count`, at display row `ty` + // from display column `tx0`; clipped to the display window. + if y >= lh || x0.checked_add(count).is_none_or(|end| end > lw) { + continue; + } + let Some(ty) = usize::try_from(y as i64 + self.offset.1).ok().filter(|ty| *ty < h) else { continue }; + let tx0 = x0 as i64 + self.offset.0; + let skip = usize::try_from(-tx0).unwrap_or(0).min(count); + let tx = usize::try_from(tx0.max(0)).unwrap_or(usize::MAX); + let take = (count - skip).min(w.saturating_sub(tx)); + let start = ty.checked_mul(w).and_then(|row| row.checked_add(tx)); + let Some(dst) = start.filter(|_| take > 0).and_then(|s| plane.get_mut(s..s.checked_add(take)?)) else { continue }; + match kind { + SampleType::F16 => { + self.f16s.resize(count, exr::prelude::f16::ZERO); + line.read_samples_into_slice(&mut self.f16s)?; + for (d, s) in dst.iter_mut().zip(self.f16s.iter().skip(skip)) { + *d = s.to_f32(); + } + } + SampleType::F32 => { + self.f32s.resize(count, 0.0); + line.read_samples_into_slice(&mut self.f32s)?; + for (d, s) in dst.iter_mut().zip(self.f32s.iter().skip(skip)) { + *d = *s; + } + } + SampleType::U32 => { + self.u32s.resize(count, 0); + line.read_samples_into_slice(&mut self.u32s)?; + for (d, s) in dst.iter_mut().zip(self.u32s.iter().skip(skip)) { + *d = *s as f32; + } + } + } + } + Ok(()) + } + + fn into_channels(self) -> PartPlanes { + // Positions of the kept channels among the kept planes. + let mut at = vec![None; self.planes.len()]; + let mut planes = Vec::with_capacity(self.planes.len()); + for (i, p) in self.planes.into_iter().enumerate() { + if let Some((name, _, data)) = p { + if let Some(slot) = at.get_mut(i) { + *slot = Some(planes.len()); + } + planes.push((name, data)); + } + } + let kept = |i: usize| at.get(i).copied().flatten(); + let has_rgb = self.rgb.is_some(); + let rgb = self.rgb.and_then(|([r, g, b], a)| { + let a = match a { + Some(a) => Some(kept(a)?), + None => None, + }; + Some(([kept(r)?, kept(g)?, kept(b)?], a)) + }); + PartPlanes { planes, rgb, has_rgb, exact: !self.subsampled && !self.own_display } + } +} + +/// How the `image` crate's OpenEXR decoder would show a file: the first part with exact `R`, `G` +/// and `B` channels (alpha `A` when it has one, else opaque), placed in the display window with +/// 0 outside the data window. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Rgb { + /// No part has R, G and B: the decoder refuses the file (a multi-layer file, shown with + /// [`layered_image`]'s rule). + None, + /// The channels (indices in [`AuxChannels::channels`]) hold exactly what it would show. + Planes([usize; 3], Option), + /// The planes can't stand in for it (subsampled channels, a depth-filled plane, another + /// display window): decode with the `image` crate. + Unsupported, +} + +/// A decoded file: its channels and how its picture is made of them. +pub(crate) struct Decoded { + pub(crate) aux: Arc, + rgb: Rgb, +} + +impl Decoded { + /// Bytes held (the planes; names and manifests are small). + fn bytes(&self) -> usize { + self.aux.channels.iter().map(|(n, p)| n.len() + std::mem::size_of_val(p.as_slice())).sum() + } +} + +/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). `None` when it +/// is not a readable EXR. +fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let display = meta.headers.first()?.shared_attributes.display_window; + let read = ReadImage::new(ReadAllLayers { read_channels: Planes { keep, display } }, |_| {}); + let img = read.from_buffered(std::io::Cursor::new(bytes)).ok()?; + let dw = img.attributes.display_window; + let mut aux = AuxChannels::new(dw.size.0 as u32, dw.size.1 as u32, 1.0); let mut crypto: HashMap, Option)> = HashMap::new(); - let mut collect_meta = |other: &HashMap| { + let mut collect_meta = |other: &HashMap| { for (k, v) in other { let key = k.to_string(); let Some(rest) = key.strip_prefix("cryptomatte/") else { continue }; let Some((id, field)) = rest.split_once('/') else { continue }; - let AttributeValue::Text(t) = v else { continue }; + let exr::meta::attribute::AttributeValue::Text(t) = v else { continue }; let e = crypto.entry(id.to_string()).or_default(); match field { "name" => e.0 = Some(t.to_string()), @@ -79,24 +283,22 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } }; collect_meta(&img.attributes.other); - // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a - // multi-gigabyte allocation per channel (#2141). - let n = w.checked_mul(h).filter(|n| *n <= MAX_DISPLAY_PIXELS)?; - // The decoded image is consumed channel by channel, so each channel's samples are freed (or - // moved) as its plane is made instead of the whole file being held twice (#474). + let mut rgb = Rgb::None; for layer in img.layer_data { collect_meta(&layer.attributes.other); - let prefix = layer.attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()); - let size = (layer.size.0, layer.size.1); - let pos = layer.attributes.layer_position; - let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); - for ch in layer.channel_data.list { - let name = channel_name(prefix.as_deref(), &ch.name.to_string()); - let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; - if let Some(plane) = place(ch.sample_data, size, offset, (w, h), n, fill) { - aux.channels.push((name, plane)); - } + let part = layer.channel_data; + if rgb == Rgb::None && part.has_rgb { + let base = aux.channels.len(); + rgb = match part.rgb { + Some(([r, g, b], a)) if part.exact => { + let picked = [Some(r), Some(g), Some(b), a]; + let depth = picked.iter().flatten().any(|i| part.planes.get(*i).is_some_and(|(n, _)| is_depth(n))); + if depth { Rgb::Unsupported } else { Rgb::Planes([base + r, base + g, base + b], a.map(|a| base + a)) } + } + _ => Rgb::Unsupported, + }; } + aux.channels.extend(part.planes); } for (_, (name, manifest)) in crypto { if let (Some(n), Some(m)) = (name, manifest) { @@ -104,46 +306,13 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } } aux.manifests.sort(); - Some(aux) + Some(Decoded { aux: Arc::new(aux), rgb }) } -/// One channel's samples (`size`, at `offset` in the `display` window of `n` pixels) as a -/// display-window plane, `fill` outside its data window. A channel covering exactly the display -/// window is converted in one pass, and an f32 one is moved without a copy. `None` when the -/// channel has fewer samples than its size. -fn place(samples: exr::image::FlatSamples, size: (usize, usize), offset: (i64, i64), display: (usize, usize), n: usize, fill: f32) -> Option> { - use exr::image::FlatSamples; - let (lw, lh) = size; - if samples.len() < lw.checked_mul(lh)? { - return None; - } - if size == display && offset == (0, 0) { - return Some(match samples { - FlatSamples::F32(mut v) => { - v.truncate(n); - v - } - FlatSamples::F16(v) => v.iter().take(n).map(|s| s.to_f32()).collect(), - FlatSamples::U32(v) => v.iter().take(n).map(|s| *s as f32).collect(), - }); - } - let value = |i: usize| match &samples { - FlatSamples::F32(v) => v.get(i).copied(), - FlatSamples::F16(v) => v.get(i).map(|s| s.to_f32()), - FlatSamples::U32(v) => v.get(i).map(|s| *s as f32), - }; - let (w, h) = display; - let mut plane = vec![fill; n]; - for y in 0..lh { - let Some(ty) = usize::try_from(y as i64 + offset.1).ok().filter(|ty| *ty < h) else { continue }; - for x in 0..lw { - let Some(tx) = usize::try_from(x as i64 + offset.0).ok().filter(|tx| *tx < w) else { continue }; - if let (Some(v), Some(p)) = (value(y * lw + x), plane.get_mut(ty * w + tx)) { - *p = v; - } - } - } - Some(plane) +/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. +pub fn read_exr_channels(bytes: &[u8]) -> Option { + let aux = decode(bytes, &|_| true)?.aux; + Some(Arc::try_unwrap(aux).unwrap_or_else(|a| (*a).clone())) } /// Names of the beauty pass, most preferred first, matched against the last part of a layer's @@ -186,6 +355,36 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { .copied() } +/// The planes a multi-layer file's picture is made of (see [`layered_image`]): red, green, blue +/// (one channel thrice for grey) and alpha, by name. Only the names of `aux` are read. +fn picture_names(aux: &AuxChannels) -> Option<([String; 3], Option)> { + let layers = aux.layers(); + let colour = picture_layer(aux, &layers)?; + let names = aux.layer_rgba(colour); + // A layer without R, G and B channels shows its first channel as grey. + let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); + let pick = |k: usize| names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first).to_string(); + let a = names.get(3).filter(|n| !n.is_empty() && aux.get(n).is_some()).cloned(); + Some(([pick(0), pick(1), pick(2)], a)) +} + +/// The channels of the picture of a multi-layer file and nothing else (names first from the +/// headers, then only those channels decoded; in a multi-part file the other parts' pixels are +/// not even decompressed), with the names of its planes. +fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let mut names = AuxChannels::new(0, 0, 1.0); + for header in meta.headers.iter() { + let prefix = part_name(header); + names.channels.extend(header.channels.list.iter().map(|ch| (channel_name(prefix.as_deref(), &ch.name.to_string()), vec![]))); + } + let (rgb, a) = picture_names(&names)?; + // Planes are looked up by name ignoring case, so every channel that could answer is kept. + let keep = |n: &str| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(n)); + let decoded = decode(bytes, &keep)?; + Some((decoded, rgb, a)) +} + /// The picture of a multi-layer OpenEXR file without an unnamed RGB layer (Blender's /// `ViewLayer.Combined.R`, the compositor's `Image.R`, Nuke's `beauty.R`, …), which the image /// decoder rejects. After Effects shows a file's main RGBA channels and leaves the other layers @@ -193,26 +392,13 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { /// that is not a data pass (never Cryptomatte, depth or normals, #412), else the first channel /// as grey. Linear, straight RGBA as the file stores it. `None` when it is not a readable EXR. pub fn layered_image(bytes: &[u8]) -> Option { - picture_of(&read_exr_channels(bytes)?) -} - -/// [`layered_image`] of the EXR at `path` from the channel cache, so the picture and EXtractoR / -/// Cryptomatte share one decode of the file (#474). -pub(crate) fn cached_layered_image(path: &str, bytes: Arc<[u8]>) -> Option { - let aux = cached(path, || Some(bytes))?; - picture_of(&aux) -} - -/// The picture of decoded channels (see [`layered_image`]). -fn picture_of(aux: &AuxChannels) -> Option { - let layers = aux.layers(); - let colour = picture_layer(aux, &layers)?; - let names = aux.layer_rgba(colour); - // A layer without R, G and B channels shows its first channel as grey. - let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); - let plane = |k: usize| -> Option<&[f32]> { aux.get(names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first)) }; - let (r, g, b) = (plane(0)?, plane(1)?, plane(2)?); - let a = names.get(3).filter(|n| !n.is_empty()).and_then(|n| aux.get(n)); + let (d, rgb, a) = picture_only(bytes)?; + let aux = &d.aux; + let [r, g, b] = [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?]; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; let n = (aux.width as usize).checked_mul(aux.height as usize)?; let mut data = Vec::with_capacity(n.checked_mul(4)?); for i in 0..n { @@ -227,40 +413,214 @@ fn picture_of(aux: &AuxChannels) -> Option { } } -type Cache = Mutex>)>>; +/// The picture made of planes `rgb` and `a` (opaque without one) of `aux`, as +/// [`crate::convert::dynamic_to_image`] makes it of the same values. `None` when a plane is +/// missing or short. +fn planes_picture(aux: &AuxChannels, rgb: [&[f32]; 3], a: Option<&[f32]>, op: AlphaOp) -> Option { + let n = (aux.width as usize).checked_mul(aux.height as usize)?; + if rgb.iter().chain(&a).any(|p| p.len() < n) { + return None; + } + let [r, g, b] = rgb; + let at = |p: &[f32], i: usize| p.get(i).copied().unwrap_or(0.0); + Some(float_to_image(aux.width, aux.height, op, |i| [at(r, i), at(g, i), at(b, i), a.map_or(1.0, |a| at(a, i))])) +} -fn cache() -> &'static Cache { - static CACHE: OnceLock = OnceLock::new(); - CACHE.get_or_init(|| Mutex::new(Vec::new())) +/// The picture of a decoded file by [`layered_image`]'s rule. +fn layered_picture_of(aux: &AuxChannels, op: AlphaOp) -> Option { + let (rgb, a) = picture_names(aux)?; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the multi-layer EXR at `path` ([`layered_image`]'s rule) as an [`Image`]: +/// from the channel cache when `shared` (an effect reads the footage's channels, so they are +/// decoded anyway), else of the picture's channels alone. +pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Option { + if shared { + let d = cache().get(path, || Some(bytes))?; + return layered_picture_of(&d.aux, op); + } + let (d, rgb, a) = picture_only(&bytes)?; + let aux = &d.aux; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the EXR at `path` from the channel cache (#474: EXtractoR / Cryptomatte read +/// this footage's channels, so the picture shares their decode): what the `image` crate's +/// decoder shows for a file it reads, [`layered_image`]'s rule for one it refuses. `None` when +/// the planes can't stand in for the `image` crate's picture (decode with it), or the file is +/// not a readable EXR. +pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Option { + let d = cache().get(path, || Some(bytes))?; + let aux = &d.aux; + match d.rgb { + Rgb::None => layered_picture_of(aux, op), + Rgb::Planes([r, g, b], a) => { + let plane = |i: usize| aux.channels.get(i).map(|(_, p)| p.as_slice()); + let a = match a { + Some(a) => Some(plane(a)?), + None => None, + }; + planes_picture(aux, [plane(r)?, plane(g)?, plane(b)?], a, op) + } + Rgb::Unsupported => None, + } +} + +/// How many bytes of decoded channels the cache keeps, like the frame cache's default +/// ([`crate::DEFAULT_BUDGET`]): two 2560×1440 files of 32 channels (470 MB each, as every plane +/// is f32). The newest file is kept whatever its size, so a frame's picture and its effects +/// share its decode. (Measured on such a sequence decoded one frame at a time: 1 GiB held peak +/// memory about 1 GB lower than 2 GiB, at the same speed.) +#[cfg(not(target_arch = "wasm32"))] +pub const CACHE_BUDGET: usize = 1 << 30; +#[cfg(target_arch = "wasm32")] +pub const CACHE_BUDGET: usize = 512 << 20; + +/// At most this many files are cached (unreadable ones hold no bytes). +const CACHE_FILES: usize = 64; + +/// A file's decode: done once, by the first caller; the others wait for it. +type Slot = Arc>>>; + +struct Entry { + path: String, + slot: Slot, + /// Bytes of the finished decode (0 while decoding). + bytes: usize, + /// When it was last asked for (a counter). + used: u64, +} + +/// Decoded EXR files by path, bounded by bytes, least recently used out first. +pub(crate) struct ChannelCache { + entries: Mutex<(Vec, u64)>, + budget: usize, + /// Files decoded (for tests). + decodes: AtomicU64, +} + +impl ChannelCache { + pub(crate) const fn new(budget: usize) -> Self { + Self { entries: Mutex::new((Vec::new(), 0)), budget, decodes: AtomicU64::new(0) } + } + + /// The decoded file at `path`, read with `read` by the first caller; callers asking for the + /// same file meanwhile wait for that decode instead of decoding it again. + pub(crate) fn get(&self, path: &str, read: impl FnOnce() -> Option>) -> Option> { + let slot = { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + match entries.iter_mut().find(|e| e.path == path) { + Some(e) => { + e.used = *clock; + e.slot.clone() + } + None => { + let slot = Slot::default(); + entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: *clock }); + slot + } + } + }; + let mut decoded_here = false; + let v = slot + .get_or_init(|| { + decoded_here = true; + self.decodes.fetch_add(1, Ordering::Relaxed); + read().and_then(|b| decode(&b, &|_| true)).map(Arc::new) + }) + .clone(); + if decoded_here { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + // (gone when the file was forgotten meanwhile) Used as of now: a file asked for + // before another but decoded after it is not the older one, or concurrent frames + // would evict each other's files before their effects read them. + if let Some(e) = entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, &slot)) { + e.bytes = v.as_ref().map_or(0, |d| d.bytes()); + e.used = *clock; + } + Self::evict(entries, self.budget, &slot); + } + v + } + + /// Drop least recently used finished files until the rest fit the budget, keeping `newest`. + fn evict(entries: &mut Vec, budget: usize, newest: &Slot) { + while entries.iter().map(|e| e.bytes).sum::() > budget || entries.len() > CACHE_FILES { + let oldest = entries + .iter() + .enumerate() + .filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && e.slot.get().is_some()) + .min_by_key(|(_, e)| e.used) + .map(|(i, _)| i); + let Some(i) = oldest else { break }; + entries.remove(i); + } + } + + pub(crate) fn forget(&self, path: &str) { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.retain(|e| e.path != path); + } + + #[cfg(test)] + fn decodes(&self) -> u64 { + self.decodes.load(Ordering::Relaxed) + } + + #[cfg(test)] + fn cached(&self) -> Vec { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.iter().map(|e| e.path.clone()).collect() + } +} + +fn cache() -> &'static ChannelCache { + static CACHE: ChannelCache = ChannelCache::new(CACHE_BUDGET); + &CACHE } /// Drop the cached channels of `path` (the file changed on disk). pub(crate) fn forget(path: &str) { - if let Ok(mut c) = cache().lock() { - c.retain(|(p, _)| p != path); - } + cache().forget(path); } -/// Read (and cache, a few files) the channels of the EXR at `path` via `read`. +/// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { - let cache = cache(); - if let Some(v) = cache.lock().ok().and_then(|c| c.iter().find(|(p, _)| p == path).map(|(_, v)| v.clone())) { - return v; - } - let v = read().and_then(|b| read_exr_channels(&b)).map(Arc::new); - if let Ok(mut c) = cache.lock() { - if c.len() >= 4 { - c.remove(0); - } - c.push((path.to_string(), v.clone())); - } - v + cache().get(path, read).map(|d| d.aux.clone()) } #[cfg(test)] mod tests { use super::*; + /// An EXR file of `layers` (with their display window `display`). + fn file(layers: Vec>>, display: exr::prelude::IntegerBounds) -> Vec { + use exr::prelude::*; + let image = Image::from_layers(ImageAttributes::new(display), layers); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() + } + + /// A single-part file of float channels named `names`, `w`×`h`, every sample `v`. + fn flat(names: &[&str], (w, h): (usize, usize), v: f32) -> Vec { + use exr::prelude::*; + let list: Vec> = names.iter().map(|n| AnyChannel::new(*n, FlatSamples::F32(vec![v; w * h]))).collect(); + let layer = Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())); + file(vec![layer], IntegerBounds::from_dimensions((w, h))) + } + #[test] fn reads_layered_channels_and_cryptomatte_metadata() { use exr::prelude::*; @@ -294,53 +654,229 @@ mod tests { assert!(read_exr_channels(b"not an exr").is_none()); } - /// #474: a full-window f32 channel (Cryptomatte ranks, …) becomes its plane without a copy, - /// so a decode no longer holds the file's samples twice; other channels are placed in the - /// display window as before. + /// #474: each line of samples goes straight into its display-window plane: a data window + /// inside the display window is placed with the fill around it (the background depth for + /// depth), one partly outside is clipped, never out of bounds; half and uint samples are + /// converted as the file stores them. #[test] - fn channels_become_planes_without_an_extra_copy() { - use exr::image::FlatSamples; - let v = vec![0.25f32; 6]; - let ptr = v.as_ptr(); - let plane = place(FlatSamples::F32(v), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(plane.as_ptr(), ptr, "moved, not copied"); - assert_eq!(plane, [0.25; 6]); - let half = |x: f32| exr::prelude::f16::from_f32(x); - let full = place(FlatSamples::F16(vec![half(0.5); 6]), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(full, [0.5; 6]); - // A 2×1 data window at (1, 1) of a 3×2 display window; the rest is the fill. - let part = place(FlatSamples::F16(vec![half(1.0), half(2.0)]), (2, 1), (1, 1), (3, 2), 6, 9.0).unwrap(); - assert_eq!(part, [9.0, 9.0, 9.0, 9.0, 1.0, 2.0]); - // Partly outside the display window: clipped, never out of bounds. - let off = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, -1), (3, 2), 6, 0.0).unwrap(); - assert_eq!(off, [0.0; 6]); - let edge = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(edge, [0.0, 0.0, 7.0, 0.0, 0.0, 0.0]); - assert!(place(FlatSamples::F32(vec![1.0; 5]), (3, 2), (0, 0), (3, 2), 6, 0.0).is_none(), "too few samples"); + fn channels_are_placed_in_the_display_window() { + use exr::prelude::*; + let half = |x: f32| f16::from_f32(x); + let part = |pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = LayerAttributes::default(); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::from_dimensions((3, 2)); + // A 2×1 data window at (1, 1). + let inside = file( + vec![part((1, 1), (2, 1), vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))])], + display, + ); + let aux = read_exr_channels(&inside).unwrap(); + assert_eq!(aux.get("G").unwrap(), [0.0, 0.0, 0.0, 0.0, 1.0, 2.0]); + let bg = BACKGROUND_DEPTH; + assert_eq!(aux.get("Z").unwrap(), [bg, bg, bg, bg, 7.0, 8.0]); + // A 2×2 data window at (2, -1): only its bottom-left sample lands at (2, 0). + let off = file(vec![part((2, -1), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0, 2.0, 3.0, 4.0]))])], display); + assert_eq!(read_exr_channels(&off).unwrap().get("G").unwrap(), [0.0, 0.0, 3.0, 0.0, 0.0, 0.0]); + // Entirely outside: all fill. + let away = file(vec![part((-5, 0), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0; 4]))])], display); + assert_eq!(read_exr_channels(&away).unwrap().get("G").unwrap(), [0.0; 6]); + } + + /// #474: concurrent requests for a file that isn't cached decode it once; the others wait + /// for that decode and share its planes. + #[test] + fn concurrent_requests_decode_a_file_once() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "Z"], (64, 64), 0.5).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let reads = AtomicU64::new(0); + let barrier = std::sync::Barrier::new(8); + let got: Vec> = std::thread::scope(|s| { + let jobs: Vec<_> = (0..8) + .map(|_| { + s.spawn(|| { + barrier.wait(); + cache + .get("/474-concurrent.exr", || { + reads.fetch_add(1, Ordering::Relaxed); + // Long enough for the others to arrive while this one decodes. + std::thread::sleep(std::time::Duration::from_millis(50)); + Some(bytes.clone()) + }) + .unwrap() + }) + }) + .collect(); + jobs.into_iter().map(|j| j.join().unwrap()).collect() + }); + assert_eq!(cache.decodes(), 1, "one decode"); + assert_eq!(reads.load(Ordering::Relaxed), 1, "the file is read once"); + assert!(got.iter().all(|d| Arc::ptr_eq(d, &got[0])), "every caller has the same planes"); + assert_eq!(got[0].aux.get("Z").and_then(|z| z.first()).copied(), Some(0.5)); + } + + /// #474: the cache is bounded by the bytes of its planes, least recently used out first; the + /// newest file stays even when it alone is over the budget, and unreadable files are cached + /// (as such) without counting bytes. + #[test] + fn cache_evicts_least_recently_used_by_bytes() { + // 4 float planes of 32×32: 16 KiB of samples (plus a few bytes of names) per file. + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + let get = |p: &str| cache.get(p, || Some(bytes.clone())).unwrap(); + assert_eq!(get("/a.exr").bytes(), one); + get("/b.exr"); + get("/a.exr"); // a is now more recent than b + get("/c.exr"); + assert_eq!(cache.cached(), ["/a.exr", "/c.exr"], "b was least recently used"); + assert_eq!(cache.decodes(), 3); + get("/a.exr"); + assert_eq!(cache.decodes(), 3, "a hit"); + // A file over the budget on its own is still kept while it is the newest. + let small = ChannelCache::new(one / 2); + assert!(small.get("/big.exr", || Some(bytes.clone())).is_some()); + assert_eq!(small.cached(), ["/big.exr"]); + small.get("/big2.exr", || Some(bytes.clone())); + assert_eq!(small.cached(), ["/big2.exr"]); + // An unreadable file is remembered (not read again) without bytes, and forgotten on request. + assert!(cache.get("/broken.exr", || Some(Arc::from(&b"not an exr"[..]))).is_none()); + assert!(cache.get("/broken.exr", || panic!("not read again")).is_none()); + cache.forget("/broken.exr"); + assert!(!cache.cached().contains(&"/broken.exr".to_string())); + // At most CACHE_FILES files, however small. + for i in 0..CACHE_FILES + 10 { + cache.get(&format!("/none{i}.exr"), || None); + } + assert!(cache.cached().len() <= CACHE_FILES); + } + + /// #474: a file counts as used when its decode finishes: one asked for first but decoded + /// after another is the newer of the two, so concurrent frames don't evict each other's + /// files before their effects read them. + #[test] + fn a_file_is_used_as_of_its_decode() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + cache.get("/first.exr", || { + // Another frame's file is asked for and decoded while this one decodes. + cache.get("/second.exr", || Some(bytes.clone())); + Some(bytes.clone()) + }); + cache.get("/third.exr", || Some(bytes.clone())); + assert_eq!(cache.cached(), ["/first.exr", "/third.exr"], "the file decoded first went first"); + } + + /// #474: a file forgotten (changed on disk) while it decodes isn't cached under its path when + /// that decode finishes. + #[test] + fn a_file_forgotten_while_it_decodes_is_not_cached() { + let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let d = cache.get("/changed.exr", || { + cache.forget("/changed.exr"); + Some(bytes.clone()) + }); + assert!(d.is_some()); + assert!(cache.cached().is_empty()); } /// #295: a multi-layer EXR whose colour is only in named layers (`diffuse.R`, as Blender and /// Nuke write them) failed to import ("does not contain non-deep rgb channels"); it now /// shows its colour layer and keeps every channel for EXtractoR. - /// #474: the picture and EXtractoR / Cryptomatte share one decode of a multi-layer file: - /// after the frame is decoded, the aux channels come from the cache without reading the file. + /// #474: once an effect has read a footage item's channels, the picture of each later frame + /// shares one decode with them: the frame's decode is cached and the channels come from it + /// without reading the file again. #[test] fn picture_and_aux_channels_share_one_decode() { use exr::prelude::*; let (w, h) = (2usize, 2usize); - let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); - let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", 0.25)].into()); - let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); - let mut bytes = std::io::Cursor::new(Vec::new()); - image.write().to_buffered(&mut bytes).unwrap(); - let b: Arc<[u8]> = bytes.into_inner().into(); - let path = "/474-shared-decode.exr"; + let frame = |v: f32| -> Arc<[u8]> { + let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); + let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", v)].into()); + let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner().into() + }; + let paths = ["/474-shared-decode_0001.exr", "/474-shared-decode_0002.exr"]; let pool = crate::MediaPool::new(); - pool.add_bytes(path, b.clone()); - let f = crate::probe_bytes(path, b).unwrap(); + pool.add_bytes(paths[0], frame(0.25)); + pool.add_bytes(paths[1], frame(0.75)); + let mut f = crate::probe_bytes(paths[0], frame(0.25)).unwrap(); + f.kind = effectcraft_project::FootageKind::Sequence; + f.sequence = paths.iter().map(|p| p.to_string()).collect(); + let t1 = f.frame_rate.tick_of(1); + use effectcraft_render::FootageSource; + // The first frame: the picture, then an effect reads the channels. pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap(); - let aux = cached(path, || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + let aux = pool.aux(effectcraft_project::ItemId(1), &f, effectcraft_time::Tick::ZERO).unwrap(); assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); + // The next frame's picture decodes the file into the cache, so its channels are not read. + let px = pool.frame_at(&f, t1).unwrap().get(0, 0); + let aux = cached(paths[1], || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.75)); + assert_eq!(px, pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap().get(0, 0), "the same picture"); + } + + /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the + /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 + /// around it), in every alpha interpretation and float curve; a file it can't stand in for + /// (a subsampled channel) is left to the `image` crate. + #[test] + fn picture_from_planes_is_the_image_crates() { + use effectcraft_project::{AlphaMode, Footage}; + use effectcraft_render::FloatCurve; + use exr::prelude::*; + let half = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F16((0..len).map(|i| f16::from_f32(v + i as f32 * 0.37 - 1.0)).collect())); + let float = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F32((0..len).map(|i| ((i * 7) % 11) as f32 * v).collect())); + let part = |name: Option<&str>, pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::SMALL_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::new((1, -1), (5, 4)); + let n = 20; + let files = [ + file(vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], display), + file(vec![part(None, (0, 0), (3, 2), vec![float("R", 0.4, 6), float("G", 0.5, 6), float("B", 0.6, 6), half("Image.R", 0.1, 6)])], display), + file( + vec![ + part(Some("diffuse"), (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n)]), + part(Some("beauty"), (2, 0), (2, 2), vec![half("R", 0.4, 4), half("G", 0.5, 4), half("B", 0.6, 4), half("A", 0.5, 4)]), + ], + display, + ), + ]; + let bits = |i: &effectcraft_raster::Image| i.data.iter().flat_map(|p| p.map(f32::to_bits)).collect::>(); + for (k, bytes) in files.iter().enumerate() { + let path = format!("/474-planes-{k}.exr"); + let d = cache().get(&path, || Some(bytes.as_slice().into())).unwrap(); + assert!(matches!(d.rgb, Rgb::Planes(..)), "file {k}: {:?}", d.rgb); + let img = crate::probe::load_oriented(bytes, Some(image::ImageFormat::OpenExr)).unwrap(); + for alpha in [AlphaMode::Straight, AlphaMode::Premultiplied, AlphaMode::Ignore] { + for curve in [FloatCurve::Srgb, FloatCurve::Linear, FloatCurve::Gamma24] { + let op = AlphaOp::of(&Footage { alpha, premul_color: [0.2, 0.4, 0.6], ..Default::default() }, curve); + let want = crate::convert::dynamic_to_image(&img, op); + let got = cached_picture(&path, bytes.as_slice().into(), op).unwrap(); + assert_eq!((got.width, got.height), (want.width, want.height)); + assert_eq!(bits(&got), bits(&want), "file {k} {alpha:?} {curve:?}"); + } + } + } + // A subsampled channel in the picture's part: the `image` crate decodes it. + let mut sub = part(None, (0, 0), (4, 2), vec![half("R", 0.1, 8), half("G", 0.2, 8), half("B", 0.3, 8), half("C", 0.3, 2)]); + if let Some(c) = sub.channel_data.list.iter_mut().find(|c| c.name == *"C") { + c.sampling = Vec2(2, 2); + } + let mut bytes = std::io::Cursor::new(Vec::new()); + if Image::from_layer(sub).write().to_buffered(&mut bytes).is_ok() { + let d = cache().get("/474-subsampled.exr", || Some(bytes.into_inner().into())).map(|d| d.rgb); + assert!(d.is_none_or(|r| r == Rgb::Unsupported), "{d:?}"); + } } /// #2141: one pixel of data in a 60000² display window is refused, not allocated per channel. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 655ca4ed8..9f1f2aa23 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -245,6 +245,9 @@ struct Inner { /// Movies that failed to open or decode (path to a readable reason; see /// [`FootageSource::problem`]). The first failure of a file is logged, later ones are not. problems: Mutex, String>>, + /// OpenEXR footage (by path) whose channels an effect has read (EXtractoR, Cryptomatte, the 3D + /// Channel effects): its pictures come from the same cached decode (#474). + aux_footage: Mutex>, } /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus @@ -315,6 +318,7 @@ impl MediaPool { conforming: Mutex::default(), conform_failed: Mutex::default(), problems: Mutex::default(), + aux_footage: Mutex::default(), }), } } @@ -856,13 +860,19 @@ impl Inner { { return Err(MediaError::Decode(format!("{path}: {why}"))); } - let loaded = crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)); - let img = match loaded { - Ok(img) => img, + // An effect reads this footage's channels: the picture is made of the same decode (#474). + let shared = lock(&self.aux_footage).contains(&footage.path); + if shared + && path.to_ascii_lowercase().ends_with(".exr") + && let Some(img) = crate::exr_channels::cached_picture(path, bytes.clone(), op) + { + return Ok(img); + } + match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { + Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. - Err(e) => crate::exr_channels::cached_layered_image(path, bytes).ok_or_else(|| MediaError::Decode(format!("{path}: {e}")))?, - }; - Ok(dynamic_to_image(&img, op)) + Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared).ok_or_else(|| MediaError::Decode(format!("{path}: {e}"))), + } } fn decode_media(&self, loc: &Loc, footage: &Footage) -> Result { @@ -962,6 +972,14 @@ impl FootageSource for MediaPool { return None; } let path = loc.key.path.clone(); + { + let mut wanted = lock(&self.inner.aux_footage); + // (a bound: names of footage items, never cleared otherwise) + if wanted.len() >= 4096 { + wanted.clear(); + } + wanted.insert(footage.path.clone()); + } crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) } diff --git a/docs/effects-parity.md b/docs/effects-parity.md index bd81a09df..ce4debd01 100644 --- a/docs/effects-parity.md +++ b/docs/effects-parity.md @@ -26,7 +26,7 @@ corpus compares renders, each effect is assumed to differ somewhere, and users k | Category | Effects | On the GPU | Known differences or bugs | |---|---:|---:|---| -| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR decode shared (#474 open: f16 storage and cache sizing remain) | +| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache bounded at 1 GB (#474 open: f16 storage remains) | | Audio | 13 | 0 | CPU by design (applied in the mixdown) | | Blur & Sharpen | 16 | 15 | Radial Blur keeps the layer edges as After Effects 2026 (713 now pixel-identical); Gaussian Blur interaction and output to be measured first ([fidelity/ae-2026-acceptance.md](fidelity/ae-2026-acceptance.md)) | | Channel | 13 | 13 | — | diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 6eeef1719..741582d9e 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); EXRs without an unnamed RGB layer decode twice (#474) | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, footage alone only its picture's channels (#474; decoded channels are held as f32) | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index f7f1df21b..0ebbeeecc 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -120,6 +120,11 @@ Nuke's `beauty`) show the beauty pass: the layer named Combined, Beauty, Image, Color, else the first colour layer that isn't a data pass. Cryptomatte, depth, mist, normals, vectors, positions and IDs are never the picture. EXtractoR reads every other layer and channel. +Footage on its own decodes only the channels of its picture. Once an effect reads an item's +channels (EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for +the picture and the effects together, and kept for a while (up to 1 GB of decoded channels); +several frames asking for the same file at once wait for one decode. + ## Replacing and reloading Importing a layered file as a composition refreshes the decoded sources used by its layers, diff --git a/docs/gaps.md b/docs/gaps.md index 016aeff22..46a96983a 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -65,7 +65,7 @@ are grouped by area; the issue tracker has the rest. | 6 | **Linux and Windows interaction bugs** | Open: #488 rename arrows (hardening landed, reporter asked for details), #492 anchor drag, #268 file drops on Wayland (winit), #613 Direct3D 12 compositor. Now closed: #491, #493, #467, #320, #302, #596 | Many reports come from Linux; some Windows GPUs fail to start the compositor | 25–45 h | G2 · [ui-parity.md](ui-parity.md) | | 7 | **Animation presets: 76 against 621** | After Effects ships 621 `.ffx` presets (Behaviors, Backgrounds, Shapes, Synthetics, Text, Transitions…); we ship 76 original presets (8 text-animator; 23 Transitions: wipes, clock and fan reveals, blinds, dissolves, card flips, CC wipes, fades, #757; 23 Backgrounds: drifting gradients, fog, cloud and smoke, plasma and magma, cells, hexagons, scrolling grids and checks, pulse rings, bokeh, light sweep, #1096, driven by expressions so they keep moving, not seamless loops; 22 Behaviors: wiggle, shake, float, orbit, bounce, drift, pendulum, spin, pulse, heartbeat, blink, strobe, flicker on transform and opacity, #1510) and read only our `.ecpreset`. Missing: Shapes, Synthetics, Image and more Text | Presets are how many designers start; tutorials assume them | 15–35 h (original presets; `.ffx` reading is out of clean-room scope) | G8 · [effects-parity.md](effects-parity.md) | | 8 | **Mask and shape bugs** | Open: #1004 mask vertex multi-select, #999 auto-pan, #1042 / #1033 vertex nodes under the bounding box, #959 / #960 shape group toggles and operators. Now closed: #510, #513, #527, #509, #303, #553, #932 (Convert To Bezier Path on a shape layer's parametric path) | Masks are core compositing | 10–20 h | G2 · [target-app-parity.md](target-app-parity.md) | -| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (decode shared, faffad3a; f16 storage and cache sizing open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | +| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | | 10 | **Colour pipeline differences** | #340 ProRes 4444 tagged bt709 but sRGB-encoded (open); ACES 2.0 / OCIO 2.5 configs (26.5) missing; now closed: #494 32-bpc blend modes, #495 and #750 Working Gamma 2.4, #546 / #547 viewer colour | Wrong colour in delivery is a client-visible failure | 15–30 h | G1 · [file-format-parity.md](file-format-parity.md) | | 11 | **Text rendering and layout** | #453 CJK text in comps: the Han fallback now reaches the installed Chinese faces; open: #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) |