diff --git a/crates/aep/src/container.rs b/crates/aep/src/container.rs index 531d74843..5f2e3b2bc 100644 --- a/crates/aep/src/container.rs +++ b/crates/aep/src/container.rs @@ -23,6 +23,7 @@ mod effects; mod error; mod footage; mod header; +pub mod index; mod keys; mod masks; mod matte; @@ -46,6 +47,7 @@ pub use effects::{DefinitionSource, Effect, EffectDefinitions, EffectParam, Laye pub use error::AepError; pub use footage::{Footage, SequenceInfo}; pub use header::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Quality, Rational, Renderer}; +pub use index::{Node, NodeWalk, Nodes, TreeIndex}; pub use keys::{ExpressionFlags, Interpolation, KeyEase, KeyframeKind, SpatialKey}; pub use masks::{Mask, MaskAtom, MaskMode, PathVertex, ShapePath}; pub use matte::{MatteMode, TrackMatte}; diff --git a/crates/aep/src/container/aepx.rs b/crates/aep/src/container/aepx.rs index a45cbae1d..6ca729807 100644 --- a/crates/aep/src/container/aepx.rs +++ b/crates/aep/src/container/aepx.rs @@ -227,7 +227,7 @@ impl Reader<'_> { let at = self.pos; self.pos = self.pos.saturating_add(1); let name = self.name()?; - let mut attrs: Vec<((usize, usize), Vec)> = Vec::new(); + let mut attrs = Vec::new(); let self_closing = loop { self.skip_ws(); let rest = self.rest(); @@ -259,8 +259,15 @@ impl Reader<'_> { if raw.contains(&b'<') { return Err(xml(vstart, "`<` inside an attribute value")); } - let mut value = Vec::new(); - decode_entities(raw, vstart, &mut value)?; + // A value without `&` is its own decoding (most of them, every `bdata` After Effects + // writes): borrow it instead of copying it. + let value = if raw.contains(&b'&') { + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + Cow::Owned(value) + } else { + Cow::Borrowed(raw) + }; self.pos = vend.saturating_add(1); attrs.push((key, value)); }; @@ -466,6 +473,15 @@ fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), A /// Decodes a `bdata` value (hex digits, two per byte). fn decode_hex(text: &[u8], offset: usize) -> Result, AepError> { + decode_hex_at_most(text, offset, MAX_BDATA) +} + +/// The most bytes one `bdata` may decode to: what the 32-bit size field of a chunk can hold (a +/// larger body could not be written back as an `.aep` either). +const MAX_BDATA: usize = u32::MAX as usize; + +/// [`decode_hex`] with the decoded-size limit as a parameter (so tests can reach it). +fn decode_hex_at_most(text: &[u8], offset: usize, max: usize) -> Result, AepError> { if text.len() % 2 == 1 { return Err(xml(offset, "bdata has an odd number of hex digits")); } @@ -476,11 +492,25 @@ fn decode_hex(text: &[u8], offset: usize) -> Result, AepError> { _ => None, }; let (pairs, _) = text.as_chunks::<2>(); - pairs - .iter() - .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) - .collect::>>() - .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) + if pairs.len() > max { + return Err(AepError::TooLarge); + } + // Nothing is reserved for a value that does not decode: one cheap pass checks every digit first. + // Blocks without an early exit inside, so the check vectorises; it stops at the first bad block. + if !text.chunks(4096).all(|block| block.iter().fold(true, |ok, b| ok & b.is_ascii_hexdigit())) { + return Err(xml(offset, "bdata holds a character that is not a hex digit")); + } + // Sized once (collecting into `Option` has no size hint and regrows step by step), and + // without aborting when the memory is not there. + let mut out = Vec::new(); + out.try_reserve_exact(pairs.len()).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; + for [hi, lo] in pairs { + match (digit(*hi), digit(*lo)) { + (Some(h), Some(l)) => out.push((h << 4) | l), + _ => return Err(xml(offset, "bdata holds a character that is not a hex digit")), + } + } + Ok(out) } /// The JSON body of an `alas` chunk from the attributes of a `fileReference`, keys in document @@ -628,6 +658,92 @@ mod tests { assert!(matches!(read_aepx(b"\xff"), Err(AepError::Xml { .. }))); } + /// Attribute values decode entities only where they have them. + #[test] + fn attribute_values_decode_entities_only_where_they_have_them() { + // An entity inside `bdata` still decodes before the hex does. + let f = read_aepx(doc("").as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&[0x01, 0xab][..])); + // A bad entity in any attribute, even of an element that is skipped, is still an error. + for bad in ["", "", ""] { + assert!(matches!(read_aepx(doc(bad).as_bytes()), Err(AepError::Xml { .. })), "{bad}"); + } + // Without `&` the value is taken as written. + let f = read_aepx(doc("").as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&[0x0a, 0xff][..])); + } + + /// A large bdata with a bad digit is refused before it is sized. + #[test] + fn a_large_bdata_with_a_bad_digit_is_refused_before_it_is_sized() { + // 10 MB of hex that is bad from the first pair (and once more at the very end). + let mut text = b"zz".to_vec(); + text.extend(std::iter::repeat_n(*b"00", 5_000_000).flatten()); + assert_eq!(decode_hex(&text, 3), Err(xml(3, "bdata holds a character that is not a hex digit"))); + let mut late = vec![b'0'; 10_000_000]; + late.extend_from_slice(b"0g"); + assert_eq!(decode_hex(&late, 3), Err(xml(3, "bdata holds a character that is not a hex digit"))); + let x = doc(&format!("", std::str::from_utf8(&text).unwrap())); + assert!(matches!(read_aepx(x.as_bytes()), Err(AepError::Xml { .. }))); + } + + /// Bdata decodes up to the size limit and not past it. + #[test] + fn bdata_decodes_up_to_the_size_limit_and_not_past_it() { + assert_eq!(decode_hex_at_most(b"00ff10", 0, 3), Ok(vec![0x00, 0xff, 0x10])); + assert_eq!(decode_hex_at_most(b"00ff10", 0, 2), Err(AepError::TooLarge)); + assert_eq!(decode_hex_at_most(b"", 0, 0), Ok(Vec::new())); + assert_eq!(decode_hex_at_most(b"00", 0, 0), Err(AepError::TooLarge)); + // The odd-length error still comes first. + assert_eq!(decode_hex_at_most(b"001", 0, 0), Err(xml(0, "bdata has an odd number of hex digits"))); + // The real limit is what a chunk's 32-bit size field holds. + assert_eq!(MAX_BDATA as u64, u64::from(u32::MAX)); + } + + /// Decode hex matches the collecting version. + #[test] + fn decode_hex_matches_the_collecting_version() { + /// The collecting decoder this one replaced, as the reference. + fn old(text: &[u8], offset: usize) -> Result, AepError> { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let digit = |b: u8| match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + }; + let (pairs, _) = text.as_chunks::<2>(); + pairs + .iter() + .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) + .collect::>>() + .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) + } + let alphabet = b"0123456789abcdefABCDEFgG \0"; + let mut seed = 0x2545_f491_4f6c_dd1du64; + for len in 0..200usize { + for _ in 0..20 { + let text: Vec = (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + // Mostly hex digits, now and then a byte that is not one. + let i = (seed % 1000) as usize; + alphabet[if i < 990 { i % 22 } else { 22 + i % 4 }] + }) + .collect(); + let new = decode_hex(&text, 7); + assert_eq!(new, old(&text, 7)); + if let Ok(v) = new { + assert_eq!(v.capacity(), v.len()); + } + } + } + } + #[test] fn limits_hold() { let deep = format!("{}{}", "".repeat(100_000), "".repeat(100_000)); diff --git a/crates/aep/src/container/arbitrary.rs b/crates/aep/src/container/arbitrary.rs index e4916cfe8..1fb8e8d86 100644 --- a/crates/aep/src/container/arbitrary.rs +++ b/crates/aep/src/container/arbitrary.rs @@ -7,9 +7,9 @@ //! Hue/Saturation (444-446) or a Levels value (512-516) changes only bytes of this `aRbp` (and stale //! UI bytes of `pard`); the visible parameters of the same names keep no record. -use crate::container::Chunk; use crate::container::bytes::{be_f32, be_i32, be_u16, be_u32}; use crate::container::effects::Effect; +use crate::container::index::Node; impl<'f, 'a> Effect<'f, 'a> { /// The arbitrary data stored for the parameter with this match name (`-NNNN`): the @@ -17,13 +17,13 @@ impl<'f, 'a> Effect<'f, 'a> { /// none. pub fn arbitrary_data(&self, match_name: &str) -> Option<&'f [u8]> { let values = self.chunk().child_list(b"tdgp")?; - let kids = values.children(); - let at = kids.iter().position(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; - // The value record, then the data list (no match name of its own). - let rest = kids.get(at.checked_add(1)?..)?; - let mut it = rest.iter().take(3); + let mut kids = values.children(); + // Up to the parameter's match name, then the value record, then the data list (no match + // name of its own). + kids.find(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; + let mut it = kids.take(3); let _record = it.next().filter(|c| c.is_list(b"tdbs"))?; - it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Chunk::bytes) + it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Node::bytes) } } diff --git a/crates/aep/src/container/effects.rs b/crates/aep/src/container/effects.rs index 2fa77d017..6ccfa3af6 100644 --- a/crates/aep/src/container/effects.rs +++ b/crates/aep/src/container/effects.rs @@ -20,8 +20,8 @@ use std::collections::{HashMap, HashSet}; -use crate::container::Chunk; use crate::container::bytes::{array, be_f32, be_f64, be_i32, be_u16, be_u32, cstr, text, until_nul}; +use crate::container::index::Node; use crate::container::typed::{KeyList, Layer, Project, PropGroup, PropNode, Property}; /// The parameter type, the `u32` at byte 12 of `pard`. The numbering matches the plug-in @@ -132,7 +132,7 @@ pub enum LayerRef { #[derive(Debug, Clone)] pub struct Effect<'f, 'a> { match_name: String, - chunk: &'f Chunk<'a>, + chunk: Node<'f, 'a>, } /// Where the parameter definitions of an effect instance come from. @@ -178,13 +178,13 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The underlying `LIST sspc` chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } /// The instance name shown in the effect controls (`fnam`). pub fn name(&self) -> Option { - self.chunk.child(b"fnam")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text) + self.chunk.child(b"fnam")?.child(b"Utf8")?.bytes().map(text) } /// The name the user gave the instance: the `tdsn` of the instance's value group when it is @@ -192,7 +192,7 @@ impl<'f, 'a> Effect<'f, 'a> { /// `None` while the instance shows the effect's name. pub fn user_name(&self) -> Option { let values = self.chunk.child_list(b"tdgp")?; - let name = values.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; + let name = values.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (name != "-_0_/-" && !name.is_empty()).then_some(name) } @@ -250,7 +250,7 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The parameters described by `part` (a `LIST parT`) with the values of this instance. - fn params_from(&self, part: &'f Chunk<'a>, borrowed: bool) -> Vec> { + fn params_from(&self, part: Node<'f, 'a>, borrowed: bool) -> Vec> { let mut out: Vec> = Vec::new(); // The stored values by match name: the first entry of a name, as `PropGroup::get` finds it // (a group of that name means no stored value). Indexed once: a lookup per parameter @@ -280,7 +280,7 @@ impl<'f, 'a> Effect<'f, 'a> { }); } else if c.is(b"pdnm") && let Some(last) = out.last_mut() - && let Some(u) = c.children().iter().find(|u| u.is(b"Utf8")) + && let Some(u) = c.child(b"Utf8") && let Some(b) = u.bytes() // An empty `pdnm` names nothing (EffectCraft's exports write one: the effect // supplies its items). @@ -293,8 +293,9 @@ impl<'f, 'a> Effect<'f, 'a> { } } -fn has_pard(part: &Chunk<'_>) -> bool { - part.children().iter().any(|c| c.is(b"pard")) +/// True when a `LIST parT` holds parameter definitions (a `pard`). +fn has_pard(part: Node<'_, '_>) -> bool { + part.children().any(|c| c.is(b"pard")) } impl<'f, 'a> Project<'f, 'a> { @@ -302,18 +303,18 @@ impl<'f, 'a> Project<'f, 'a> { /// name and the `LIST parT` of the copied instance, in file order (the stack order of the /// types, 365/366). The `EfDC` count before them is little-endian and is not needed to read /// them. A type whose copy has no definitions is left out. - pub fn effect_definitions(&self) -> Vec<(String, &'f Chunk<'a>)> { - let mut out: Vec<(String, &'f Chunk<'a>)> = Vec::new(); + pub fn effect_definitions(&self) -> Vec<(String, Node<'f, 'a>)> { + let mut out: Vec<(String, Node<'f, 'a>)> = Vec::new(); let Some(defs) = self.effect_defs else { return out }; let mut seen: HashSet = HashSet::new(); - for df in defs.children().iter().filter(|c| c.is_list(b"EfDf")) { + for df in defs.children().filter(|c| c.is_list(b"EfDf")) { let mut name: Option = None; for c in df.children() { if c.is(b"tdmn") { name = c.bytes().map(|b| text(until_nul(b))); } else if c.is_list(b"sspc") && let Some(n) = name.take() - && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(p)) + && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(*p)) && seen.insert(n.clone()) { out.push((n, part)); @@ -328,8 +329,8 @@ impl<'f, 'a> Project<'f, 'a> { /// For a type without a project-level copy it also keeps the own `parT` of the first instance /// in the file (items, layers and effects in file order) that has one. pub fn effect_definition_index(&self) -> EffectDefinitions<'f, 'a> { - let by_name: HashMap> = self.effect_definitions().into_iter().collect(); - let mut instances: HashMap> = HashMap::new(); + let by_name: HashMap> = self.effect_definitions().into_iter().collect(); + let mut instances: HashMap> = HashMap::new(); for item in self.all_items() { let Some(comp) = item.comp() else { continue }; for layer in comp.layers() { @@ -337,7 +338,7 @@ impl<'f, 'a> Project<'f, 'a> { if by_name.contains_key(&fx.match_name) || instances.contains_key(&fx.match_name) { continue; } - if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(p)) { + if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(*p)) { instances.insert(fx.match_name, part); } } @@ -363,9 +364,9 @@ impl<'f, 'a> Project<'f, 'a> { #[derive(Debug, Clone, Default)] pub struct EffectDefinitions<'f, 'a> { /// The project-level copies (`EfdG`). - by_name: HashMap>, + by_name: HashMap>, /// For a type without a project-level copy: the own `parT` of the first instance that has one. - instances: HashMap>, + instances: HashMap>, } impl<'f, 'a> EffectDefinitions<'f, 'a> { @@ -392,7 +393,7 @@ impl<'f, 'a> EffectDefinitions<'f, 'a> { return effect.params(); } match self.by_name.get(effect.match_name()).or_else(|| self.instances.get(effect.match_name())) { - Some(part) => effect.params_from(part, true), + Some(part) => effect.params_from(*part, true), None => Vec::new(), } } @@ -638,6 +639,7 @@ impl<'f, 'a> Layer<'f, 'a> { #[cfg(test)] mod tests { use super::*; + use crate::container::Chunk; fn tdmn(name: &str) -> Chunk<'static> { let mut b = name.as_bytes().to_vec(); @@ -670,11 +672,11 @@ mod tests { ("A".to_string(), Chunk::list(*b"tdbs", vec![Chunk::data(*b"cdat", vec![0u8; 8])])), ]; let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["A", "B", "C"]); let first = params.first().and_then(|p| p.stored).expect("A stored"); - assert!(first.chunk.children().is_empty(), "the first A (the second has a cdat)"); + assert!(first.chunk.children().next().is_none(), "the first A (the second has a cdat)"); assert!(params.get(1).is_some_and(|p| p.stored.is_none()), "a group of that name is no stored value"); assert!(params.get(2).is_some_and(|p| p.stored.is_none())); } @@ -687,7 +689,7 @@ mod tests { let names: Vec = (0..n).map(|i| format!("P{i}")).collect(); let stored: Vec<(String, Chunk<'static>)> = names.iter().map(|m| (m.clone(), tdbs())).collect(); let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.len(), n); assert!(params.iter().all(|p| p.stored.is_some())); @@ -707,19 +709,19 @@ mod tests { } let efdg = Chunk::list(*b"EfdG", defs); let fold = Chunk::list(*b"Fold", Vec::new()); - let project = Project { fold: &fold, effect_defs: Some(&efdg) }; + let project = Project { fold: Node::from(&fold), effect_defs: Some(Node::from(&efdg)) }; let all = project.effect_definitions(); assert_eq!(all.len(), n); let index = project.effect_definition_index(); let bare = Chunk::list(*b"sspc", vec![Chunk::list(*b"parT", Vec::new())]); for i in (0..n).step_by(50) { - let fx = Effect { match_name: format!("FX{i}"), chunk: &bare }; + let fx = Effect { match_name: format!("FX{i}"), chunk: Node::from(&bare) }; assert_eq!(index.source(&fx), DefinitionSource::Project); let params = index.params(&fx); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["R0"], "the first copy's definitions"); assert!(params.iter().all(|p| p.borrowed_definition)); } - let unknown = Effect { match_name: "nope".into(), chunk: &bare }; + let unknown = Effect { match_name: "nope".into(), chunk: Node::from(&bare) }; assert_eq!(index.source(&unknown), DefinitionSource::Missing); assert!(index.params(&unknown).is_empty()); assert_eq!(project.definition_source(&unknown), DefinitionSource::Missing); diff --git a/crates/aep/src/container/footage.rs b/crates/aep/src/container/footage.rs index c1007fce5..ce7c8e6c5 100644 --- a/crates/aep/src/container/footage.rs +++ b/crates/aep/src/container/footage.rs @@ -2,8 +2,8 @@ //! of a layer's source. use crate::container::bytes::{be_f32, be_f64, be_u32, cstr, le_u32, text}; -use crate::container::chunk::Chunk; use crate::container::header::Rational; +use crate::container::index::Node; use crate::container::typed::{Item, ItemKind, Layer, Project}; /// The footage data of an item (`LIST Pin ` with `sspc`, `opti` and, for files, `Als2/alas`). @@ -201,14 +201,13 @@ impl SequenceInfo { /// is unreadable (empty). A name that is not a plain file name (holding `/`, `\`, `:` or a control /// character such as NUL, or being `.` or `..`) makes the whole list unreadable (empty): the names /// are joined to the folder. -fn sequence_file_list(list: &Chunk<'_>) -> Vec { +fn sequence_file_list(list: Node<'_, '_>) -> Vec { // After Effects writes the count first (the one sample); a list without a readable count is // not read rather than guessed. let Some(count) = list.child(b"StVS").and_then(|c| c.bytes()).and_then(|b| le_u32(b, 0)).and_then(|n| usize::try_from(n).ok()) else { return Vec::new(); }; - let names: Vec = - list.children().iter().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).take(count).map(text).filter(|n| !n.is_empty()).collect(); + let names: Vec = list.children().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).take(count).map(text).filter(|n| !n.is_empty()).collect(); if names.iter().any(|n| n.contains(|c: char| matches!(c, '/' | '\\' | ':') || c.is_control()) || n == "." || n == "..") { return Vec::new(); } @@ -240,9 +239,8 @@ impl<'f, 'a> Item<'f, 'a> { } let pin = self.chunk.child_list(b"Pin ")?; let sspc = pin.child(b"sspc")?.bytes()?; - let kids = pin.children(); - let after = kids.iter().position(|c| c.is_list(b"Als2"))?; - let naming = || kids.iter().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); + let after = pin.children().position(|c| c.is_list(b"Als2"))?; + let naming = || pin.children().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); let (start, end, digits) = (be_u32(sspc, 172)?, be_u32(sspc, 176)?, be_u32(sspc, 180)?); // Force alphabetical order: the file list replaces the numbering. if let Some(list) = naming().find(|c| c.is_list(b"StVc")) { @@ -252,7 +250,7 @@ impl<'f, 'a> Item<'f, 'a> { } return Some(SequenceInfo { prefix: String::new(), suffix: String::new(), start, end, digits, files }); } - let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).map(text).take(2).collect(); + let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).map(text).take(2).collect(); let (prefix, suffix) = match names.as_slice() { [prefix, suffix] => (prefix.clone(), suffix.clone()), // An empty prefix is not stored (`0001.png` has only `.png`). diff --git a/crates/aep/src/container/index.rs b/crates/aep/src/container/index.rs new file mode 100644 index 000000000..30d7fcb9d --- /dev/null +++ b/crates/aep/src/container/index.rs @@ -0,0 +1,978 @@ +//! A read-only, flat index of the chunk tree for readers that only look (the importer). +//! +//! [`TreeIndex`] holds every chunk of a file in one `Vec` in depth-first order (a chunk, then its +//! whole subtree, then its next sibling). Each entry keeps its id, list type, pad byte, the bytes of +//! its body (borrowed from the input) and the index one past its subtree, so: +//! - a chunk's subtree is the index range `at + 1 .. end`, +//! - its direct children are found by stepping from `at + 1` to each child's `end`, +//! - the whole tree is one array, grown a few times while it is built and freed at once; it never +//! holds more slots than the input has room for chunk headers or the chunk limit allows. +//! +//! [`crate::container::parse`] builds the owned [`Chunk`] tree instead (one `Vec` per list), which +//! the exporter and editors need. [`TreeIndex::to_file`] and [`Node::to_chunk`] convert the index +//! into that tree where an owned one is needed. +//! +//! [`TreeIndex::parse_with_limits`] walks the input exactly as the [`Chunk`] parser does: the same +//! chunks are counted against [`Limits::max_chunks`] in the same order, the same lists are kept +//! opaque at [`Limits::max_depth`] or when damaged, and the same errors come back. A test compares +//! the two on the corpus and on generated and damaged trees. Two errors are its own, and both are +//! stricter than the [`Chunk`] parser (which would abort or could not address the tree): +//! [`AepError::TooLarge`] when the array cannot grow (a failed `try_reserve`), and +//! [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. +//! +//! [`Node`] is the handle the typed view ([`crate::container::Project`] and the types below it) +//! reads through: a chunk of either tree. The view works the same on both. + +use std::borrow::Cow; +use std::fmt; + +use crate::container::bytes::be_u32; +use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS, Walk}; +use crate::container::error::AepError; +use crate::container::read::{Limits, container_end, id4}; + +/// What an entry's body is (the [`Body`] variants). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + Data, + /// [`Body::Chunks`]. + Nested, + List, + OpaqueList, +} + +/// One chunk of a [`TreeIndex`]. +#[derive(Debug, Clone, Copy)] +struct Entry<'a> { + /// The body bytes of a data chunk, or the bytes after the list type of an opaque list; empty + /// for a list or nested chunk (their content is their children). + data: &'a [u8], + id: [u8; 4], + /// The list type of a `LIST` (parsed or opaque); zero otherwise. + list_type: [u8; 4], + /// One past the last entry of this chunk's subtree. + end: u32, + kind: Kind, + pad: Option, +} + +impl<'a> Entry<'a> { + /// An entry for a chunk whose body is not read yet. + fn placeholder(id: [u8; 4]) -> Self { + Self { data: &[], id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None } + } +} + +/// The chunk tree of a file as one flat, read-only array (see the module documentation). +#[derive(Clone)] +pub struct TreeIndex<'a> { + nodes: Vec>, + trailer: &'a [u8], +} + +impl fmt::Debug for TreeIndex<'_> { + /// The sizes only: the chunk count and the trailer length. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TreeIndex").field("chunks", &self.nodes.len()).field("trailer", &self.trailer.len()).finish() + } +} + +/// An index that does not fit a `u32` (more than 4 billion chunks) is refused as too many chunks. +fn index_u32(at: usize) -> Result { + u32::try_from(at).map_err(|_| AepError::TooManyChunks) +} + +impl<'a> TreeIndex<'a> { + /// Indexes an `.aep` with the default [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::container::parse`], for the same inputs. + pub fn parse(data: &'a [u8]) -> Result { + Self::parse_with_limits(data, Limits::default()) + } + + /// Indexes an `.aep` with explicit [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::container::parse_with_limits`], for the same inputs and limits; also + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks and + /// [`AepError::TooLarge`] when the array cannot be allocated. + pub fn parse_with_limits(data: &'a [u8], limits: Limits) -> Result { + let end = container_end(data)?; + // Every chunk header takes 8 bytes of its own and is counted once, so the tree never has + // more chunks than this. + let bound = (end.saturating_sub(12) / 8).min(limits.max_chunks); + let mut builder = Builder { data, limits, count: 0, nodes: Vec::new(), bound }; + // A small start; [`Builder::push`] doubles it as chunks arrive, never past the bound, so + // the array stays proportional to the chunk count (a file of a few large chunks gets a few + // slots, not one per 32 bytes). `try_reserve_exact` so it fails nothing here. + let _ = builder.nodes.try_reserve_exact(bound.min(INITIAL_SLOTS)); + builder.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(Self { nodes: builder.nodes, trailer }) + } + + /// Indexes an already parsed (or built, or `.aepx`) chunk tree. The index borrows the file's + /// bytes. + /// + /// # Errors + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. + pub fn from_file(file: &'a AepFile<'_>) -> Result { + // Sized once, to the chunks the tree holds. + let mut nodes: Vec> = Vec::new(); + nodes.try_reserve_exact(file.walk().count()).map_err(|_| AepError::TooLarge)?; + // Each frame: the siblings still to visit and the entry whose subtree they fill. + let mut stack: Vec<(std::slice::Iter<'a, Chunk<'_>>, Option)> = vec![(file.chunks.iter(), None)]; + while let Some((iter, parent)) = stack.last_mut() { + let parent = *parent; + match iter.next() { + Some(chunk) => { + let at = nodes.len(); + // The entry after this one must still have a `u32` index. + let next = index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + let (kind, data, list_type) = match &chunk.body { + Body::Data(d) => (Kind::Data, &d[..], [0; 4]), + Body::Chunks(_) => (Kind::Nested, &[][..], [0; 4]), + Body::List { list_type, .. } => (Kind::List, &[][..], *list_type), + Body::OpaqueList { list_type, data } => (Kind::OpaqueList, &data[..], *list_type), + }; + nodes.push(Entry { data, id: chunk.id, list_type, end: next, kind, pad: chunk.pad }); + let kids = chunk.children(); + if !kids.is_empty() { + stack.push((kids.iter(), Some(at))); + } + } + None => { + stack.pop(); + if let Some(p) = parent { + let end = index_u32(nodes.len())?; + if let Some(e) = nodes.get_mut(p) { + e.end = end; + } + } + } + } + } + Ok(Self { nodes, trailer: &file.trailer[..] }) + } + + /// The number of chunks in the tree (at every depth). + pub fn len(&self) -> usize { + self.nodes.len() + } + + /// True when the container holds no chunks. + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// The bytes after the container's declared end (see [`AepFile::trailer`]). + pub fn trailer(&self) -> &'a [u8] { + self.trailer + } + + /// The top-level chunks. + pub fn chunks(&self) -> Nodes<'_, 'a> { + Nodes(NodesRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX) }) + } + + /// The first top-level chunk with this id. + pub fn chunk(&self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub fn list(&self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } + + /// Visits every chunk depth-first with its nesting depth (the top level is 1), as + /// [`AepFile::walk`] does. + pub fn walk(&self) -> NodeWalk<'_, 'a> { + NodeWalk(WalkRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX), open: Vec::new() }) + } + + /// The owned chunk tree (the bodies stay borrowed from the input): what + /// [`crate::container::parse`] gives for the same file. + pub fn to_file(&self) -> AepFile<'a> { + AepFile { chunks: self.build(0, u32::try_from(self.nodes.len()).unwrap_or(u32::MAX)), trailer: Cow::Borrowed(self.trailer) } + } + + /// The entry at `at`, `None` past the end. + fn entry(&self, at: u32) -> Option<&Entry<'a>> { + self.nodes.get(usize::try_from(at).ok()?) + } + + /// The end of the subtree at `at`; never before `at + 1`, so walks always move on. + fn end_of(&self, at: u32) -> u32 { + self.entry(at).map_or(u32::MAX, |e| e.end).max(at.saturating_add(1)) + } + + /// The owned chunks for the siblings that start at `next` and fill `next..end`, without + /// recursion. + fn build(&self, next: u32, end: u32) -> Vec> { + struct Frame<'a> { + /// The list or nested chunk being filled; `None` for the outermost range. + at: Option, + kids: Vec>, + next: u32, + end: u32, + } + let mut stack = vec![Frame { at: None, kids: Vec::new(), next, end }]; + loop { + let Some(top) = stack.last_mut() else { return Vec::new() }; + if top.next < top.end { + let at = top.next; + top.next = self.end_of(at); + let Some(e) = self.entry(at).copied() else { + top.next = top.end; + continue; + }; + match e.kind { + Kind::List | Kind::Nested => { + let sub_end = self.end_of(at); + stack.push(Frame { at: Some(at), kids: Vec::new(), next: at.saturating_add(1), end: sub_end }); + } + Kind::Data => top.kids.push(Chunk { id: e.id, body: Body::Data(Cow::Borrowed(e.data)), pad: e.pad }), + Kind::OpaqueList => { + top.kids.push(Chunk { id: e.id, body: Body::OpaqueList { list_type: e.list_type, data: Cow::Borrowed(e.data) }, pad: e.pad }); + } + } + } else { + let Some(frame) = stack.pop() else { return Vec::new() }; + let Some(at) = frame.at else { return frame.kids }; + let Some(e) = self.entry(at).copied() else { continue }; + let body = match e.kind { + Kind::Nested => Body::Chunks(frame.kids), + Kind::List | Kind::Data | Kind::OpaqueList => Body::List { list_type: e.list_type, children: frame.kids }, + }; + let chunk = Chunk { id: e.id, body, pad: e.pad }; + match stack.last_mut() { + Some(parent) => parent.kids.push(chunk), + None => return vec![chunk], + } + } + } + } +} + +/// The slots a [`TreeIndex`] starts with (32 KiB); it doubles from there. +const INITIAL_SLOTS: usize = 1024; + +/// Builds a [`TreeIndex`]; mirrors `read::Parser` step for step (see the module documentation). +struct Builder<'a> { + data: &'a [u8], + limits: Limits, + count: usize, + nodes: Vec>, + /// The most chunks the input can hold under the limits: `nodes` never reserves past it. + bound: usize, +} + +impl<'a> Builder<'a> { + /// Appends an entry. A full array grows by its length (at least 16), but never past + /// [`Self::bound`]; an allocation the target cannot make is an error, not an abort. + fn push(&mut self, entry: Entry<'a>) -> Result<(), AepError> { + let len = self.nodes.len(); + if len == self.nodes.capacity() { + let more = len.max(16).min(self.bound.saturating_sub(len)).max(1); + self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?; + } + self.nodes.push(entry); + Ok(()) + } + + /// Appends the sibling chunks that exactly fill `start..end`, each followed by its subtree. + fn sequence(&mut self, start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let mut pos = start; + while pos < end { + let short = AepError::Malformed { offset: pos, what: "bytes left over that are too short for a chunk header" }; + let body_start = pos.checked_add(8).filter(|b| *b <= end).ok_or(short)?; + let id = id4(self.data, pos).ok_or(AepError::Malformed { offset: pos, what: "short read" })?; + let size = be_u32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; + let avail = end - body_start; + if size > avail { + return Err(AepError::ChunkTooLarge { offset: pos, size: size as u64, available: avail as u64 }); + } + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + let at = self.nodes.len(); + // This entry and the one after it must have `u32` indices. + index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + self.push(Entry::placeholder(id))?; + let body_end = body_start + size; + self.body(at, id, body_start, body_end, depth)?; + let mut next = body_end; + let mut pad = None; + if size % 2 == 1 && next < end { + pad = Some(self.data.get(next).copied().ok_or(AepError::Malformed { offset: next, what: "missing pad byte" })?); + next += 1; + } + let subtree_end = index_u32(self.nodes.len())?; + if let Some(e) = self.nodes.get_mut(at) { + e.pad = pad; + e.end = subtree_end; + } + pos = next; + } + Ok(()) + } + + /// Like [`Self::sequence`] but a structural problem yields `false` and removes what the + /// attempt appended (the caller keeps bytes). The chunk count keeps the attempt's chunks, as + /// the [`Chunk`] parser's does. + fn tentative(&mut self, start: usize, end: usize, depth: usize) -> Result { + let mark = self.nodes.len(); + match self.sequence(start, end, depth) { + Ok(()) => Ok(true), + Err(e) if e.is_structural() => { + self.nodes.truncate(mark); + Ok(false) + } + Err(e) => Err(e), + } + } + + /// The input bytes `start..end` (empty when out of range). + fn slice(&self, start: usize, end: usize) -> &'a [u8] { + self.data.get(start..end).unwrap_or(&[]) + } + + /// Fills in the body of the entry at `at` once the parser knows what it is. + fn set(&mut self, at: usize, kind: Kind, data: &'a [u8], list_type: [u8; 4]) { + if let Some(e) = self.nodes.get_mut(at) { + e.kind = kind; + e.data = data; + e.list_type = list_type; + } + } + + /// Fills in the entry at `at` for the body `start..end` and appends its subtree. + fn body(&mut self, at: usize, id: [u8; 4], start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let len = end - start; + if id == LIST && len >= 4 { + let list_type = id4(self.data, start).ok_or(AepError::Malformed { offset: start, what: "short read" })?; + if depth < self.limits.max_depth && self.tentative(start + 4, end, depth + 1)? { + self.set(at, Kind::List, &[], list_type); + } else { + let data = self.slice(start + 4, end); + self.set(at, Kind::OpaqueList, data, list_type); + } + return Ok(()); + } + if NESTED_IDS.contains(&id) && len >= 8 && depth < self.limits.max_depth { + let mark = self.nodes.len(); + if self.tentative(start, end, depth + 1)? && self.nodes.len() > mark { + self.set(at, Kind::Nested, &[], [0; 4]); + return Ok(()); + } + } + let data = self.slice(start, end); + self.set(at, Kind::Data, data, [0; 4]); + Ok(()) + } +} + +/// A chunk of either tree: an owned [`Chunk`] or an entry of a [`TreeIndex`]. The typed view +/// reads through it, so it works the same on both. Cheap to copy. +#[derive(Clone, Copy)] +pub struct Node<'f, 'a>(NodeRepr<'f, 'a>); + +#[derive(Clone, Copy)] +enum NodeRepr<'f, 'a> { + Chunk(&'f Chunk<'a>), + Index(&'f TreeIndex<'a>, u32), +} + +impl<'f, 'a> From<&'f Chunk<'a>> for Node<'f, 'a> { + /// The handle for this value. + fn from(chunk: &'f Chunk<'a>) -> Self { + Node(NodeRepr::Chunk(chunk)) + } +} + +/// Prints the chunk as [`Chunk`]'s `Debug` does, whichever tree it is in, without copying the +/// subtree. +impl fmt::Debug for Node<'_, '_> { + /// The chunk as `Chunk`'s `Debug` prints it. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + NodeRepr::Chunk(c) => fmt::Debug::fmt(c, f), + NodeRepr::Index(..) => f.debug_struct("Chunk").field("id", &self.id()).field("body", &BodyDebug(*self)).field("pad", &self.pad()).finish(), + } + } +} + +/// The body of an index [`Node`], printed as [`Body`]'s `Debug` prints it. +struct BodyDebug<'f, 'a>(Node<'f, 'a>); + +/// The children of a [`Node`], printed as a list. +struct ChildrenDebug<'f, 'a>(Node<'f, 'a>); + +impl fmt::Debug for BodyDebug<'_, '_> { + /// The body as `Body`'s `Debug` prints it. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Some(e) = self.0.entry() else { return f.write_str("?") }; + match e.kind { + Kind::Data => f.debug_tuple("Data").field(&e.data).finish(), + Kind::Nested => f.debug_tuple("Chunks").field(&ChildrenDebug(self.0)).finish(), + Kind::List => f.debug_struct("List").field("list_type", &e.list_type).field("children", &ChildrenDebug(self.0)).finish(), + Kind::OpaqueList => f.debug_struct("OpaqueList").field("list_type", &e.list_type).field("data", &e.data).finish(), + } + } +} + +impl fmt::Debug for ChildrenDebug<'_, '_> { + /// The children as a list. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.0.children()).finish() + } +} + +impl<'f, 'a> Node<'f, 'a> { + /// The entry at `at`, `None` past the end. + fn entry(self) -> Option<&'f Entry<'a>> { + match self.0 { + NodeRepr::Index(tree, at) => tree.entry(at), + NodeRepr::Chunk(_) => None, + } + } + + /// The [`Chunk`] when this node is one of an owned tree; `None` for an index entry (see + /// [`Self::to_chunk`]). + pub fn as_chunk(self) -> Option<&'f Chunk<'a>> { + match self.0 { + NodeRepr::Chunk(c) => Some(c), + NodeRepr::Index(..) => None, + } + } + + /// The chunk and its subtree as an owned [`Chunk`] (bodies stay borrowed). + pub fn to_chunk(self) -> Chunk<'a> { + match self.0 { + NodeRepr::Chunk(c) => c.clone(), + NodeRepr::Index(tree, at) => tree.build(at, tree.end_of(at)).into_iter().next().unwrap_or_else(|| Chunk::data(self.id(), &[][..])), + } + } + + /// The 4-byte id (see [`Chunk::id`]). + pub fn id(self) -> [u8; 4] { + match self.0 { + NodeRepr::Chunk(c) => c.id, + NodeRepr::Index(..) => self.entry().map_or([0; 4], |e| e.id), + } + } + + /// The pad byte (see [`Chunk::pad`]). + pub fn pad(self) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.pad, + NodeRepr::Index(..) => self.entry().and_then(|e| e.pad), + } + } + + /// The raw bytes of a data chunk, or the bytes after the list type of an opaque list (see + /// [`Chunk::bytes`]). + pub fn bytes(self) -> Option<&'f [u8]> { + match self.0 { + NodeRepr::Chunk(c) => c.bytes(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::Data | Kind::OpaqueList)).map(|e| e.data), + } + } + + /// The child chunks of a list or nested chunk; none for data. + pub fn children(self) -> Nodes<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => Nodes(NodesRepr::Slice(c.children().iter())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + Nodes(NodesRepr::Index { tree, next: at.saturating_add(1), end }) + } + } + } + + /// The list type when this is a `LIST` (parsed or opaque). + pub fn list_type(self) -> Option<[u8; 4]> { + match self.0 { + NodeRepr::Chunk(c) => c.list_type(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::List | Kind::OpaqueList)).map(|e| e.list_type), + } + } + + /// True when this is a `LIST` of the given type. + pub fn is_list(self, list_type: &[u8; 4]) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_list(list_type), + NodeRepr::Index(..) => self.entry().is_some_and(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type), + } + } + + /// True when this is a `LIST` whose content could not be read as chunks. + pub fn is_opaque_list(self) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_opaque_list(), + NodeRepr::Index(..) => self.entry().is_some_and(|e| e.kind == Kind::OpaqueList), + } + } + + /// True when this chunk has the given id. + pub fn is(self, id: &[u8; 4]) -> bool { + self.id() == *id + } + + /// The first child chunk with this id. + pub fn child(self, id: &[u8; 4]) -> Option { + self.children().find(|c| c.is(id)) + } + + /// The first child `LIST` of this type. + pub fn child_list(self, list_type: &[u8; 4]) -> Option { + self.children().find(|c| c.is_list(list_type)) + } + + /// Every chunk below this one depth-first (children before later siblings), with the depth + /// counted from 1 for the children of this chunk. + pub fn descendants(self) -> NodeWalk<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => NodeWalk(WalkRepr::Chunk(c.descendants())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + NodeWalk(WalkRepr::Index { tree, next: at.saturating_add(1), end, open: Vec::new() }) + } + } + } + + /// The first descendant `LIST` of this type, searching depth-first. + pub fn find_list(self, list_type: &[u8; 4]) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.find_list(list_type).map(Node::from), + // The subtree is a contiguous range in depth-first order: a plain scan. + NodeRepr::Index(tree, at) => { + let first = usize::try_from(at).ok()?.checked_add(1)?; + let end = usize::try_from(tree.end_of(at)).ok()?.min(tree.nodes.len()); + let range = tree.nodes.get(first..end)?; + let i = range.iter().position(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type)?; + let found = u32::try_from(first.checked_add(i)?).ok()?; + Some(Node(NodeRepr::Index(tree, found))) + } + } + } +} + +/// The children of a [`Node`] (or the top-level chunks of a [`TreeIndex`]), in file order. +#[derive(Clone)] +pub struct Nodes<'f, 'a>(NodesRepr<'f, 'a>); + +#[derive(Clone)] +enum NodesRepr<'f, 'a> { + Slice(std::slice::Iter<'f, Chunk<'a>>), + Index { tree: &'f TreeIndex<'a>, next: u32, end: u32 }, +} + +impl<'f, 'a> Iterator for Nodes<'f, 'a> { + type Item = Node<'f, 'a>; + + /// The next item, `None` at the end of the range. + fn next(&mut self) -> Option { + match &mut self.0 { + NodesRepr::Slice(it) => it.next().map(Node::from), + NodesRepr::Index { tree, next, end } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + *next = tree.end_of(at); + Some(Node(NodeRepr::Index(tree, at))) + } + } + } +} + +/// A depth-first walk over [`Node`]s with their depth; see [`Node::descendants`] and +/// [`TreeIndex::walk`]. +pub struct NodeWalk<'f, 'a>(WalkRepr<'f, 'a>); + +enum WalkRepr<'f, 'a> { + Chunk(Walk<'f, 'a>), + /// `open` holds the subtree ends of the entries above the next one (its ancestors inside the + /// walked range); their count is the next entry's depth minus one. + Index { + tree: &'f TreeIndex<'a>, + next: u32, + end: u32, + open: Vec, + }, +} + +impl<'f, 'a> Iterator for NodeWalk<'f, 'a> { + type Item = (usize, Node<'f, 'a>); + + /// The next item, `None` at the end of the range. + fn next(&mut self) -> Option { + match &mut self.0 { + WalkRepr::Chunk(w) => w.next().map(|(d, c)| (d, Node::from(c))), + WalkRepr::Index { tree, next, end, open } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + while open.last().is_some_and(|e| *e <= at) { + open.pop(); + } + let depth = open.len().saturating_add(1); + let sub_end = tree.end_of(at); + if sub_end > at.saturating_add(1) { + open.push(sub_end); + } + *next = at.saturating_add(1); + Some((depth, Node(NodeRepr::Index(tree, at)))) + } + } + } +} + +/// The top level of either tree: what the project settings and the version are read from. +#[derive(Clone, Copy)] +pub(crate) enum Root<'f, 'a> { + File(&'f AepFile<'a>), + Index(&'f TreeIndex<'a>), +} + +impl<'f, 'a> From<&'f AepFile<'a>> for Root<'f, 'a> { + /// The handle for this value. + fn from(file: &'f AepFile<'a>) -> Self { + Root::File(file) + } +} + +impl<'f, 'a> From<&'f TreeIndex<'a>> for Root<'f, 'a> { + /// The handle for this value. + fn from(index: &'f TreeIndex<'a>) -> Self { + Root::Index(index) + } +} + +impl<'f, 'a> Root<'f, 'a> { + /// The top-level chunks. + pub(crate) fn chunks(self) -> Nodes<'f, 'a> { + match self { + Root::File(f) => Nodes(NodesRepr::Slice(f.chunks.iter())), + Root::Index(t) => t.chunks(), + } + } + + /// The first top-level chunk with this id. + pub(crate) fn chunk(self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub(crate) fn list(self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::container::{parse_with_limits, write}; + + /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number (xorshift64*). + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: u64) -> u64 { + if n == 0 { 0 } else { self.next() % n } + } + } + + /// A random tree: lists, nested-id chunks holding a `Utf8`, data of odd and even sizes, opaque + /// lists, empty lists, deep chains. + fn tree(rng: &mut Rng, depth: usize, budget: &mut usize) -> Vec> { + let n = rng.below(6); + let mut out = Vec::new(); + for _ in 0..n { + if *budget == 0 { + break; + } + *budget -= 1; + let pick = rng.below(10); + let chunk = if depth < 12 && pick < 4 { + let types: [&[u8; 4]; 4] = [b"Fold", b"tdgp", b"tdbs", b"Layr"]; + let t = *types[usize::try_from(rng.below(4)).expect("small")]; + Chunk::list(t, tree(rng, depth + 1, budget)) + } else if pick == 4 { + let ids: [&[u8; 4]; 3] = [b"tdsn", b"pdnm", b"fnam"]; + let id = *ids[usize::try_from(rng.below(3)).expect("small")]; + let text = vec![b'a'; usize::try_from(rng.below(5)).expect("small")]; + Chunk { id, body: Body::Chunks(vec![Chunk::data(*b"Utf8", text)]), pad: None } + } else if pick == 5 { + let raw = vec![0x5a; usize::try_from(rng.below(9)).expect("small")]; + Chunk { id: LIST, body: Body::OpaqueList { list_type: *b"btdk", data: raw.into() }, pad: None } + } else { + let ids: [&[u8; 4]; 4] = [b"cdat", b"tdmn", b"Utf8", b"idta"]; + let id = *ids[usize::try_from(rng.below(4)).expect("small")]; + Chunk::data(id, vec![u8::try_from(rng.below(256)).expect("byte"); usize::try_from(rng.below(13)).expect("small")]) + }; + out.push(chunk); + } + out + } + + /// Both readers on the same bytes and limits: the same result, errors included; and the + /// index's walk visits the same chunks at the same depths as the tree's. + fn same(bytes: &[u8], limits: Limits) { + let old = parse_with_limits(bytes, limits); + let new = TreeIndex::parse_with_limits(bytes, limits); + match (&old, &new) { + (Ok(file), Ok(index)) => { + assert_eq!(&index.to_file(), file, "the index converts to the parsed tree"); + let a: Vec<(usize, Chunk<'_>)> = file.walk().map(|(d, c)| (d, c.clone())).collect(); + let b: Vec<(usize, Chunk<'_>)> = index.walk().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(a, b, "the walks agree"); + assert_eq!(index.len(), a.len()); + // Indexing the parsed tree gives the same index. + let again = TreeIndex::from_file(file).expect("index"); + assert_eq!(again.to_file(), *file); + } + (Err(a), Err(b)) => assert_eq!(a, b, "the same error"), + _ => panic!("one reader failed: {:?} vs {:?}", old.as_ref().err(), new.as_ref().err()), + } + } + + /// The default limits and four tight ones (depth 1 and 3, 7 and 40 chunks). + fn limits() -> [Limits; 5] { + [ + Limits::default(), + Limits { max_depth: 1, max_chunks: 10_000_000 }, + Limits { max_depth: 3, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 7 }, + Limits { max_depth: 64, max_chunks: 40 }, + ] + } + + /// The index matches the chunk parser on generated and damaged trees. + #[test] + fn the_index_matches_the_chunk_parser_on_generated_and_damaged_trees() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + for _ in 0..400 { + let mut budget = 120; + let chunks = tree(&mut rng, 0, &mut budget); + let file = AepFile { chunks, trailer: Cow::Borrowed(b"x") }; + let bytes = write(&file).expect("write"); + for l in limits() { + same(&bytes, l); + } + // Damage: overwrite a few bytes past the container header (sizes, ids, list types). + for _ in 0..4 { + let mut bad = bytes.clone(); + for _ in 0..1 + rng.below(3) { + let at = 12 + usize::try_from(rng.below(u64::try_from(bad.len().saturating_sub(12)).expect("len"))).expect("index"); + if let Some(b) = bad.get_mut(at) { + *b = u8::try_from(rng.below(256)).expect("byte"); + } + } + for l in limits() { + same(&bad, l); + } + } + // Truncated inputs (the container size then no longer fits). + let cut = bytes.len().saturating_sub(usize::try_from(rng.below(8)).expect("small") + 1); + same(&bytes[..cut], Limits::default()); + } + } + + /// Header errors are the same. + #[test] + fn header_errors_are_the_same() { + for bytes in [&b""[..], b"RIFX", b"RIFX\0\0\0\x04Egg", b"RIFX\0\0\0\x04Fnord", b"RIFX\0\0\0\x08Egg!", b"RIFX\0\0\0\x03Egg!", b"RIFF\0\0\0\x04Egg!"] { + same(bytes, Limits::default()); + } + } + + /// The typed accessors give the same answers through either tree. + #[test] + fn nodes_answer_alike_on_both_trees() { + let mut rng = Rng(42); + for _ in 0..100 { + let mut budget = 200; + let file = AepFile { chunks: tree(&mut rng, 0, &mut budget), trailer: Cow::Borrowed(&[]) }; + let bytes = write(&file).expect("write"); + let parsed = crate::container::parse(&bytes).expect("parse"); + let index = TreeIndex::parse(&bytes).expect("index"); + let a: Vec> = parsed.walk().map(|(_, c)| Node::from(c)).collect(); + let b: Vec> = index.walk().map(|(_, n)| n).collect(); + assert_eq!(a.len(), b.len()); + for (x, y) in a.iter().zip(&b) { + assert_eq!((x.id(), x.pad(), x.bytes(), x.list_type(), x.is_opaque_list()), (y.id(), y.pad(), y.bytes(), y.list_type(), y.is_opaque_list())); + assert_eq!(x.children().map(Node::to_chunk).collect::>(), y.children().map(Node::to_chunk).collect::>()); + for t in [b"Fold", b"tdgp", b"tdbs", b"Layr", b"btdk"] { + assert_eq!(x.find_list(t).map(Node::to_chunk), y.find_list(t).map(Node::to_chunk)); + assert_eq!(x.child_list(t).map(Node::to_chunk), y.child_list(t).map(Node::to_chunk)); + assert_eq!(x.is_list(t), y.is_list(t)); + } + for id in [b"cdat", b"Utf8", b"tdsn"] { + assert_eq!(x.child(id).map(Node::to_chunk), y.child(id).map(Node::to_chunk)); + } + let dx: Vec<(usize, Chunk<'_>)> = x.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + let dy: Vec<(usize, Chunk<'_>)> = y.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(dx, dy); + assert_eq!(format!("{x:?}"), format!("{y:?}")); + } + } + } + + /// The index never reserves more than the input or the limit allows. + #[test] + fn the_index_never_reserves_more_than_the_input_or_the_limit_allows() { + // 1000 empty headers: the most chunks the bytes can hold. + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("index"); + assert_eq!(index.len(), 1000); + // The start (1024 slots) is cut to what the input can hold. + assert!(index.nodes.capacity() <= 1000, "{} slots for at most 1000 headers", index.nodes.capacity()); + // A short tail or a header that runs past the end does not raise the bound. + let mut big = data.clone(); + big[16..20].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(TreeIndex::parse(&big).is_err() && crate::container::parse(&big).is_err()); + // Under a tighter limit the parse fails as the chunk parser does, and the guess never + // exceeds the limit. + let tight = Limits { max_depth: 64, max_chunks: 10 }; + assert_eq!(TreeIndex::parse_with_limits(&data, tight).err(), Some(AepError::TooManyChunks)); + let small = Limits { max_depth: 64, max_chunks: 1000 }; + let index = TreeIndex::parse_with_limits(&data, small).expect("exactly at the limit"); + assert_eq!(index.len(), 1000); + // A limit below what the input holds bounds the reservation: 100 chunks, never more slots. + let mut wide = b"RIFX\0\0\0\0Egg!".to_vec(); + wide.extend_from_slice(b"LIST"); + wide.extend_from_slice(&u32::try_from(4 + 8 * 99).expect("small").to_be_bytes()); + wide.extend_from_slice(b"Fold"); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 99).flatten()); + let declared = u32::try_from(wide.len() - 8).expect("small"); + wide[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse_with_limits(&wide, Limits { max_depth: 64, max_chunks: 100 }).expect("100 chunks"); + assert_eq!(index.len(), 100); + assert!(index.nodes.capacity() <= 100); + // `from_file` is sized exactly. + let file = crate::container::parse(&wide).expect("parse"); + assert_eq!(TreeIndex::from_file(&file).expect("index").nodes.capacity(), 100); + } + + /// A file of `n` data chunks of `size` bytes each. + fn flat(n: usize, size: usize) -> Vec { + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + for _ in 0..n { + data.extend_from_slice(b"cdat"); + data.extend_from_slice(&u32::try_from(size).expect("small").to_be_bytes()); + data.extend(std::iter::repeat_n(0u8, size)); + } + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + data + } + + /// The index grows with the chunk count not the byte size. + #[test] + fn the_index_grows_with_the_chunk_count_not_the_byte_size() { + // 4 MB in 4 chunks: a few slots, not one per 32 bytes (that would be 131 072 slots, 4 MB). + let big = flat(4, 1 << 20); + let index = TreeIndex::parse(&big).expect("index"); + assert_eq!(index.len(), 4); + assert!(index.nodes.capacity() <= INITIAL_SLOTS, "{} slots for 4 chunks", index.nodes.capacity()); + // Many small chunks: at most twice the count (it doubles). + let many = flat(10_000, 2); + let index = TreeIndex::parse(&many).expect("index"); + assert_eq!(index.len(), 10_000); + assert!(index.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", index.nodes.capacity()); + same(&big, Limits::default()); + same(&many, Limits::default()); + } + + /// `LIST Fold` around `inner`, followed by 6 bytes too short for a header, so the list's + /// sequence fails and the list is kept opaque. + fn broken_level(inner: &[u8]) -> Vec { + let mut body = b"Fold".to_vec(); + body.extend_from_slice(inner); + body.extend_from_slice(&[0xee; 6]); + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(body.len()).expect("small").to_be_bytes()); + out.extend_from_slice(&body); + out + } + + /// Failed attempts count once and leave nothing behind. + #[test] + fn failed_attempts_count_once_and_leave_nothing_behind() { + let mut wide = b"Fold".to_vec(); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let mut level = b"LIST".to_vec(); + level.extend_from_slice(&u32::try_from(wide.len()).expect("small").to_be_bytes()); + level.extend_from_slice(&wide); + for _ in 0..50 { + level = broken_level(&level); + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let headers = 1 + 50 + 1000; + let index = TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers }).expect("parse"); + assert_eq!(TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers - 1 }).err(), Some(AepError::TooManyChunks)); + // Only the outermost list is left, opaque: every failed attempt was rolled back. + assert_eq!(index.len(), 1); + let top = index.chunks().next().expect("one top-level chunk"); + assert!(top.is_opaque_list() && top.bytes().map(<[u8]>::len) == Some(level.len() - 12)); + assert_eq!(write(&index.to_file()).expect("write"), data); + same(&data, Limits::default()); + } + + /// Depth beyond the limit stays opaque without recursion past it. + #[test] + fn depth_beyond_the_limit_stays_opaque_without_recursion_past_it() { + // 5000 nested lists: far deeper than the limit; everything below depth 64 stays bytes. + let mut level = b"LIST\0\0\0\x04tdgp".to_vec(); + for _ in 0..5000 { + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(level.len() + 4).expect("small").to_be_bytes()); + out.extend_from_slice(b"tdgp"); + out.extend_from_slice(&level); + level = out; + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("parse"); + assert_eq!(index.len(), 64); + assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64)); + assert!(index.walk().last().is_some_and(|(_, n)| n.is_opaque_list())); + same(&data, Limits::default()); + // Converting and indexing a deep owned tree uses no recursion either. + let file = index.to_file(); + assert_eq!(TreeIndex::from_file(&file).expect("index").len(), 64); + } +} diff --git a/crates/aep/src/container/masks.rs b/crates/aep/src/container/masks.rs index 6246fc1f5..7f00c07eb 100644 --- a/crates/aep/src/container/masks.rs +++ b/crates/aep/src/container/masks.rs @@ -6,8 +6,8 @@ //! property (`tdbs`, with a key list when animated) and `omks` with one `LIST shap` per keyframe (a //! single one when the path is static). -use crate::container::Chunk; use crate::container::bytes::{be_f32, be_u32, until_nul}; +use crate::container::index::Node; use crate::container::typed::{KeyList, Layer, PropGroup, PropNode, Property}; use crate::container::values::{Tracked, read}; @@ -93,8 +93,10 @@ pub struct ShapePath { } impl ShapePath { - /// Decodes a `LIST shap`; `None` when `shph` or the point list is malformed. - pub fn from_chunk(shap: &Chunk<'_>) -> Option { + /// Decodes a `LIST shap` (a [`Chunk`](crate::container::Chunk) or a [`Node`]); `None` when + /// `shph` or the point list is malformed. + pub fn from_chunk<'f, 'a: 'f>(shap: impl Into>) -> Option { + let shap = shap.into(); let shph = shap.child(b"shph")?.bytes()?; let bbox = [be_f32(shph, 4)?, be_f32(shph, 8)?, be_f32(shph, 12)?, be_f32(shph, 16)?]; let closed_flag = *shph.get(20)?; @@ -187,7 +189,7 @@ fn mask_from_atom(atom: &PropGroup<'_, '_>, mkif: Option<&[u8]>) -> Mask { } } if let Some(omks) = oms.child_list(b"omks") { - mask.shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); + mask.shapes = omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); } } mask @@ -236,7 +238,7 @@ impl<'f, 'a> Layer<'f, 'a> { if let Some(PropNode::Other(oms)) = atom.get("ADBE Mask Shape") { path = oms.child_list(b"tdbs").and_then(Property::from_chunk); if let Some(omks) = oms.child_list(b"omks") { - path_shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); + path_shapes = omks.children().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); } } masks.push(MaskAtom { mask, atom, path, path_shapes }); diff --git a/crates/aep/src/container/read.rs b/crates/aep/src/container/read.rs index abae015ba..200b3d84a 100644 --- a/crates/aep/src/container/read.rs +++ b/crates/aep/src/container/read.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; +use crate::container::bytes::{array, be_u32}; use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; use crate::container::error::AepError; @@ -36,10 +37,20 @@ pub fn parse(data: &[u8]) -> Result, AepError> { /// # Errors /// See [`parse`]; also fails with [`AepError::TooManyChunks`]. pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + let end = container_end(data)?; + let mut parser = Parser { data, limits, count: 0 }; + let chunks = parser.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) +} + +/// Checks the `RIFX`/`Egg!` header and returns where the container ends (its chunks fill +/// `12..end`; the trailer follows). Shared with [`crate::container::index::TreeIndex::parse`]. +pub(super) fn container_end(data: &[u8]) -> Result { if id4(data, 0) != Some(*b"RIFX") { return Err(AepError::NotRifx); } - let declared = u64::from(be32(data, 4).ok_or(AepError::NotRifx)?); + let declared = u64::from(be_u32(data, 4).ok_or(AepError::NotRifx)?); let form = id4(data, 8).ok_or(AepError::NotRifx)?; if &form != b"Egg!" { return Err(AepError::BadFormType(form)); @@ -49,20 +60,12 @@ pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, Aep return Err(AepError::Truncated { declared, available }); } // `declared <= available < data.len()`, so the end is inside the input. - let end = usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge)?; - let mut parser = Parser { data, limits, count: 0 }; - let chunks = parser.sequence(12, end, 1)?; - let trailer = data.get(end..).unwrap_or(&[]); - Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) + usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge) } -fn be32(data: &[u8], at: usize) -> Option { - id4(data, at).map(u32::from_be_bytes) -} - -fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { - let b = data.get(at..at.checked_add(4)?)?; - Some([*b.first()?, *b.get(1)?, *b.get(2)?, *b.get(3)?]) +/// The 4-byte id at `at`, `None` when the input is too short. +pub(super) fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { + array::<4>(data, at) } struct Parser<'a> { @@ -74,13 +77,16 @@ struct Parser<'a> { impl<'a> Parser<'a> { /// Parses the sibling chunks that exactly fill `start..end`. fn sequence(&mut self, start: usize, end: usize, depth: usize) -> Result>, AepError> { + // Sized once: a well-formed sequence fills exactly this many slots. `try_reserve` so a + // limit raised past what the target can allocate fails nothing here (the Vec then grows). let mut out = Vec::new(); + let _ = out.try_reserve_exact(self.capacity_for(start, end)); let mut pos = start; while pos < end { let short = AepError::Malformed { offset: pos, what: "bytes left over that are too short for a chunk header" }; let body_start = pos.checked_add(8).filter(|b| *b <= end).ok_or(short)?; let id = id4(self.data, pos).ok_or(AepError::Malformed { offset: pos, what: "short read" })?; - let size = be32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; + let size = be_u32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; let avail = end - body_start; if size > avail { return Err(AepError::ChunkTooLarge { offset: pos, size: size as u64, available: avail as u64 }); @@ -103,6 +109,26 @@ impl<'a> Parser<'a> { Ok(out) } + /// The slots to reserve for the sequence `start..end`: its header count, bounded by the input + /// (one per 8 bytes at most) and by the chunks the limit still allows. + fn capacity_for(&self, start: usize, end: usize) -> usize { + self.count_siblings(start, end).min(self.limits.max_chunks.saturating_sub(self.count)) + } + + /// The number of chunk headers that tile `start..end` the way [`Self::sequence`] walks them, + /// stopping at the first that does not fit. Never more than `(end - start) / 8`. + fn count_siblings(&self, start: usize, end: usize) -> usize { + let mut n = 0usize; + let mut pos = start; + while let Some(body_start) = pos.checked_add(8).filter(|b| *b <= end) { + let Some(size) = be_u32(self.data, pos.saturating_add(4)) else { break }; + let Some(next) = usize::try_from(size).ok().and_then(|s| body_start.checked_add(s)).filter(|e| *e <= end) else { break }; + n = n.saturating_add(1); + pos = if size % 2 == 1 && next < end { next.saturating_add(1) } else { next }; + } + n + } + /// Like [`Self::sequence`] but a structural problem yields `None` (the caller keeps bytes). fn tentative(&mut self, start: usize, end: usize, depth: usize) -> Result>>, AepError> { match self.sequence(start, end, depth) { @@ -140,3 +166,113 @@ impl<'a> Parser<'a> { Ok(Body::Data(Cow::Borrowed(self.slice(start, end)))) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Every children `Vec` in the tree, depth first. + fn child_vecs<'f, 'a>(chunks: &'f [Chunk<'a>], out: &mut Vec<&'f Vec>>) { + for c in chunks { + match &c.body { + Body::List { children, .. } | Body::Chunks(children) => { + out.push(children); + child_vecs(children, out); + } + Body::Data(_) | Body::OpaqueList { .. } => {} + } + } + } + + /// Every sibling list is allocated once at its exact length. + #[test] + fn every_sibling_list_is_allocated_once_at_its_exact_length() { + let inner: Vec> = (0..37u8).map(|i| Chunk::data(*b"cdat", vec![i; usize::from(i % 5)])).collect(); + let middle: Vec> = (0..9).map(|_| Chunk::list(*b"tdbs", inner.clone())).collect(); + let file = AepFile { chunks: vec![Chunk::data(*b"head", vec![0; 3]), Chunk::list(*b"Fold", middle)], trailer: Cow::Borrowed(&[]) }; + let bytes = crate::container::write(&file).expect("write"); + let parsed = parse(&bytes).expect("parse"); + assert_eq!(crate::container::write(&parsed).expect("rewrite"), bytes); + let mut vecs = vec![&parsed.chunks]; + child_vecs(&parsed.chunks, &mut vecs); + assert_eq!(vecs.len(), 11); + // `try_reserve_exact` gives exactly the requested capacity with std's allocator today; a + // Vec grown push by push would have rounded up to a power of two (37 -> 64, 9 -> 16). + for v in vecs { + assert_eq!(v.capacity(), v.len(), "a sibling Vec grew past its length"); + } + } + + /// The preallocation is bounded by the input and the chunk limit. + #[test] + fn the_preallocation_is_bounded_by_the_input_and_the_chunk_limit() { + // 1000 empty headers: the most chunks the bytes can hold. + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let parser = Parser { data: &data, limits: Limits::default(), count: 0 }; + assert_eq!(parser.count_siblings(12, data.len()), 1000); + assert!(parser.count_siblings(12, data.len()) <= (data.len() - 12) / 8); + // A header whose size runs past the end stops the count; a stray tail is not counted. + assert_eq!(parser.count_siblings(12, data.len() - 3), 999); + let mut big = data.clone(); + big[16..20].copy_from_slice(&u32::MAX.to_be_bytes()); + let parser = Parser { data: &big, limits: Limits::default(), count: 0 }; + assert_eq!(parser.count_siblings(12, big.len()), 0); + let f = parse(&data).expect("parse"); + assert_eq!((f.chunks.len(), f.chunks.capacity()), (1000, 1000)); + // Under a tighter limit the parse fails as before, without sizing for more than allowed. + let tight = Limits { max_depth: 64, max_chunks: 10 }; + assert_eq!(parse_with_limits(&data, tight), Err(AepError::TooManyChunks)); + // The reservation never exceeds what the chunk limit still allows. + let mut parser = Parser { data: &data, limits: tight, count: 0 }; + assert_eq!(parser.capacity_for(12, data.len()), 10); + parser.count = 7; + assert_eq!(parser.capacity_for(12, data.len()), 3); + parser.count = 11; + assert_eq!(parser.capacity_for(12, data.len()), 0); + let parser = Parser { data: &data, limits: Limits::default(), count: 0 }; + assert_eq!(parser.capacity_for(12, data.len()), 1000); + } + + /// `LIST Fold` around `inner`, followed by 6 bytes too short for a header, so the list's + /// sequence fails and the list is kept opaque. + fn broken_level(inner: &[u8]) -> Vec { + let mut body = b"Fold".to_vec(); + body.extend_from_slice(inner); + body.extend_from_slice(&[0xee; 6]); + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(body.len()).expect("small").to_be_bytes()); + out.extend_from_slice(&body); + out + } + + /// Nested lists with broken tails parse each header once. + #[test] + fn nested_lists_with_broken_tails_parse_each_header_once() { + // A wide, well-formed list at the bottom, then 50 levels that each fail and fall back to + // opaque: every level re-parses its child before giving up. + let mut wide = b"Fold".to_vec(); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let mut level = b"LIST".to_vec(); + level.extend_from_slice(&u32::try_from(wide.len()).expect("small").to_be_bytes()); + level.extend_from_slice(&wide); + for _ in 0..50 { + level = broken_level(&level); + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + // 1 + 50 + 1000 headers in the file: the parse counts each exactly once (the count spans + // every attempt), so the work stays linear in the input. + let headers = 1 + 50 + 1000; + let f = parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers }).expect("parse"); + assert_eq!(parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers - 1 }), Err(AepError::TooManyChunks)); + // The outermost list keeps its bytes, and the file writes back unchanged. + let [top] = f.chunks.as_slice() else { panic!("one top-level chunk") }; + assert!(matches!(&top.body, Body::OpaqueList { list_type, data } if list_type == b"Fold" && data.len() + 12 == level.len())); + assert_eq!(crate::container::write(&f).expect("write"), data); + } +} diff --git a/crates/aep/src/container/settings.rs b/crates/aep/src/container/settings.rs index b44588232..a443df874 100644 --- a/crates/aep/src/container/settings.rs +++ b/crates/aep/src/container/settings.rs @@ -2,8 +2,10 @@ //! linear-light switches, the time display and the frame count start. Cases 620-634: one change //! each against the base project 620; every field below is the only byte (or chunk) that changes. +use crate::container::bytes::be_u32; use crate::container::chunk::AepFile; use crate::container::footage::json_string; +use crate::container::index::{Node, Root, TreeIndex}; /// The time display style (`nhed` byte 8, feet + frames byte 11, film type byte 13). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -54,7 +56,17 @@ pub struct ProjectSettings { impl ProjectSettings { /// Reads the settings from the top-level chunks. pub fn read(file: &AepFile<'_>) -> ProjectSettings { - let nhed = file.chunk(b"nhed").and_then(|c| c.bytes()); + Self::of(Root::from(file)) + } + + /// Reads the settings from the top-level chunks of an indexed file. + pub fn read_index(index: &TreeIndex<'_>) -> ProjectSettings { + Self::of(Root::from(index)) + } + + /// Reads the settings from the top-level chunks of either tree. + pub(crate) fn of(file: Root<'_, '_>) -> ProjectSettings { + let nhed = file.chunk(b"nhed").and_then(Node::bytes); let byte = |i: usize| nhed.and_then(|b| b.get(i).copied()); let bits_per_channel = byte(15).and_then(|v| match v { 0 => Some(8), @@ -73,9 +85,9 @@ impl ProjectSettings { 1 => Some(1), _ => None, }); - let flag = |id: &[u8; 4]| file.chunk(id).and_then(|c| c.bytes()).and_then(|b| b.first().copied()) == Some(1); - let working_gamma_24 = file.chunk(b"dwga").and_then(|c| c.bytes()).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); - let scene_compensation = file.chunk(b"acer").and_then(|c| c.bytes()).and_then(|b| b.first().copied()).map(|v| v != 0); + let flag = |id: &[u8; 4]| file.chunk(id).and_then(Node::bytes).and_then(|b| b.first().copied()) == Some(1); + let working_gamma_24 = file.chunk(b"dwga").and_then(Node::bytes).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); + let scene_compensation = file.chunk(b"acer").and_then(Node::bytes).and_then(|b| b.first().copied()).map(|v| v != 0); ProjectSettings { bits_per_channel, working_space: working_space(file), @@ -90,9 +102,10 @@ impl ProjectSettings { } /// The working space: the `Utf8` right after `PwCs`. -fn working_space(file: &AepFile<'_>) -> Option { - let at = file.chunks.iter().position(|c| c.is(b"PwCs"))?; - let json = file.chunks.get(at.checked_add(1)?).filter(|c| c.is(b"Utf8"))?.bytes()?; +fn working_space(file: Root<'_, '_>) -> Option { + let mut chunks = file.chunks(); + chunks.find(|c| c.is(b"PwCs"))?; + let json = chunks.next().filter(|c| c.is(b"Utf8"))?.bytes()?; let trimmed: Vec = json.iter().copied().filter(|b| !b.is_ascii_whitespace() && *b != 0).collect(); if trimmed == b"{}" { return Some(WorkingSpace::None); @@ -130,7 +143,7 @@ fn base64_decode(text: &str) -> Option> { /// The profile description (ICC.1 tag `desc`): a v2 `desc` (ASCII) or a v4 `mluc` (its first /// record, UTF-16BE). `None` when the profile or the tag is cut short. fn icc_description(icc: &[u8]) -> Option { - let be32 = |at: usize| icc.get(at..at.checked_add(4)?).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes); + let be32 = |at: usize| be_u32(icc, at); let count = be32(128)?; let tag = (0..count.min(1024)).find_map(|i| { let at = 132usize.checked_add(usize::try_from(i).ok()?.checked_mul(12)?)?; @@ -141,13 +154,13 @@ fn icc_description(icc: &[u8]) -> Option { let body = icc.get(off..off.checked_add(len)?)?; let text = match body.get(..4)? { b"desc" => { - let n = usize::try_from(u32::from_be_bytes(body.get(8..12)?.try_into().ok()?)).ok()?; + let n = usize::try_from(be_u32(body, 8)?).ok()?; let ascii = body.get(12..12usize.checked_add(n)?)?; String::from_utf8_lossy(ascii.split(|b| *b == 0).next().unwrap_or_default()).into_owned() } b"mluc" => { - let n = usize::try_from(u32::from_be_bytes(body.get(20..24)?.try_into().ok()?)).ok()?; - let at = usize::try_from(u32::from_be_bytes(body.get(24..28)?.try_into().ok()?)).ok()?; + let n = usize::try_from(be_u32(body, 20)?).ok()?; + let at = usize::try_from(be_u32(body, 24)?).ok()?; let units: Vec = body.get(at..at.checked_add(n)?)?.as_chunks::<2>().0.iter().map(|p| u16::from_be_bytes(*p)).collect(); String::from_utf16_lossy(&units) } diff --git a/crates/aep/src/container/shapes.rs b/crates/aep/src/container/shapes.rs index 8aad83d3f..dee01bb22 100644 --- a/crates/aep/src/container/shapes.rs +++ b/crates/aep/src/container/shapes.rs @@ -140,7 +140,7 @@ fn number(group: &PropGroup<'_, '_>, name: &str) -> Option> { fn paths_of(group: &PropGroup<'_, '_>) -> Vec { let Some(PropNode::Other(oms)) = group.get("ADBE Vector Shape") else { return Vec::new() }; let Some(omks) = oms.child_list(b"omks") else { return Vec::new() }; - omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() + omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() } fn item(match_name: String, group: &PropGroup<'_, '_>) -> ShapeItem { diff --git a/crates/aep/src/container/textdoc.rs b/crates/aep/src/container/textdoc.rs index 566a48534..e673e63cf 100644 --- a/crates/aep/src/container/textdoc.rs +++ b/crates/aep/src/container/textdoc.rs @@ -5,8 +5,11 @@ //! as `(` + BOM `fe ff` + UTF-16BE + `)` with `\` escapes. [`parse_text_dict`] reads it (depth //! limited, never panics) and [`Layer::text_document`] pulls out the verified fields. -use crate::container::Chunk; +use std::borrow::Cow; +use std::sync::LazyLock; + use crate::container::bytes::{be_u32, text}; +use crate::container::index::Node; use crate::container::typed::{Comp, Layer, Property}; /// How deep dictionaries and arrays may nest before the parser gives up. @@ -73,15 +76,99 @@ impl TextValue { _ => None, } } +} - fn key_text(&self) -> String { +/// The text of a key the way the dictionary stores it: a `/name` as written, any other value in +/// its canonical form (`06` and `+6` are `6`, `1.0` is `1`, a string its decoded text). +fn owned_key(value: &V<'_>) -> String { + match value { + V::Name(s) | V::Token(s) => text(s), + V::Str(utf16) => decode_utf16be(utf16), + V::Int(i) => i.to_string(), + V::Float(f) => f.to_string(), + V::Bool(b) => b.to_string(), + V::Bytes(b) => text(b), + V::Dict(_) | V::Array(_) => String::new(), + } +} + +/// A value of the text dictionary as [`TextDocument::parse`] reads it: it borrows names, bare +/// words and strings from the body, so a document is read without a `String` per token. A key is +/// borrowed when it is a `/name` (every key After Effects writes); its text is the same as the +/// [`TextValue`] key's. [`V::into_owned`] gives the public [`TextValue`]. +#[derive(Debug, Clone)] +enum V<'b> { + Dict(Vec<(Cow<'b, str>, V<'b>)>), + Array(Vec>), + /// A string that started with the byte order mark: the UTF-16BE bytes after it, unescaped. + Str(Cow<'b, [u8]>), + Bytes(Cow<'b, [u8]>), + Name(&'b [u8]), + Int(i64), + Float(f64), + Bool(bool), + Token(&'b [u8]), +} + +impl<'b> V<'b> { + /// The value of a dictionary key (the last one when a key repeats). + fn get(&self, key: &str) -> Option<&V<'b>> { match self { - Self::Name(s) | Self::Str(s) | Self::Token(s) => s.clone(), - Self::Int(i) => i.to_string(), - Self::Float(f) => f.to_string(), - Self::Bool(b) => b.to_string(), - Self::Bytes(b) => text(b), - Self::Dict(_) | Self::Array(_) => String::new(), + Self::Dict(pairs) => pairs.iter().rev().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + + fn at(&self, index: usize) -> Option<&V<'b>> { + match self { + Self::Array(items) => items.get(index), + _ => None, + } + } + + /// The decoded text of a string value. + fn string(&self) -> Option { + match self { + Self::Str(utf16) => Some(decode_utf16be(utf16)), + _ => None, + } + } + + fn as_f64(&self) -> Option { + match self { + Self::Int(i) => Some(*i as f64), + Self::Float(f) => Some(*f), + _ => None, + } + } + + fn as_i64(&self) -> Option { + match self { + Self::Int(i) => Some(*i), + _ => None, + } + } + + /// The key a value makes when it is written in a key position. + fn into_key(self) -> Cow<'b, str> { + match self { + Self::Name(s) => String::from_utf8_lossy(s), + other => Cow::Owned(owned_key(&other)), + } + } + + /// The public form. Recursion is bounded: a parsed value nests at most [`MAX_DEPTH`] + 1 deep. + fn into_owned(self) -> TextValue { + match self { + Self::Dict(pairs) => TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.into_owned(), v.into_owned())).collect()), + Self::Array(items) => TextValue::Array(items.into_iter().map(V::into_owned).collect()), + Self::Str(utf16) => TextValue::Str(decode_utf16be(&utf16)), + Self::Bytes(b) => TextValue::Bytes(b.into_owned()), + Self::Name(s) => TextValue::Name(text(s)), + Self::Int(i) => TextValue::Int(i), + Self::Float(f) => TextValue::Float(f), + Self::Bool(b) => TextValue::Bool(b), + Self::Token(s) => TextValue::Token(text(s)), } } } @@ -89,32 +176,44 @@ impl TextValue { struct Parser<'b> { b: &'b [u8], i: usize, + /// The pairs of the dictionaries being read, innermost last: a dictionary's pairs are moved + /// into a vector of their exact size when it closes (one allocation per dictionary instead of + /// one per doubling). + pairs: Vec<(Cow<'b, str>, V<'b>)>, + /// The same for the items of arrays. + items: Vec>, } +/// Whitespace between tokens. fn is_ws(c: u8) -> bool { matches!(c, b' ' | b'\t' | b'\r' | b'\n') } +/// A byte that ends a bare word. fn is_delimiter(c: u8) -> bool { is_ws(c) || matches!(c, b'/' | b'[' | b']' | b'<' | b'>' | b'(' | b')') } -impl Parser<'_> { +impl<'b> Parser<'b> { + /// The next byte, `None` at the end. fn peek(&self) -> Option { self.b.get(self.i).copied() } + /// Moves past whitespace. fn skip_ws(&mut self) { while self.peek().is_some_and(is_ws) { self.i = self.i.saturating_add(1); } } + /// True when the unread bytes start with `s`. fn starts_with(&self, s: &[u8]) -> bool { self.b.get(self.i..).is_some_and(|rest| rest.starts_with(s)) } - fn word(&mut self) -> &[u8] { + /// The bare word at the position (up to the next delimiter), borrowed from the body. + fn word(&mut self) -> &'b [u8] { let start = self.i; while self.peek().is_some_and(|c| !is_delimiter(c)) { self.i = self.i.saturating_add(1); @@ -122,79 +221,106 @@ impl Parser<'_> { self.b.get(start..self.i).unwrap_or(&[]) } - fn parse(&mut self, depth: usize) -> Option { + /// The body of a `( ... )` string after the `(`, with `\` escapes resolved: borrowed when it + /// has none. `None` when the string or an escape is not closed. + fn string_body(&mut self) -> Option> { + let rest = self.b.get(self.i..)?; + let stop = rest.iter().position(|c| matches!(c, b')' | b'\\'))?; + let head = rest.get(..stop)?; + self.i = self.i.saturating_add(stop); + if self.peek()? == b')' { + self.i = self.i.saturating_add(1); + return Some(Cow::Borrowed(head)); + } + let mut out = head.to_vec(); + loop { + let c = self.peek()?; + if c == b'\\' { + out.push(*self.b.get(self.i.saturating_add(1))?); + self.i = self.i.saturating_add(2); + } else if c == b')' { + self.i = self.i.saturating_add(1); + return Some(Cow::Owned(out)); + } else { + out.push(c); + self.i = self.i.saturating_add(1); + } + } + } + + /// The value at the position, `None` when it is malformed or nested deeper than the limit. + fn parse(&mut self, depth: usize) -> Option> { self.skip_ws(); if depth > MAX_DEPTH { return None; } if self.starts_with(b"<<") { self.i = self.i.saturating_add(2); - let mut pairs = Vec::new(); + // On an error the pairs read so far stay on the stack: the error ends the whole parse. + let mark = self.pairs.len(); loop { self.skip_ws(); if self.starts_with(b">>") { self.i = self.i.saturating_add(2); - return Some(TextValue::Dict(pairs)); + return Some(V::Dict(self.pairs.drain(mark..).collect())); } let key = self.parse(depth.saturating_add(1))?; let value = self.parse(depth.saturating_add(1))?; - pairs.push((key.key_text(), value)); + self.pairs.push((key.into_key(), value)); } } match self.peek()? { b'[' => { self.i = self.i.saturating_add(1); - let mut items = Vec::new(); + let mark = self.items.len(); loop { self.skip_ws(); if self.peek()? == b']' { self.i = self.i.saturating_add(1); - return Some(TextValue::Array(items)); + return Some(V::Array(self.items.drain(mark..).collect())); } - items.push(self.parse(depth.saturating_add(1))?); + let item = self.parse(depth.saturating_add(1))?; + self.items.push(item); } } b'(' => { self.i = self.i.saturating_add(1); - let mut out = Vec::new(); - loop { - let c = self.peek()?; - if c == b'\\' { - out.push(*self.b.get(self.i.saturating_add(1))?); - self.i = self.i.saturating_add(2); - } else if c == b')' { - self.i = self.i.saturating_add(1); - break; - } else { - out.push(c); - self.i = self.i.saturating_add(1); - } - } - Some(match out.strip_prefix(&[0xfe, 0xff]) { - Some(utf16) => TextValue::Str(decode_utf16be(utf16)), - None => TextValue::Bytes(out), + let body = self.string_body()?; + Some(if body.starts_with(&[0xfe, 0xff]) { + V::Str(match body { + Cow::Borrowed(b) => Cow::Borrowed(b.get(2..).unwrap_or(&[])), + Cow::Owned(mut v) => { + let n = v.len().min(2); + v.drain(..n); + Cow::Owned(v) + } + }) + } else { + V::Bytes(body) }) } b'/' => { self.i = self.i.saturating_add(1); - Some(TextValue::Name(text(self.word()))) + Some(V::Name(self.word())) } _ => { let word = self.word(); if word.is_empty() { return None; } - let w = text(word); - Some(match w.as_str() { - "true" => TextValue::Bool(true), - "false" => TextValue::Bool(false), + // A word that is not UTF-8 reads as text with replacement characters, which is + // neither a boolean nor a number: a token. + let Ok(w) = std::str::from_utf8(word) else { return Some(V::Token(word)) }; + Some(match w { + "true" => V::Bool(true), + "false" => V::Bool(false), _ => match w.parse::() { - Ok(i) => TextValue::Int(i), + Ok(i) => V::Int(i), // `parse` also takes `nan`, `inf` and overflowing exponents; those are not // numbers a text document holds, so they stay tokens and read as missing. Err(_) => match w.parse::() { - Ok(f) if f.is_finite() => TextValue::Float(f), - _ => TextValue::Token(w), + Ok(f) if f.is_finite() => V::Float(f), + _ => V::Token(word), }, }, }) @@ -203,6 +329,7 @@ impl Parser<'_> { } } +/// A big-endian UTF-16 string as text (unpaired surrogates become U+FFFD). fn decode_utf16be(bytes: &[u8]) -> String { let (pairs, rest) = bytes.as_chunks::<2>(); let units = pairs.iter().map(|p| u16::from_be_bytes(*p)); @@ -213,11 +340,9 @@ fn decode_utf16be(bytes: &[u8]) -> String { s } -/// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). -/// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was -/// read before is returned. -pub fn parse_text_dict(body: &[u8]) -> TextValue { - let mut parser = Parser { b: body, i: 0 }; +/// The top-level `key value` pairs of a `btdk` body, borrowed (see [`parse_text_dict`]). +fn parse_dict(body: &[u8]) -> V<'_> { + let mut parser = Parser { b: body, i: 0, pairs: Vec::new(), items: Vec::new() }; let mut pairs = Vec::new(); loop { parser.skip_ws(); @@ -226,13 +351,20 @@ pub fn parse_text_dict(body: &[u8]) -> TextValue { } let Some(key) = parser.parse(0) else { break }; let Some(value) = parser.parse(0) else { break }; - pairs.push((key.key_text(), value)); + pairs.push((key.into_key(), value)); } - TextValue::Dict(pairs) + V::Dict(pairs) +} + +/// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). +/// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was +/// read before is returned. +pub fn parse_text_dict(body: &[u8]) -> TextValue { + parse_dict(body).into_owned() } /// The run style slots (`RUN/6/0[n]/0/0/6/`) that are decoded; every other slot is compared -/// with its default (`style_defaults`) and listed in +/// with its default (`STYLE_DEFAULTS`) and listed in /// [`TextDocument::undecoded_style_fields`] when it differs. pub const DECODED_STYLE_SLOTS: &[&str] = &["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12", "13", "18", "35", "36", "52", "53", "54", "56", "57", "58", "62", "63", "70"]; @@ -241,24 +373,27 @@ pub const DECODED_STYLE_SLOTS: &[&str] = pub const DECODED_PARAGRAPH_SLOTS: &[&str] = &["0", "1", "2", "3", "4", "5", "8", "9", "21", "29", "33"]; /// A number slot. -fn val(v: f64) -> TextValue { - TextValue::Float(v) +fn val(v: f64) -> V<'static> { + V::Float(v) } -fn dict(pairs: Vec<(&str, TextValue)>) -> TextValue { - TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) +/// A dictionary value with these keys, for the built-in defaults. +fn dict(pairs: Vec<(&'static str, V<'static>)>) -> V<'static> { + V::Dict(pairs.into_iter().map(|(k, v)| (Cow::Borrowed(k), v)).collect()) } -fn numbers(v: &[f64]) -> TextValue { - TextValue::Array(v.iter().map(|x| val(*x)).collect()) +/// An array of numbers, for the built-in defaults. +fn numbers(v: &[f64]) -> V<'static> { + V::Array(v.iter().map(|x| val(*x)).collect()) } /// The default of every run style slot that is not decoded and not neutral (zero, `false`, /// empty, `/nil`): the value an untouched point text holds in every sample (110, 243-245). A slot /// with a neutral default is left out; [`undecoded_fields`] treats a missing default as neutral. -fn style_defaults() -> TextValue { - let paint = |a: f64| dict(vec![("99", TextValue::Name("SimplePaint".into())), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); - let class = |name: &str| dict(vec![("99", TextValue::Name(name.into()))]); +/// Built once. +static STYLE_DEFAULTS: LazyLock> = LazyLock::new(|| { + let paint = |a: f64| dict(vec![("99", V::Name(b"SimplePaint")), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); + let class = |name: &'static [u8]| dict(vec![("99", V::Name(name))]); dict(vec![ ("37", val(2.0)), ("41", val(2.0)), @@ -268,24 +403,24 @@ fn style_defaults() -> TextValue { ("46", val(7.0)), ("49", val(-1.0)), ("50", val(-1.0)), - ("55", class("SimpleBlender")), + ("55", class(b"SimpleBlender")), ("64", val(4.0)), ("71", val(4.0)), - ("77", TextValue::Bool(true)), - ("78", TextValue::Bool(true)), + ("77", V::Bool(true)), + ("78", V::Bool(true)), ("79", paint(0.0)), ("82", val(3.0)), ("83", val(3.0)), - ("86", class("SimpleCustomFeature")), + ("86", class(b"SimpleCustomFeature")), ("87", val(100.0)), - ("88", TextValue::Bool(true)), + ("88", V::Bool(true)), ("95", dict(vec![("0", val(0.0)), ("1", val(0.0))])), - ("96", TextValue::Bool(true)), + ("96", V::Bool(true)), ]) -} +}); -/// The same for the paragraph slots (110, 243-245). -fn paragraph_defaults() -> TextValue { +/// The same for the paragraph slots (110, 243-245). Built once. +static PARAGRAPH_DEFAULTS: LazyLock> = LazyLock::new(|| { dict(vec![ ("6", val(1.0)), ("7", val(1.2)), @@ -293,30 +428,35 @@ fn paragraph_defaults() -> TextValue { ("11", val(2.0)), ("12", val(2.0)), ("14", val(36.0)), - ("15", TextValue::Bool(true)), + ("15", V::Bool(true)), ("16", val(0.5)), ("17", numbers(&[0.8, 1.0, 1.33])), ("18", numbers(&[0.0, 0.0, 0.0])), ("19", numbers(&[1.0, 1.0, 1.0])), ("20", val(6.0)), ("22", val(1.0)), - ("23", TextValue::Bool(true)), + ("23", V::Bool(true)), ("24", val(1.0)), - ("26", TextValue::Bool(true)), + ("26", V::Bool(true)), ("31", val(36.0)), ("35", val(1.0)), ("40", val(2.0)), ]) -} +}); -/// Equal values; numbers compare by value (`0` equals `0.0`). -fn same(a: &TextValue, b: &TextValue) -> bool { +/// Equal values; numbers compare by value (`0` equals `0.0`), names and words by their text, +/// strings by their decoded text. +fn same(a: &V<'_>, b: &V<'_>) -> bool { match (a, b) { - (TextValue::Dict(x), TextValue::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), - (TextValue::Array(x), TextValue::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + (V::Dict(x), V::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), + (V::Array(x), V::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + (V::Str(x), V::Str(y)) => x == y || decode_utf16be(x) == decode_utf16be(y), + (V::Bytes(x), V::Bytes(y)) => x == y, + (V::Name(x), V::Name(y)) | (V::Token(x), V::Token(y)) => x == y || text(x) == text(y), + (V::Bool(x), V::Bool(y)) => x == y, _ => match (a.as_f64(), b.as_f64()) { (Some(p), Some(q)) => p == q, - _ => a == b, + _ => false, }, } } @@ -325,12 +465,12 @@ fn same(a: &TextValue, b: &TextValue) -> bool { /// listed when it holds something else (a neutral value carries nothing that could be lost). /// `/nil` is how an untouched document stores a slot without a value (244: paragraph `/27`, `/28`, /// `/36`). -fn is_neutral(v: &TextValue) -> bool { +fn is_neutral(v: &V<'_>) -> bool { match v { - TextValue::Name(n) => n == "nil", - TextValue::Bool(b) => !b, - TextValue::Array(a) => a.is_empty(), - TextValue::Dict(d) => d.is_empty(), + V::Name(n) => *n == b"nil", + V::Bool(b) => !b, + V::Array(a) => a.is_empty(), + V::Dict(d) => d.is_empty(), other => other.as_f64() == Some(0.0), } } @@ -355,13 +495,13 @@ pub const MAX_UNDECODED_FIELDS: usize = 64; /// Adds to `out` the slots of `actual` that are not decoded and differ from `sampled`, each once /// and at most [`MAX_UNDECODED_FIELDS`] in all. -fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { - let TextValue::Dict(pairs) = actual else { return }; +fn undecoded_fields(actual: &V<'_>, sampled: Option<&V<'_>>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { + let V::Dict(pairs) = actual else { return }; for (key, value) in pairs { if out.len() >= MAX_UNDECODED_FIELDS { return; } - if decoded.contains(&key.as_str()) || out.contains(key) { + if decoded.contains(&key.as_ref()) || out.iter().any(|o| o == key) { continue; } if neutral.iter().any(|(slot, values)| slot == key && value.as_i64().is_some_and(|v| values.contains(&v))) { @@ -372,7 +512,7 @@ fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[ None => !is_neutral(value), }; if differs { - out.push(key.clone()); + out.push(key.to_string()); } } } @@ -432,13 +572,13 @@ pub struct KernRun { /// The manual kerning list of a text run dictionary (`RUN/8/0`): entries `<< /0 << /0 amount >> /1 /// length >>` (554 kerns each of `HHHH` by 50; the closing `\r` has an entry without an amount). -fn kern_runs(run: &TextValue) -> Vec { - let Some(TextValue::Array(items)) = run.get("8").and_then(|v| v.get("0")) else { return Vec::new() }; +fn kern_runs(run: &V<'_>) -> Vec { + let Some(V::Array(items)) = run.get("8").and_then(|v| v.get("0")) else { return Vec::new() }; let runs: Vec = items .iter() .filter_map(|e| { let length = e.get("1").and_then(usize_of)?; - let amount = e.get("0").and_then(|v| v.get("0")).and_then(TextValue::as_f64).filter(|v| v.is_finite()); + let amount = e.get("0").and_then(|v| v.get("0")).and_then(V::as_f64).filter(|v| v.is_finite()); Some(KernRun { length, amount }) }) .collect(); @@ -644,24 +784,26 @@ pub struct TextBox { pub height: f64, } -fn colour_of(style: &TextValue, slot: &str) -> Option<[f64; 4]> { +/// The RGBA colour in a style slot (`slot/0/1`), `None` when it is not four numbers. +fn colour_of(style: &V<'_>, slot: &str) -> Option<[f64; 4]> { let c = style.get(slot)?.get("0")?.get("1")?; Some([c.at(0)?.as_f64()?, c.at(1)?.as_f64()?, c.at(2)?.as_f64()?, c.at(3)?.as_f64()?]) } -fn usize_of(v: &TextValue) -> Option { +/// A non-negative integer value as `usize`. +fn usize_of(v: &V<'_>) -> Option { usize::try_from(v.as_i64()?).ok() } /// A finite number slot of a style, `default` when it is missing or not a finite number. -fn num(style: &TextValue, slot: &str, default: f64) -> f64 { - style.get(slot).and_then(TextValue::as_f64).filter(|v| v.is_finite()).unwrap_or(default) +fn num(style: &V<'_>, slot: &str, default: f64) -> f64 { + style.get(slot).and_then(V::as_f64).filter(|v| v.is_finite()).unwrap_or(default) } /// A boolean slot of a style, `default` when it is missing or not a boolean. -fn flag(style: &TextValue, slot: &str, default: bool) -> bool { +fn flag(style: &V<'_>, slot: &str, default: bool) -> bool { match style.get(slot) { - Some(TextValue::Bool(b)) => *b, + Some(V::Bool(b)) => *b, _ => default, } } @@ -675,9 +817,9 @@ struct RunNotes { } /// The paragraph runs of a text run dictionary (`RUN/5/0`). -fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { - let Some(TextValue::Array(items)) = run.get("5").and_then(|v| v.get("0")) else { return Vec::new() }; - let sampled = Some(paragraph_defaults()); +fn paragraph_runs(run: &V<'_>, notes: &mut RunNotes) -> Vec { + let Some(V::Array(items)) = run.get("5").and_then(|v| v.get("0")) else { return Vec::new() }; + let sampled = Some(&*PARAGRAPH_DEFAULTS); let mut out: Vec = Vec::with_capacity(items.len()); for e in items { let Some(length) = e.get("1").and_then(usize_of) else { @@ -685,7 +827,7 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { continue; }; let para = e.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("5")); - let justification = match para.and_then(|p| p.get("0")).and_then(TextValue::as_i64) { + let justification = match para.and_then(|p| p.get("0")).and_then(V::as_i64) { Some(0) => Justification::Left, Some(1) => Justification::Right, Some(2) => Justification::Center, @@ -700,12 +842,12 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { } }; if let Some(p) = para { - undecoded_fields(p, sampled.as_ref(), DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES, &mut notes.paragraph_fields); + undecoded_fields(p, sampled, DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES, &mut notes.paragraph_fields); } - let num = |slot: &str| para.and_then(|p| p.get(slot)).and_then(TextValue::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0); - let int = |slot: &str, default: i64| para.and_then(|p| p.get(slot)).and_then(TextValue::as_i64).unwrap_or(default); + let num = |slot: &str| para.and_then(|p| p.get(slot)).and_then(V::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0); + let int = |slot: &str, default: i64| para.and_then(|p| p.get(slot)).and_then(V::as_i64).unwrap_or(default); let flag = |slot: &str, default: bool| match para.and_then(|p| p.get(slot)) { - Some(TextValue::Bool(b)) => *b, + Some(V::Bool(b)) => *b, _ => default, }; out.push(ParagraphRun { @@ -727,9 +869,9 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { } /// The style runs of a text run dictionary (`RUN/6/0`). -fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { - let Some(TextValue::Array(items)) = run.get("6").and_then(|v| v.get("0")) else { return Vec::new() }; - let sampled = Some(style_defaults()); +fn style_runs(run: &V<'_>, notes: &mut RunNotes) -> Vec { + let Some(V::Array(items)) = run.get("6").and_then(|v| v.get("0")) else { return Vec::new() }; + let sampled = Some(&*STYLE_DEFAULTS); let mut out: Vec = Vec::with_capacity(items.len()); for e in items { let Some(length) = e.get("1").and_then(usize_of) else { @@ -742,31 +884,31 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { out.push(StyleRun { length, ..prev }); continue; }; - let font_size = style.get("1").and_then(TextValue::as_f64); + let font_size = style.get("1").and_then(V::as_f64); let fill_argb = colour_of(style, "53"); let font_index = style.get("0").and_then(usize_of); if font_size.is_none() || fill_argb.is_none() || font_index.is_none() { notes.incomplete = notes.incomplete.saturating_add(1); } - undecoded_fields(style, sampled.as_ref(), DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES, &mut notes.style_fields); + undecoded_fields(style, sampled, DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES, &mut notes.style_fields); out.push(StyleRun { length, font_size: font_size.unwrap_or(prev.font_size), fill_argb: fill_argb.unwrap_or(prev.fill_argb), stroke_argb: colour_of(style, "54"), - tracking: style.get("8").and_then(TextValue::as_f64).unwrap_or(0.0), - faux_bold: matches!(style.get("2"), Some(TextValue::Bool(true))), - faux_italic: matches!(style.get("3"), Some(TextValue::Bool(true))), - caps: match style.get("12").and_then(TextValue::as_i64) { + tracking: style.get("8").and_then(V::as_f64).unwrap_or(0.0), + faux_bold: matches!(style.get("2"), Some(V::Bool(true))), + faux_italic: matches!(style.get("3"), Some(V::Bool(true))), + caps: match style.get("12").and_then(V::as_i64) { None | Some(0) => Caps::Normal, Some(1) => Caps::Small, Some(2) => Caps::All, Some(other) => Caps::Other(other), }, - auto_leading: matches!(style.get("4"), Some(TextValue::Bool(true))), - leading: style.get("5").and_then(TextValue::as_f64), + auto_leading: matches!(style.get("4"), Some(V::Bool(true))), + leading: style.get("5").and_then(V::as_f64), font_index: font_index.unwrap_or(prev.font_index), - baseline: match style.get("13").and_then(TextValue::as_i64) { + baseline: match style.get("13").and_then(V::as_i64) { None | Some(0) => FontBaseline::Normal, Some(1) => FontBaseline::Superscript, Some(2) => FontBaseline::Subscript, @@ -775,22 +917,22 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { horizontal_scale: num(style, "6", 1.0), vertical_scale: num(style, "7", 1.0), baseline_shift: num(style, "9", 0.0), - auto_kern: match style.get("11").and_then(TextValue::as_i64) { + auto_kern: match style.get("11").and_then(V::as_i64) { Some(0) => AutoKern::None, None | Some(1) => AutoKern::Metrics, Some(2) => AutoKern::Optical, Some(other) => AutoKern::Other(other), }, ligatures: flag(style, "18", false), - baseline_direction: style.get("35").and_then(TextValue::as_i64).unwrap_or(1), + baseline_direction: style.get("35").and_then(V::as_i64).unwrap_or(1), tsume: num(style, "36", 0.0), no_break: flag(style, "52", false), apply_fill: flag(style, "56", true), apply_stroke: flag(style, "57", false), stroke_over_fill: flag(style, "58", true), - line_join: style.get("62").and_then(TextValue::as_i64).unwrap_or(0), + line_join: style.get("62").and_then(V::as_i64).unwrap_or(0), stroke_width: num(style, "63", 1.0), - digit_set: style.get("70").and_then(TextValue::as_i64).unwrap_or(0), + digit_set: style.get("70").and_then(V::as_i64).unwrap_or(0), }); } out @@ -798,13 +940,13 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { /// The box of paragraph text from the frame dictionary `/0/8/0[0]/0`: key `/1/0` holds 16 x, y pairs /// that trace the rectangle, and `/2/0` is 1; both keys are absent for point text. -fn text_box(top: &TextValue) -> Option { +fn text_box(top: &V<'_>) -> Option { let frame = top.get("0")?.get("8")?.get("0")?.at(0)?.get("0")?; if frame.get("2")?.get("0")?.as_i64()? != 1 { return None; } - let TextValue::Array(coords) = frame.get("1")?.get("0")? else { return None }; - let numbers: Vec = coords.iter().map(TextValue::as_f64).collect::>()?; + let V::Array(coords) = frame.get("1")?.get("0")? else { return None }; + let numbers: Vec = coords.iter().map(V::as_f64).collect::>()?; let (pairs, _) = numbers.as_chunks::<2>(); if pairs.is_empty() { return None; @@ -818,18 +960,15 @@ fn text_box(top: &TextValue) -> Option { impl TextDocument { /// Decodes the body of a `btdk` list. pub fn parse(body: &[u8]) -> Option { - let top = parse_text_dict(body); + let top = parse_dict(body); let run = top.get("1")?.get("1")?.at(0)?.get("0")?; - let raw_text = run.get("0")?.as_str()?.to_string(); + let raw_text = run.get("0")?.string()?; let style = run.get("6")?.get("0")?.at(0)?.get("0")?.get("0")?.get("6")?; let fill_argb = colour_of(style, "53")?; let font_index = usize::try_from(style.get("0")?.as_i64()?).ok()?; let mut fonts = Vec::new(); - if let Some(TextValue::Array(list)) = top.get("0").and_then(|v| v.get("1")).and_then(|v| v.get("0")) { - fonts = list - .iter() - .map(|f| f.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("0")).and_then(TextValue::as_str).unwrap_or_default().to_string()) - .collect(); + if let Some(V::Array(list)) = top.get("0").and_then(|v| v.get("1")).and_then(|v| v.get("0")) { + fonts = list.iter().map(|f| f.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("0")).and_then(V::string).unwrap_or_default()).collect(); } let mut notes = RunNotes::default(); let paragraphs = paragraph_runs(run, &mut notes); @@ -857,7 +996,7 @@ impl TextDocument { .and_then(|v| v.get("0")) .and_then(|v| v.get("2")) .and_then(|v| v.get("1")) - .and_then(TextValue::as_i64), + .and_then(V::as_i64), }) } @@ -881,23 +1020,32 @@ impl TextDocument { } impl<'f, 'a> Layer<'f, 'a> { - /// The text document of a text layer (`LIST btdk` below `ADBE Text Properties`). + /// The Source Text record `LIST btds` (`ADBE Text Document`). It is a direct child of the + /// `ADBE Text Properties` group, which is a direct child of the layer's root `tdgp` + /// (`Layr/tdgp/tdgp/btds` in every sample, and where the exporter writes it), so only that + /// level is searched rather than the whole layer. + fn source_text_record(&self) -> Option> { + let root = self.chunk.child_list(b"tdgp")?; + root.children().filter(|g| g.is_list(b"tdgp")).find_map(|g| g.child_list(b"btds")) + } + + /// The text document of a text layer: the first `LIST btdk` in the Source Text record `btds` + /// (the same documents [`Layer::text_document_count`] counts). pub fn text_document(&self) -> Option { - TextDocument::parse(self.chunk.find_list(b"btdk")?.bytes()?) + TextDocument::parse(self.source_text_record()?.find_list(b"btdk")?.bytes()?) } /// The Source Text property record: the `tdbs` in `btds[ADBE Text Document]` (a placeholder /// in every sample: no keys, no expression). Source Text keyframes and expressions were not /// sampled; a `list` (keys) or an expression in it marks them. pub fn source_text_property(&self) -> Option> { - let btds = self.chunk.descendants().map(|(_, c)| c).find(|c| c.is_list(b"btds"))?; - Property::from_chunk(btds.child_list(b"tdbs")?) + Property::from_chunk(self.source_text_record()?.child_list(b"tdbs")?) } - /// The number of text documents (`LIST btdk`) of the layer: 1 in every sample; more would be - /// one per Source Text keyframe (not sampled). + /// The number of text documents (`LIST btdk`) in the Source Text record: 1 in every sample; + /// more would be one per Source Text keyframe (not sampled). pub fn text_document_count(&self) -> usize { - self.chunk.descendants().filter(|(_, c)| c.is_list(b"btdk")).count() + self.source_text_record().map_or(0, |btds| btds.descendants().filter(|(_, c)| c.is_list(b"btdk")).count()) } } @@ -942,10 +1090,24 @@ pub enum CueType { Other(u32), } +impl<'f, 'a> Node<'f, 'a> { + /// The marker list `LIST mrst` of a marker holder: a layer (`LIST Layr`) or a composition's + /// `SecL` list. A layer's `mrst` is a direct child of its root `tdgp` (`ADBE Marker`, where + /// After Effects and the exporter write it), so only that place is looked at; most layers have + /// none, and searching their whole property tree for it was a large part of an import. Any + /// other holder is searched depth-first. + pub(crate) fn marker_list(self) -> Option> { + if self.is_list(b"Layr") { + return self.child_list(b"tdgp")?.child_list(b"mrst"); + } + self.find_list(b"mrst") + } +} + /// Reads the markers below `holder` (a layer or the composition's `SecL` list): the `mrst` list has a /// property whose 16-byte records give the times and one `Nmrd` list per marker with the strings. -fn markers_in(holder: &Chunk<'_>) -> Vec { - let Some(mrst) = holder.find_list(b"mrst") else { return Vec::new() }; +fn markers_in(holder: Node<'_, '_>) -> Vec { + let Some(mrst) = holder.marker_list() else { return Vec::new() }; let Some(prop) = mrst.child_list(b"tdbs").and_then(Property::from_chunk) else { return Vec::new() }; let Some(keys) = prop.keys() else { return Vec::new() }; let scale = prop.time_scale().unwrap_or(0); @@ -955,11 +1117,11 @@ fn markers_in(holder: &Chunk<'_>) -> Vec { .filter_map(|(k, nmrd)| { let time_units = k.time_units()?; // The five strings, in the order comment, chapter, URL, frame target, cue point name. - let mut strings = nmrd.children().iter().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); + let mut strings = nmrd.children().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); let mut next = || strings.next().unwrap_or_default(); let (comment, chapter, url, frame_target, cue_point_name) = (next(), next(), next(), next(), next()); let extras: Vec = strings.collect(); - let header = nmrd.child(b"NmHd").and_then(Chunk::bytes); + let header = nmrd.child(b"NmHd").and_then(Node::bytes); let duration_seconds = header.and_then(|h| { let (duration, per_second) = (be_u32(h, 8)?, be_u32(h, 12)?); (per_second != 0).then(|| f64::from(duration) / f64::from(per_second)) @@ -1002,9 +1164,631 @@ impl<'f, 'a> Comp<'f, 'a> { } } +/// The dictionary parser and the defaults as they were before they borrowed from the body (#3276), +/// kept as the reference the differential tests compare with. +#[cfg(test)] +mod oracle { + use super::{MAX_DEPTH, MAX_UNDECODED_FIELDS, TextValue}; + use crate::container::bytes::text; + + pub(super) fn same(a: &TextValue, b: &TextValue) -> bool { + match (a, b) { + (TextValue::Dict(x), TextValue::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), + (TextValue::Array(x), TextValue::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + _ => match (a.as_f64(), b.as_f64()) { + (Some(p), Some(q)) => p == q, + _ => a == b, + }, + } + } + + pub(super) fn is_neutral(v: &TextValue) -> bool { + match v { + TextValue::Name(n) => n == "nil", + TextValue::Bool(b) => !b, + TextValue::Array(a) => a.is_empty(), + TextValue::Dict(d) => d.is_empty(), + other => other.as_f64() == Some(0.0), + } + } + + pub(super) fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { + let TextValue::Dict(pairs) = actual else { return }; + for (key, value) in pairs { + if out.len() >= MAX_UNDECODED_FIELDS { + return; + } + if decoded.contains(&key.as_str()) || out.contains(key) { + continue; + } + if neutral.iter().any(|(slot, values)| slot == key && value.as_i64().is_some_and(|v| values.contains(&v))) { + continue; + } + let differs = match sampled.and_then(|s| s.get(key)) { + Some(default) => !same(value, default), + None => !is_neutral(value), + }; + if differs { + out.push(key.clone()); + } + } + } + + impl TextValue { + /// The canonical text of a key, as the owning parser built it. + fn key_text(&self) -> String { + match self { + Self::Name(s) | Self::Str(s) | Self::Token(s) => s.clone(), + Self::Int(i) => i.to_string(), + Self::Float(f) => f.to_string(), + Self::Bool(b) => b.to_string(), + Self::Bytes(b) => text(b), + Self::Dict(_) | Self::Array(_) => String::new(), + } + } + } + + struct Parser<'b> { + b: &'b [u8], + i: usize, + } + + /// Whitespace between tokens. + fn is_ws(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\r' | b'\n') + } + + /// A byte that ends a bare word. + fn is_delimiter(c: u8) -> bool { + is_ws(c) || matches!(c, b'/' | b'[' | b']' | b'<' | b'>' | b'(' | b')') + } + + impl Parser<'_> { + /// The next byte, `None` at the end. + fn peek(&self) -> Option { + self.b.get(self.i).copied() + } + + /// Moves past whitespace. + fn skip_ws(&mut self) { + while self.peek().is_some_and(is_ws) { + self.i = self.i.saturating_add(1); + } + } + + /// True when the unread bytes start with `s`. + fn starts_with(&self, s: &[u8]) -> bool { + self.b.get(self.i..).is_some_and(|rest| rest.starts_with(s)) + } + + /// The bare word at the position (up to the next delimiter), borrowed from the body. + fn word(&mut self) -> &[u8] { + let start = self.i; + while self.peek().is_some_and(|c| !is_delimiter(c)) { + self.i = self.i.saturating_add(1); + } + self.b.get(start..self.i).unwrap_or(&[]) + } + + /// The value at the position, `None` when it is malformed or nested deeper than the limit. + fn parse(&mut self, depth: usize) -> Option { + self.skip_ws(); + if depth > MAX_DEPTH { + return None; + } + if self.starts_with(b"<<") { + self.i = self.i.saturating_add(2); + let mut pairs = Vec::new(); + loop { + self.skip_ws(); + if self.starts_with(b">>") { + self.i = self.i.saturating_add(2); + return Some(TextValue::Dict(pairs)); + } + let key = self.parse(depth.saturating_add(1))?; + let value = self.parse(depth.saturating_add(1))?; + pairs.push((key.key_text(), value)); + } + } + match self.peek()? { + b'[' => { + self.i = self.i.saturating_add(1); + let mut items = Vec::new(); + loop { + self.skip_ws(); + if self.peek()? == b']' { + self.i = self.i.saturating_add(1); + return Some(TextValue::Array(items)); + } + items.push(self.parse(depth.saturating_add(1))?); + } + } + b'(' => { + self.i = self.i.saturating_add(1); + let mut out = Vec::new(); + loop { + let c = self.peek()?; + if c == b'\\' { + out.push(*self.b.get(self.i.saturating_add(1))?); + self.i = self.i.saturating_add(2); + } else if c == b')' { + self.i = self.i.saturating_add(1); + break; + } else { + out.push(c); + self.i = self.i.saturating_add(1); + } + } + Some(match out.strip_prefix(&[0xfe, 0xff]) { + Some(utf16) => TextValue::Str(decode_utf16be(utf16)), + None => TextValue::Bytes(out), + }) + } + b'/' => { + self.i = self.i.saturating_add(1); + Some(TextValue::Name(text(self.word()))) + } + _ => { + let word = self.word(); + if word.is_empty() { + return None; + } + let w = text(word); + Some(match w.as_str() { + "true" => TextValue::Bool(true), + "false" => TextValue::Bool(false), + _ => match w.parse::() { + Ok(i) => TextValue::Int(i), + // `parse` also takes `nan`, `inf` and overflowing exponents; those are not + // numbers a text document holds, so they stay tokens and read as missing. + Err(_) => match w.parse::() { + Ok(f) if f.is_finite() => TextValue::Float(f), + _ => TextValue::Token(w), + }, + }, + }) + } + } + } + } + + /// A big-endian UTF-16 string as text (unpaired surrogates become U+FFFD). + fn decode_utf16be(bytes: &[u8]) -> String { + let (pairs, rest) = bytes.as_chunks::<2>(); + let units = pairs.iter().map(|p| u16::from_be_bytes(*p)); + let mut s: String = char::decode_utf16(units).map(|r| r.unwrap_or(char::REPLACEMENT_CHARACTER)).collect(); + if !rest.is_empty() { + s.push(char::REPLACEMENT_CHARACTER); + } + s + } + + /// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). + /// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was + /// read before is returned. + pub(super) fn parse_text_dict(body: &[u8]) -> TextValue { + let mut parser = Parser { b: body, i: 0 }; + let mut pairs = Vec::new(); + loop { + parser.skip_ws(); + if parser.i >= body.len() { + break; + } + let Some(key) = parser.parse(0) else { break }; + let Some(value) = parser.parse(0) else { break }; + pairs.push((key.key_text(), value)); + } + TextValue::Dict(pairs) + } + + fn val(v: f64) -> TextValue { + TextValue::Float(v) + } + + /// A dictionary value with these keys, for the built-in defaults. + fn dict(pairs: Vec<(&str, TextValue)>) -> TextValue { + TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) + } + + /// An array of numbers, for the built-in defaults. + fn numbers(v: &[f64]) -> TextValue { + TextValue::Array(v.iter().map(|x| val(*x)).collect()) + } + + /// The default of every run style slot that is not decoded and not neutral (zero, `false`, + /// empty, `/nil`): the value an untouched point text holds in every sample (110, 243-245). A slot + /// with a neutral default is left out; [`undecoded_fields`] treats a missing default as neutral. + pub(super) fn style_defaults() -> TextValue { + let paint = |a: f64| dict(vec![("99", TextValue::Name("SimplePaint".into())), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); + let class = |name: &str| dict(vec![("99", TextValue::Name(name.into()))]); + dict(vec![ + ("37", val(2.0)), + ("41", val(2.0)), + ("43", dict(vec![("0", val(0.5))])), + ("44", val(2.0)), + ("45", val(2.0)), + ("46", val(7.0)), + ("49", val(-1.0)), + ("50", val(-1.0)), + ("55", class("SimpleBlender")), + ("64", val(4.0)), + ("71", val(4.0)), + ("77", TextValue::Bool(true)), + ("78", TextValue::Bool(true)), + ("79", paint(0.0)), + ("82", val(3.0)), + ("83", val(3.0)), + ("86", class("SimpleCustomFeature")), + ("87", val(100.0)), + ("88", TextValue::Bool(true)), + ("95", dict(vec![("0", val(0.0)), ("1", val(0.0))])), + ("96", TextValue::Bool(true)), + ]) + } + + /// The same for the paragraph slots (110, 243-245). + pub(super) fn paragraph_defaults() -> TextValue { + dict(vec![ + ("6", val(1.0)), + ("7", val(1.2)), + ("10", val(6.0)), + ("11", val(2.0)), + ("12", val(2.0)), + ("14", val(36.0)), + ("15", TextValue::Bool(true)), + ("16", val(0.5)), + ("17", numbers(&[0.8, 1.0, 1.33])), + ("18", numbers(&[0.0, 0.0, 0.0])), + ("19", numbers(&[1.0, 1.0, 1.0])), + ("20", val(6.0)), + ("22", val(1.0)), + ("23", TextValue::Bool(true)), + ("24", val(1.0)), + ("26", TextValue::Bool(true)), + ("31", val(36.0)), + ("35", val(1.0)), + ("40", val(2.0)), + ]) + } +} + #[cfg(test)] mod tests { use super::*; + use crate::container::Chunk; + + /// A `LIST` of this type holding `kids`. + fn list(t: &[u8; 4], kids: Vec>) -> Chunk<'static> { + Chunk::list(*t, kids) + } + + /// A document with every kind of value, close to what After Effects writes: names, numbers, + /// a UTF-16 string with escapes, a plain string, nested dictionaries and arrays, an undecoded + /// style slot, a font list and a box frame. + const DOC: &[u8] = b"/0 << /1 << /0 [ << /0 << /0 << /0 (\xfe\xff\0M\0y\0r) >> /1 1 >> >> << /0 << /0 << /0 (\xfe\xff\0A\0\\)\0b) >> >> >> ] >> \ + /8 << /0 [ << /0 << /1 << /0 [ 0 0 10 0 10 5 0 5 ] >> /2 << /0 1 /1 2 >> >> >> ] >> >> \ + /1 << /1 [ << /0 << /0 (\xfe\xff\0H\0i\0\r) /5 << /0 [ << /0 << /0 << /5 << /0 2 /9 false /24 0 /34 7 /17 [ .8 1.0 1.33 ] >> >> >> /1 3 >> ] >> \ + /6 << /0 [ << /0 << /0 << /6 << /0 1 /1 36.5 /53 << /0 << /0 1 /1 [ 1 0.5 0 1 ] >> >> /79 << /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 0 ] >> >> /68 1 /90 /x /91 (plain) /92 word >> >> >> /1 3 >> ] >> \ + /8 << /0 [ << /0 << /0 50 >> /1 2 >> << /1 1 >> ] >> >> >> ] >>"; + + /// A deterministic xorshift generator: the tests need varied input, not a dependency. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number (xorshift). + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: usize) -> usize { + usize::try_from(self.next() % (n as u64)).unwrap_or(0) + } + } + + /// The borrowing parser reads what the owning one read. + #[test] + fn the_borrowing_parser_reads_what_the_owning_one_read() { + // Every key form the owned keys canonicalise (`06`, `+6`, `-0`, `1.0`, `1e2`, strings, + // booleans, containers), names that are not UTF-8, escapes, unclosed strings and escapes, + // non-finite numbers, and bare words. + let parts: &[&[u8]] = &[ + b"<<", + b">>", + b"[", + b"]", + b"/0", + b"/1", + b"/nil", + b"/SimplePaint", + b"/\xff\xfe", + b"/\xc3\xa9", + b"/", + b"06", + b"+6", + b"-0", + b"1.0", + b"1.50", + b"1e2", + b"-2.5", + b".8", + b"inf", + b"nan", + b"1e999", + b"9223372036854775808", + b"true", + b"false", + b"(abc)", + b"(\xfe\xff\0A\0B)", + b"(\xfe\xff\0A\0)", + b"(a\\)b)", + b"(\\", + b"(\xfe\xff\\)\0)", + b"(\\\xfe\xff\0C)", + b"()", + b"(\xfe\xff)", + b"(unclosed", + b"word", + b"\xc3\x28", + b"\xff", + b" ", + b"\n", + b"\t\r", + b"<", + b">", + b")", + ]; + // A value: mostly well formed, with a random part in its place now and then. + /// Appends one generated value to `out`: mostly well formed, now and then a random part. + fn value(rng: &mut Rng, parts: &[&[u8]], depth: usize, out: &mut Vec) { + let pick = |rng: &mut Rng| parts.get(rng.below(parts.len())).copied().unwrap_or_default(); + match rng.below(if depth > 6 { 5 } else { 10 }) { + 0 => out.extend_from_slice(pick(rng)), + 1..=4 => out.extend_from_slice([&b"/0"[..], b"1", b"(\xfe\xff\0z)", b"0.5", b"/nil"].get(rng.below(5)).copied().unwrap_or_default()), + 5..=7 => { + out.extend_from_slice(b"<< "); + for _ in 0..rng.below(5) { + let key = if rng.below(4) == 0 { pick(rng) } else { [&b"/0"[..], b"/1", b"/6", b"/53"].get(rng.below(4)).copied().unwrap_or_default() }; + out.extend_from_slice(key); + out.push(b' '); + value(rng, parts, depth + 1, out); + out.push(b' '); + } + if rng.below(20) != 0 { + out.extend_from_slice(b">>"); + } + } + _ => { + out.extend_from_slice(b"[ "); + for _ in 0..rng.below(5) { + value(rng, parts, depth + 1, out); + out.push(b' '); + } + if rng.below(20) != 0 { + out.push(b']'); + } + } + } + } + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let (mut read, mut nested) = (0, 0); + for case in 0..40_000 { + let mut body = Vec::new(); + if case % 2 == 0 { + for _ in 0..rng.below(60) { + body.extend_from_slice(parts.get(rng.below(parts.len())).copied().unwrap_or_default()); + if rng.below(3) != 0 { + body.push(b' '); + } + } + } else { + for _ in 0..1 + rng.below(4) { + body.extend_from_slice(b"/0 "); + value(&mut rng, parts, 0, &mut body); + body.push(b'\n'); + } + } + let parsed = parse_text_dict(&body); + assert_eq!(parsed, oracle::parse_text_dict(&body), "case {case}: {}", String::from_utf8_lossy(&body)); + let _ = TextDocument::parse(&body); + if let TextValue::Dict(pairs) = &parsed { + read += usize::from(!pairs.is_empty()); + nested += usize::from(pairs.iter().any(|(_, v)| matches!(v, TextValue::Dict(d) if !d.is_empty()))); + } + } + // The input is not all rejected at the first token. + assert!(read > 25_000 && nested > 3_000, "{read} cases read something, {nested} a dictionary"); + // Every cut of a realistic document, and nesting past the limit. + for end in 0..=DOC.len() { + let body = DOC.get(..end).unwrap_or_default(); + assert_eq!(parse_text_dict(body), oracle::parse_text_dict(body), "cut at {end}"); + let _ = TextDocument::parse(body); + } + for depth in [63, 64, 65, 66, 200] { + let body = [b"/k ".to_vec(), b"<< /a ".repeat(depth), b"1".to_vec(), b" >>".repeat(depth), b" /z 2".to_vec()].concat(); + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "depth {depth}"); + let body = [b"/k ".to_vec(), b"[ ".repeat(depth), b"]".repeat(depth)].concat(); + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "array depth {depth}"); + } + // Dictionaries and arrays mixed (`<< /a [ << /a [ ...`), closed, and left open exactly + // at the limit. + for depth in 60..=68 { + let open: Vec = (0..depth).flat_map(|i| if i % 2 == 0 { &b"<< /a "[..] } else { &b"[ "[..] }).copied().collect(); + let close: Vec = (0..depth).rev().flat_map(|i| if i % 2 == 0 { &b" >>"[..] } else { &b" ]"[..] }).copied().collect(); + for body in [[b"/k ".to_vec(), open.clone(), b"1".to_vec(), close, b" /z 2".to_vec()].concat(), [b"/k ".to_vec(), open, b"1".to_vec()].concat()] { + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "mixed depth {depth}"); + let _ = TextDocument::parse(&body); + } + } + } + + /// A realistic document is read in full. + #[test] + fn a_realistic_document_is_read_in_full() { + let doc = TextDocument::parse(DOC).expect("a document"); + assert_eq!((doc.text.as_str(), doc.font_size, doc.fill_argb, doc.font.as_deref()), ("Hi", 36.5, [1.0, 0.5, 0.0, 1.0], Some("A)b"))); + assert_eq!(doc.fonts, ["Myr", "A)b"]); + assert_eq!(doc.paragraphs.first().map(|p| (p.length, p.justification, p.auto_hyphenate)), Some((3, Justification::Center, false))); + // `/68 1` and `/34 7` are neutral; `/90 /x`, `/91 (plain)` and `/92 word` differ from the + // (missing, neutral) default; `/79` equals its default. + assert_eq!(doc.undecoded_style_fields, ["90", "91", "92"]); + assert!(doc.undecoded_paragraph_fields.is_empty()); + assert_eq!(doc.kerning, [KernRun { length: 2, amount: Some(50.0) }, KernRun { length: 1, amount: None }]); + assert_eq!(doc.text_box, Some(TextBox { x: 0.0, y: 0.0, width: 10.0, height: 5.0 })); + assert_eq!(doc.line_orientation, Some(2)); + } + + /// The defaults are the sampled ones and built once. + #[test] + fn the_defaults_are_the_sampled_ones_and_built_once() { + assert_eq!(STYLE_DEFAULTS.clone().into_owned(), oracle::style_defaults()); + assert_eq!(PARAGRAPH_DEFAULTS.clone().into_owned(), oracle::paragraph_defaults()); + assert!(std::ptr::eq(&*STYLE_DEFAULTS, &*STYLE_DEFAULTS)); + } + + /// Names keys and strings are borrowed from the body. + #[test] + fn names_keys_and_strings_are_borrowed_from_the_body() { + let V::Dict(pairs) = parse_dict(b"/a /b /c (\xfe\xff\0x) /d (plain) /e word /f 1.5 /g (\\(esc) 07 1") else { panic!("a dict") }; + // Every `/name` key, name, string without escapes and bare word points into the body. + assert!(pairs.iter().take(6).all(|(k, _)| matches!(k, Cow::Borrowed(_)))); + assert!(matches!(pairs.first().map(|p| &p.1), Some(V::Name(b"b")))); + assert!(matches!(pairs.get(1).map(|p| &p.1), Some(V::Str(Cow::Borrowed(b"\0x"))))); + assert!(matches!(pairs.get(2).map(|p| &p.1), Some(V::Bytes(Cow::Borrowed(b"plain"))))); + assert!(matches!(pairs.get(3).map(|p| &p.1), Some(V::Token(b"word")))); + // An escaped string is copied; a number key is written in its canonical form. + assert!(matches!(pairs.get(5).map(|p| &p.1), Some(V::Bytes(Cow::Owned(b))) if b == b"(esc")); + assert_eq!(pairs.get(6).map(|p| p.0.as_ref()), Some("7")); + // Containers hold what was read. + let V::Dict(outer) = parse_dict(b"/0 << /a 1 /b 2 /c 3 /d 4 /e 5 >> /1 [ 1 2 3 4 5 6 7 8 9 ]") else { panic!("a dict") }; + assert!(matches!(outer.first().map(|p| &p.1), Some(V::Dict(d)) if d.len() == 5 && d.capacity() >= d.len())); + assert!(matches!(outer.get(1).map(|p| &p.1), Some(V::Array(a)) if a.len() == 9 && a.capacity() >= a.len())); + } + + /// Slot checks agree with the owned ones. + #[test] + fn slot_checks_agree_with_the_owned_ones() { + // Slots: defaults, decoded, neutral-listed and unknown ones. Values: each default as + // stored (also as `Int` where it is `2.0`), near misses, neutral values and every kind. + let slots = ["0", "6", "7", "17", "24", "34", "35", "37", "43", "55", "68", "77", "79", "86", "95", "96", "200", "x"]; + let values: &[&[u8]] = &[ + b"2", + b"2.0", + b"7", + b"1", + b"0", + b"-1", + b"0.5", + b"1.2", + b"36", + b"true", + b"false", + b"/nil", + b"/x", + b"/SimpleBlender", + b"[ ]", + b"<< >>", + b"[ .8 1 1.33 ]", + b"[ 0.8 1.0 ]", + b"[ 1 1 1 ]", + b"<< /0 .5 >>", + b"<< /0 0 /1 0 >>", + b"<< /1 0 /0 0 >>", + b"<< /99 /SimpleBlender >>", + b"<< /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 0 ] >> >>", + b"<< /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 1 ] >> >>", + b"(plain)", + b"(\xfe\xff\0a)", + b"word", + b"/\xff", + ]; + let mut rng = Rng(0x2545_f491_4f6c_dd1d); + let mut listed = 0; + for case in 0..20_000 { + let mut body = b"/s << ".to_vec(); + for _ in 0..rng.below(12) { + body.push(b'/'); + body.extend_from_slice(slots.get(rng.below(slots.len())).copied().unwrap_or_default().as_bytes()); + body.push(b' '); + body.extend_from_slice(values.get(rng.below(values.len())).copied().unwrap_or_default()); + body.push(b' '); + } + body.extend_from_slice(b">>"); + let (new, old) = (parse_dict(&body), oracle::parse_text_dict(&body)); + let (Some(new), Some(old)) = (new.get("s"), old.get("s")) else { panic!("case {case}: a dict") }; + for (sampled, old_sampled, decoded, neutral) in [ + (&*STYLE_DEFAULTS, oracle::style_defaults(), DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES), + (&*PARAGRAPH_DEFAULTS, oracle::paragraph_defaults(), DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES), + ] { + let (mut a, mut b) = (Vec::new(), Vec::new()); + undecoded_fields(new, Some(sampled), decoded, neutral, &mut a); + oracle::undecoded_fields(old, Some(&old_sampled), decoded, neutral, &mut b); + assert_eq!(a, b, "case {case}: {}", String::from_utf8_lossy(&body)); + listed += a.len(); + } + // Every pair of values compares and reads as neutral alike. + if let (V::Dict(x), TextValue::Dict(y)) = (new, old) { + for ((_, p), (_, q)) in x.iter().zip(y) { + assert_eq!(is_neutral(p), oracle::is_neutral(q), "case {case}"); + for ((_, r), (_, t)) in x.iter().zip(y) { + assert_eq!(same(p, r), oracle::same(q, t), "case {case}"); + } + } + } + } + assert!(listed > 10_000, "{listed} slots listed"); + } + + /// Values compare by their text. + #[test] + fn values_compare_by_their_text() { + // Two names that are not UTF-8 read as the same text with replacement characters, as the + // owned values did. + assert!(same(&V::Name(b"\xff"), &V::Name(b"\xfe"))); + assert!(!same(&V::Name(b"a"), &V::Name(b"b"))); + assert!(same(&V::Str(Cow::Borrowed(b"\xd8\x00")), &V::Str(Cow::Owned(b"\xd8\x01".to_vec())))); + assert!(same(&V::Int(0), &V::Float(0.0)) && !same(&V::Int(0), &V::Bool(false)) && !same(&V::Name(b"x"), &V::Token(b"x"))); + assert!(is_neutral(&V::Name(b"nil")) && !is_neutral(&V::Token(b"nil")) && is_neutral(&V::Float(0.0)) && !is_neutral(&V::Str(Cow::Borrowed(b"")))); + } + + /// A layers marker list is the one in its root group. + #[test] + fn a_layers_marker_list_is_the_one_in_its_root_group() { + let marked = |tag: u8| list(b"mrst", vec![Chunk::data(*b"test", vec![tag])]); + // The stored place: `Layr/tdgp/mrst`. + let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), marked(1)])]); + assert_eq!(Node::from(&layer).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[1u8][..])); + // A list deeper in the layer is not the layer's marker list (the old depth-first search + // found it); a layer without a marker list in its root group has no markers. + let deep = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(2)])])]); + assert!(Node::from(&deep).marker_list().is_none()); + assert!(Layer { chunk: Node::from(&deep) }.markers().is_empty()); + // Any other holder (a composition's `SecL`) is still searched depth-first. + let secl = list(b"SecL", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(3)])])]); + assert_eq!(Node::from(&secl).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[3u8][..])); + } + + /// The source text record is found in the text group. + #[test] + fn the_source_text_record_is_found_in_the_text_group() { + let doc = b"/1 << /1 [ << /0 << /0 (\xfe\xff\0H\0\r) /6 << /0 [ << /0 << /0 << /6 << /0 0 /1 50 /53 << /0 << /1 [ 1 1 0 0 ] >> >> >> >> >> /1 2 >> ] >> >> >> ] >>"; + // `btdk` is an opaque list (its body is not a chunk sequence), as the reader makes it. + let btdk = || Chunk { id: *b"LIST", body: crate::container::Body::OpaqueList { list_type: *b"btdk", data: doc.to_vec().into() }, pad: None }; + let btds_real = |n: usize| list(b"btds", std::iter::once(list(b"tdbs", vec![])).chain((0..n).map(|_| btdk())).collect()); + let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), list(b"tdgp", vec![btds_real(2)])])]); + let view = Layer { chunk: Node::from(&layer) }; + assert_eq!(view.text_document().map(|d| d.text), Some("H".to_string())); + assert!(view.source_text_property().is_some()); + assert_eq!(view.text_document_count(), 2); + // A record anywhere else in the layer is not its Source Text. + let elsewhere = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![list(b"tdgp", vec![btds_real(1)])])])]); + let view = Layer { chunk: Node::from(&elsewhere) }; + assert!(view.text_document().is_none() && view.source_text_property().is_none()); + assert_eq!(view.text_document_count(), 0); + } #[test] fn non_finite_numbers_are_not_numbers() { @@ -1017,13 +1801,13 @@ mod tests { fn undecoded_fields_are_capped_and_linear() { // 300 000 distinct undecoded slots that differ from a sampled default: before, each was // checked against every slot recorded so far (4.5e10 compares) and all were listed. - let actual = TextValue::Dict((0..300_000).map(|i| (format!("{}", 9000 + i), TextValue::Int(1))).collect()); + let actual = V::Dict((0..300_000).map(|i| (Cow::Owned(format!("{}", 9000 + i)), V::Int(1))).collect()); let mut out = Vec::new(); undecoded_fields(&actual, None, &["0"], &[], &mut out); assert_eq!(out.len(), MAX_UNDECODED_FIELDS); assert_eq!(out.first().map(String::as_str), Some("9000")); // A slot listed once is not listed again. - let again = TextValue::Dict(vec![("9000".into(), TextValue::Int(1))]); + let again = V::Dict(vec![(Cow::Borrowed("9000"), V::Int(1))]); let mut short = vec!["9000".to_string()]; undecoded_fields(&again, None, &[], &[], &mut short); assert_eq!(short, ["9000"]); diff --git a/crates/aep/src/container/typed.rs b/crates/aep/src/container/typed.rs index 09567fd84..7e7bfd0ec 100644 --- a/crates/aep/src/container/typed.rs +++ b/crates/aep/src/container/typed.rs @@ -4,42 +4,57 @@ //! //! Only facts that were established from our own one-change-per-file corpus and are checked by //! tests against it are decoded (see `docs/aep-format.md` for the evidence). Everything else -//! stays reachable through the raw [`Chunk`] tree. Views borrow the file and never fail loudly: -//! a chunk that does not have the expected shape yields `None` or is skipped. +//! stays reachable through the raw chunk tree ([`Node`]). Views borrow the file and never fail +//! loudly: a chunk that does not have the expected shape yields `None` or is skipped. +//! +//! A view reads either tree: the owned [`Chunk`](crate::container::Chunk) tree +//! ([`Project::new`]) or the flat [`TreeIndex`] the importer builds ([`Project::from_index`]); +//! both give the same answers. use crate::container::bytes::{be_f64, be_u16, be_u32, text, until_nul}; -use crate::container::chunk::{AepFile, Chunk}; +use crate::container::chunk::AepFile; +use crate::container::index::{Node, Root, TreeIndex}; /// The name a user gave a property or group: the `Utf8` inside its `tdsn` chunk. `None` for the /// sentinel `-_0_/-` ("no custom name") and for empty names. -pub(crate) fn user_name(chunk: &Chunk<'_>) -> Option { - let name = chunk.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; +pub(crate) fn user_name(chunk: Node<'_, '_>) -> Option { + let name = chunk.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (!name.is_empty() && name != "-_0_/-").then_some(name) } /// The content of a `Utf8` child as text (not trimmed). -pub(crate) fn utf8_child(chunk: &Chunk<'_>) -> Option { - chunk.child(b"Utf8").and_then(Chunk::bytes).map(text) +pub(crate) fn utf8_child(chunk: Node<'_, '_>) -> Option { + chunk.child(b"Utf8").and_then(Node::bytes).map(text) } /// The project: the top-level `LIST Fold`, which holds the project items. #[derive(Debug, Clone, Copy)] pub struct Project<'f, 'a> { - pub(crate) fold: &'f Chunk<'a>, + pub(crate) fold: Node<'f, 'a>, /// The top-level `LIST EfdG` (project-level effect definitions), when there is one. - pub(crate) effect_defs: Option<&'f Chunk<'a>>, + pub(crate) effect_defs: Option>, } impl<'f, 'a> Project<'f, 'a> { /// The project view of a file, or `None` when it has no `LIST Fold`. pub fn new(file: &'f AepFile<'a>) -> Option { - file.list(b"Fold").map(|fold| Self { fold, effect_defs: file.list(b"EfdG") }) + Self::of(Root::from(file)) + } + + /// The project view of an indexed file ([`TreeIndex`]), or `None` when it has no `LIST Fold`. + pub fn from_index(index: &'f TreeIndex<'a>) -> Option { + Self::of(Root::from(index)) + } + + /// The project view of either tree: its `LIST Fold` and `LIST EfdG`. + pub(crate) fn of(root: Root<'f, 'a>) -> Option { + root.list(b"Fold").map(|fold| Self { fold, effect_defs: root.list(b"EfdG") }) } /// The top-level items (the project panel's root; folders hold their children, see /// [`Item::children`]). pub fn items(&self) -> impl Iterator> + 'f { - self.fold.children().iter().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.fold.children().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// True when the `LIST Fold` could not be read as chunks (damaged): its items cannot be seen. @@ -78,12 +93,12 @@ pub enum ItemKind { /// A project item (`LIST Item`). #[derive(Debug, Clone, Copy)] pub struct Item<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Item<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -106,7 +121,7 @@ impl<'f, 'a> Item<'f, 'a> { /// The items inside a folder (`LIST Sfdr`). pub fn children(&self) -> impl Iterator> + 'f { - self.chunk.child_list(b"Sfdr").into_iter().flat_map(|sfdr| sfdr.children().iter()).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.chunk.child_list(b"Sfdr").into_iter().flat_map(Node::children).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// Whether the item has the `LIST Sfdr` of a folder's contents (After Effects writes an empty @@ -117,7 +132,7 @@ impl<'f, 'a> Item<'f, 'a> { /// True when the folder's `LIST Sfdr` is there but could not be read as chunks (damaged). pub fn folder_contents_unreadable(&self) -> bool { - self.chunk.child_list(b"Sfdr").is_some_and(Chunk::is_opaque_list) + self.chunk.child_list(b"Sfdr").is_some_and(Node::is_opaque_list) } /// The composition view when this item has composition data (`cdta`). @@ -158,7 +173,7 @@ impl TimeBase { /// A composition item (an [`Item`] with `cdta`). #[derive(Debug, Clone, Copy)] pub struct Comp<'f, 'a> { - pub(crate) item: &'f Chunk<'a>, + pub(crate) item: Node<'f, 'a>, } impl<'f, 'a> Comp<'f, 'a> { @@ -172,19 +187,19 @@ impl<'f, 'a> Comp<'f, 'a> { /// The layers, top layer first (`LIST Layr`; the `DLay`/`SLay`/`CLay`/`SecL` lists next to /// them are not layers). pub fn layers(&self) -> impl Iterator> + 'f { - self.item.children().iter().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) + self.item.children().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) } } /// A layer (`LIST Layr`). #[derive(Debug, Clone, Copy)] pub struct Layer<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Layer<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -214,12 +229,12 @@ pub enum PropNode<'f, 'a> { /// A property (`LIST tdbs`). Property(Property<'f, 'a>), /// Any other list that follows a match name (observed: `otst`, the effect instance `sspc`). - Other(&'f Chunk<'a>), + Other(Node<'f, 'a>), } impl<'f, 'a> PropNode<'f, 'a> { /// The node a `LIST` after a match name is. - fn of(c: &'f Chunk<'a>) -> Self { + fn of(c: Node<'f, 'a>) -> Self { if c.is_list(b"tdgp") { PropNode::Group(PropGroup { chunk: c }) } else if c.is_list(b"tdbs") { @@ -242,17 +257,18 @@ pub struct PropEntry<'f, 'a> { /// `ADBE Group End` closes the group and names nothing. #[derive(Debug, Clone, Copy)] pub struct PropGroup<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> PropGroup<'f, 'a> { - /// A group view of a `LIST tdgp` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A group view of a `LIST tdgp` chunk (a [`Chunk`](crate::container::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdgp").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -273,7 +289,7 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes().map(match_name); - } else if c.id == crate::container::chunk::LIST + } else if c.is(&crate::container::chunk::LIST) && let Some(name) = pending.take() { out.push(PropEntry { match_name: name, node: PropNode::of(c) }); @@ -294,9 +310,9 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes(); - } else if c.id == crate::container::chunk::LIST + } else if c.is(&crate::container::chunk::LIST) && let Some(body) = pending.take() - && match_name(body) == name + && match_name_is(body, name) { return Some(PropNode::of(c)); } @@ -323,21 +339,30 @@ fn match_name(body: &[u8]) -> String { text(until_nul(body)) } +/// `match_name(body) == name` without building the `String`: equal bytes match; otherwise only +/// a body that is not valid UTF-8 can still match (through its U+FFFD replacements), and only a +/// name that holds U+FFFD. +fn match_name_is(body: &[u8], name: &str) -> bool { + let bytes = until_nul(body); + bytes == name.as_bytes() || (name.contains('\u{FFFD}') && String::from_utf8_lossy(bytes) == name) +} + /// A property (`LIST tdbs`): `tdsb`, `tdsn`, the 124-byte header `tdb4`, then either the static /// value `cdat` or a keyframe `LIST list`. #[derive(Debug, Clone, Copy)] pub struct Property<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Property<'f, 'a> { - /// A property view of a `LIST tdbs` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A property view of a `LIST tdbs` chunk (a [`Chunk`](crate::container::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdbs").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -445,12 +470,13 @@ pub struct KeyLayout { impl<'f> KeyList<'f> { /// A view of a `LIST list` chunk. `None` when `lhd3` is not 52 bytes or `ldat` is not exactly /// `count * record_size` bytes (a list without `ldat` must have no records). - pub fn from_chunk<'a>(chunk: &'f Chunk<'a>) -> Option { + pub fn from_chunk<'a: 'f>(chunk: impl Into>) -> Option { + let chunk = chunk.into(); if !chunk.is_list(b"list") { return None; } let header = ListHeader::parse(chunk.child(b"lhd3")?.bytes()?)?; - let ldat: &[u8] = chunk.child(b"ldat").and_then(Chunk::bytes).unwrap_or(&[]); + let ldat: &[u8] = chunk.child(b"ldat").and_then(Node::bytes).unwrap_or(&[]); let expected = usize::try_from(header.count).ok()?.checked_mul(usize::try_from(header.record_size).ok()?)?; (ldat.len() == expected).then_some(Self { header, ldat, layout: None }) } @@ -509,6 +535,7 @@ impl<'f> Keyframe<'f> { #[cfg(test)] mod tests { use super::*; + use crate::container::Chunk; fn key_list(count: u32, record_size: u32, ldat: &[u8]) -> Chunk<'static> { let mut lhd3 = vec![0u8; 52]; @@ -545,4 +572,22 @@ mod tests { assert!(g.get("C").is_none()); assert_eq!(g.names(), ["A", "B", "A"]); } + + /// Match name is agrees with comparing the decoded name. + #[test] + fn match_name_is_agrees_with_comparing_the_decoded_name() { + let bodies: [&[u8]; 9] = + [b"ADBE Position\0\0\0", b"ADBE Position", b"ADBE Pos", b"", b"\0ADBE", b"ADBE \xff Bad\0", b"\xff", b"\xef\xbf\xbd", "Gr\u{fc}n\0".as_bytes()]; + let names = ["ADBE Position", "ADBE Pos", "", "ADBE \u{FFFD} Bad", "\u{FFFD}", "Gr\u{fc}n", "ADBE Position\0", "ADBE \u{FFFD}"]; + for body in bodies { + for name in names { + assert_eq!(match_name_is(body, name), match_name(body) == name, "{body:?} vs {name:?}"); + } + } + // Names read back from a group find their entries, a damaged one included. + let group = Chunk::list(*b"tdgp", vec![Chunk::data(*b"tdmn", b"X\xffY\0".to_vec()), Chunk::list(*b"tdbs", Vec::new())]); + let g = PropGroup::from_chunk(&group).expect("group"); + let names = g.names(); + assert!(g.get(&names[0]).is_some()); + } } diff --git a/crates/aep/src/container/values.rs b/crates/aep/src/container/values.rs index a690751a2..ddd5caa75 100644 --- a/crates/aep/src/container/values.rs +++ b/crates/aep/src/container/values.rs @@ -8,6 +8,7 @@ //! 100 %. These defaults are verified for AV layers (solids); cameras and lights use other //! conventions (see the `camera` module). +use crate::container::bytes::be_f64; use crate::container::typed::{Layer, Project, PropGroup, PropNode}; /// Where a [`Tracked`] value came from. @@ -192,8 +193,7 @@ impl<'f, 'a> Layer<'f, 'a> { let otda = otst.child_list(b"otky")?.child(b"otda")?.bytes()?; let mut degrees = [0.0; 3]; for (i, slot) in degrees.iter_mut().enumerate() { - let b = otda.get(i * 8..i * 8 + 8)?; - *slot = f64::from_be_bytes(b.try_into().ok()?); + *slot = be_f64(otda, i * 8)?; } Some(Orientation { degrees, @@ -223,12 +223,10 @@ impl<'f, 'a> Layer<'f, 'a> { .child_list(b"otky") .map(|k| { k.children() - .iter() .filter(|c| c.is(b"otda")) .map(|c| { let b = c.bytes()?; - let at = |i: usize| b.get(i * 8..i * 8 + 8).and_then(|s| s.try_into().ok()).map(f64::from_be_bytes); - Some([at(0)?, at(1)?, at(2)?]) + Some([be_f64(b, 0)?, be_f64(b, 8)?, be_f64(b, 16)?]) }) .collect() }) diff --git a/crates/aep/src/container/version.rs b/crates/aep/src/container/version.rs index 1c859cdac..407c7046c 100644 --- a/crates/aep/src/container/version.rs +++ b/crates/aep/src/container/version.rs @@ -10,6 +10,7 @@ use std::fmt; use crate::container::bytes::be_u32; use crate::container::chunk::AepFile; +use crate::container::index::{Root, TreeIndex}; /// An After Effects version as packed in `head` / `svap`: `26.2.1x2` is major 26, minor 2, bug-fix /// 1, build 2 (the form of the scripting API's `app.version`). @@ -71,13 +72,35 @@ impl AepFile<'_> { /// version there, and the older version for File ▸ Save a Copy As 25.x / 24.x. `None` when the /// chunk is missing, short or holds no version. pub fn file_version(&self) -> Option { - AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + Root::from(self).file_version() } /// The version of the After Effects that wrote the file: `svap` bytes 0 to 3 (the same word in a /// down-saved copy as in a normal save). `None` when the chunk is missing (After Effects opens such /// a file), short or holds no version. pub fn saved_by_version(&self) -> Option { + Root::from(self).saved_by_version() + } +} + +impl TreeIndex<'_> { + /// [`AepFile::file_version`] of the indexed file. + pub fn file_version(&self) -> Option { + Root::from(self).file_version() + } + + /// [`AepFile::saved_by_version`] of the indexed file. + pub fn saved_by_version(&self) -> Option { + Root::from(self).saved_by_version() + } +} + +impl Root<'_, '_> { + pub(crate) fn file_version(self) -> Option { + AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + } + + pub(crate) fn saved_by_version(self) -> Option { AeVersion::from_word(be_u32(self.chunk(b"svap")?.bytes()?, 0)?) } } diff --git a/crates/aep/src/container/write.rs b/crates/aep/src/container/write.rs index 3ccab50a4..9f401b3d4 100644 --- a/crates/aep/src/container/write.rs +++ b/crates/aep/src/container/write.rs @@ -17,6 +17,18 @@ const MAX_WRITE_DEPTH: usize = 256; /// [`AepError::TooDeep`] for an absurdly deep hand-built tree. pub fn write(file: &AepFile<'_>) -> Result, AepError> { let mut out = Vec::new(); + // Sized once to the exact output. A tree past the 32-bit container size fails at once (the + // same error the write below would give after serialising it). `try_reserve` so a hand-built + // tree that repeats one large borrowed body many times cannot abort on the allocation; the + // writer then grows as it goes, as before. + if let Some(len) = sequence_len(&file.chunks, 1) { + if u32::try_from(len.saturating_add(4)).is_err() { + return Err(AepError::TooLarge); + } + if let Some(total) = len.checked_add(12).and_then(|n| n.checked_add(file.trailer.len())) { + let _ = out.try_reserve_exact(total); + } + } out.extend_from_slice(b"RIFX"); out.extend_from_slice(&[0; 4]); out.extend_from_slice(b"Egg!"); @@ -56,6 +68,28 @@ fn write_sequence(out: &mut Vec, chunks: &[Chunk<'_>], depth: usize) -> Resu Ok(()) } +/// The number of bytes [`write_sequence`] produces for `chunks`; `None` past the depth limit or +/// on overflow (both make the write fail). +fn sequence_len(chunks: &[Chunk<'_>], depth: usize) -> Option { + if depth > MAX_WRITE_DEPTH { + return None; + } + let count = chunks.len(); + let mut total = 0usize; + for (i, chunk) in chunks.iter().enumerate() { + let body = match &chunk.body { + Body::Data(d) => d.len(), + Body::Chunks(c) => sequence_len(c, depth + 1)?, + Body::List { children, .. } => sequence_len(children, depth + 1)?.checked_add(4)?, + Body::OpaqueList { data, .. } => data.len().checked_add(4)?, + }; + // The same pad rule as `write_chunk`. + let pad = usize::from(body % 2 == 1 && (chunk.pad.is_some() || i + 1 != count)); + total = total.checked_add(8)?.checked_add(body)?.checked_add(pad)?; + } + Some(total) +} + fn write_chunk(out: &mut Vec, chunk: &Chunk<'_>, is_last: bool, depth: usize) -> Result<(), AepError> { out.extend_from_slice(&chunk.id); let size_at = out.len(); @@ -84,3 +118,62 @@ fn write_chunk(out: &mut Vec, chunk: &Chunk<'_>, is_last: bool, depth: usize } Ok(()) } + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + + use super::*; + + /// The output is sized once to its exact length. + #[test] + fn the_output_is_sized_once_to_its_exact_length() { + let odd = |n: u8| Chunk::data(*b"cdat", vec![n; usize::from(n)]); + let mut padded = odd(3); + padded.pad = Some(0x7f); + let trees = [ + AepFile { chunks: Vec::new(), trailer: Cow::Borrowed(&[]) }, + AepFile { + chunks: vec![ + odd(1), + Chunk::list(*b"Fold", vec![odd(2), odd(3), Chunk::list(*b"tdgp", vec![odd(5)])]), + Chunk { id: *b"LIST", body: Body::OpaqueList { list_type: *b"btdk", data: Cow::Owned(vec![1, 2, 3]) }, pad: None }, + Chunk { id: *b"tdmn", body: Body::Chunks(vec![odd(7), padded]), pad: None }, + odd(9), + ], + trailer: Cow::Borrowed(b""), + }, + ]; + for file in &trees { + let out = write(file).expect("write"); + assert_eq!(sequence_len(&file.chunks, 1).map(|n| n + 12 + file.trailer.len()), Some(out.len())); + // Exact with std's allocator today; grown push by push it would be rounded up. + assert_eq!(out.capacity(), out.len(), "the output grew past its size"); + } + } + + /// A tree past the container size fails before serialising. + #[cfg(target_pointer_width = "64")] + #[test] + fn a_tree_past_the_container_size_fails_before_serialising() { + // 4097 chunks borrowing one 1 MiB body: over 4 GiB of output, none of it written. + let body = vec![0u8; 1 << 20]; + let chunks: Vec> = (0..4097).map(|_| Chunk::data(*b"cdat", &body[..])).collect(); + let file = AepFile { chunks, trailer: Cow::Borrowed(&[]) }; + assert_eq!(write(&file), Err(AepError::TooLarge)); + // Just under the limit still sizes, and fails nothing up front. + assert!(sequence_len(&file.chunks[..4000], 1).is_some_and(|n| u32::try_from(n + 4).is_ok())); + } + + /// A tree past the depth limit has no size. + #[test] + fn a_tree_past_the_depth_limit_has_no_size() { + let mut deep = Chunk::data(*b"cdat", vec![0]); + for _ in 0..=MAX_WRITE_DEPTH { + deep = Chunk::list(*b"tdgp", vec![deep]); + } + let file = AepFile { chunks: vec![deep], trailer: Cow::Borrowed(&[]) }; + assert_eq!(sequence_len(&file.chunks, 1), None); + assert_eq!(write(&file), Err(AepError::TooDeep)); + } +} diff --git a/crates/aep/src/export/base.rs b/crates/aep/src/export/base.rs index dfaca7321..05aa15891 100644 --- a/crates/aep/src/export/base.rs +++ b/crates/aep/src/export/base.rs @@ -15,6 +15,7 @@ pub(crate) fn bytes(len: usize, fields: &[(usize, &[u8])]) -> Vec { layout(len, fields).finish(|| format!("base::bytes ({len} bytes, fields at {:?})", fields.iter().map(|(at, v)| (*at, v.len())).collect::>())) } +/// The fixed layout of `len` zero bytes with `fields` written at their offsets. fn layout(len: usize, fields: &[(usize, &[u8])]) -> Fixed { let mut b = Fixed::zeroed(len); for (at, v) in fields { @@ -280,6 +281,7 @@ pub(crate) fn solid_item() -> Chunk<'static> { mod tests { use super::*; + /// A field out of range is an error. #[test] fn a_field_out_of_range_is_an_error() { let ok = layout(4, &[(0, &[1]), (3, &[2])]); diff --git a/crates/aep/src/export/emit.rs b/crates/aep/src/export/emit.rs index f3a4859be..f7b0ec6b8 100644 --- a/crates/aep/src/export/emit.rs +++ b/crates/aep/src/export/emit.rs @@ -6,9 +6,10 @@ use std::collections::HashMap; use crate::{AepFile, Chunk}; use effectcraft_project::{ItemId, LayerId}; +use crate::export::keys::fill_key_list; use crate::export::plan::{CompPlan, Entry, ItemPlan, KeyPlan, LayerPlan, LayerSrc, ProjectPlan, PropKind, PropPlan, SolidPlan, natural_key}; -use crate::export::synth::tdmn_name; -use crate::export::template::Template; +use crate::export::synth::{is_tdmn, tdmn_name}; +use crate::export::template::{self, Template}; use crate::export::tree::{ R, child_data, data_mut, kids_mut, list_mut, put, put_bits, put_cstr, put_f32, put_f64, put_i32, put_u8, put_u16, put_u32, set_utf8, tpl, }; @@ -59,10 +60,14 @@ struct Cx<'t> { type Sorted = (String, u64, Vec>); pub(crate) fn emit(plan: &ProjectPlan, extra: &[u8]) -> R> { - let t = Template::load()?; + emit_with(template::shared()?, plan, extra) +} + +/// [`emit`] with the prototypes of `t`. +fn emit_with(t: &Template, plan: &ProjectPlan, extra: &[u8]) -> R> { let special = plan.root.iter().find(|e| e.name == "Solids" && matches!(e.plan, ItemPlan::Folder(_))).map(|e| e.ec); let mut cx = Cx { - t: &t, + t, ids: Ids { next: FIRST_ID }, solid_ids: HashMap::new(), comp_ids: HashMap::new(), @@ -78,16 +83,14 @@ pub(crate) fn emit(plan: &ProjectPlan, extra: &[u8]) -> R> { let mut root: Vec = Vec::new(); build_level(&plan.root, &mut cx, &ids_by_entry, &mut |name, key, chunks| root.push((name, key, chunks)))?; - // The Solids folder: root-level solids and the hidden solids of null layers. - let mut solids_children: Vec = Vec::new(); - for (name, key, chunks) in root.iter().filter(|(_, k, _)| *k & ROOT_SOLID != 0) { - solids_children.push((name.clone(), *key, chunks.clone())); - } - root.retain(|(_, k, c)| *k & ROOT_SOLID == 0 && !c.is_empty()); - for (i, (name, chunk)) in cx.extra_solids.iter().enumerate() { - let mut v = vec![chunk.clone()]; + // The Solids folder: root-level solids and the hidden solids of null layers (moved, not + // copied: the root keeps neither). + let (mut solids_children, mut root): (Vec, Vec) = root.into_iter().partition(|(_, k, _)| *k & ROOT_SOLID != 0); + root.retain(|(_, _, c)| !c.is_empty()); + for (i, (name, chunk)) in std::mem::take(&mut cx.extra_solids).into_iter().enumerate() { + let mut v = vec![chunk]; v.extend(cx.t.fset.iter().cloned()); - solids_children.push((name.clone(), i as u64, v)); + solids_children.push((name, i as u64, v)); } let mut folder = match cx.special_slot.take() { Some((item, inner)) => { @@ -126,26 +129,57 @@ pub(crate) fn emit(plan: &ProjectPlan, extra: &[u8]) -> R> { /// Marks a root entry that goes into the `Solids` folder. const ROOT_SOLID: u64 = 1 << 63; +/// Sorts by name in natural order, then by the tie-break key. Each name's sort key is built once, +/// not on every comparison; the sort is stable, as `sort_by` is. fn sort_named(v: &mut [Sorted]) { - v.sort_by(|a, b| natural_cmp(&a.0, &b.0).then(a.1.cmp(&b.1))); + v.sort_by_cached_key(|(name, key, _)| (natural_bytes(name), *key)); } -fn natural_cmp(a: &str, b: &str) -> std::cmp::Ordering { - let (ka, kb) = (natural_key(a), natural_key(b)); - for (x, y) in ka.iter().zip(kb.iter()) { - let c = match (x.0, y.0) { - (true, true) => x.2.cmp(&y.2).then_with(|| x.1.cmp(&y.1)), - (false, false) => x.1.cmp(&y.1), - (true, false) => std::cmp::Ordering::Less, - (false, true) => std::cmp::Ordering::Greater, - }; - if c != std::cmp::Ordering::Equal { - return c; +/// The [`natural_key`] of `name` as one byte string whose plain byte order is the natural order: +/// segment by segment, a number before text, numbers by value and then by their digit strings +/// byte by byte (`01` before `1`), text by its lowercase string, and a key that is a prefix of +/// the other first. +/// +/// - A number: `0x01`, the count of significant bytes of its value, those bytes big-endian, its +/// digits, `0x00` (digits are never `0x00`, so a digit string that is a prefix of another, as +/// `0` of `00`, sorts first). +/// - Text: `0x02`, its lowercase UTF-8 with each `0x00` written as `0x00 0x01`, then `0x00 0x00`. +/// +/// At most 5 bytes per byte of `name` (`NATURAL_BYTES_PER_NAME_BYTE`), so the cached keys stay +/// within a small multiple of the names they sort (one segment as a `(bool, String, u128)` takes +/// 48 bytes plus its string, which is why the segments themselves are not cached). +fn natural_bytes(name: &str) -> Vec { + let mut out = Vec::with_capacity(name.len().saturating_add(4)); + for (number, text, value) in natural_key(name) { + if number { + out.push(0x01); + let be = value.to_be_bytes(); + let zeros = be.iter().take_while(|b| **b == 0).count(); + let significant = be.get(zeros..).unwrap_or_default(); + out.push(significant.len() as u8); + out.extend_from_slice(significant); + out.extend_from_slice(text.as_bytes()); + out.push(0x00); + } else { + out.push(0x02); + for b in text.bytes() { + out.push(b); + if b == 0 { + out.push(0x01); + } + } + out.extend_from_slice(&[0x00, 0x00]); } } - ka.len().cmp(&kb.len()) + out } +/// The bound on [`natural_bytes`]: a run of `d` digits takes at most `3 + 2d` bytes (its value +/// never needs more bytes than digits), a text run of `b` bytes at most `3 + 2b` (lowercasing and +/// the NUL escape at most double a character), and every run is at least one byte long. +#[cfg(test)] +const NATURAL_BYTES_PER_NAME_BYTE: usize = 5; + fn assign_ids(entries: &[Entry], cx: &mut Cx<'_>, map: &mut HashMap) -> R<()> { // Iterative to bound the stack by the folder depth the plan already limits. let mut stack: Vec<&Entry> = entries.iter().rev().collect(); @@ -405,8 +439,7 @@ fn put_used_comps(cx: &Cx<'_>, item: &mut Chunk<'static>, cp: &CompPlan) -> R<() fn put_time_remap_max(layr: &mut Chunk<'static>, seconds: f64) -> R<()> { let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = - kids.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Time Remapping")).ok_or_else(|| tpl("layer without Time Remapping"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Time Remapping")).ok_or_else(|| tpl("layer without Time Remapping"))?; let tdbs = kids.get_mut(at + 1).ok_or_else(|| tpl("Time Remapping without a record"))?; put_f64(child_data(tdbs, b"tduM")?, 0, seconds) } @@ -421,8 +454,7 @@ fn put_time_remap_max(layr: &mut Chunk<'static>, seconds: f64) -> R<()> { fn put_time_remap(cx: &Cx<'_>, layr: &mut Chunk<'static>, remap: &PropPlan) -> R<()> { let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = - kids.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Time Remapping")).ok_or_else(|| tpl("layer without Time Remapping"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Time Remapping")).ok_or_else(|| tpl("layer without Time Remapping"))?; let tdbs = kids.get_mut(at + 1).ok_or_else(|| tpl("Time Remapping without a record"))?; put_u32(child_data(tdbs, b"tdsb")?, 0, 1)?; { @@ -455,21 +487,12 @@ fn put_time_remap(cx: &Cx<'_>, layr: &mut Chunk<'static>, remap: &PropPlan) -> R .find(|k| matches!(&k.body, crate::Body::List { list_type, .. } if list_type == b"list")) .cloned() .ok_or_else(|| tpl("keyed prototype without a list"))?; - let n = u32::try_from(remap.keys.len()).map_err(|_| tpl("too many keys"))?; let size = 48usize; - { - let lhd3 = child_data(&mut list, b"lhd3")?; - crate::export::synth::put_key_counts(lhd3, n, 48)?; - } let mut ldat = Vec::with_capacity(size * remap.keys.len()); for k in &remap.keys { ldat.extend(plain_record(1, k, size)?); } - let ld = kids_mut(&mut list)?; - match ld.iter_mut().find(|c| c.is(b"ldat")) { - Some(c) => *c = Chunk::data(*b"ldat", ldat), - None => ld.push(Chunk::data(*b"ldat", ldat)), - } + fill_key_list(&mut list, ldat, remap.keys.len(), size)?; if let Some(slot) = rk.get_mut(cdat_at) { *slot = list; } @@ -617,10 +640,7 @@ fn put_masks(cx: &Cx<'_>, layr: &mut Chunk<'static>, lp: &LayerPlan) -> R<()> { let parade = crate::export::masks::build_parade(&cx.t.masks, &lp.masks, lp.source_size)?; let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = kids - .iter() - .position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Transform Group")) - .ok_or_else(|| tpl("layer without a Transform group"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Transform Group")).ok_or_else(|| tpl("layer without a Transform group"))?; kids.splice(at..at, parade); Ok(()) } @@ -634,10 +654,7 @@ fn put_effects(cx: &mut Cx<'_>, layr: &mut Chunk<'static>, lp: &LayerPlan, layer let parade = crate::export::effects::build_parade(&cx.t.effects, &lp.fx, layer_id, layer_ids, lp.source_size, &mut cx.fx_use)?; let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = kids - .iter() - .position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Transform Group")) - .ok_or_else(|| tpl("layer without a Transform group"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Transform Group")).ok_or_else(|| tpl("layer without a Transform group"))?; kids.splice(at..at, parade); Ok(()) } @@ -667,20 +684,19 @@ fn put_properties(cx: &Cx<'_>, layr: &mut Chunk<'static>, lp: &LayerPlan, cp: &C } let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = kids - .iter() - .position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Transform Group")) - .ok_or_else(|| tpl("layer without a Transform group"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Transform Group")).ok_or_else(|| tpl("layer without a Transform group"))?; let group = kids.get_mut(at + 1).ok_or_else(|| tpl("Transform group without a list"))?; let gk = kids_mut(group)?; // Split the group into its header (up to the first match name) and (tdmn, node) pairs. let first = gk.iter().position(|c| c.is(b"tdmn")).ok_or_else(|| tpl("empty Transform group"))?; let header: Vec> = gk.drain(..first).collect(); - let mut pairs: Vec<(String, Vec>)> = Vec::new(); + let mut pairs: Vec<(Vec, Vec>)> = Vec::new(); let mut it = std::mem::take(gk).into_iter(); while let Some(m) = it.next() { - let name = tdmn_name(&m).unwrap_or_default(); - if name == "ADBE Group End" { + // The match name as the shared decoder reads it (lossy UTF-8 up to the first NUL). Every + // name looked up below is ASCII, so a name matches exactly when its bytes do. + let name = tdmn_name(&m).unwrap_or_default().into_bytes(); + if name == b"ADBE Group End" { pairs.push((name, vec![m])); continue; } @@ -690,22 +706,19 @@ fn put_properties(cx: &Cx<'_>, layr: &mut Chunk<'static>, lp: &LayerPlan, cp: &C for p in &lp.props { let name = p.kind.match_name(); let built = build_prop(cx, p, lp, cp)?; - let tdmn = pairs.iter().find(|(n, _)| n == name).and_then(|(_, v)| v.first().cloned()).unwrap_or_else(|| { - cx.t.stat - .iter() - .chain(cx.t.keyd.iter()) - .find(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some(name)) - .cloned() - .unwrap_or_else(|| Chunk::data(*b"tdmn", Vec::new())) + let tdmn = pairs.iter().find(|(n, _)| n == name.as_bytes()).and_then(|(_, v)| v.first().cloned()).unwrap_or_else(|| { + cx.t.stat.iter().chain(cx.t.keyd.iter()).find(|c| is_tdmn(c, name)).cloned().unwrap_or_else(|| Chunk::data(*b"tdmn", Vec::new())) }); let entry = vec![tdmn, built]; - if let Some(slot) = pairs.iter_mut().find(|(n, _)| n == name) { + if let Some(slot) = pairs.iter_mut().find(|(n, _)| n == name.as_bytes()) { slot.1 = entry; } else { - pairs.push((name.to_string(), entry)); + pairs.push((name.as_bytes().to_vec(), entry)); } } - let rank = |n: &str| if n == "ADBE Group End" { usize::MAX } else { GROUP_ORDER.iter().position(|g| *g == n).unwrap_or(GROUP_ORDER.len()) }; + let rank = |n: &[u8]| { + if n == b"ADBE Group End" { usize::MAX } else { GROUP_ORDER.iter().position(|g| g.as_bytes() == n).unwrap_or(GROUP_ORDER.len()) } + }; pairs.sort_by_key(|(n, _)| rank(n)); gk.extend(header); gk.extend(pairs.into_iter().flat_map(|(_, v)| v)); @@ -739,22 +752,12 @@ fn build_prop(cx: &Cx<'_>, p: &PropPlan, lp: &LayerPlan, cp: &CompPlan) -> R *c = Chunk::data(*b"ldat", ldat), - None => ld.push(Chunk::data(*b"ldat", ldat)), - } + fill_key_list(list_mut(&mut tdbs, b"list")?, ldat, p.keys.len(), size)?; } if let Some((text, _)) = &p.expr { let kids = kids_mut(&mut tdbs)?; @@ -814,3 +817,283 @@ pub(crate) fn plain_record(n: usize, k: &KeyPlan, size: usize) -> R> { } Ok(r) } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + use effectcraft_color::Label; + use effectcraft_keyframe::{Keyframe, Value}; + use effectcraft_project::{Comp, Expression, ItemKind, LayerSource, Project, Property, Solid, build}; + use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; + + /// Seconds as ticks. + fn secs(s: f64) -> Tick { + Tick((s * TICKS_PER_SECOND as f64).round() as i64) + } + + /// Adds a layer to `comp`, changed by `edit` first. + fn add_layer(p: &mut Project, comp: ItemId, name: &str, source: LayerSource, edit: impl FnOnce(&mut effectcraft_project::Layer)) { + let mut c = p.comp(comp).cloned().expect("comp"); + let mut l = build::layer(p, &c, name, source, (640, 360), None); + l.name = name.to_string(); + edit(&mut l); + c.layers.insert(0, l); + if let Some(i) = p.item_mut(comp) { + i.kind = ItemKind::Comp(Arc::new(c)); + } + } + + /// Gives the property two keys (0 s and 1.5 s) holding its value. + fn two_keys(prop: Option<&mut Property>) { + let prop = prop.expect("property"); + let v = prop.value.clone(); + prop.keys = vec![Keyframe::new(secs(0.0), v.clone()), Keyframe::new(secs(1.5), v)]; + } + + /// A project that reaches every part of `emit`: folders (one named `Solids`), root and nested + /// solids with names that sort naturally, a precomposition, null layers (hidden solids), keyed + /// Transform properties, an expression and a keyed time remap. + fn varied(tag: &str) -> Project { + let mut p = Project::default(); + let solid = |w| ItemKind::Solid(Solid { color: [0.2, 0.4, 0.6], width: w, height: 360, pixel_aspect: 1.0 }); + let solids = p.add_item("Solids", Label::None, None, ItemKind::Folder); + let nested = p.add_item("Folder 10", Label::None, None, ItemKind::Folder); + p.add_item("folder 9", Label::None, Some(nested), ItemKind::Folder); + let mut ids = Vec::new(); + for name in ["Solid 10", "Solid 2", "solid 2", "Solid 02", "Solid", "Ölung 1", ""] { + ids.push(p.add_item(&format!("{name}{tag}"), Label::Red, None, solid(640))); + } + p.add_item("In Solids 3", Label::None, Some(solids), solid(320)); + p.add_item("In folder", Label::None, Some(nested), solid(100)); + let pre = p.add_item("Pre 1", Label::None, Some(nested), ItemKind::Comp(Arc::new(Comp::new(640, 360, FrameRate::new(25, 1), secs(3.0))))); + add_layer(&mut p, pre, "S", LayerSource::Solid { item: ids[0] }, |_| {}); + let main = p.add_item("Main", Label::None, None, ItemKind::Comp(Arc::new(Comp::new(1920, 1080, FrameRate::new(30, 1), secs(5.0))))); + add_layer(&mut p, main, "Null A", LayerSource::Null, |_| {}); + add_layer(&mut p, main, "Null B", LayerSource::Null, |l| two_keys(l.props.prop_mut("transform/position"))); + add_layer(&mut p, main, "Solid layer", LayerSource::Solid { item: ids[1] }, |l| { + two_keys(l.props.prop_mut("transform/opacity")); + two_keys(l.props.prop_mut("transform/anchor")); + if let Some(r) = l.props.prop_mut("transform/rotation") { + r.expr = Some(Expression { text: "time * 10".into(), enabled: true }); + } + }); + add_layer(&mut p, main, "Precomp", LayerSource::Comp { item: pre }, |l| { + let mut r = Property::new(9_000, "timeRemap", "Time Remap", Value::Scalar(0.0)); + r.keys = vec![Keyframe::new(secs(0.0), Value::Scalar(3.0)), Keyframe::new(secs(3.0), Value::Scalar(0.0))]; + l.props.children.insert(0, r.into()); + }); + p + } + + /// The export plan of `p`. + fn plan_of(p: &Project) -> ProjectPlan { + let mut col = crate::export::report::Collector::default(); + crate::export::plan::plan(p, &mut col) + } + + /// The template is built once and every later export uses the same one. + #[test] + fn the_template_is_built_once() { + let a = template::shared().expect("template"); + let b = template::shared().expect("template"); + assert!(std::ptr::eq(a, b)); + } + + /// Exports with the shared template give the bytes a freshly built template gives, also after + /// other exports have used it (nothing an export does reaches the template). + #[test] + fn the_shared_template_exports_what_a_fresh_one_does() { + let (a, b) = (plan_of(&varied("")), plan_of(&varied(" x"))); + let fresh = |plan: &ProjectPlan| emit_with(&Template::load().expect("template"), plan, b"trailer").expect("emit"); + let (want_a, want_b) = (fresh(&a), fresh(&b)); + assert!(want_a != want_b); + // The project exports without losses and the remap is written. + let r = crate::export::export(&varied(""), &crate::export::ExportOptions { allow_losses: false }).expect("lossless"); + assert_eq!(r.report.exported.time_remaps, 1); + // The null layers' hidden solids and the root solids went into the `Solids` folder. + let file = crate::parse(&want_a).expect("parse"); + let folder = file.walk().map(|(_, c)| c).filter(|c| c.is_list(b"Sfdr")).map(|c| c.children().iter().filter(|k| k.is_list(b"Item")).count()).max(); + assert_eq!(folder, Some(10)); + for _ in 0..3 { + assert!(emit(&a, b"trailer").expect("emit") == want_a); + assert!(emit(&b, b"trailer").expect("emit") == want_b); + } + } + + /// Exports running at the same time on several threads all give the single-threaded bytes. + /// This checks concurrent exports from a template that is already built: other tests in this + /// binary usually fill the shared template first, so the race on the first build is not + /// reliably exercised here (the `static OnceLock` makes that race safe by construction). + #[test] + fn concurrent_exports_give_the_same_bytes() { + let plans: Vec = ["", " x", " y", " z"].iter().map(|t| plan_of(&varied(t))).collect(); + let want: Vec> = plans.iter().map(|plan| emit_with(&Template::load().expect("template"), plan, &[]).expect("emit")).collect(); + let got: Vec>> = std::thread::scope(|s| { + let handles: Vec<_> = (0..8) + .map(|i| { + let plans = &plans; + s.spawn(move || (0..4).map(|k| emit(&plans[(i + k) % plans.len()], &[]).expect("emit")).collect::>()) + }) + .collect(); + handles.into_iter().map(|h| h.join().expect("thread")).collect() + }); + for (i, bytes) in got.iter().enumerate() { + for (k, b) in bytes.iter().enumerate() { + assert!(b == &want[(i + k) % want.len()], "thread {i}, export {k}"); + } + } + } + + /// The comparator `sort_named` used before its keys were cached: `natural_key` built on every + /// comparison. + fn old_sort_named(v: &mut [Sorted]) { + /// The natural name order the cached sort keys replaced, as the reference. + fn natural_cmp(a: &str, b: &str) -> std::cmp::Ordering { + let (ka, kb) = (natural_key(a), natural_key(b)); + for (x, y) in ka.iter().zip(kb.iter()) { + let c = match (x.0, y.0) { + (true, true) => x.2.cmp(&y.2).then_with(|| x.1.cmp(&y.1)), + (false, false) => x.1.cmp(&y.1), + (true, false) => std::cmp::Ordering::Less, + (false, true) => std::cmp::Ordering::Greater, + }; + if c != std::cmp::Ordering::Equal { + return c; + } + } + ka.len().cmp(&kb.len()) + } + v.sort_by(|a, b| natural_cmp(&a.0, &b.0).then(a.1.cmp(&b.1))); + } + + /// The cached-key sort orders names exactly as the comparator it replaced, and keeps entries + /// with the same name and key in their order (each carries its position as a marker chunk). + #[test] + fn sort_named_orders_as_the_old_comparator() { + let parts = [ + "", + "0", + "00", + "1", + "01", + "2", + "10", + "a", + "A", + "b", + " ", + "-", + ".", + "é", + "É", + "ß", + "İ", + "\u{0}", + "\u{1}", + "a\u{0}", + "Ⱥ", + "K", + "ẞ", + "Solid ", + "solid ", + "x9", + "9x", + "340282366920938463463374607431768211455", + "340282366920938463463374607431768211456", + "999999999999999999999999999999999999999999", + ]; + // A small deterministic generator (splitmix64). + let mut state = 0x2545_f491_4f6c_dd1d_u64; + let mut next = move || { + state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + (z ^ (z >> 31)) as usize + }; + for round in 0..2000 { + let n = 1 + next() % 40; + let v: Vec = (0..n) + .map(|i| { + let name: String = (0..next() % 4).map(|_| parts[next() % parts.len()]).collect(); + let key = (next() % 3) as u64 | if next() % 2 == 0 { ROOT_SOLID } else { 0 }; + (name, key, vec![Chunk::data(*b"mark", (i as u32).to_le_bytes().to_vec())]) + }) + .collect(); + let (mut old, mut new) = (v.clone(), v); + old_sort_named(&mut old); + sort_named(&mut new); + assert_eq!(old, new, "round {round}"); + } + } + + /// The cached sort keys stay within [`NATURAL_BYTES_PER_NAME_BYTE`] bytes per name byte, also + /// for the worst name (255 bytes of alternating one-character text and digit runs: 255 + /// segments, which as `(bool, String, u128)` segments took about 14 KB), and the per-character + /// premise of the bound holds for every character. + #[test] + fn sort_keys_are_bounded_by_the_name() { + for c in (0..=0x10_ffff_u32).filter_map(char::from_u32) { + let lower: usize = c.to_lowercase().map(char::len_utf8).sum::() + usize::from(c == '\0'); + assert!(lower <= 2 * c.len_utf8(), "{c:?}"); + } + let worst: String = (0..255).map(|i| if i % 2 == 0 { 'a' } else { '1' }).collect(); + let others = ["", "1", "a", "\u{0}", "İ", "Ⱥ1", "99999999999999999999999999999999999999999999", &"\u{0}".repeat(255), &"Ⱥ".repeat(127)]; + for name in std::iter::once(worst.as_str()).chain(others) { + let k = natural_bytes(name); + assert!(k.len() <= NATURAL_BYTES_PER_NAME_BYTE * name.len(), "{name:?}: {} bytes", k.len()); + } + assert_eq!(natural_key(&worst).len(), 255); + assert!(natural_bytes(&worst).len() <= 1_275); + } + + /// The shared `is_tdmn` / `tdmn_name` the emitter uses agree with the Latin-1 decoding it used + /// before for every name it looks up, which holds because those names are ASCII. + #[test] + fn match_names_compare_as_the_latin1_decoding_did() { + /// The Latin-1 decoding the emitter used before. + fn old_tdmn_name(c: &Chunk<'_>) -> Option { + let d = c.bytes()?; + Some(d.iter().take_while(|b| **b != 0).map(|b| char::from(*b)).collect()) + } + let mut wanted: Vec<&str> = GROUP_ORDER.to_vec(); + wanted.extend(PropKind::ALL.iter().map(|k| k.match_name())); + wanted.extend(["ADBE Time Remapping", "ADBE Transform Group", "ADBE Group End"]); + assert!(wanted.iter().all(|n| n.is_ascii())); + let mut chunks: Vec> = wanted.iter().map(|n| Chunk::data(*b"tdmn", format!("{n}\0\0junk").into_bytes())).collect(); + chunks.extend([ + Chunk::data(*b"tdmn", Vec::new()), + Chunk::data(*b"tdmn", b"ADBE Position".to_vec()), + Chunk::data(*b"tdmn", b"ADBE Position_0\0".to_vec()), + Chunk::data(*b"tdmn", b"ADBE Opacit\xe9\0".to_vec()), + Chunk::data(*b"tdmn", b"\xc3\x84DBE Scale\0".to_vec()), + Chunk::data(*b"tdmn", b"\0ADBE Scale".to_vec()), + Chunk::list(*b"tdmn", Vec::new()), + ]); + for c in &chunks { + for n in &wanted { + assert_eq!(is_tdmn(c, n), old_tdmn_name(c).as_deref() == Some(*n), "{c:?} {n}"); + let key = tdmn_name(c).unwrap_or_default().into_bytes(); + assert_eq!(key == n.as_bytes(), old_tdmn_name(c).as_deref() == Some(*n), "{c:?} {n}"); + } + } + } + + /// `fill_key_list`, which the time remap and the Transform keys use, writes the counts and + /// replaces the records, or adds them when the list has none. + #[test] + fn fill_key_list_writes_counts_and_records() { + let lhd3 = || Chunk::data(*b"lhd3", vec![0; 52]); + let mut want_lhd3 = vec![0; 52]; + crate::export::synth::put_key_counts(&mut want_lhd3, 3, 48).expect("counts"); + for start in [vec![lhd3(), Chunk::data(*b"ldat", vec![9; 4])], vec![lhd3()]] { + let mut list = Chunk::list(*b"list", start); + fill_key_list(&mut list, vec![7; 144], 3, 48).expect("set"); + assert_eq!(list.children(), &[Chunk::data(*b"lhd3", want_lhd3.clone()), Chunk::data(*b"ldat", vec![7; 144])]); + } + let mut no_header = Chunk::list(*b"list", Vec::new()); + assert!(fill_key_list(&mut no_header, Vec::new(), 0, 48).is_err()); + } +} diff --git a/crates/aep/src/export/fxsynth.rs b/crates/aep/src/export/fxsynth.rs index 8b9eff5dc..5824e17af 100644 --- a/crates/aep/src/export/fxsynth.rs +++ b/crates/aep/src/export/fxsynth.rs @@ -61,6 +61,7 @@ pub(crate) fn pard(p: &PDef, popup_items: Option) -> Vec { pard_layout(p, popup_items).finish(|| format!("pard (type {})", p.code)) } +/// The 148-byte `pard` layout of a parameter definition: type, name and defaults. fn pard_layout(p: &PDef, popup_items: Option) -> Fixed { let mut b = Fixed::zeroed(148); b.put(12, &u32::from(p.code).to_be_bytes()); diff --git a/crates/aep/src/export/keys.rs b/crates/aep/src/export/keys.rs index b3cd1158c..a19187418 100644 --- a/crates/aep/src/export/keys.rs +++ b/crates/aep/src/export/keys.rs @@ -90,6 +90,7 @@ mod tests { use super::*; use crate::export::synth::key_list; + /// Keys are timed sorted merged and reported. #[test] fn keys_are_timed_sorted_merged_and_reported() { use effectcraft_keyframe::Value; @@ -122,6 +123,7 @@ mod tests { assert!(col.finish().losses.is_empty()); } + /// A key list is filled and replaces the value. #[test] fn a_key_list_is_filled_and_replaces_the_value() { let mut list = Chunk::list(*b"list", vec![Chunk::data(*b"lhd3", vec![0; 52])]); @@ -142,6 +144,7 @@ mod tests { assert!(set_key_list(&mut Chunk::list(*b"tdbs", Vec::new()), Vec::new(), 0, 48).is_err()); } + /// A spatial record holds the point layout. #[test] fn a_spatial_record_holds_the_point_layout() { let k = KeyPlan { diff --git a/crates/aep/src/export/masks.rs b/crates/aep/src/export/masks.rs index b46811500..9b235eb28 100644 --- a/crates/aep/src/export/masks.rs +++ b/crates/aep/src/export/masks.rs @@ -368,6 +368,7 @@ pub(crate) fn shph() -> Vec { shph_layout().finish(|| "shph".to_string()) } +/// The 24-byte `shph` layout of a closed path with a unit box. fn shph_layout() -> Fixed { let mut b = Fixed::zeroed(24); b.put(0, &[0xb3, 0xde, 0x02, 0x01]); @@ -634,6 +635,7 @@ mod tests { use super::*; use crate::export::synth::is_tdmn; + /// A closed three-point mask. fn mask() -> MaskPlan { let shape = ShapePlan { bbox: [0.0, 0.0, 1.0, 1.0], open: false, points: vec![[0.0, 0.0]; 3] }; MaskPlan { @@ -649,6 +651,7 @@ mod tests { } } + /// An atom without its path is an error. #[test] fn an_atom_without_its_path_is_an_error() { let mut t = MaskProtos::build(); diff --git a/crates/aep/src/export/report.rs b/crates/aep/src/export/report.rs index 309f564e8..b6094e791 100644 --- a/crates/aep/src/export/report.rs +++ b/crates/aep/src/export/report.rs @@ -262,6 +262,7 @@ impl Collector { mod tests { use super::*; + /// A report with losses in two categories, as the collector finishes it. fn collected() -> ExportReport { let mut col = Collector::default(); col.exported.layers = 2; @@ -286,6 +287,7 @@ mod tests { assert_eq!(back, collected()); } + /// The queries read the losses. #[test] fn the_queries_read_the_losses() { let report = collected(); @@ -296,6 +298,7 @@ mod tests { assert_eq!((empty.total_losses(), empty.is_lossless(), empty.summary()), (0, true, String::new())); } + /// Details stop at the cap. #[test] fn details_stop_at_the_cap() { let mut col = Collector::default(); diff --git a/crates/aep/src/export/synth.rs b/crates/aep/src/export/synth.rs index 80678baa3..8af515f24 100644 --- a/crates/aep/src/export/synth.rs +++ b/crates/aep/src/export/synth.rs @@ -126,6 +126,7 @@ pub(crate) fn tdb4(spec: &RecSpec, keyed: bool, time_scale: u32) -> Vec { tdb4_layout(spec, keyed, time_scale).finish(|| "tdb4".to_string()) } +/// The 124-byte `tdb4` property header layout for a record spec. fn tdb4_layout(spec: &RecSpec, keyed: bool, time_scale: u32) -> Fixed { let mut b = Fixed::zeroed(124); b.put(0, &[0xdb, 0x99, 0x00, spec.dims]); @@ -161,6 +162,7 @@ pub(crate) fn key_list(size: u32, count: u32, records: Vec) -> Chunk<'static Chunk::list(*b"list", vec![Chunk::data(*b"lhd3", lhd3), Chunk::data(*b"ldat", records)]) } +/// The 52-byte `lhd3` key list header for `count` records of `size` bytes. fn key_list_header(size: u32, count: u32) -> Fixed { let mut lhd3 = Fixed::zeroed(52); lhd3.put(0, &0x00d0_0bee_u32.to_be_bytes()); @@ -229,6 +231,7 @@ mod tests { assert_eq!(rec.child(b"cdat").and_then(|c| c.bytes()).map(<[u8]>::len), Some(40)); } + /// Every record header fits. #[test] fn every_record_header_fits() { for dims in 0..=u8::MAX { @@ -246,6 +249,7 @@ mod tests { } } + /// A match name reads as the importer reads it. #[test] fn a_match_name_reads_as_the_importer_reads_it() { let m = tdmn("ADBE Group End"); diff --git a/crates/aep/src/export/template.rs b/crates/aep/src/export/template.rs index 91daba636..f26be4c84 100644 --- a/crates/aep/src/export/template.rs +++ b/crates/aep/src/export/template.rs @@ -81,6 +81,24 @@ fn transform_records(keyed: bool) -> Vec> { .collect() } +/// The template every export clones from: built by the first export of the process and kept, as +/// [`Template::load`] always builds the same prototypes and nothing changes them afterwards (the +/// exporter only reads them through `&Template` and clones what it edits). +/// +/// Concurrent exports are safe: the `static` requires `Template: Send + Sync`, which the compiler +/// checks. Exports that race on the first call each build a template; one is kept and the others +/// are dropped, and all of them are equal. A failed build is not kept, so the next export tries +/// again and reports the same error. +pub(crate) fn shared() -> R<&'static Template> { + static SHARED: std::sync::OnceLock