diff --git a/crates/aep/src/container.rs b/crates/aep/src/container.rs index b6c3b4490..b8a6d1957 100644 --- a/crates/aep/src/container.rs +++ b/crates/aep/src/container.rs @@ -23,6 +23,7 @@ mod effects; mod error; mod footage; mod header; +pub mod index; mod keys; mod masks; mod matte; @@ -44,6 +45,7 @@ pub use effects::{DefinitionSource, Effect, EffectDefinitions, EffectParam, Laye pub use error::AepError; pub use footage::{Footage, SequenceInfo}; pub use header::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Quality, Rational, Renderer}; +pub use index::{Node, NodeWalk, Nodes, TreeIndex}; pub use keys::{ExpressionFlags, Interpolation, KeyEase, KeyframeKind, SpatialKey}; pub use masks::{Mask, MaskAtom, MaskMode, PathVertex, ShapePath}; pub use matte::{MatteMode, TrackMatte}; diff --git a/crates/aep/src/container/arbitrary.rs b/crates/aep/src/container/arbitrary.rs index 5762da111..c3322ebdd 100644 --- a/crates/aep/src/container/arbitrary.rs +++ b/crates/aep/src/container/arbitrary.rs @@ -7,9 +7,9 @@ //! Hue/Saturation (444-446) or a Levels value (512-516) changes only bytes of this `aRbp` (and stale //! UI bytes of `pard`); the visible parameters of the same names keep no record. -use crate::container::Chunk; use crate::container::bytes::{be_f32, be_i32, be_u16, be_u32}; use crate::container::effects::Effect; +use crate::container::index::Node; impl<'f, 'a> Effect<'f, 'a> { /// The arbitrary data stored for the parameter with this match name (`-NNNN`): the @@ -17,13 +17,13 @@ impl<'f, 'a> Effect<'f, 'a> { /// none. pub fn arbitrary_data(&self, match_name: &str) -> Option<&'f [u8]> { let values = self.chunk().child_list(b"tdgp")?; - let kids = values.children(); - let at = kids.iter().position(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; - // The value record, then the data list (no match name of its own). - let rest = kids.get(at.checked_add(1)?..)?; - let mut it = rest.iter().take(3); + let mut kids = values.children(); + // Up to the parameter's match name, then the value record, then the data list (no match + // name of its own). + kids.find(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; + let mut it = kids.take(3); let _record = it.next().filter(|c| c.is_list(b"tdbs"))?; - it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Chunk::bytes) + it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Node::bytes) } } diff --git a/crates/aep/src/container/effects.rs b/crates/aep/src/container/effects.rs index 2fa77d017..09a5b1ed6 100644 --- a/crates/aep/src/container/effects.rs +++ b/crates/aep/src/container/effects.rs @@ -20,8 +20,8 @@ use std::collections::{HashMap, HashSet}; -use crate::container::Chunk; use crate::container::bytes::{array, be_f32, be_f64, be_i32, be_u16, be_u32, cstr, text, until_nul}; +use crate::container::index::Node; use crate::container::typed::{KeyList, Layer, Project, PropGroup, PropNode, Property}; /// The parameter type, the `u32` at byte 12 of `pard`. The numbering matches the plug-in @@ -132,7 +132,7 @@ pub enum LayerRef { #[derive(Debug, Clone)] pub struct Effect<'f, 'a> { match_name: String, - chunk: &'f Chunk<'a>, + chunk: Node<'f, 'a>, } /// Where the parameter definitions of an effect instance come from. @@ -178,13 +178,13 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The underlying `LIST sspc` chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } /// The instance name shown in the effect controls (`fnam`). pub fn name(&self) -> Option { - self.chunk.child(b"fnam")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text) + self.chunk.child(b"fnam")?.child(b"Utf8")?.bytes().map(text) } /// The name the user gave the instance: the `tdsn` of the instance's value group when it is @@ -192,7 +192,7 @@ impl<'f, 'a> Effect<'f, 'a> { /// `None` while the instance shows the effect's name. pub fn user_name(&self) -> Option { let values = self.chunk.child_list(b"tdgp")?; - let name = values.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; + let name = values.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (name != "-_0_/-" && !name.is_empty()).then_some(name) } @@ -250,7 +250,7 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The parameters described by `part` (a `LIST parT`) with the values of this instance. - fn params_from(&self, part: &'f Chunk<'a>, borrowed: bool) -> Vec> { + fn params_from(&self, part: Node<'f, 'a>, borrowed: bool) -> Vec> { let mut out: Vec> = Vec::new(); // The stored values by match name: the first entry of a name, as `PropGroup::get` finds it // (a group of that name means no stored value). Indexed once: a lookup per parameter @@ -280,7 +280,7 @@ impl<'f, 'a> Effect<'f, 'a> { }); } else if c.is(b"pdnm") && let Some(last) = out.last_mut() - && let Some(u) = c.children().iter().find(|u| u.is(b"Utf8")) + && let Some(u) = c.child(b"Utf8") && let Some(b) = u.bytes() // An empty `pdnm` names nothing (EffectCraft's exports write one: the effect // supplies its items). @@ -293,8 +293,8 @@ impl<'f, 'a> Effect<'f, 'a> { } } -fn has_pard(part: &Chunk<'_>) -> bool { - part.children().iter().any(|c| c.is(b"pard")) +fn has_pard(part: Node<'_, '_>) -> bool { + part.children().any(|c| c.is(b"pard")) } impl<'f, 'a> Project<'f, 'a> { @@ -302,18 +302,18 @@ impl<'f, 'a> Project<'f, 'a> { /// name and the `LIST parT` of the copied instance, in file order (the stack order of the /// types, 365/366). The `EfDC` count before them is little-endian and is not needed to read /// them. A type whose copy has no definitions is left out. - pub fn effect_definitions(&self) -> Vec<(String, &'f Chunk<'a>)> { - let mut out: Vec<(String, &'f Chunk<'a>)> = Vec::new(); + pub fn effect_definitions(&self) -> Vec<(String, Node<'f, 'a>)> { + let mut out: Vec<(String, Node<'f, 'a>)> = Vec::new(); let Some(defs) = self.effect_defs else { return out }; let mut seen: HashSet = HashSet::new(); - for df in defs.children().iter().filter(|c| c.is_list(b"EfDf")) { + for df in defs.children().filter(|c| c.is_list(b"EfDf")) { let mut name: Option = None; for c in df.children() { if c.is(b"tdmn") { name = c.bytes().map(|b| text(until_nul(b))); } else if c.is_list(b"sspc") && let Some(n) = name.take() - && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(p)) + && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(*p)) && seen.insert(n.clone()) { out.push((n, part)); @@ -328,8 +328,8 @@ impl<'f, 'a> Project<'f, 'a> { /// For a type without a project-level copy it also keeps the own `parT` of the first instance /// in the file (items, layers and effects in file order) that has one. pub fn effect_definition_index(&self) -> EffectDefinitions<'f, 'a> { - let by_name: HashMap> = self.effect_definitions().into_iter().collect(); - let mut instances: HashMap> = HashMap::new(); + let by_name: HashMap> = self.effect_definitions().into_iter().collect(); + let mut instances: HashMap> = HashMap::new(); for item in self.all_items() { let Some(comp) = item.comp() else { continue }; for layer in comp.layers() { @@ -337,7 +337,7 @@ impl<'f, 'a> Project<'f, 'a> { if by_name.contains_key(&fx.match_name) || instances.contains_key(&fx.match_name) { continue; } - if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(p)) { + if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(*p)) { instances.insert(fx.match_name, part); } } @@ -363,9 +363,9 @@ impl<'f, 'a> Project<'f, 'a> { #[derive(Debug, Clone, Default)] pub struct EffectDefinitions<'f, 'a> { /// The project-level copies (`EfdG`). - by_name: HashMap>, + by_name: HashMap>, /// For a type without a project-level copy: the own `parT` of the first instance that has one. - instances: HashMap>, + instances: HashMap>, } impl<'f, 'a> EffectDefinitions<'f, 'a> { @@ -392,7 +392,7 @@ impl<'f, 'a> EffectDefinitions<'f, 'a> { return effect.params(); } match self.by_name.get(effect.match_name()).or_else(|| self.instances.get(effect.match_name())) { - Some(part) => effect.params_from(part, true), + Some(part) => effect.params_from(*part, true), None => Vec::new(), } } @@ -638,6 +638,7 @@ impl<'f, 'a> Layer<'f, 'a> { #[cfg(test)] mod tests { use super::*; + use crate::container::Chunk; fn tdmn(name: &str) -> Chunk<'static> { let mut b = name.as_bytes().to_vec(); @@ -670,11 +671,11 @@ mod tests { ("A".to_string(), Chunk::list(*b"tdbs", vec![Chunk::data(*b"cdat", vec![0u8; 8])])), ]; let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["A", "B", "C"]); let first = params.first().and_then(|p| p.stored).expect("A stored"); - assert!(first.chunk.children().is_empty(), "the first A (the second has a cdat)"); + assert!(first.chunk.children().next().is_none(), "the first A (the second has a cdat)"); assert!(params.get(1).is_some_and(|p| p.stored.is_none()), "a group of that name is no stored value"); assert!(params.get(2).is_some_and(|p| p.stored.is_none())); } @@ -687,7 +688,7 @@ mod tests { let names: Vec = (0..n).map(|i| format!("P{i}")).collect(); let stored: Vec<(String, Chunk<'static>)> = names.iter().map(|m| (m.clone(), tdbs())).collect(); let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.len(), n); assert!(params.iter().all(|p| p.stored.is_some())); @@ -707,19 +708,19 @@ mod tests { } let efdg = Chunk::list(*b"EfdG", defs); let fold = Chunk::list(*b"Fold", Vec::new()); - let project = Project { fold: &fold, effect_defs: Some(&efdg) }; + let project = Project { fold: Node::from(&fold), effect_defs: Some(Node::from(&efdg)) }; let all = project.effect_definitions(); assert_eq!(all.len(), n); let index = project.effect_definition_index(); let bare = Chunk::list(*b"sspc", vec![Chunk::list(*b"parT", Vec::new())]); for i in (0..n).step_by(50) { - let fx = Effect { match_name: format!("FX{i}"), chunk: &bare }; + let fx = Effect { match_name: format!("FX{i}"), chunk: Node::from(&bare) }; assert_eq!(index.source(&fx), DefinitionSource::Project); let params = index.params(&fx); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["R0"], "the first copy's definitions"); assert!(params.iter().all(|p| p.borrowed_definition)); } - let unknown = Effect { match_name: "nope".into(), chunk: &bare }; + let unknown = Effect { match_name: "nope".into(), chunk: Node::from(&bare) }; assert_eq!(index.source(&unknown), DefinitionSource::Missing); assert!(index.params(&unknown).is_empty()); assert_eq!(project.definition_source(&unknown), DefinitionSource::Missing); diff --git a/crates/aep/src/container/footage.rs b/crates/aep/src/container/footage.rs index c1007fce5..ce7c8e6c5 100644 --- a/crates/aep/src/container/footage.rs +++ b/crates/aep/src/container/footage.rs @@ -2,8 +2,8 @@ //! of a layer's source. use crate::container::bytes::{be_f32, be_f64, be_u32, cstr, le_u32, text}; -use crate::container::chunk::Chunk; use crate::container::header::Rational; +use crate::container::index::Node; use crate::container::typed::{Item, ItemKind, Layer, Project}; /// The footage data of an item (`LIST Pin ` with `sspc`, `opti` and, for files, `Als2/alas`). @@ -201,14 +201,13 @@ impl SequenceInfo { /// is unreadable (empty). A name that is not a plain file name (holding `/`, `\`, `:` or a control /// character such as NUL, or being `.` or `..`) makes the whole list unreadable (empty): the names /// are joined to the folder. -fn sequence_file_list(list: &Chunk<'_>) -> Vec { +fn sequence_file_list(list: Node<'_, '_>) -> Vec { // After Effects writes the count first (the one sample); a list without a readable count is // not read rather than guessed. let Some(count) = list.child(b"StVS").and_then(|c| c.bytes()).and_then(|b| le_u32(b, 0)).and_then(|n| usize::try_from(n).ok()) else { return Vec::new(); }; - let names: Vec = - list.children().iter().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).take(count).map(text).filter(|n| !n.is_empty()).collect(); + let names: Vec = list.children().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).take(count).map(text).filter(|n| !n.is_empty()).collect(); if names.iter().any(|n| n.contains(|c: char| matches!(c, '/' | '\\' | ':') || c.is_control()) || n == "." || n == "..") { return Vec::new(); } @@ -240,9 +239,8 @@ impl<'f, 'a> Item<'f, 'a> { } let pin = self.chunk.child_list(b"Pin ")?; let sspc = pin.child(b"sspc")?.bytes()?; - let kids = pin.children(); - let after = kids.iter().position(|c| c.is_list(b"Als2"))?; - let naming = || kids.iter().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); + let after = pin.children().position(|c| c.is_list(b"Als2"))?; + let naming = || pin.children().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); let (start, end, digits) = (be_u32(sspc, 172)?, be_u32(sspc, 176)?, be_u32(sspc, 180)?); // Force alphabetical order: the file list replaces the numbering. if let Some(list) = naming().find(|c| c.is_list(b"StVc")) { @@ -252,7 +250,7 @@ impl<'f, 'a> Item<'f, 'a> { } return Some(SequenceInfo { prefix: String::new(), suffix: String::new(), start, end, digits, files }); } - let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).map(text).take(2).collect(); + let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).map(text).take(2).collect(); let (prefix, suffix) = match names.as_slice() { [prefix, suffix] => (prefix.clone(), suffix.clone()), // An empty prefix is not stored (`0001.png` has only `.png`). diff --git a/crates/aep/src/container/index.rs b/crates/aep/src/container/index.rs new file mode 100644 index 000000000..cb9621e94 --- /dev/null +++ b/crates/aep/src/container/index.rs @@ -0,0 +1,955 @@ +//! A read-only, flat index of the chunk tree for readers that only look (the importer). +//! +//! [`TreeIndex`] holds every chunk of a file in one `Vec` in depth-first order (a chunk, then its +//! whole subtree, then its next sibling). Each entry keeps its id, list type, pad byte, the bytes of +//! its body (borrowed from the input) and the index one past its subtree, so: +//! - a chunk's subtree is the index range `at + 1 .. end`, +//! - its direct children are found by stepping from `at + 1` to each child's `end`, +//! - the whole tree is one array, grown a few times while it is built and freed at once; it never +//! holds more slots than the input has room for chunk headers or the chunk limit allows. +//! +//! [`crate::container::parse`] builds the owned [`Chunk`] tree instead (one `Vec` per list), which +//! the exporter and editors need. [`TreeIndex::to_file`] and [`Node::to_chunk`] convert the index +//! into that tree where an owned one is needed. +//! +//! [`TreeIndex::parse_with_limits`] walks the input exactly as the [`Chunk`] parser does: the same +//! chunks are counted against [`Limits::max_chunks`] in the same order, the same lists are kept +//! opaque at [`Limits::max_depth`] or when damaged, and the same errors come back. A test compares +//! the two on the corpus and on generated and damaged trees. Two errors are its own, and both are +//! stricter than the [`Chunk`] parser (which would abort or could not address the tree): +//! [`AepError::TooLarge`] when the array cannot grow (a failed `try_reserve`), and +//! [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. +//! +//! [`Node`] is the handle the typed view ([`crate::container::Project`] and the types below it) +//! reads through: a chunk of either tree. The view works the same on both. + +use std::borrow::Cow; +use std::fmt; + +use crate::container::bytes::be_u32; +use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS, Walk}; +use crate::container::error::AepError; +use crate::container::read::{Limits, container_end, id4}; + +/// What an entry's body is (the [`Body`] variants). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + Data, + /// [`Body::Chunks`]. + Nested, + List, + OpaqueList, +} + +/// One chunk of a [`TreeIndex`]. +#[derive(Debug, Clone, Copy)] +struct Entry<'a> { + /// The body bytes of a data chunk, or the bytes after the list type of an opaque list; empty + /// for a list or nested chunk (their content is their children). + data: &'a [u8], + id: [u8; 4], + /// The list type of a `LIST` (parsed or opaque); zero otherwise. + list_type: [u8; 4], + /// One past the last entry of this chunk's subtree. + end: u32, + kind: Kind, + pad: Option, +} + +impl<'a> Entry<'a> { + fn placeholder(id: [u8; 4]) -> Self { + Self { data: &[], id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None } + } +} + +/// The chunk tree of a file as one flat, read-only array (see the module documentation). +#[derive(Clone)] +pub struct TreeIndex<'a> { + nodes: Vec>, + trailer: &'a [u8], +} + +impl fmt::Debug for TreeIndex<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TreeIndex").field("chunks", &self.nodes.len()).field("trailer", &self.trailer.len()).finish() + } +} + +/// An index that does not fit a `u32` (more than 4 billion chunks) is refused as too many chunks. +fn index_u32(at: usize) -> Result { + u32::try_from(at).map_err(|_| AepError::TooManyChunks) +} + +impl<'a> TreeIndex<'a> { + /// Indexes an `.aep` with the default [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::container::parse`], for the same inputs. + pub fn parse(data: &'a [u8]) -> Result { + Self::parse_with_limits(data, Limits::default()) + } + + /// Indexes an `.aep` with explicit [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::container::parse_with_limits`], for the same inputs and limits; also + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks and + /// [`AepError::TooLarge`] when the array cannot be allocated. + pub fn parse_with_limits(data: &'a [u8], limits: Limits) -> Result { + let end = container_end(data)?; + // Every chunk header takes 8 bytes of its own and is counted once, so the tree never has + // more chunks than this. + let bound = (end.saturating_sub(12) / 8).min(limits.max_chunks); + let mut builder = Builder { data, limits, count: 0, nodes: Vec::new(), bound }; + // A small start; [`Builder::push`] doubles it as chunks arrive, never past the bound, so + // the array stays proportional to the chunk count (a file of a few large chunks gets a few + // slots, not one per 32 bytes). `try_reserve_exact` so it fails nothing here. + let _ = builder.nodes.try_reserve_exact(bound.min(INITIAL_SLOTS)); + builder.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(Self { nodes: builder.nodes, trailer }) + } + + /// Indexes an already parsed (or built, or `.aepx`) chunk tree. The index borrows the file's + /// bytes. + /// + /// # Errors + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. + pub fn from_file(file: &'a AepFile<'_>) -> Result { + // Sized once, to the chunks the tree holds. + let mut nodes: Vec> = Vec::new(); + nodes.try_reserve_exact(file.walk().count()).map_err(|_| AepError::TooLarge)?; + // Each frame: the siblings still to visit and the entry whose subtree they fill. + let mut stack: Vec<(std::slice::Iter<'a, Chunk<'_>>, Option)> = vec![(file.chunks.iter(), None)]; + while let Some((iter, parent)) = stack.last_mut() { + let parent = *parent; + match iter.next() { + Some(chunk) => { + let at = nodes.len(); + // The entry after this one must still have a `u32` index. + let next = index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + let (kind, data, list_type) = match &chunk.body { + Body::Data(d) => (Kind::Data, &d[..], [0; 4]), + Body::Chunks(_) => (Kind::Nested, &[][..], [0; 4]), + Body::List { list_type, .. } => (Kind::List, &[][..], *list_type), + Body::OpaqueList { list_type, data } => (Kind::OpaqueList, &data[..], *list_type), + }; + nodes.push(Entry { data, id: chunk.id, list_type, end: next, kind, pad: chunk.pad }); + let kids = chunk.children(); + if !kids.is_empty() { + stack.push((kids.iter(), Some(at))); + } + } + None => { + stack.pop(); + if let Some(p) = parent { + let end = index_u32(nodes.len())?; + if let Some(e) = nodes.get_mut(p) { + e.end = end; + } + } + } + } + } + Ok(Self { nodes, trailer: &file.trailer[..] }) + } + + /// The number of chunks in the tree (at every depth). + pub fn len(&self) -> usize { + self.nodes.len() + } + + /// True when the container holds no chunks. + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// The bytes after the container's declared end (see [`AepFile::trailer`]). + pub fn trailer(&self) -> &'a [u8] { + self.trailer + } + + /// The top-level chunks. + pub fn chunks(&self) -> Nodes<'_, 'a> { + Nodes(NodesRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX) }) + } + + /// The first top-level chunk with this id. + pub fn chunk(&self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub fn list(&self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } + + /// Visits every chunk depth-first with its nesting depth (the top level is 1), as + /// [`AepFile::walk`] does. + pub fn walk(&self) -> NodeWalk<'_, 'a> { + NodeWalk(WalkRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX), open: Vec::new() }) + } + + /// The owned chunk tree (the bodies stay borrowed from the input): what + /// [`crate::container::parse`] gives for the same file. + pub fn to_file(&self) -> AepFile<'a> { + AepFile { chunks: self.build(0, u32::try_from(self.nodes.len()).unwrap_or(u32::MAX)), trailer: Cow::Borrowed(self.trailer) } + } + + fn entry(&self, at: u32) -> Option<&Entry<'a>> { + self.nodes.get(usize::try_from(at).ok()?) + } + + /// The end of the subtree at `at`; never before `at + 1`, so walks always move on. + fn end_of(&self, at: u32) -> u32 { + self.entry(at).map_or(u32::MAX, |e| e.end).max(at.saturating_add(1)) + } + + /// The owned chunks for the siblings that start at `next` and fill `next..end`, without + /// recursion. + fn build(&self, next: u32, end: u32) -> Vec> { + struct Frame<'a> { + /// The list or nested chunk being filled; `None` for the outermost range. + at: Option, + kids: Vec>, + next: u32, + end: u32, + } + let mut stack = vec![Frame { at: None, kids: Vec::new(), next, end }]; + loop { + let Some(top) = stack.last_mut() else { return Vec::new() }; + if top.next < top.end { + let at = top.next; + top.next = self.end_of(at); + let Some(e) = self.entry(at).copied() else { + top.next = top.end; + continue; + }; + match e.kind { + Kind::List | Kind::Nested => { + let sub_end = self.end_of(at); + stack.push(Frame { at: Some(at), kids: Vec::new(), next: at.saturating_add(1), end: sub_end }); + } + Kind::Data => top.kids.push(Chunk { id: e.id, body: Body::Data(Cow::Borrowed(e.data)), pad: e.pad }), + Kind::OpaqueList => { + top.kids.push(Chunk { id: e.id, body: Body::OpaqueList { list_type: e.list_type, data: Cow::Borrowed(e.data) }, pad: e.pad }); + } + } + } else { + let Some(frame) = stack.pop() else { return Vec::new() }; + let Some(at) = frame.at else { return frame.kids }; + let Some(e) = self.entry(at).copied() else { continue }; + let body = match e.kind { + Kind::Nested => Body::Chunks(frame.kids), + Kind::List | Kind::Data | Kind::OpaqueList => Body::List { list_type: e.list_type, children: frame.kids }, + }; + let chunk = Chunk { id: e.id, body, pad: e.pad }; + match stack.last_mut() { + Some(parent) => parent.kids.push(chunk), + None => return vec![chunk], + } + } + } + } +} + +/// The slots a [`TreeIndex`] starts with (32 KiB); it doubles from there. +const INITIAL_SLOTS: usize = 1024; + +/// Builds a [`TreeIndex`]; mirrors `read::Parser` step for step (see the module documentation). +struct Builder<'a> { + data: &'a [u8], + limits: Limits, + count: usize, + nodes: Vec>, + /// The most chunks the input can hold under the limits: `nodes` never reserves past it. + bound: usize, +} + +impl<'a> Builder<'a> { + /// Appends an entry. A full array grows by its length (at least 16), but never past + /// [`Self::bound`]; an allocation the target cannot make is an error, not an abort. + fn push(&mut self, entry: Entry<'a>) -> Result<(), AepError> { + let len = self.nodes.len(); + if len == self.nodes.capacity() { + let more = len.max(16).min(self.bound.saturating_sub(len)).max(1); + self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?; + } + self.nodes.push(entry); + Ok(()) + } + + /// Appends the sibling chunks that exactly fill `start..end`, each followed by its subtree. + fn sequence(&mut self, start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let mut pos = start; + while pos < end { + let short = AepError::Malformed { offset: pos, what: "bytes left over that are too short for a chunk header" }; + let body_start = pos.checked_add(8).filter(|b| *b <= end).ok_or(short)?; + let id = id4(self.data, pos).ok_or(AepError::Malformed { offset: pos, what: "short read" })?; + let size = be_u32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; + let avail = end - body_start; + if size > avail { + return Err(AepError::ChunkTooLarge { offset: pos, size: size as u64, available: avail as u64 }); + } + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + let at = self.nodes.len(); + // This entry and the one after it must have `u32` indices. + index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + self.push(Entry::placeholder(id))?; + let body_end = body_start + size; + self.body(at, id, body_start, body_end, depth)?; + let mut next = body_end; + let mut pad = None; + if size % 2 == 1 && next < end { + pad = Some(self.data.get(next).copied().ok_or(AepError::Malformed { offset: next, what: "missing pad byte" })?); + next += 1; + } + let subtree_end = index_u32(self.nodes.len())?; + if let Some(e) = self.nodes.get_mut(at) { + e.pad = pad; + e.end = subtree_end; + } + pos = next; + } + Ok(()) + } + + /// Like [`Self::sequence`] but a structural problem yields `false` and removes what the + /// attempt appended (the caller keeps bytes). The chunk count keeps the attempt's chunks, as + /// the [`Chunk`] parser's does. + fn tentative(&mut self, start: usize, end: usize, depth: usize) -> Result { + let mark = self.nodes.len(); + match self.sequence(start, end, depth) { + Ok(()) => Ok(true), + Err(e) if e.is_structural() => { + self.nodes.truncate(mark); + Ok(false) + } + Err(e) => Err(e), + } + } + + fn slice(&self, start: usize, end: usize) -> &'a [u8] { + self.data.get(start..end).unwrap_or(&[]) + } + + fn set(&mut self, at: usize, kind: Kind, data: &'a [u8], list_type: [u8; 4]) { + if let Some(e) = self.nodes.get_mut(at) { + e.kind = kind; + e.data = data; + e.list_type = list_type; + } + } + + /// Fills in the entry at `at` for the body `start..end` and appends its subtree. + fn body(&mut self, at: usize, id: [u8; 4], start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let len = end - start; + if id == LIST && len >= 4 { + let list_type = id4(self.data, start).ok_or(AepError::Malformed { offset: start, what: "short read" })?; + if depth < self.limits.max_depth && self.tentative(start + 4, end, depth + 1)? { + self.set(at, Kind::List, &[], list_type); + } else { + let data = self.slice(start + 4, end); + self.set(at, Kind::OpaqueList, data, list_type); + } + return Ok(()); + } + if NESTED_IDS.contains(&id) && len >= 8 && depth < self.limits.max_depth { + let mark = self.nodes.len(); + if self.tentative(start, end, depth + 1)? && self.nodes.len() > mark { + self.set(at, Kind::Nested, &[], [0; 4]); + return Ok(()); + } + } + let data = self.slice(start, end); + self.set(at, Kind::Data, data, [0; 4]); + Ok(()) + } +} + +/// A chunk of either tree: an owned [`Chunk`] or an entry of a [`TreeIndex`]. The typed view +/// reads through it, so it works the same on both. Cheap to copy. +#[derive(Clone, Copy)] +pub struct Node<'f, 'a>(NodeRepr<'f, 'a>); + +#[derive(Clone, Copy)] +enum NodeRepr<'f, 'a> { + Chunk(&'f Chunk<'a>), + Index(&'f TreeIndex<'a>, u32), +} + +impl<'f, 'a> From<&'f Chunk<'a>> for Node<'f, 'a> { + fn from(chunk: &'f Chunk<'a>) -> Self { + Node(NodeRepr::Chunk(chunk)) + } +} + +/// Prints the chunk as [`Chunk`]'s `Debug` does, whichever tree it is in, without copying the +/// subtree. +impl fmt::Debug for Node<'_, '_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + NodeRepr::Chunk(c) => fmt::Debug::fmt(c, f), + NodeRepr::Index(..) => f.debug_struct("Chunk").field("id", &self.id()).field("body", &BodyDebug(*self)).field("pad", &self.pad()).finish(), + } + } +} + +/// The body of an index [`Node`], printed as [`Body`]'s `Debug` prints it. +struct BodyDebug<'f, 'a>(Node<'f, 'a>); + +/// The children of a [`Node`], printed as a list. +struct ChildrenDebug<'f, 'a>(Node<'f, 'a>); + +impl fmt::Debug for BodyDebug<'_, '_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Some(e) = self.0.entry() else { return f.write_str("?") }; + match e.kind { + Kind::Data => f.debug_tuple("Data").field(&e.data).finish(), + Kind::Nested => f.debug_tuple("Chunks").field(&ChildrenDebug(self.0)).finish(), + Kind::List => f.debug_struct("List").field("list_type", &e.list_type).field("children", &ChildrenDebug(self.0)).finish(), + Kind::OpaqueList => f.debug_struct("OpaqueList").field("list_type", &e.list_type).field("data", &e.data).finish(), + } + } +} + +impl fmt::Debug for ChildrenDebug<'_, '_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.0.children()).finish() + } +} + +impl<'f, 'a> Node<'f, 'a> { + fn entry(self) -> Option<&'f Entry<'a>> { + match self.0 { + NodeRepr::Index(tree, at) => tree.entry(at), + NodeRepr::Chunk(_) => None, + } + } + + /// The [`Chunk`] when this node is one of an owned tree; `None` for an index entry (see + /// [`Self::to_chunk`]). + pub fn as_chunk(self) -> Option<&'f Chunk<'a>> { + match self.0 { + NodeRepr::Chunk(c) => Some(c), + NodeRepr::Index(..) => None, + } + } + + /// The chunk and its subtree as an owned [`Chunk`] (bodies stay borrowed). + pub fn to_chunk(self) -> Chunk<'a> { + match self.0 { + NodeRepr::Chunk(c) => c.clone(), + NodeRepr::Index(tree, at) => tree.build(at, tree.end_of(at)).into_iter().next().unwrap_or_else(|| Chunk::data(self.id(), &[][..])), + } + } + + /// The 4-byte id (see [`Chunk::id`]). + pub fn id(self) -> [u8; 4] { + match self.0 { + NodeRepr::Chunk(c) => c.id, + NodeRepr::Index(..) => self.entry().map_or([0; 4], |e| e.id), + } + } + + /// The pad byte (see [`Chunk::pad`]). + pub fn pad(self) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.pad, + NodeRepr::Index(..) => self.entry().and_then(|e| e.pad), + } + } + + /// The raw bytes of a data chunk, or the bytes after the list type of an opaque list (see + /// [`Chunk::bytes`]). + pub fn bytes(self) -> Option<&'f [u8]> { + match self.0 { + NodeRepr::Chunk(c) => c.bytes(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::Data | Kind::OpaqueList)).map(|e| e.data), + } + } + + /// The child chunks of a list or nested chunk; none for data. + pub fn children(self) -> Nodes<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => Nodes(NodesRepr::Slice(c.children().iter())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + Nodes(NodesRepr::Index { tree, next: at.saturating_add(1), end }) + } + } + } + + /// The list type when this is a `LIST` (parsed or opaque). + pub fn list_type(self) -> Option<[u8; 4]> { + match self.0 { + NodeRepr::Chunk(c) => c.list_type(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::List | Kind::OpaqueList)).map(|e| e.list_type), + } + } + + /// True when this is a `LIST` of the given type. + pub fn is_list(self, list_type: &[u8; 4]) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_list(list_type), + NodeRepr::Index(..) => self.entry().is_some_and(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type), + } + } + + /// True when this is a `LIST` whose content could not be read as chunks. + pub fn is_opaque_list(self) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_opaque_list(), + NodeRepr::Index(..) => self.entry().is_some_and(|e| e.kind == Kind::OpaqueList), + } + } + + /// True when this chunk has the given id. + pub fn is(self, id: &[u8; 4]) -> bool { + self.id() == *id + } + + /// The first child chunk with this id. + pub fn child(self, id: &[u8; 4]) -> Option { + self.children().find(|c| c.is(id)) + } + + /// The first child `LIST` of this type. + pub fn child_list(self, list_type: &[u8; 4]) -> Option { + self.children().find(|c| c.is_list(list_type)) + } + + /// Every chunk below this one depth-first (children before later siblings), with the depth + /// counted from 1 for the children of this chunk. + pub fn descendants(self) -> NodeWalk<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => NodeWalk(WalkRepr::Chunk(c.descendants())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + NodeWalk(WalkRepr::Index { tree, next: at.saturating_add(1), end, open: Vec::new() }) + } + } + } + + /// The first descendant `LIST` of this type, searching depth-first. + pub fn find_list(self, list_type: &[u8; 4]) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.find_list(list_type).map(Node::from), + // The subtree is a contiguous range in depth-first order: a plain scan. + NodeRepr::Index(tree, at) => { + let first = usize::try_from(at).ok()?.checked_add(1)?; + let end = usize::try_from(tree.end_of(at)).ok()?.min(tree.nodes.len()); + let range = tree.nodes.get(first..end)?; + let i = range.iter().position(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type)?; + let found = u32::try_from(first.checked_add(i)?).ok()?; + Some(Node(NodeRepr::Index(tree, found))) + } + } + } +} + +/// The children of a [`Node`] (or the top-level chunks of a [`TreeIndex`]), in file order. +#[derive(Clone)] +pub struct Nodes<'f, 'a>(NodesRepr<'f, 'a>); + +#[derive(Clone)] +enum NodesRepr<'f, 'a> { + Slice(std::slice::Iter<'f, Chunk<'a>>), + Index { tree: &'f TreeIndex<'a>, next: u32, end: u32 }, +} + +impl<'f, 'a> Iterator for Nodes<'f, 'a> { + type Item = Node<'f, 'a>; + + fn next(&mut self) -> Option { + match &mut self.0 { + NodesRepr::Slice(it) => it.next().map(Node::from), + NodesRepr::Index { tree, next, end } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + *next = tree.end_of(at); + Some(Node(NodeRepr::Index(tree, at))) + } + } + } +} + +/// A depth-first walk over [`Node`]s with their depth; see [`Node::descendants`] and +/// [`TreeIndex::walk`]. +pub struct NodeWalk<'f, 'a>(WalkRepr<'f, 'a>); + +enum WalkRepr<'f, 'a> { + Chunk(Walk<'f, 'a>), + /// `open` holds the subtree ends of the entries above the next one (its ancestors inside the + /// walked range); their count is the next entry's depth minus one. + Index { + tree: &'f TreeIndex<'a>, + next: u32, + end: u32, + open: Vec, + }, +} + +impl<'f, 'a> Iterator for NodeWalk<'f, 'a> { + type Item = (usize, Node<'f, 'a>); + + fn next(&mut self) -> Option { + match &mut self.0 { + WalkRepr::Chunk(w) => w.next().map(|(d, c)| (d, Node::from(c))), + WalkRepr::Index { tree, next, end, open } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + while open.last().is_some_and(|e| *e <= at) { + open.pop(); + } + let depth = open.len().saturating_add(1); + let sub_end = tree.end_of(at); + if sub_end > at.saturating_add(1) { + open.push(sub_end); + } + *next = at.saturating_add(1); + Some((depth, Node(NodeRepr::Index(tree, at)))) + } + } + } +} + +/// The top level of either tree: what the project settings and the version are read from. +#[derive(Clone, Copy)] +pub(crate) enum Root<'f, 'a> { + File(&'f AepFile<'a>), + Index(&'f TreeIndex<'a>), +} + +impl<'f, 'a> From<&'f AepFile<'a>> for Root<'f, 'a> { + fn from(file: &'f AepFile<'a>) -> Self { + Root::File(file) + } +} + +impl<'f, 'a> From<&'f TreeIndex<'a>> for Root<'f, 'a> { + fn from(index: &'f TreeIndex<'a>) -> Self { + Root::Index(index) + } +} + +impl<'f, 'a> Root<'f, 'a> { + /// The top-level chunks. + pub(crate) fn chunks(self) -> Nodes<'f, 'a> { + match self { + Root::File(f) => Nodes(NodesRepr::Slice(f.chunks.iter())), + Root::Index(t) => t.chunks(), + } + } + + /// The first top-level chunk with this id. + pub(crate) fn chunk(self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub(crate) fn list(self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::container::{parse_with_limits, write}; + + /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate. + struct Rng(u64); + + impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + fn below(&mut self, n: u64) -> u64 { + if n == 0 { 0 } else { self.next() % n } + } + } + + /// A random tree: lists, nested-id chunks holding a `Utf8`, data of odd and even sizes, opaque + /// lists, empty lists, deep chains. + fn tree(rng: &mut Rng, depth: usize, budget: &mut usize) -> Vec> { + let n = rng.below(6); + let mut out = Vec::new(); + for _ in 0..n { + if *budget == 0 { + break; + } + *budget -= 1; + let pick = rng.below(10); + let chunk = if depth < 12 && pick < 4 { + let types: [&[u8; 4]; 4] = [b"Fold", b"tdgp", b"tdbs", b"Layr"]; + let t = *types[usize::try_from(rng.below(4)).expect("small")]; + Chunk::list(t, tree(rng, depth + 1, budget)) + } else if pick == 4 { + let ids: [&[u8; 4]; 3] = [b"tdsn", b"pdnm", b"fnam"]; + let id = *ids[usize::try_from(rng.below(3)).expect("small")]; + let text = vec![b'a'; usize::try_from(rng.below(5)).expect("small")]; + Chunk { id, body: Body::Chunks(vec![Chunk::data(*b"Utf8", text)]), pad: None } + } else if pick == 5 { + let raw = vec![0x5a; usize::try_from(rng.below(9)).expect("small")]; + Chunk { id: LIST, body: Body::OpaqueList { list_type: *b"btdk", data: raw.into() }, pad: None } + } else { + let ids: [&[u8; 4]; 4] = [b"cdat", b"tdmn", b"Utf8", b"idta"]; + let id = *ids[usize::try_from(rng.below(4)).expect("small")]; + Chunk::data(id, vec![u8::try_from(rng.below(256)).expect("byte"); usize::try_from(rng.below(13)).expect("small")]) + }; + out.push(chunk); + } + out + } + + /// Both readers on the same bytes and limits: the same result, errors included; and the + /// index's walk visits the same chunks at the same depths as the tree's. + fn same(bytes: &[u8], limits: Limits) { + let old = parse_with_limits(bytes, limits); + let new = TreeIndex::parse_with_limits(bytes, limits); + match (&old, &new) { + (Ok(file), Ok(index)) => { + assert_eq!(&index.to_file(), file, "the index converts to the parsed tree"); + let a: Vec<(usize, Chunk<'_>)> = file.walk().map(|(d, c)| (d, c.clone())).collect(); + let b: Vec<(usize, Chunk<'_>)> = index.walk().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(a, b, "the walks agree"); + assert_eq!(index.len(), a.len()); + // Indexing the parsed tree gives the same index. + let again = TreeIndex::from_file(file).expect("index"); + assert_eq!(again.to_file(), *file); + } + (Err(a), Err(b)) => assert_eq!(a, b, "the same error"), + _ => panic!("one reader failed: {:?} vs {:?}", old.as_ref().err(), new.as_ref().err()), + } + } + + fn limits() -> [Limits; 5] { + [ + Limits::default(), + Limits { max_depth: 1, max_chunks: 10_000_000 }, + Limits { max_depth: 3, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 7 }, + Limits { max_depth: 64, max_chunks: 40 }, + ] + } + + #[test] + fn the_index_matches_the_chunk_parser_on_generated_and_damaged_trees() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + for _ in 0..400 { + let mut budget = 120; + let chunks = tree(&mut rng, 0, &mut budget); + let file = AepFile { chunks, trailer: Cow::Borrowed(b"x") }; + let bytes = write(&file).expect("write"); + for l in limits() { + same(&bytes, l); + } + // Damage: overwrite a few bytes past the container header (sizes, ids, list types). + for _ in 0..4 { + let mut bad = bytes.clone(); + for _ in 0..1 + rng.below(3) { + let at = 12 + usize::try_from(rng.below(u64::try_from(bad.len().saturating_sub(12)).expect("len"))).expect("index"); + if let Some(b) = bad.get_mut(at) { + *b = u8::try_from(rng.below(256)).expect("byte"); + } + } + for l in limits() { + same(&bad, l); + } + } + // Truncated inputs (the container size then no longer fits). + let cut = bytes.len().saturating_sub(usize::try_from(rng.below(8)).expect("small") + 1); + same(&bytes[..cut], Limits::default()); + } + } + + #[test] + fn header_errors_are_the_same() { + for bytes in [&b""[..], b"RIFX", b"RIFX\0\0\0\x04Egg", b"RIFX\0\0\0\x04Fnord", b"RIFX\0\0\0\x08Egg!", b"RIFX\0\0\0\x03Egg!", b"RIFF\0\0\0\x04Egg!"] { + same(bytes, Limits::default()); + } + } + + /// The typed accessors give the same answers through either tree. + #[test] + fn nodes_answer_alike_on_both_trees() { + let mut rng = Rng(42); + for _ in 0..100 { + let mut budget = 200; + let file = AepFile { chunks: tree(&mut rng, 0, &mut budget), trailer: Cow::Borrowed(&[]) }; + let bytes = write(&file).expect("write"); + let parsed = crate::container::parse(&bytes).expect("parse"); + let index = TreeIndex::parse(&bytes).expect("index"); + let a: Vec> = parsed.walk().map(|(_, c)| Node::from(c)).collect(); + let b: Vec> = index.walk().map(|(_, n)| n).collect(); + assert_eq!(a.len(), b.len()); + for (x, y) in a.iter().zip(&b) { + assert_eq!((x.id(), x.pad(), x.bytes(), x.list_type(), x.is_opaque_list()), (y.id(), y.pad(), y.bytes(), y.list_type(), y.is_opaque_list())); + assert_eq!(x.children().map(Node::to_chunk).collect::>(), y.children().map(Node::to_chunk).collect::>()); + for t in [b"Fold", b"tdgp", b"tdbs", b"Layr", b"btdk"] { + assert_eq!(x.find_list(t).map(Node::to_chunk), y.find_list(t).map(Node::to_chunk)); + assert_eq!(x.child_list(t).map(Node::to_chunk), y.child_list(t).map(Node::to_chunk)); + assert_eq!(x.is_list(t), y.is_list(t)); + } + for id in [b"cdat", b"Utf8", b"tdsn"] { + assert_eq!(x.child(id).map(Node::to_chunk), y.child(id).map(Node::to_chunk)); + } + let dx: Vec<(usize, Chunk<'_>)> = x.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + let dy: Vec<(usize, Chunk<'_>)> = y.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(dx, dy); + assert_eq!(format!("{x:?}"), format!("{y:?}")); + } + } + } + + #[test] + fn the_index_never_reserves_more_than_the_input_or_the_limit_allows() { + // 1000 empty headers: the most chunks the bytes can hold. + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("index"); + assert_eq!(index.len(), 1000); + // The start (1024 slots) is cut to what the input can hold. + assert!(index.nodes.capacity() <= 1000, "{} slots for at most 1000 headers", index.nodes.capacity()); + // A short tail or a header that runs past the end does not raise the bound. + let mut big = data.clone(); + big[16..20].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(TreeIndex::parse(&big).is_err() && crate::container::parse(&big).is_err()); + // Under a tighter limit the parse fails as the chunk parser does, and the guess never + // exceeds the limit. + let tight = Limits { max_depth: 64, max_chunks: 10 }; + assert_eq!(TreeIndex::parse_with_limits(&data, tight).err(), Some(AepError::TooManyChunks)); + let small = Limits { max_depth: 64, max_chunks: 1000 }; + let index = TreeIndex::parse_with_limits(&data, small).expect("exactly at the limit"); + assert_eq!(index.len(), 1000); + // A limit below what the input holds bounds the reservation: 100 chunks, never more slots. + let mut wide = b"RIFX\0\0\0\0Egg!".to_vec(); + wide.extend_from_slice(b"LIST"); + wide.extend_from_slice(&u32::try_from(4 + 8 * 99).expect("small").to_be_bytes()); + wide.extend_from_slice(b"Fold"); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 99).flatten()); + let declared = u32::try_from(wide.len() - 8).expect("small"); + wide[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse_with_limits(&wide, Limits { max_depth: 64, max_chunks: 100 }).expect("100 chunks"); + assert_eq!(index.len(), 100); + assert!(index.nodes.capacity() <= 100); + // `from_file` is sized exactly. + let file = crate::container::parse(&wide).expect("parse"); + assert_eq!(TreeIndex::from_file(&file).expect("index").nodes.capacity(), 100); + } + + /// A file of `n` data chunks of `size` bytes each. + fn flat(n: usize, size: usize) -> Vec { + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + for _ in 0..n { + data.extend_from_slice(b"cdat"); + data.extend_from_slice(&u32::try_from(size).expect("small").to_be_bytes()); + data.extend(std::iter::repeat_n(0u8, size)); + } + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + data + } + + #[test] + fn the_index_grows_with_the_chunk_count_not_the_byte_size() { + // 4 MB in 4 chunks: a few slots, not one per 32 bytes (that would be 131 072 slots, 4 MB). + let big = flat(4, 1 << 20); + let index = TreeIndex::parse(&big).expect("index"); + assert_eq!(index.len(), 4); + assert!(index.nodes.capacity() <= INITIAL_SLOTS, "{} slots for 4 chunks", index.nodes.capacity()); + // Many small chunks: at most twice the count (it doubles). + let many = flat(10_000, 2); + let index = TreeIndex::parse(&many).expect("index"); + assert_eq!(index.len(), 10_000); + assert!(index.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", index.nodes.capacity()); + same(&big, Limits::default()); + same(&many, Limits::default()); + } + + /// `LIST Fold` around `inner`, followed by 6 bytes too short for a header, so the list's + /// sequence fails and the list is kept opaque. + fn broken_level(inner: &[u8]) -> Vec { + let mut body = b"Fold".to_vec(); + body.extend_from_slice(inner); + body.extend_from_slice(&[0xee; 6]); + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(body.len()).expect("small").to_be_bytes()); + out.extend_from_slice(&body); + out + } + + #[test] + fn failed_attempts_count_once_and_leave_nothing_behind() { + let mut wide = b"Fold".to_vec(); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let mut level = b"LIST".to_vec(); + level.extend_from_slice(&u32::try_from(wide.len()).expect("small").to_be_bytes()); + level.extend_from_slice(&wide); + for _ in 0..50 { + level = broken_level(&level); + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let headers = 1 + 50 + 1000; + let index = TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers }).expect("parse"); + assert_eq!(TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers - 1 }).err(), Some(AepError::TooManyChunks)); + // Only the outermost list is left, opaque: every failed attempt was rolled back. + assert_eq!(index.len(), 1); + let top = index.chunks().next().expect("one top-level chunk"); + assert!(top.is_opaque_list() && top.bytes().map(<[u8]>::len) == Some(level.len() - 12)); + assert_eq!(write(&index.to_file()).expect("write"), data); + same(&data, Limits::default()); + } + + #[test] + fn depth_beyond_the_limit_stays_opaque_without_recursion_past_it() { + // 5000 nested lists: far deeper than the limit; everything below depth 64 stays bytes. + let mut level = b"LIST\0\0\0\x04tdgp".to_vec(); + for _ in 0..5000 { + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(level.len() + 4).expect("small").to_be_bytes()); + out.extend_from_slice(b"tdgp"); + out.extend_from_slice(&level); + level = out; + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("parse"); + assert_eq!(index.len(), 64); + assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64)); + assert!(index.walk().last().is_some_and(|(_, n)| n.is_opaque_list())); + same(&data, Limits::default()); + // Converting and indexing a deep owned tree uses no recursion either. + let file = index.to_file(); + assert_eq!(TreeIndex::from_file(&file).expect("index").len(), 64); + } +} diff --git a/crates/aep/src/container/masks.rs b/crates/aep/src/container/masks.rs index 6246fc1f5..7f00c07eb 100644 --- a/crates/aep/src/container/masks.rs +++ b/crates/aep/src/container/masks.rs @@ -6,8 +6,8 @@ //! property (`tdbs`, with a key list when animated) and `omks` with one `LIST shap` per keyframe (a //! single one when the path is static). -use crate::container::Chunk; use crate::container::bytes::{be_f32, be_u32, until_nul}; +use crate::container::index::Node; use crate::container::typed::{KeyList, Layer, PropGroup, PropNode, Property}; use crate::container::values::{Tracked, read}; @@ -93,8 +93,10 @@ pub struct ShapePath { } impl ShapePath { - /// Decodes a `LIST shap`; `None` when `shph` or the point list is malformed. - pub fn from_chunk(shap: &Chunk<'_>) -> Option { + /// Decodes a `LIST shap` (a [`Chunk`](crate::container::Chunk) or a [`Node`]); `None` when + /// `shph` or the point list is malformed. + pub fn from_chunk<'f, 'a: 'f>(shap: impl Into>) -> Option { + let shap = shap.into(); let shph = shap.child(b"shph")?.bytes()?; let bbox = [be_f32(shph, 4)?, be_f32(shph, 8)?, be_f32(shph, 12)?, be_f32(shph, 16)?]; let closed_flag = *shph.get(20)?; @@ -187,7 +189,7 @@ fn mask_from_atom(atom: &PropGroup<'_, '_>, mkif: Option<&[u8]>) -> Mask { } } if let Some(omks) = oms.child_list(b"omks") { - mask.shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); + mask.shapes = omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); } } mask @@ -236,7 +238,7 @@ impl<'f, 'a> Layer<'f, 'a> { if let Some(PropNode::Other(oms)) = atom.get("ADBE Mask Shape") { path = oms.child_list(b"tdbs").and_then(Property::from_chunk); if let Some(omks) = oms.child_list(b"omks") { - path_shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); + path_shapes = omks.children().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); } } masks.push(MaskAtom { mask, atom, path, path_shapes }); diff --git a/crates/aep/src/container/read.rs b/crates/aep/src/container/read.rs index 8c43d9314..a47d06684 100644 --- a/crates/aep/src/container/read.rs +++ b/crates/aep/src/container/read.rs @@ -37,6 +37,16 @@ pub fn parse(data: &[u8]) -> Result, AepError> { /// # Errors /// See [`parse`]; also fails with [`AepError::TooManyChunks`]. pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + let end = container_end(data)?; + let mut parser = Parser { data, limits, count: 0 }; + let chunks = parser.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) +} + +/// Checks the `RIFX`/`Egg!` header and returns where the container ends (its chunks fill +/// `12..end`; the trailer follows). Shared with [`crate::container::index::TreeIndex::parse`]. +pub(super) fn container_end(data: &[u8]) -> Result { if id4(data, 0) != Some(*b"RIFX") { return Err(AepError::NotRifx); } @@ -50,14 +60,10 @@ pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, Aep return Err(AepError::Truncated { declared, available }); } // `declared <= available < data.len()`, so the end is inside the input. - let end = usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge)?; - let mut parser = Parser { data, limits, count: 0 }; - let chunks = parser.sequence(12, end, 1)?; - let trailer = data.get(end..).unwrap_or(&[]); - Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) + usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge) } -fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { +pub(super) fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { array::<4>(data, at) } diff --git a/crates/aep/src/container/settings.rs b/crates/aep/src/container/settings.rs index a39f09dff..3079d5c7d 100644 --- a/crates/aep/src/container/settings.rs +++ b/crates/aep/src/container/settings.rs @@ -5,6 +5,7 @@ use crate::container::bytes::be_u32; use crate::container::chunk::AepFile; use crate::container::footage::json_string; +use crate::container::index::{Node, Root, TreeIndex}; /// The time display style (`nhed` byte 8, feet + frames byte 11, film type byte 13). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -55,7 +56,16 @@ pub struct ProjectSettings { impl ProjectSettings { /// Reads the settings from the top-level chunks. pub fn read(file: &AepFile<'_>) -> ProjectSettings { - let nhed = file.chunk(b"nhed").and_then(|c| c.bytes()); + Self::of(Root::from(file)) + } + + /// Reads the settings from the top-level chunks of an indexed file. + pub fn read_index(index: &TreeIndex<'_>) -> ProjectSettings { + Self::of(Root::from(index)) + } + + pub(crate) fn of(file: Root<'_, '_>) -> ProjectSettings { + let nhed = file.chunk(b"nhed").and_then(Node::bytes); let byte = |i: usize| nhed.and_then(|b| b.get(i).copied()); let bits_per_channel = byte(15).and_then(|v| match v { 0 => Some(8), @@ -74,9 +84,9 @@ impl ProjectSettings { 1 => Some(1), _ => None, }); - let flag = |id: &[u8; 4]| file.chunk(id).and_then(|c| c.bytes()).and_then(|b| b.first().copied()) == Some(1); - let working_gamma_24 = file.chunk(b"dwga").and_then(|c| c.bytes()).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); - let scene_compensation = file.chunk(b"acer").and_then(|c| c.bytes()).and_then(|b| b.first().copied()).map(|v| v != 0); + let flag = |id: &[u8; 4]| file.chunk(id).and_then(Node::bytes).and_then(|b| b.first().copied()) == Some(1); + let working_gamma_24 = file.chunk(b"dwga").and_then(Node::bytes).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); + let scene_compensation = file.chunk(b"acer").and_then(Node::bytes).and_then(|b| b.first().copied()).map(|v| v != 0); ProjectSettings { bits_per_channel, working_space: working_space(file), @@ -91,9 +101,10 @@ impl ProjectSettings { } /// The working space: the `Utf8` right after `PwCs`. -fn working_space(file: &AepFile<'_>) -> Option { - let at = file.chunks.iter().position(|c| c.is(b"PwCs"))?; - let json = file.chunks.get(at.checked_add(1)?).filter(|c| c.is(b"Utf8"))?.bytes()?; +fn working_space(file: Root<'_, '_>) -> Option { + let mut chunks = file.chunks(); + chunks.find(|c| c.is(b"PwCs"))?; + let json = chunks.next().filter(|c| c.is(b"Utf8"))?.bytes()?; let trimmed: Vec = json.iter().copied().filter(|b| !b.is_ascii_whitespace() && *b != 0).collect(); if trimmed == b"{}" { return Some(WorkingSpace::None); diff --git a/crates/aep/src/container/shapes.rs b/crates/aep/src/container/shapes.rs index 8aad83d3f..dee01bb22 100644 --- a/crates/aep/src/container/shapes.rs +++ b/crates/aep/src/container/shapes.rs @@ -140,7 +140,7 @@ fn number(group: &PropGroup<'_, '_>, name: &str) -> Option> { fn paths_of(group: &PropGroup<'_, '_>) -> Vec { let Some(PropNode::Other(oms)) = group.get("ADBE Vector Shape") else { return Vec::new() }; let Some(omks) = oms.child_list(b"omks") else { return Vec::new() }; - omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() + omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() } fn item(match_name: String, group: &PropGroup<'_, '_>) -> ShapeItem { diff --git a/crates/aep/src/container/textdoc.rs b/crates/aep/src/container/textdoc.rs index 28e581909..2de5d8eb2 100644 --- a/crates/aep/src/container/textdoc.rs +++ b/crates/aep/src/container/textdoc.rs @@ -8,8 +8,8 @@ use std::borrow::Cow; use std::sync::LazyLock; -use crate::container::Chunk; use crate::container::bytes::{be_u32, text}; +use crate::container::index::Node; use crate::container::typed::{Comp, Layer, Property}; /// How deep dictionaries and arrays may nest before the parser gives up. @@ -1012,9 +1012,9 @@ impl<'f, 'a> Layer<'f, 'a> { /// `ADBE Text Properties` group, which is a direct child of the layer's root `tdgp` /// (`Layr/tdgp/tdgp/btds` in every sample, and where the exporter writes it), so only that /// level is searched rather than the whole layer. - fn source_text_record(&self) -> Option<&'f Chunk<'a>> { + fn source_text_record(&self) -> Option> { let root = self.chunk.child_list(b"tdgp")?; - root.children().iter().filter(|g| g.is_list(b"tdgp")).find_map(|g| g.child_list(b"btds")) + root.children().filter(|g| g.is_list(b"tdgp")).find_map(|g| g.child_list(b"btds")) } /// The text document of a text layer: the first `LIST btdk` in the Source Text record `btds` @@ -1078,13 +1078,13 @@ pub enum CueType { Other(u32), } -impl<'a> Chunk<'a> { +impl<'f, 'a> Node<'f, 'a> { /// The marker list `LIST mrst` of a marker holder: a layer (`LIST Layr`) or a composition's /// `SecL` list. A layer's `mrst` is a direct child of its root `tdgp` (`ADBE Marker`, where /// After Effects and the exporter write it), so only that place is looked at; most layers have /// none, and searching their whole property tree for it was a large part of an import. Any /// other holder is searched depth-first. - pub(crate) fn marker_list(&self) -> Option<&Chunk<'a>> { + pub(crate) fn marker_list(self) -> Option> { if self.is_list(b"Layr") { return self.child_list(b"tdgp")?.child_list(b"mrst"); } @@ -1094,7 +1094,7 @@ impl<'a> Chunk<'a> { /// Reads the markers below `holder` (a layer or the composition's `SecL` list): the `mrst` list has a /// property whose 16-byte records give the times and one `Nmrd` list per marker with the strings. -fn markers_in(holder: &Chunk<'_>) -> Vec { +fn markers_in(holder: Node<'_, '_>) -> Vec { let Some(mrst) = holder.marker_list() else { return Vec::new() }; let Some(prop) = mrst.child_list(b"tdbs").and_then(Property::from_chunk) else { return Vec::new() }; let Some(keys) = prop.keys() else { return Vec::new() }; @@ -1105,11 +1105,11 @@ fn markers_in(holder: &Chunk<'_>) -> Vec { .filter_map(|(k, nmrd)| { let time_units = k.time_units()?; // The five strings, in the order comment, chapter, URL, frame target, cue point name. - let mut strings = nmrd.children().iter().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); + let mut strings = nmrd.children().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); let mut next = || strings.next().unwrap_or_default(); let (comment, chapter, url, frame_target, cue_point_name) = (next(), next(), next(), next(), next()); let extras: Vec = strings.collect(); - let header = nmrd.child(b"NmHd").and_then(Chunk::bytes); + let header = nmrd.child(b"NmHd").and_then(Node::bytes); let duration_seconds = header.and_then(|h| { let (duration, per_second) = (be_u32(h, 8)?, be_u32(h, 12)?); (per_second != 0).then(|| f64::from(duration) / f64::from(per_second)) @@ -1432,6 +1432,7 @@ mod oracle { #[cfg(test)] mod tests { use super::*; + use crate::container::Chunk; fn list(t: &[u8; 4], kids: Vec>) -> Chunk<'static> { Chunk::list(*t, kids) @@ -1725,15 +1726,15 @@ mod tests { let marked = |tag: u8| list(b"mrst", vec![Chunk::data(*b"test", vec![tag])]); // The stored place: `Layr/tdgp/mrst`. let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), marked(1)])]); - assert_eq!(layer.marker_list().and_then(|m| m.child(b"test")).and_then(Chunk::bytes), Some(&[1u8][..])); + assert_eq!(Node::from(&layer).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[1u8][..])); // A list deeper in the layer is not the layer's marker list (the old depth-first search // found it); a layer without a marker list in its root group has no markers. let deep = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(2)])])]); - assert!(deep.marker_list().is_none()); - assert!(Layer { chunk: &deep }.markers().is_empty()); + assert!(Node::from(&deep).marker_list().is_none()); + assert!(Layer { chunk: Node::from(&deep) }.markers().is_empty()); // Any other holder (a composition's `SecL`) is still searched depth-first. let secl = list(b"SecL", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(3)])])]); - assert_eq!(secl.marker_list().and_then(|m| m.child(b"test")).and_then(Chunk::bytes), Some(&[3u8][..])); + assert_eq!(Node::from(&secl).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[3u8][..])); } #[test] @@ -1743,13 +1744,13 @@ mod tests { let btdk = || Chunk { id: *b"LIST", body: crate::container::Body::OpaqueList { list_type: *b"btdk", data: doc.to_vec().into() }, pad: None }; let btds_real = |n: usize| list(b"btds", std::iter::once(list(b"tdbs", vec![])).chain((0..n).map(|_| btdk())).collect()); let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), list(b"tdgp", vec![btds_real(2)])])]); - let view = Layer { chunk: &layer }; + let view = Layer { chunk: Node::from(&layer) }; assert_eq!(view.text_document().map(|d| d.text), Some("H".to_string())); assert!(view.source_text_property().is_some()); assert_eq!(view.text_document_count(), 2); // A record anywhere else in the layer is not its Source Text. let elsewhere = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![list(b"tdgp", vec![btds_real(1)])])])]); - let view = Layer { chunk: &elsewhere }; + let view = Layer { chunk: Node::from(&elsewhere) }; assert!(view.text_document().is_none() && view.source_text_property().is_none()); assert_eq!(view.text_document_count(), 0); } diff --git a/crates/aep/src/container/typed.rs b/crates/aep/src/container/typed.rs index 592a074d5..8cd35aced 100644 --- a/crates/aep/src/container/typed.rs +++ b/crates/aep/src/container/typed.rs @@ -4,42 +4,56 @@ //! //! Only facts that were established from our own one-change-per-file corpus and are checked by //! tests against it are decoded (see `docs/aep-format.md` for the evidence). Everything else -//! stays reachable through the raw [`Chunk`] tree. Views borrow the file and never fail loudly: -//! a chunk that does not have the expected shape yields `None` or is skipped. +//! stays reachable through the raw chunk tree ([`Node`]). Views borrow the file and never fail +//! loudly: a chunk that does not have the expected shape yields `None` or is skipped. +//! +//! A view reads either tree: the owned [`Chunk`](crate::container::Chunk) tree +//! ([`Project::new`]) or the flat [`TreeIndex`] the importer builds ([`Project::from_index`]); +//! both give the same answers. use crate::container::bytes::{be_f64, be_u16, be_u32, text, until_nul}; -use crate::container::chunk::{AepFile, Chunk}; +use crate::container::chunk::AepFile; +use crate::container::index::{Node, Root, TreeIndex}; /// The name a user gave a property or group: the `Utf8` inside its `tdsn` chunk. `None` for the /// sentinel `-_0_/-` ("no custom name") and for empty names. -pub(crate) fn user_name(chunk: &Chunk<'_>) -> Option { - let name = chunk.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; +pub(crate) fn user_name(chunk: Node<'_, '_>) -> Option { + let name = chunk.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (!name.is_empty() && name != "-_0_/-").then_some(name) } /// The content of a `Utf8` child as text (not trimmed). -pub(crate) fn utf8_child(chunk: &Chunk<'_>) -> Option { - chunk.child(b"Utf8").and_then(Chunk::bytes).map(text) +pub(crate) fn utf8_child(chunk: Node<'_, '_>) -> Option { + chunk.child(b"Utf8").and_then(Node::bytes).map(text) } /// The project: the top-level `LIST Fold`, which holds the project items. #[derive(Debug, Clone, Copy)] pub struct Project<'f, 'a> { - pub(crate) fold: &'f Chunk<'a>, + pub(crate) fold: Node<'f, 'a>, /// The top-level `LIST EfdG` (project-level effect definitions), when there is one. - pub(crate) effect_defs: Option<&'f Chunk<'a>>, + pub(crate) effect_defs: Option>, } impl<'f, 'a> Project<'f, 'a> { /// The project view of a file, or `None` when it has no `LIST Fold`. pub fn new(file: &'f AepFile<'a>) -> Option { - file.list(b"Fold").map(|fold| Self { fold, effect_defs: file.list(b"EfdG") }) + Self::of(Root::from(file)) + } + + /// The project view of an indexed file ([`TreeIndex`]), or `None` when it has no `LIST Fold`. + pub fn from_index(index: &'f TreeIndex<'a>) -> Option { + Self::of(Root::from(index)) + } + + pub(crate) fn of(root: Root<'f, 'a>) -> Option { + root.list(b"Fold").map(|fold| Self { fold, effect_defs: root.list(b"EfdG") }) } /// The top-level items (the project panel's root; folders hold their children, see /// [`Item::children`]). pub fn items(&self) -> impl Iterator> + 'f { - self.fold.children().iter().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.fold.children().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// True when the `LIST Fold` could not be read as chunks (damaged): its items cannot be seen. @@ -78,12 +92,12 @@ pub enum ItemKind { /// A project item (`LIST Item`). #[derive(Debug, Clone, Copy)] pub struct Item<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Item<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -106,7 +120,7 @@ impl<'f, 'a> Item<'f, 'a> { /// The items inside a folder (`LIST Sfdr`). pub fn children(&self) -> impl Iterator> + 'f { - self.chunk.child_list(b"Sfdr").into_iter().flat_map(|sfdr| sfdr.children().iter()).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.chunk.child_list(b"Sfdr").into_iter().flat_map(Node::children).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// Whether the item has the `LIST Sfdr` of a folder's contents (After Effects writes an empty @@ -117,7 +131,7 @@ impl<'f, 'a> Item<'f, 'a> { /// True when the folder's `LIST Sfdr` is there but could not be read as chunks (damaged). pub fn folder_contents_unreadable(&self) -> bool { - self.chunk.child_list(b"Sfdr").is_some_and(Chunk::is_opaque_list) + self.chunk.child_list(b"Sfdr").is_some_and(Node::is_opaque_list) } /// The composition view when this item has composition data (`cdta`). @@ -158,7 +172,7 @@ impl TimeBase { /// A composition item (an [`Item`] with `cdta`). #[derive(Debug, Clone, Copy)] pub struct Comp<'f, 'a> { - pub(crate) item: &'f Chunk<'a>, + pub(crate) item: Node<'f, 'a>, } impl<'f, 'a> Comp<'f, 'a> { @@ -172,19 +186,19 @@ impl<'f, 'a> Comp<'f, 'a> { /// The layers, top layer first (`LIST Layr`; the `DLay`/`SLay`/`CLay`/`SecL` lists next to /// them are not layers). pub fn layers(&self) -> impl Iterator> + 'f { - self.item.children().iter().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) + self.item.children().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) } } /// A layer (`LIST Layr`). #[derive(Debug, Clone, Copy)] pub struct Layer<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Layer<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -214,12 +228,12 @@ pub enum PropNode<'f, 'a> { /// A property (`LIST tdbs`). Property(Property<'f, 'a>), /// Any other list that follows a match name (observed: `otst`, the effect instance `sspc`). - Other(&'f Chunk<'a>), + Other(Node<'f, 'a>), } impl<'f, 'a> PropNode<'f, 'a> { /// The node a `LIST` after a match name is. - fn of(c: &'f Chunk<'a>) -> Self { + fn of(c: Node<'f, 'a>) -> Self { if c.is_list(b"tdgp") { PropNode::Group(PropGroup { chunk: c }) } else if c.is_list(b"tdbs") { @@ -242,17 +256,18 @@ pub struct PropEntry<'f, 'a> { /// `ADBE Group End` closes the group and names nothing. #[derive(Debug, Clone, Copy)] pub struct PropGroup<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> PropGroup<'f, 'a> { - /// A group view of a `LIST tdgp` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A group view of a `LIST tdgp` chunk (a [`Chunk`](crate::container::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdgp").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -273,7 +288,7 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes().map(match_name); - } else if c.id == crate::container::chunk::LIST + } else if c.is(&crate::container::chunk::LIST) && let Some(name) = pending.take() { out.push(PropEntry { match_name: name, node: PropNode::of(c) }); @@ -294,7 +309,7 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes(); - } else if c.id == crate::container::chunk::LIST + } else if c.is(&crate::container::chunk::LIST) && let Some(body) = pending.take() && match_name_is(body, name) { @@ -335,17 +350,18 @@ fn match_name_is(body: &[u8], name: &str) -> bool { /// value `cdat` or a keyframe `LIST list`. #[derive(Debug, Clone, Copy)] pub struct Property<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Property<'f, 'a> { - /// A property view of a `LIST tdbs` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A property view of a `LIST tdbs` chunk (a [`Chunk`](crate::container::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdbs").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -453,12 +469,13 @@ pub struct KeyLayout { impl<'f> KeyList<'f> { /// A view of a `LIST list` chunk. `None` when `lhd3` is not 52 bytes or `ldat` is not exactly /// `count * record_size` bytes (a list without `ldat` must have no records). - pub fn from_chunk<'a>(chunk: &'f Chunk<'a>) -> Option { + pub fn from_chunk<'a: 'f>(chunk: impl Into>) -> Option { + let chunk = chunk.into(); if !chunk.is_list(b"list") { return None; } let header = ListHeader::parse(chunk.child(b"lhd3")?.bytes()?)?; - let ldat: &[u8] = chunk.child(b"ldat").and_then(Chunk::bytes).unwrap_or(&[]); + let ldat: &[u8] = chunk.child(b"ldat").and_then(Node::bytes).unwrap_or(&[]); let expected = usize::try_from(header.count).ok()?.checked_mul(usize::try_from(header.record_size).ok()?)?; (ldat.len() == expected).then_some(Self { header, ldat, layout: None }) } @@ -517,6 +534,7 @@ impl<'f> Keyframe<'f> { #[cfg(test)] mod tests { use super::*; + use crate::container::Chunk; fn key_list(count: u32, record_size: u32, ldat: &[u8]) -> Chunk<'static> { let mut lhd3 = vec![0u8; 52]; diff --git a/crates/aep/src/container/values.rs b/crates/aep/src/container/values.rs index 4a001ee1e..ddd5caa75 100644 --- a/crates/aep/src/container/values.rs +++ b/crates/aep/src/container/values.rs @@ -223,7 +223,6 @@ impl<'f, 'a> Layer<'f, 'a> { .child_list(b"otky") .map(|k| { k.children() - .iter() .filter(|c| c.is(b"otda")) .map(|c| { let b = c.bytes()?; diff --git a/crates/aep/src/container/version.rs b/crates/aep/src/container/version.rs index 1c859cdac..407c7046c 100644 --- a/crates/aep/src/container/version.rs +++ b/crates/aep/src/container/version.rs @@ -10,6 +10,7 @@ use std::fmt; use crate::container::bytes::be_u32; use crate::container::chunk::AepFile; +use crate::container::index::{Root, TreeIndex}; /// An After Effects version as packed in `head` / `svap`: `26.2.1x2` is major 26, minor 2, bug-fix /// 1, build 2 (the form of the scripting API's `app.version`). @@ -71,13 +72,35 @@ impl AepFile<'_> { /// version there, and the older version for File ▸ Save a Copy As 25.x / 24.x. `None` when the /// chunk is missing, short or holds no version. pub fn file_version(&self) -> Option { - AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + Root::from(self).file_version() } /// The version of the After Effects that wrote the file: `svap` bytes 0 to 3 (the same word in a /// down-saved copy as in a normal save). `None` when the chunk is missing (After Effects opens such /// a file), short or holds no version. pub fn saved_by_version(&self) -> Option { + Root::from(self).saved_by_version() + } +} + +impl TreeIndex<'_> { + /// [`AepFile::file_version`] of the indexed file. + pub fn file_version(&self) -> Option { + Root::from(self).file_version() + } + + /// [`AepFile::saved_by_version`] of the indexed file. + pub fn saved_by_version(&self) -> Option { + Root::from(self).saved_by_version() + } +} + +impl Root<'_, '_> { + pub(crate) fn file_version(self) -> Option { + AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + } + + pub(crate) fn saved_by_version(self) -> Option { AeVersion::from_word(be_u32(self.chunk(b"svap")?.bytes()?, 0)?) } } diff --git a/crates/aep/src/import.rs b/crates/aep/src/import.rs index 956e3b1c5..8662e182c 100644 --- a/crates/aep/src/import.rs +++ b/crates/aep/src/import.rs @@ -40,6 +40,8 @@ mod time; use effectcraft_project::{ItemId, Project, ProjectSettings}; +use crate::container::index::{Root, TreeIndex}; + pub use effects::supported_effects; pub use report::{AeVersionInfo, ImportReport, Imported, LIMITATIONS, Loss, LossCategory, MAX_DETAILS, TESTED_MAJORS, UNKNOWN}; @@ -113,17 +115,29 @@ pub struct ImportResult { /// [`AepImportError::Refused`] when something would be lost and `allow_losses` is not set. On any /// error `project` is unchanged. pub fn import(project: &mut Project, bytes: &[u8], options: &ImportOptions<'_>) -> Result { - let file = crate::parse_any(bytes)?; - let view = crate::Project::new(&file).ok_or(AepImportError::NoProject)?; + // The importer only reads: an `.aep` is indexed in place (one flat array that borrows the + // bytes); an `.aepx` is read into the chunk tree, which the view reads directly. + if crate::is_aepx(bytes) { + let file = crate::read_aepx(bytes)?; + import_root(project, Root::from(&file), options) + } else { + let index = TreeIndex::parse(bytes)?; + import_root(project, Root::from(&index), options) + } +} + +/// [`import`] of a file read into either tree. +fn import_root(project: &mut Project, file: Root<'_, '_>, options: &ImportOptions<'_>) -> Result { + let view = crate::Project::of(file).ok_or(AepImportError::NoProject)?; // Work on a copy so a refusal leaves the project untouched (compositions are shared, cheap). let mut work = project.clone(); let (folder, comps, items, mut report) = build::run(&mut work, &view, options); - report.ae_version = report::AeVersionInfo::of(&file); + report.ae_version = report::AeVersionInfo::of_root(file); if let Some(line) = report.ae_version.limitation() { report.limitations.push(line); } let mut col = report::Collector::default(); - let settings = settings::map(&file, &work.settings, &mut col); + let settings = settings::map(file, &work.settings, &mut col); for loss in col.finish().losses.into_iter().filter(|_| options.settings) { let at = report.losses.iter().position(|l| l.category > loss.category).unwrap_or(report.losses.len()); report.losses.insert(at, loss); @@ -155,3 +169,74 @@ pub fn analyze_with(bytes: &[u8], options: &ImportOptions<'_>) -> Result { + let a = import_root(&mut Project::default(), Root::from(&file), &options); + let b = import_root(&mut Project::default(), Root::from(&index), &options); + assert!(a == b, "{name}: the import differs through the index"); + } + (Err(a), Err(b)) => assert_eq!(a, b, "{name}"), + (a, b) => panic!("{name}: {:?} vs {:?}", a.err(), b.err()), + } + } + + fn tdmn(name: &str) -> Chunk<'static> { + let mut b = name.as_bytes().to_vec(); + b.resize(40, 0); + Chunk::data(*b"tdmn", b) + } + + #[test] + fn a_synthetic_project_imports_the_same_through_the_index() { + // A composition with a layer whose effect stores arbitrary data and a named value group. + let utf8 = |id: [u8; 4], t: &str| Chunk { id, body: Body::Chunks(vec![Chunk::data(*b"Utf8", t.as_bytes().to_vec())]), pad: None }; + let record = Chunk::list(*b"tdbs", vec![Chunk::data(*b"tdsb", vec![0, 0, 0, 1]), Chunk::data(*b"cdat", vec![0, 0, 0, 1])]); + let values = Chunk::list( + *b"tdgp", + vec![ + utf8(*b"tdsn", "Mine"), + tdmn("ADBE Easy Levels2-0001"), + record, + Chunk::list(*b"aRbs", vec![Chunk::data(*b"aRbp", vec![0u8; 108])]), + tdmn("ADBE Group End"), + ], + ); + let part = Chunk::list(*b"parT", vec![Chunk::data(*b"parn", vec![0, 0, 0, 2]), tdmn("ADBE Easy Levels2-0001"), Chunk::data(*b"pard", vec![0u8; 148])]); + let sspc = Chunk::list(*b"sspc", vec![utf8(*b"fnam", "Levels"), part, values]); + let parade = Chunk::list(*b"tdgp", vec![tdmn("ADBE Easy Levels2"), sspc, tdmn("ADBE Group End")]); + let root = Chunk::list(*b"tdgp", vec![tdmn("ADBE Effect Parade"), parade, tdmn("ADBE Group End")]); + let layer = Chunk::list(*b"Layr", vec![Chunk::data(*b"ldta", vec![0u8; 160]), Chunk::data(*b"Utf8", b"Layer".to_vec()), root]); + let mut idta = vec![0u8; 64]; + idta[..2].copy_from_slice(&4u16.to_be_bytes()); + let mut cdta = vec![0u8; 200]; + cdta[4..12].copy_from_slice(&[0, 0, 0, 1, 0, 0, 0, 30]); + let item = Chunk::list(*b"Item", vec![Chunk::data(*b"idta", idta), Chunk::data(*b"Utf8", b"Comp".to_vec()), Chunk::data(*b"cdta", cdta), layer]); + let file = AepFile { chunks: vec![Chunk::data(*b"head", vec![0u8; 20]), Chunk::list(*b"Fold", vec![item])], trailer: std::borrow::Cow::Borrowed(&[]) }; + both(&write(&file).expect("write"), "synthetic"); + } + + /// Every `.aep` of the fixture directory (skipped without `EFFECTCRAFT_AEP_FIXTURES`). + #[test] + fn every_fixture_imports_the_same_through_the_index() { + let Some(dir) = std::env::var_os("EFFECTCRAFT_AEP_FIXTURES").filter(|d| !d.is_empty()) else { return }; + let Ok(entries) = std::fs::read_dir(dir) else { return }; + let mut paths: Vec = + entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aep")).collect(); + paths.sort(); + for path in paths { + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + both(&bytes, &path.display().to_string()); + } + } +} diff --git a/crates/aep/src/import/markers.rs b/crates/aep/src/import/markers.rs index 93faea90a..7767bfca4 100644 --- a/crates/aep/src/import/markers.rs +++ b/crates/aep/src/import/markers.rs @@ -11,7 +11,8 @@ //! EffectCraft's `timeRemap` property (seconds, keys in layer time) at the top of the layer's //! properties, where the engine's Enable Time Remapping puts it. -use crate::{Chunk, CueType, Layer as AepLayer, Marker as AepMarker, PropNode, Property as AepProp}; +use crate::container::index::Node as AepNode; +use crate::{CueType, Layer as AepLayer, Marker as AepMarker, PropNode, Property as AepProp}; use effectcraft_color::Label; use effectcraft_keyframe::Value; use effectcraft_project::build::Ids; @@ -23,12 +24,12 @@ use crate::import::report::{Collector, LossCategory}; use crate::import::time::TimeMap; /// The time scale of the marker list below `holder` (the layer, or the composition's `SecL`). -fn marker_scale(holder: &Chunk<'_>) -> Option { +fn marker_scale(holder: AepNode<'_, '_>) -> Option { holder.marker_list()?.child_list(b"tdbs").and_then(AepProp::from_chunk)?.time_scale().filter(|s| *s != 0) } /// The markers of a layer (`holder` = the layer chunk) or a composition (`holder` = its `SecL`). -pub(crate) fn import_markers(list: Vec, holder: &Chunk<'_>, what: &str, tm: &TimeMap, col: &mut Collector) -> Vec { +pub(crate) fn import_markers(list: Vec, holder: AepNode<'_, '_>, what: &str, tm: &TimeMap, col: &mut Collector) -> Vec { let scale = marker_scale(holder).unwrap_or(tm.timescale); let mut out: Vec = Vec::with_capacity(list.len()); for m in list { diff --git a/crates/aep/src/import/report.rs b/crates/aep/src/import/report.rs index d7dbfe898..c72fb8528 100644 --- a/crates/aep/src/import/report.rs +++ b/crates/aep/src/import/report.rs @@ -8,6 +8,7 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; use crate::container::AepFile; +use crate::container::index::Root; /// Most example lines kept per loss category (the count is always exact). pub const MAX_DETAILS: usize = 20; @@ -247,6 +248,10 @@ pub const UNKNOWN: &str = "unknown"; impl AeVersionInfo { /// Reads the versions of a parsed file. pub fn of(file: &AepFile<'_>) -> Self { + Self::of_root(Root::from(file)) + } + + pub(crate) fn of_root(file: Root<'_, '_>) -> Self { let file_version = file.file_version(); Self { file_version: file_version.map_or_else(|| UNKNOWN.to_string(), |v| v.text()), diff --git a/crates/aep/src/import/settings.rs b/crates/aep/src/import/settings.rs index 29d520c39..70e245d78 100644 --- a/crates/aep/src/import/settings.rs +++ b/crates/aep/src/import/settings.rs @@ -1,7 +1,8 @@ //! Project settings (colour depth, working space, working gamma, linear-light switches, time //! display, frame count start) from [`crate::ProjectSettings`] (cases 620-634). -use crate::{AepFile, TimeDisplay, WorkingSpace}; +use crate::container::index::{Node, Root}; +use crate::{TimeDisplay, WorkingSpace}; use effectcraft_color::ColorSpace; use effectcraft_project::{BitDepth, ProjectSettings, TimeDisplayStyle}; @@ -22,9 +23,9 @@ pub(crate) fn color_space(name: &str) -> Option { /// holds in an unknown form) keeps the current value. The two linear-light switches are always /// held: After Effects writes their chunk only when on (626, 627), so a file without it has them off. A working space EffectCraft does not have is /// reported. -pub(crate) fn map(file: &AepFile<'_>, current: &ProjectSettings, col: &mut Collector) -> ProjectSettings { - let s = crate::ProjectSettings::read(file); - let nhed = file.chunk(b"nhed").and_then(|c| c.bytes()); +pub(crate) fn map(file: Root<'_, '_>, current: &ProjectSettings, col: &mut Collector) -> ProjectSettings { + let s = crate::ProjectSettings::of(file); + let nhed = file.chunk(b"nhed").and_then(Node::bytes); let mut out = current.clone(); match s.bits_per_channel { Some(8) => out.bit_depth = BitDepth::Bpc8, diff --git a/crates/aep/src/import/shapes.rs b/crates/aep/src/import/shapes.rs index 345346a6f..aa06201ac 100644 --- a/crates/aep/src/import/shapes.rs +++ b/crates/aep/src/import/shapes.rs @@ -394,7 +394,7 @@ fn path(g: &AepGroup<'_, '_>, cx: &mut Cx<'_, '_>) -> Option { return None; }; let shapes: Vec> = - oms.child_list(b"omks").map(|k| k.children().iter().filter(|c| c.is_list(b"shap")).map(AepPath::from_chunk).collect()).unwrap_or_default(); + oms.child_list(b"omks").map(|k| k.children().filter(|c| c.is_list(b"shap")).map(AepPath::from_chunk).collect()).unwrap_or_default(); let first = match shapes.first() { Some(Some(s)) => to_path(s, (1, 1)), Some(None) => Err("its path is unreadable"), diff --git a/crates/aep/tests/it/common/mod.rs b/crates/aep/tests/it/common/mod.rs index 01054ba24..634df7f4c 100644 --- a/crates/aep/tests/it/common/mod.rs +++ b/crates/aep/tests/it/common/mod.rs @@ -132,7 +132,7 @@ pub fn resolve(name: &str) -> Option { /// the little-endian `u32` of each `idpi` chunk (the importer does not read them; the exporter /// writes them). pub fn used_item_ids(item: &effectcraft_aep::Item<'_, '_>) -> Vec { - item.chunk().children().iter().filter(|c| c.is(b"idpi")).filter_map(|c| Some(u32::from_le_bytes(c.bytes()?.get(..4)?.try_into().ok()?))).collect() + item.chunk().children().filter(|c| c.is(b"idpi")).filter_map(|c| Some(u32::from_le_bytes(c.bytes()?.get(..4)?.try_into().ok()?))).collect() } /// The bytes of a case (see the module comment), or `None` when it is missing. diff --git a/crates/aep/tests/it/container.rs b/crates/aep/tests/it/container.rs index 5031a11dc..4fc6d804e 100644 --- a/crates/aep/tests/it/container.rs +++ b/crates/aep/tests/it/container.rs @@ -25,5 +25,6 @@ mod shapes; mod text_fields; mod text_markers_footage; mod text_style_slots; +mod tree_index; mod typed; mod values; diff --git a/crates/aep/tests/it/container/batch3_core.rs b/crates/aep/tests/it/container/batch3_core.rs index 1813e2df8..7b9699e36 100644 --- a/crates/aep/tests/it/container/batch3_core.rs +++ b/crates/aep/tests/it/container/batch3_core.rs @@ -117,8 +117,8 @@ fn a_transform_set_to_its_default_adds_no_record() { assert_eq!(t.opacity_percent.source, ValueSource::Default); assert!(layer.properties().unwrap().find(&["ADBE Transform Group", "ADBE Position"]).is_none()); // The two cases are layers of one file: they differ in the layer id only. - let a = without_layer_id(layer.chunk()); - skip!(with_layer("299-save-noise.aep", L, |_, other| assert_eq!(a, without_layer_id(other.chunk())))); + let a = without_layer_id(&layer.chunk().to_chunk()); + skip!(with_layer("299-save-noise.aep", L, |_, other| assert_eq!(a, without_layer_id(&other.chunk().to_chunk())))); })); } diff --git a/crates/aep/tests/it/container/tree_index.rs b/crates/aep/tests/it/container/tree_index.rs new file mode 100644 index 000000000..e49d641a0 --- /dev/null +++ b/crates/aep/tests/it/container/tree_index.rs @@ -0,0 +1,212 @@ +//! The flat tree index the importer reads ([`TreeIndex`]) against the owned chunk tree, on every +//! file of the fixture directory: the same tree, and the same answers from the typed view. + +use std::fmt::Write as _; + +use effectcraft_aep::container::index::TreeIndex; +use effectcraft_aep::{ + AepFile, Chunk, CryptomatteData, CurvesData, Effect, HueSaturationData, LevelsData, Limits, MeshData, Project, PropGroup, PropNode, is_aepx, parse, + parse_with_limits, read_aepx, write, +}; + +/// Every `.aep` and `.aepx` in the fixture directory (the real projects included), sorted. +fn every_file() -> Vec { + let Some(dir) = crate::common::dir() else { + crate::common::skipped("*.aep"); + return Vec::new(); + }; + let Ok(entries) = std::fs::read_dir(&dir) else { + crate::common::skipped("*.aep"); + return Vec::new(); + }; + let mut out: Vec = + entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aep" || x == "aepx")).collect(); + out.sort(); + out +} + +/// What the typed view reads from a project, as text: items, compositions, layers with their +/// headers, transforms, masks, effects with every parameter value, text, markers, the whole +/// property tree with static values and key records, settings and versions. +fn dump(view: &Project<'_, '_>) -> String { + let mut s = String::new(); + let defs = view.effect_definition_index(); + for item in view.all_items() { + let _ = writeln!(s, "item {:?} {:?} {:?} {:?} {:?}", item.kind(), item.name(), item.display_name(), item.id(), item.label()); + let _ = writeln!(s, " footage {:?} {:?} {:?}", item.footage(), item.sequence_info(), item.conform_frame_rate()); + let _ = writeln!(s, " folder {} {}", item.has_folder_contents(), item.folder_contents_unreadable()); + let Some(comp) = item.comp() else { continue }; + let _ = writeln!(s, " comp {:?} {:?} {:?}", comp.time_base(), comp.header(), comp.markers()); + for layer in comp.layers() { + let _ = writeln!(s, " layer {:?} {:?} {:?} {:?}", layer.name(), layer.id(), layer.header(), layer.track_matte()); + let _ = writeln!(s, " transform {:?}", layer.transform(1920, 1080)); + let _ = writeln!(s, " time remap {:?} orientation {:?} {:?}", layer.time_remap(), layer.orientation(), layer.orientation_keys()); + let _ = writeln!(s, " masks {:?}", layer.mask_atoms().iter().map(|m| (&m.mask, &m.path_shapes)).collect::>()); + let _ = writeln!(s, " shapes {:?}", layer.shape_groups()); + let _ = + writeln!(s, " text {:?} {} {:?}", layer.text_document(), layer.text_document_count(), layer.source_text_property().map(|p| p.is_animated())); + let _ = writeln!(s, " markers {:?} source {:?}", layer.markers(), view.layer_source(&layer).map(|i| i.id())); + for fx in layer.effects() { + let _ = + writeln!(s, " fx {} {:?} {:?} {:?} {:?}", fx.match_name(), fx.display_name(), fx.enabled(), defs.source(&fx), fx.value_record_names()); + let _ = writeln!(s, " names {:?} {:?} {:?} {:?}", fx.name(), fx.user_name(), fx.declared_param_count(), fx.has_own_definitions()); + for p in defs.params(&fx) { + let _ = writeln!(s, " {} {:?} {:?} {:?} {:?}", p.match_name, p.kind, p.name, p.items, p.value()); + arbitrary(&fx, &p.match_name, &mut s); + } + for name in fx.value_record_names() { + arbitrary(&fx, &name, &mut s); + } + } + if let Some(root) = layer.properties() { + tree(&root, 0, &mut s); + } + } + } + s +} + +/// The arbitrary data stored for a parameter and what each decoder reads from it. +fn arbitrary(fx: &Effect<'_, '_>, name: &str, s: &mut String) { + let Some(data) = fx.arbitrary_data(name) else { return }; + let _ = writeln!(s, " arbitrary {name} {data:?}"); + let _ = writeln!(s, " {:?} {:?}", HueSaturationData::parse(data), LevelsData::parse(data)); + let _ = writeln!(s, " {:?} {:?} {:?}", CurvesData::parse(data), MeshData::parse(data), CryptomatteData::parse(data)); +} + +fn tree(group: &PropGroup<'_, '_>, depth: usize, s: &mut String) { + if depth > 32 { + return; + } + let _ = writeln!(s, "{:depth$}group {:?} {:?}", "", group.user_name(), group.flags()); + for e in group.entries() { + match e.node { + PropNode::Group(g) => tree(&g, depth + 1, s), + PropNode::Property(p) => { + let keys: Option>> = p.keys().map(|k| k.records().map(<[u8]>::to_vec).collect()); + let _ = writeln!(s, "{:depth$} {} {:?} {:?} {:?} {:?}", "", e.match_name, p.user_name(), p.header(), p.static_value(), keys); + } + PropNode::Other(c) => { + let _ = writeln!(s, "{:depth$} {} other {:?} {}", "", e.match_name, c.list_type(), c.descendants().count()); + } + } + } +} + +#[test] +fn the_index_is_the_parsed_tree_and_the_view_reads_the_same_on_every_file() { + for path in every_file() { + let name = path.display().to_string(); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{name}: {e}")); + // `.aepx` is read into the chunk tree and indexed from it; `.aep` is indexed in place. + // A file that does not read (a damaged sample) must fail the index the same way. + let file: AepFile<'_> = match if is_aepx(&bytes) { read_aepx(&bytes) } else { parse(&bytes) } { + Ok(file) => file, + Err(e) => { + if !is_aepx(&bytes) { + assert_eq!(TreeIndex::parse(&bytes).err(), Some(e), "{name}"); + } + continue; + } + }; + let index = (if is_aepx(&bytes) { TreeIndex::from_file(&file) } else { TreeIndex::parse(&bytes) }).unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(index.to_file() == file, "{name}: the index is not the parsed tree"); + assert_eq!(index.len(), file.walk().count(), "{name}"); + assert_eq!(index.file_version(), file.file_version(), "{name}"); + assert_eq!(index.saved_by_version(), file.saved_by_version(), "{name}"); + assert_eq!(effectcraft_aep::ProjectSettings::read_index(&index), effectcraft_aep::ProjectSettings::read(&file), "{name}"); + let (Some(a), Some(b)) = (Project::new(&file), Project::from_index(&index)) else { + assert!(Project::new(&file).is_none() && Project::from_index(&index).is_none(), "{name}"); + continue; + }; + let (da, db) = (dump(&a), dump(&b)); + assert!(da == db, "{name}: the view reads differently through the index"); + } +} + +#[test] +fn the_index_and_the_parser_fail_alike_under_tight_limits() { + for path in every_file().into_iter().filter(|p| p.extension().is_some_and(|x| x == "aep")) { + let name = path.display().to_string(); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{name}: {e}")); + let total = parse(&bytes).map(|f| f.walk().count()).unwrap_or(0); + for limits in [ + Limits { max_depth: 2, max_chunks: 10_000_000 }, + Limits { max_depth: 6, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: total }, + Limits { max_depth: 64, max_chunks: total.saturating_sub(1) }, + Limits { max_depth: 64, max_chunks: total / 2 }, + ] { + let old = parse_with_limits(&bytes, limits); + let new = TreeIndex::parse_with_limits(&bytes, limits); + match (old, new) { + (Ok(f), Ok(i)) => assert!(i.to_file() == f, "{name}: {limits:?}"), + (Err(a), Err(b)) => assert_eq!(a, b, "{name}: {limits:?}"), + (a, b) => panic!("{name}: {limits:?}: {:?} vs {:?}", a.err(), b.err()), + } + } + } +} + +/// A 40-byte NUL-padded match name. +fn tdmn(name: &str) -> Chunk<'static> { + let mut b = name.as_bytes().to_vec(); + b.resize(40, 0); + Chunk::data(*b"tdmn", b) +} + +fn utf8(id: [u8; 4], text: &str) -> Chunk<'static> { + Chunk { id, body: effectcraft_aep::Body::Chunks(vec![Chunk::data(*b"Utf8", text.as_bytes().to_vec())]), pad: None } +} + +/// A project with one composition, one layer and one effect whose five parameters carry arbitrary +/// data in the sizes the decoders read (Hue/Saturation, Levels, Curves, a 2 x 2 mesh, Cryptomatte), +/// so the arbitrary-data path runs without the fixture directory. +fn synthetic() -> Vec { + let mut mesh = vec![0u8; 180]; + mesh[..8].copy_from_slice(&[0, 0, 0, 2, 0, 0, 0, 2]); + let mut cry = vec![0u8; 200]; + cry[..4].copy_from_slice(b"cry1"); + cry[32..37].copy_from_slice(b"Layer"); + let mut levels = vec![0u8; 108]; + levels[8..16].copy_from_slice(&1.0f64.to_be_bytes()); + let payloads: [Vec; 5] = [vec![7u8; 180], levels, vec![0u8; CurvesData::SIZE], mesh, cry]; + let (mut part, mut values) = + (vec![Chunk::data(*b"parn", 6u32.to_be_bytes().to_vec())], vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), utf8(*b"tdsn", "Mine")]); + for (i, data) in payloads.into_iter().enumerate() { + let name = format!("FX-{:04}", i + 1); + let mut pard = vec![0u8; 148]; + pard[12..16].copy_from_slice(&11u32.to_be_bytes()); + part.extend([tdmn(&name), Chunk::data(*b"pard", pard)]); + let record = Chunk::list(*b"tdbs", vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), Chunk::data(*b"cdat", vec![0, 0, 0, 1])]); + values.extend([tdmn(&name), record, Chunk::list(*b"aRbs", vec![Chunk::data(*b"aRbp", data)])]); + } + values.push(tdmn("ADBE Group End")); + let sspc = Chunk::list(*b"sspc", vec![utf8(*b"fnam", "Effect"), Chunk::list(*b"parT", part), Chunk::list(*b"tdgp", values)]); + let parade = Chunk::list(*b"tdgp", vec![tdmn("FX"), sspc, tdmn("ADBE Group End")]); + let root = Chunk::list(*b"tdgp", vec![tdmn("ADBE Effect Parade"), parade, tdmn("ADBE Group End")]); + let layer = Chunk::list(*b"Layr", vec![Chunk::data(*b"ldta", vec![0u8; 160]), Chunk::data(*b"Utf8", b"Layer".to_vec()), root]); + let mut idta = vec![0u8; 64]; + idta[..2].copy_from_slice(&4u16.to_be_bytes()); + idta[16..20].copy_from_slice(&2u32.to_be_bytes()); + let mut cdta = vec![0u8; 200]; + cdta[4..12].copy_from_slice(&[0, 0, 0, 1, 0, 0, 0, 30]); + let item = Chunk::list(*b"Item", vec![Chunk::data(*b"idta", idta), Chunk::data(*b"Utf8", b"Comp".to_vec()), Chunk::data(*b"cdta", cdta), layer]); + let file = AepFile { chunks: vec![Chunk::data(*b"head", vec![0u8; 20]), Chunk::list(*b"Fold", vec![item])], trailer: std::borrow::Cow::Borrowed(&[]) }; + write(&file).expect("write") +} + +#[test] +fn the_view_reads_arbitrary_effect_data_the_same_through_the_index() { + let bytes = synthetic(); + let file = parse(&bytes).expect("parse"); + let index = TreeIndex::parse(&bytes).expect("index"); + let (a, b) = (Project::new(&file).expect("project"), Project::from_index(&index).expect("project")); + let (da, db) = (dump(&a), dump(&b)); + assert_eq!(da, db); + // The synthetic effect really reaches every decoder (the dump is not empty of them). + assert_eq!(da.matches("arbitrary FX-").count(), 10, "{da}"); + assert!(da.contains("Some(HueSaturationData") && da.contains("Some(LevelsData") && da.contains("Some(CurvesData"), "{da}"); + assert!(da.contains("Some(MeshData") && da.contains("Some(CryptomatteData"), "{da}"); + assert!(da.contains("Some(\"Mine\")"), "the user name is read: {da}"); +} diff --git a/crates/aep/tests/it/container/typed.rs b/crates/aep/tests/it/container/typed.rs index 381f86796..e85391e0f 100644 --- a/crates/aep/tests/it/container/typed.rs +++ b/crates/aep/tests/it/container/typed.rs @@ -111,7 +111,7 @@ fn match_names_come_from_the_tdmn_chunks() { assert!(!names.iter().any(|n| n == "ADBE Group End")); assert!(root.find(&["ADBE Transform Group", "ADBE Nonexistent"]).is_none()); // The raw tdmn body is 40 bytes, NUL padded. - let raw = root.chunk().children().iter().find(|c| c.is(b"tdmn")).unwrap(); + let raw = root.chunk().children().find(|c| c.is(b"tdmn")).unwrap(); assert_eq!(raw.bytes().unwrap().len(), 40); } @@ -150,8 +150,8 @@ fn effect_parameter_value() { assert!(instance.is_list(b"sspc")); // The effect's name is in a nested fnam chunk. let fnam = instance.child(b"fnam").unwrap(); - assert!(matches!(fnam.body, Body::Chunks(_))); - assert_eq!(fnam.children()[0].bytes(), Some(&b"Slider Control"[..])); + assert!(matches!(fnam.to_chunk().body, Body::Chunks(_))); + assert_eq!(fnam.children().next().unwrap().bytes(), Some(&b"Slider Control"[..])); // The parameters are a property group inside the instance. let params = PropGroup::from_chunk(instance.child_list(b"tdgp").unwrap()).unwrap(); assert_eq!(params.names(), ["ADBE Slider Control-0000", "ADBE Slider Control-0001", "ADBE Effect Built In Params"]); diff --git a/crates/aep/tests/it/export/effects.rs b/crates/aep/tests/it/export/effects.rs index 7a684705a..51b0226a6 100644 --- a/crates/aep/tests/it/export/effects.rs +++ b/crates/aep/tests/it/export/effects.rs @@ -390,7 +390,7 @@ fn static_records(file: &AepFile<'_>) -> std::collections::BTreeMap<(String, usi for (li, layer) in comp.layers().enumerate() { for (ei, e) in layer.effects().into_iter().enumerate() { let Some(spec) = effectcraft_aep::effect_map::by_match_name(e.match_name()) else { continue }; - let Some(values) = e.chunk().child_list(b"tdgp") else { continue }; + let Some(values) = e.chunk().child_list(b"tdgp").map(|v| v.to_chunk()) else { continue }; let k = values.children(); for (i, c) in k.iter().enumerate() { let Some(name) = diff --git a/crates/aep/tests/it/export/objects.rs b/crates/aep/tests/it/export/objects.rs index f9de6e836..692c87630 100644 --- a/crates/aep/tests/it/export/objects.rs +++ b/crates/aep/tests/it/export/objects.rs @@ -493,7 +493,7 @@ fn shape_records(bytes: &[u8]) -> std::collections::BTreeMap<(usize, String), ef for item in view.all_items() { let Some(comp) = item.comp() else { continue }; for (li, layer) in comp.layers().enumerate() { - if let Some(g) = layer.chunk().child_list(b"tdgp") { + if let Some(g) = layer.chunk().child_list(b"tdgp").map(|g| g.to_chunk()) { walk(g.children(), "", li, false, &mut out); } } @@ -618,7 +618,7 @@ fn keyed_shape_records(bytes: &[u8]) -> Vec<(String, Vec, Vec, Vec, for item in view.all_items() { let Some(comp) = item.comp() else { continue }; for layer in comp.layers() { - if let Some(g) = layer.chunk().child_list(b"tdgp") { + if let Some(g) = layer.chunk().child_list(b"tdgp").map(|g| g.to_chunk()) { walk(g.children(), "", false, &mut out); } } @@ -853,7 +853,8 @@ fn a_camera_transform_group_ends_with_its_group_end() { let view = effectcraft_aep::Project::new(&file).expect("project"); let comp = view.all_items().into_iter().find_map(|i| i.comp()).expect("comp"); let layer = comp.layers().next().expect("camera"); - let root = layer.chunk().child_list(b"tdgp").expect("tree").children(); + let tree = layer.chunk().child_list(b"tdgp").expect("tree").to_chunk(); + let root = tree.children(); let name = |c: &effectcraft_aep::Chunk<'_>| String::from_utf8_lossy(c.bytes().unwrap_or_default().split(|b| *b == 0).next().unwrap_or_default()).into_owned(); let at = root.iter().position(|c| c.is(b"tdmn") && name(c) == "ADBE Transform Group").expect("transform"); diff --git a/crates/aep/tests/it/export/styles.rs b/crates/aep/tests/it/export/styles.rs index ee3524a36..a3fca75ca 100644 --- a/crates/aep/tests/it/export/styles.rs +++ b/crates/aep/tests/it/export/styles.rs @@ -53,7 +53,7 @@ fn stored_group(bytes: &[u8]) -> Vec { { // As far as the exporter writes records (`decoded`). let mut f2 = effectcraft_aep::AepFile { - chunks: vec![crate::export::helpers::decoded(&g.chunk().clone().into_owned())], + chunks: vec![crate::export::helpers::decoded(&g.chunk().to_chunk().into_owned())], trailer: std::borrow::Cow::Owned(vec![]), }; f2.trailer = std::borrow::Cow::Owned(vec![]); diff --git a/crates/aep/tests/it/import/combined.rs b/crates/aep/tests/it/import/combined.rs index e6897e004..476c57a95 100644 --- a/crates/aep/tests/it/import/combined.rs +++ b/crates/aep/tests/it/import/combined.rs @@ -67,7 +67,7 @@ fn graft<'a>(file: &mut AepFile<'a>, pair: &[Chunk<'a>]) { /// target's project copy does not hold the donor's types. fn own_definitions<'a>(pair: &mut [Chunk<'a>], donor: &AepFile<'_>) { let defs: Vec<(String, Chunk<'static>)> = effectcraft_aep::Project::new(donor) - .map(|p| p.effect_definitions().into_iter().map(|(m, part)| (m, part.clone().into_owned())).collect()) + .map(|p| p.effect_definitions().into_iter().map(|(m, part)| (m, part.to_chunk().into_owned())).collect()) .unwrap_or_default(); let Some(effectcraft_aep::Body::List { children, .. }) = pair.get_mut(1).map(|g| &mut g.body) else { return }; let mut name: Option = None;