diff --git a/crates/aep/README.md b/crates/aep/README.md index 965bea317..db415a026 100644 --- a/crates/aep/README.md +++ b/crates/aep/README.md @@ -21,7 +21,9 @@ tree; pad bytes, odd bodies and the bytes after the container, an XMP packet, ar The XML form of a project (`.aepx`) is read into the same tree by `read_aepx` (a small bounded XML tokenizer, no dependency); `parse_any` picks the reader by content, so the typed view and the -importer take either form. The mapping is in `docs/aep-format.md`, section 1. +importer take either form. The mapping is in `docs/aep-format.md`, section 1. The importer, which +only reads, does not build the owned tree: it reads either form through the flat index +`TreeIndex`, an `.aep` in place and an `.aepx` decoded into one buffer first. Bodies that look like lists but are not chunk sequences (for example `LIST btdk`, raw text) stay opaque rather than failing the parse. Input is untrusted: sizes are checked against the remaining diff --git a/crates/aep/src/container.rs b/crates/aep/src/container.rs index 43b1cf627..1cd0fd2b7 100644 --- a/crates/aep/src/container.rs +++ b/crates/aep/src/container.rs @@ -37,6 +37,7 @@ mod values; mod version; mod write; +pub(crate) use aepx::{AepxDrafts, index_aepx}; pub use aepx::{is_aepx, parse_any, read_aepx, read_aepx_with_limits}; pub use arbitrary::{ CURVE_IDENTITY, CryptomatteData, CurvesData, ExtractorChannels, HueSaturationData, LevelsData, MeshData, curve_lut, identifier_channel_type, diff --git a/crates/aep/src/container/aepx.rs b/crates/aep/src/container/aepx.rs index 6ca729807..c4d2865ff 100644 --- a/crates/aep/src/container/aepx.rs +++ b/crates/aep/src/container/aepx.rs @@ -12,13 +12,23 @@ //! instructions. A document type declaration is refused (no entity expansion). It runs without //! recursion; the nesting depth and the element count are bounded by [`Limits`], and everything //! it allocates is bounded by the size of the input. +//! +//! There are two readers. [`read_aepx`] builds the owned tree element by element +//! (`aepx/tree.rs`). The importer, which only reads, uses `index_aepx`: it decodes every body +//! (hex, text, the alias JSON, the numbers) into one buffer and records the chunks as entries of +//! the flat index ([`Draft`]s, in depth-first order, bodies as ranges of the buffer), which +//! [`crate::container::TreeIndex::from_drafts`] turns into an index that borrows the buffer. +//! The tests hold both to the same tree and the same errors. use std::borrow::Cow; -use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::container::chunk::{AepFile, LIST, NESTED_IDS}; use crate::container::error::AepError; +use crate::container::index::{Draft, Kind as EntryKind}; use crate::container::read::{Limits, parse}; +mod tree; + /// The root element of an `.aepx`. const ROOT: &[u8] = b"AfterEffectsProject"; /// The element that holds the XMP packet (the trailer of an `.aep`). @@ -68,17 +78,59 @@ pub fn read_aepx(data: &[u8]) -> Result, AepError> { /// # Errors /// See [`read_aepx`]. pub fn read_aepx_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + tree::read(data, limits) +} + +/// An `.aepx` read for the flat index: every body decoded into one buffer, and the chunks as +/// [`Draft`]s in depth-first order with their bodies as ranges of that buffer. +/// [`crate::container::TreeIndex::from_drafts`] makes the index, which borrows the buffer. +pub(crate) struct AepxDrafts { + /// The decoded bodies (hex, text, alias JSON, numbers) and the XMP packet. + pub(crate) buffer: Vec, + pub(crate) nodes: Vec, + /// The XMP packet: a range of `buffer` (empty without one). + pub(crate) trailer: (usize, usize), +} + +/// Reads an `.aepx` into [`AepxDrafts`]: the tree [`read_aepx_with_limits`] gives, with the same +/// errors, without one allocation per chunk. +/// +/// # Errors +/// See [`read_aepx_with_limits`]. When memory runs out, two errors come up in more cases than +/// there: [`AepError::TooLarge`] when the chunk list cannot grow (the owned-tree reader would +/// abort), and [`AepError::Xml`] "bdata is too large to hold in memory" also when the shared body +/// buffer cannot grow (the owned-tree reader reserves each value on its own). +pub(crate) fn index_aepx(data: &[u8], limits: Limits) -> Result { let text = match std::str::from_utf8(data) { Ok(t) => t.as_bytes(), Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), }; - Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() + // Every counted element takes at least 3 bytes (`` in a document cut short), so there are + // never more chunks than this. + let bound = (text.len() / 3).min(limits.max_chunks); + let mut nodes = Vec::new(); + let _ = nodes.try_reserve_exact(bound.min(INITIAL_SLOTS)); + // Hex is most of an `.aepx` and decodes to half its length: one buffer of that size usually + // holds everything. A failed reservation is not an error here; the buffer grows as it goes. + let mut buffer = Vec::new(); + let _ = buffer.try_reserve_exact(text.len() / 2); + let reader = Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), root_closed: false, trailer: None, buffer, nodes, bound, attrs: Vec::new() }; + reader.run() } +/// The slots the chunk list starts with; it doubles from there (as the array of +/// [`crate::container::TreeIndex`] does). +const INITIAL_SLOTS: usize = 1024; + fn xml(offset: usize, what: &'static str) -> AepError { AepError::Xml { offset, what } } +/// The pad byte of a body of `len` bytes, as [`crate::container::Chunk::data`] sets it. +fn pad_of(len: usize) -> Option { + if len % 2 == 1 { Some(0) } else { None } +} + /// What an open element becomes when it closes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Kind { @@ -102,13 +154,14 @@ struct Frame { at: usize, /// The name as written, to match the end tag. name: (usize, usize), - /// The decoded `bdata` attribute. - bdata: Option>, + /// The chunk's entry in the list (for `Chunk`, `Utf8` and `Alias`). + entry: Option, + /// The buffer length when the element opened: its body starts here. + mark: usize, + /// The end of the decoded `bdata` attribute in the buffer. + bdata_end: Option, /// The attributes of a `fileReference`, in document order. attrs: Vec<(String, String)>, - /// Collected text (only for `string` and the XMP packet). - text: Vec, - children: Vec>, } struct Reader<'d> { @@ -117,13 +170,19 @@ struct Reader<'d> { limits: Limits, count: usize, stack: Vec, - /// The top-level chunks once the root element has closed. - top: Option>>, - trailer: Option>, + /// True once the root element has closed. + root_closed: bool, + trailer: Option<(usize, usize)>, + buffer: Vec, + nodes: Vec, + /// The most chunks the input can hold under the limits: `nodes` never reserves past it. + bound: usize, + /// The attributes of the current start tag; kept to reuse its allocation. + attrs: Vec<((usize, usize), Cow<'d, [u8]>)>, } -impl Reader<'_> { - fn run(mut self) -> Result, AepError> { +impl<'d> Reader<'d> { + fn run(mut self) -> Result { if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { self.pos = 3; } @@ -137,11 +196,13 @@ impl Reader<'_> { if let Some(open) = self.stack.last() { return Err(xml(open.at, "an element is not closed before the end of the file")); } - let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; - Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + if !self.root_closed { + return Err(xml(self.data.len(), "no AfterEffectsProject root element")); + } + Ok(AepxDrafts { buffer: self.buffer, nodes: self.nodes, trailer: self.trailer.unwrap_or((0, 0)) }) } - fn rest(&self) -> &[u8] { + fn rest(&self) -> &'d [u8] { self.data.get(self.pos..).unwrap_or(&[]) } @@ -181,14 +242,13 @@ impl Reader<'_> { self.pos = start; let end = self.skip_past(b"]]>", "a CDATA section is not closed")?; let content = self.data.get(start..end).unwrap_or(&[]); - let at = start; - match self.stack.last_mut() { + match self.stack.last() { Some(frame) => { if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { - frame.text.extend_from_slice(content); + self.buffer.extend_from_slice(content); } } - None => return Err(xml(at, "CDATA outside the root element")), + None => return Err(xml(start, "CDATA outside the root element")), } } else if rest.starts_with(b" { self.pos = self.pos.saturating_add(n); } - fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + fn slice(&self, (a, b): (usize, usize)) -> &'d [u8] { self.data.get(a..b).unwrap_or(&[]) } - fn start_tag(&mut self) -> Result<(), AepError> { - let at = self.pos; - self.pos = self.pos.saturating_add(1); - let name = self.name()?; - let mut attrs = Vec::new(); - let self_closing = loop { + /// Reads the attributes of a start tag into `attrs`; true when the tag closes itself. + fn attributes(&mut self, at: usize, attrs: &mut Vec<((usize, usize), Cow<'d, [u8]>)>) -> Result { + loop { self.skip_ws(); let rest = self.rest(); if rest.starts_with(b"/>") { self.pos = self.pos.saturating_add(2); - break true; + return Ok(true); } if rest.starts_with(b">") { self.pos = self.pos.saturating_add(1); - break false; + return Ok(false); } if rest.is_empty() { return Err(xml(at, "a start tag is not closed")); @@ -270,7 +327,16 @@ impl Reader<'_> { }; self.pos = vend.saturating_add(1); attrs.push((key, value)); - }; + } + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs = std::mem::take(&mut self.attrs); + attrs.clear(); + let self_closing = self.attributes(at, &mut attrs)?; let kind = self.kind_of(name, at)?; if kind != Kind::Root { self.count = self.count.saturating_add(1); @@ -282,14 +348,25 @@ impl Reader<'_> { return Err(AepError::TooDeep); } } - let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; - if matches!(kind, Kind::Chunk(_)) { - if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { - frame.bdata = Some(decode_hex(value, key.1)?); + let mark = self.buffer.len(); + let mut frame = Frame { kind, at, name, entry: None, mark, bdata_end: None, attrs: Vec::new() }; + match kind { + Kind::Chunk(id) => { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + decode_hex_into(value, key.1, MAX_BDATA, &mut self.buffer)?; + frame.bdata_end = Some(self.buffer.len()); + } + frame.entry = Some(self.open_entry(id)?); } - } else if kind == Kind::Alias { - frame.attrs = attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + Kind::Utf8 => frame.entry = Some(self.open_entry(*b"Utf8")?), + Kind::Alias => { + frame.attrs = + attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + frame.entry = Some(self.open_entry(*b"alas")?); + } + Kind::Root | Kind::Xmp | Kind::Skip => {} } + self.attrs = attrs; self.stack.push(frame); if self_closing { self.close()?; @@ -297,12 +374,38 @@ impl Reader<'_> { Ok(()) } + /// Appends the entry of a chunk whose element just opened; its body is filled in when it + /// closes. The list grows by its length (at least 16) but never past [`Self::bound`]; an + /// allocation the target cannot make is an error, not an abort. + fn open_entry(&mut self, id: [u8; 4]) -> Result { + let at = self.nodes.len(); + // This entry and the one after it must have `u32` indices. + u32::try_from(at.checked_add(1).ok_or(AepError::TooManyChunks)?).map_err(|_| AepError::TooManyChunks)?; + if at == self.nodes.capacity() { + // Past the bound (which only a wrong guess could reach) it keeps doubling. + let room = self.bound.saturating_sub(at); + let more = if room == 0 { at.max(16) } else { at.max(16).min(room) }; + self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?; + } + self.nodes.push(Draft::placeholder(id)); + Ok(at) + } + + /// Fills in the entry at `at` once its element has closed; its subtree is what follows it. + fn finish(&mut self, at: usize, id: [u8; 4], kind: EntryKind, list_type: [u8; 4], body: (usize, usize), pad: Option) -> Result<(), AepError> { + let end = u32::try_from(self.nodes.len()).map_err(|_| AepError::TooManyChunks)?; + if let Some(d) = self.nodes.get_mut(at) { + *d = Draft { start: body.0, len: body.1.saturating_sub(body.0), id, list_type, end, kind, pad }; + } + Ok(()) + } + /// Classifies an element by its name and where it is. fn kind_of(&self, name: (usize, usize), at: usize) -> Result { let n = self.slice(name); let parent = match self.stack.last() { None => { - if self.top.is_some() { + if self.root_closed { return Err(xml(at, "a second root element")); } if n != ROOT { @@ -346,39 +449,102 @@ impl Reader<'_> { self.close() } - /// Closes the innermost element and hands its chunk to the parent. + /// Closes the innermost element and fills in its chunk's entry. fn close(&mut self) -> Result<(), AepError> { let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; - let chunk = match frame.kind { - Kind::Root => { - self.top = Some(frame.children); - return Ok(()); - } - Kind::Skip => return Ok(()), + match frame.kind { + Kind::Root => self.root_closed = true, + Kind::Skip => {} Kind::Xmp => { + // Only the first packet is kept; a later one is dropped from the buffer (its + // children are skipped, so nothing else was written after it opened). if self.trailer.is_none() { - self.trailer = Some(frame.text); + self.trailer = Some((frame.mark, self.buffer.len())); + } else { + self.buffer.truncate(frame.mark); } - return Ok(()); } - Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), - Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), - Kind::Chunk(id) => chunk_of(id, frame)?, - }; - if let Some(parent) = self.stack.last_mut() { - parent.children.push(chunk); + Kind::Utf8 => { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + let end = self.buffer.len(); + self.finish(at, *b"Utf8", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?; + } + Kind::Alias => { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + self.buffer.extend_from_slice(alias_json(&frame.attrs).as_bytes()); + let end = self.buffer.len(); + self.finish(at, *b"alas", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?; + } + Kind::Chunk(id) => self.close_chunk(id, &frame)?, } Ok(()) } + /// The chunk of a closed element named by a chunk id (what the owned reader's `chunk_of` + /// built): `bdata` wins over children, `numS` / `ppSn` hold their number, a nested id holds its + /// children (or nothing), any other name is a `LIST` of that type. + fn close_chunk(&mut self, id: [u8; 4], frame: &Frame) -> Result<(), AepError> { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + let after = at.checked_add(1).ok_or(AepError::TooManyChunks)?; + if let Some(bdata_end) = frame.bdata_end { + // The children were read (and counted, and checked) but are not part of the chunk. + self.nodes.truncate(after); + self.buffer.truncate(bdata_end); + let body = (frame.mark, bdata_end); + let pad = pad_of(bdata_end.saturating_sub(frame.mark)); + if OPAQUE_LIST_TYPES.contains(&id) { + return self.finish(at, LIST, EntryKind::OpaqueList, id, body, pad); + } + return self.finish(at, id, EntryKind::Data, [0; 4], body, pad); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let missing = || xml(frame.at, "a numS or ppSn element without its number"); + // Exactly one child chunk, a `Utf8`: the entry after this one, whose subtree runs to the end. + let only = self.nodes.get(after).copied().filter(|c| usize::try_from(c.end).ok() == Some(self.nodes.len())).ok_or_else(missing)?; + if only.id != *b"Utf8" { + return Err(missing()); + } + let text = match only.kind { + EntryKind::Data | EntryKind::OpaqueList => only.start.checked_add(only.len).and_then(|end| self.buffer.get(only.start..end)).unwrap_or(&[]), + EntryKind::List | EntryKind::Nested => &[], + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body: [u8; 8]; + let bytes: &[u8] = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + body = u64::from(n as u32).to_be_bytes(); + body.get(4..).unwrap_or(&[]) + } else { + body = text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes(); + &body + }; + self.nodes.truncate(after); + self.buffer.truncate(frame.mark); + self.buffer.extend_from_slice(bytes); + let end = self.buffer.len(); + return self.finish(at, id, EntryKind::Data, [0; 4], (frame.mark, end), pad_of(bytes.len())); + } + let empty = (frame.mark, frame.mark); + if NESTED_IDS.contains(&id) { + if self.nodes.len() == after { + return self.finish(at, id, EntryKind::Data, [0; 4], empty, None); + } + return self.finish(at, id, EntryKind::Nested, [0; 4], empty, None); + } + self.finish(at, LIST, EntryKind::List, id, empty, None) + } + fn text(&mut self) -> Result<(), AepError> { let start = self.pos; let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); let end = start.saturating_add(len); self.pos = end; let raw = self.data.get(start..end).unwrap_or(&[]); - match self.stack.last_mut() { - Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + match self.stack.last() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut self.buffer), Some(_) => Ok(()), None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), None => Err(xml(start, "text outside the root element")), @@ -399,41 +565,6 @@ fn chunk_id(name: &[u8]) -> Option<[u8; 4]> { Some(id) } -/// The chunk of a closed element named by a chunk id. -fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { - if let Some(bytes) = frame.bdata { - if OPAQUE_LIST_TYPES.contains(&id) { - let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; - return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); - } - return Ok(Chunk::data(id, bytes)); - } - // Numbers written as text: `0` and `150.000000000`. - if &id == b"numS" || &id == b"ppSn" { - let text = match frame.children.as_slice() { - [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), - _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), - }; - let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); - let body = if &id == b"numS" { - // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. - let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); - let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; - (n as u32).to_be_bytes().to_vec() - } else { - text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() - }; - return Ok(Chunk::data(id, body)); - } - if NESTED_IDS.contains(&id) { - if frame.children.is_empty() { - return Ok(Chunk::data(id, Vec::new())); - } - return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); - } - Ok(Chunk::list(id, frame.children)) -} - /// Appends `raw` with the predefined entities and character references decoded. fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { let mut i = 0; @@ -471,7 +602,7 @@ fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), A Ok(()) } -/// Decodes a `bdata` value (hex digits, two per byte). +/// Decodes a `bdata` value (hex digits, two per byte) into a new `Vec` of exactly its size. fn decode_hex(text: &[u8], offset: usize) -> Result, AepError> { decode_hex_at_most(text, offset, MAX_BDATA) } @@ -482,35 +613,63 @@ const MAX_BDATA: usize = u32::MAX as usize; /// [`decode_hex`] with the decoded-size limit as a parameter (so tests can reach it). fn decode_hex_at_most(text: &[u8], offset: usize, max: usize) -> Result, AepError> { - if text.len() % 2 == 1 { - return Err(xml(offset, "bdata has an odd number of hex digits")); - } + // Checked before anything is reserved, then sized once (collecting into `Option` has no + // size hint and regrows step by step), and without aborting when the memory is not there. + let pairs = check_hex(text, offset, max)?; + let mut out = Vec::new(); + out.try_reserve_exact(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; + decode_checked(text, offset, &mut out)?; + Ok(out) +} + +/// Appends the bytes of a `bdata` value (hex digits, two per byte, at most `max` bytes) to `out`. +/// On an error `out` is unchanged. +fn decode_hex_into(text: &[u8], offset: usize, max: usize, out: &mut Vec) -> Result<(), AepError> { + let pairs = check_hex(text, offset, max)?; + // Grown at most once per value (amortised over a shared buffer), and without aborting when the + // memory is not there. + out.try_reserve(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; + decode_checked(text, offset, out) +} + +/// Appends the bytes of a `bdata` value that [`check_hex`] accepted. On an error (a digit the +/// check let through, which it does not) `out` is unchanged. +fn decode_checked(text: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { let digit = |b: u8| match b { b'0'..=b'9' => Some(b - b'0'), b'a'..=b'f' => Some(b - b'a' + 10), b'A'..=b'F' => Some(b - b'A' + 10), _ => None, }; - let (pairs, _) = text.as_chunks::<2>(); - if pairs.len() > max { + let mark = out.len(); + for [hi, lo] in text.as_chunks::<2>().0 { + match (digit(*hi), digit(*lo)) { + (Some(h), Some(l)) => out.push((h << 4) | l), + _ => { + out.truncate(mark); + return Err(xml(offset, "bdata holds a character that is not a hex digit")); + } + } + } + Ok(()) +} + +/// Checks a `bdata` value before anything is reserved for it: an even number of hex digits, at +/// most `max` bytes. Returns the number of bytes it decodes to. +fn check_hex(text: &[u8], offset: usize, max: usize) -> Result { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let pairs = text.len() / 2; + if pairs > max { return Err(AepError::TooLarge); } - // Nothing is reserved for a value that does not decode: one cheap pass checks every digit first. - // Blocks without an early exit inside, so the check vectorises; it stops at the first bad block. + // One cheap pass checks every digit first. Blocks without an early exit inside, so the check + // vectorises; it stops at the first bad block. if !text.chunks(4096).all(|block| block.iter().fold(true, |ok, b| ok & b.is_ascii_hexdigit())) { return Err(xml(offset, "bdata holds a character that is not a hex digit")); } - // Sized once (collecting into `Option` has no size hint and regrows step by step), and - // without aborting when the memory is not there. - let mut out = Vec::new(); - out.try_reserve_exact(pairs.len()).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; - for [hi, lo] in pairs { - match (digit(*hi), digit(*lo)) { - (Some(h), Some(l)) => out.push((h << 4) | l), - _ => return Err(xml(offset, "bdata holds a character that is not a hex digit")), - } - } - Ok(out) + Ok(pairs) } /// The JSON body of an `alas` chunk from the attributes of a `fileReference`, keys in document @@ -558,6 +717,7 @@ fn json_str(out: &mut String, s: &str) { #[cfg(test)] mod tests { use super::*; + use crate::container::index::TreeIndex; fn doc(body: &str) -> String { format!("\n{body}\n") @@ -753,4 +913,292 @@ mod tests { let many = format!("{}", "".repeat(10)); assert_eq!(read_aepx_with_limits(many.as_bytes(), Limits { max_depth: 64, max_chunks: 9 }), Err(AepError::TooManyChunks)); } + + /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number. + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: usize) -> usize { + if n == 0 { 0 } else { usize::try_from(self.next() % n as u64).unwrap() } + } + + /// One of `items`. + fn pick<'s>(&mut self, items: &[&'s str]) -> &'s str { + items[self.below(items.len())] + } + } + + /// `n` random bytes as hex, upper or lower case. + fn hex(rng: &mut Rng, n: usize) -> String { + let upper = rng.below(4) == 0; + (0..n).map(|_| if upper { format!("{:02X}", rng.below(256)) } else { format!("{:02x}", rng.below(256)) }).collect() + } + + /// Random `.aepx` content: every element form the reader knows (data, lists, nested ids, + /// strings, numbers, `btdk`, file references, skipped elements, XMP packets), `bdata` next to + /// children, bodies that read as chunk sequences, entities, CDATA, comments and stray text. + fn content(rng: &mut Rng, depth: usize, budget: &mut usize, out: &mut String) { + for _ in 0..rng.below(6) { + if *budget == 0 { + return; + } + *budget -= 1; + match rng.below(20) { + 0..=3 => { + let id = rng.pick(&["cdat", "tdmn", "head", "Utf8", "idta", "LIST", "a"]); + let n = rng.below(9); + out.push_str(&format!("<{id} bdata=\"{}\"/>", hex(rng, n))); + } + 4..=6 if depth < 10 => { + let id = rng.pick(&["Fold", "tdgp", "Layr", "Pin", "LIST", "btdk", "sspc"]); + let attr = if rng.below(5) == 0 { format!(" bdata=\"{}\"", hex(rng, 3)) } else { String::new() }; + out.push_str(&format!("<{id}{attr}>")); + content(rng, depth + 1, budget, out); + out.push_str(&format!("")); + } + 7 => { + let id = rng.pick(&["tdsn", "pdnm", "fnam"]); + match rng.below(4) { + 0 => out.push_str(&format!("<{id}/>")), + 1 => out.push_str(&format!("<{id}>name & A")), + // A body that is itself a chunk sequence (`Utf8`, 1 byte, pad): still data. + 2 => out.push_str(&format!("<{id} bdata=\"557466380000000161{}\"/>", if rng.below(2) == 0 { "00" } else { "" })), + _ => out.push_str(&format!("<{id}>ab")), + } + } + 8 => { + let text = rng.pick(&["plain", "A < B", "", "ab", "", "été", "xgoney", "\r\n"]); + out.push_str(&format!("{text}")); + } + 9 => { + let id = rng.pick(&["numS", "ppSn"]); + let inner = rng.pick(&[ + "0", + " 7 ", + "-2147483648", + "150.000000000", + "x", + "", + "12", + "", + "", + "34", + ]); + let attr = if rng.below(6) == 0 { " bdata=\"00000002\"" } else { "" }; + out.push_str(&format!("<{id}{attr}>{inner}")); + } + 10 => { + let n = rng.pick(&["", "41", "414243", "4c495354000000000000"]); + out.push_str(&format!("")); + } + 11 => { + let platform = rng.pick(&["Win", "1", "Mac"]); + out.push_str(&format!( + "", + rng.below(3) + )); + } + 12 if depth < 10 => { + out.push_str("text"); + content(rng, depth + 1, budget, out); + out.push_str(""); + } + 13 => { + let packet = rng.pick(&["x]]>", "plain & text", ""]); + out.push_str(&format!("{packet}")); + } + 14 => out.push_str(rng.pick(&["", "", "\n ", "stray"])), + _ => { + let n = rng.below(5); + out.push_str(&format!("", hex(rng, n))); + } + } + } + } + + /// The default limits and six tight ones (depth 0, 1 and 3; 0, 7 and 40 chunks). + fn limit_sets() -> [Limits; 7] { + [ + Limits { max_depth: 0, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 0 }, + Limits::default(), + Limits { max_depth: 1, max_chunks: 10_000_000 }, + Limits { max_depth: 3, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 7 }, + Limits { max_depth: 64, max_chunks: 40 }, + ] + } + + /// The index reader and the owned-tree reader give the same `Result` (the tree, or the error), + /// and the index walks like the index of the owned tree. + fn same(bytes: &[u8], limits: Limits) { + let old = read_aepx_with_limits(bytes, limits); + let new = index_aepx(bytes, limits).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned()); + assert_eq!(new, old, "{}", String::from_utf8_lossy(bytes)); + if let Ok(file) = &old { + let AepxDrafts { buffer, nodes, trailer } = index_aepx(bytes, limits).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + let expected = TreeIndex::from_file(file).unwrap(); + assert_eq!(index.len(), expected.len()); + assert_eq!(index.trailer(), expected.trailer()); + let a: Vec = index.walk().map(|(d, n)| format!("{d} {n:?}")).collect(); + let b: Vec = expected.walk().map(|(d, n)| format!("{d} {n:?}")).collect(); + assert_eq!(a, b); + } + } + + /// The two readers give the same tree and errors on generated and damaged documents. + #[test] + fn the_two_readers_agree_on_generated_and_damaged_documents() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let noise = b"<>/=\"'&;#x0aZ \n![]-?"; + for _ in 0..600 { + let mut budget = 80; + let mut body = String::new(); + content(&mut rng, 0, &mut budget, &mut body); + let x = doc(&body); + for l in limit_sets() { + same(x.as_bytes(), l); + } + for _ in 0..4 { + let mut bad = x.clone().into_bytes(); + for _ in 0..1 + rng.below(3) { + let at = rng.below(bad.len()); + bad[at] = noise[rng.below(noise.len())]; + } + for l in limit_sets() { + same(&bad, l); + } + } + let cut = rng.below(x.len()); + same(&x.as_bytes()[..cut], Limits::default()); + // Bytes that are not UTF-8, and a multi-byte character cut in half. + let mut bad = x.clone().into_bytes(); + let at = rng.below(bad.len()); + bad.insert(at, 0xff); + same(&bad, Limits::default()); + let open = format!("{}é", x.trim_end().trim_end_matches("")).into_bytes(); + same(&open[..open.len() - 1], Limits::default()); + } + } + + /// The flat chunk list never reserves more slots than the input or the chunk limit allows. + #[test] + fn the_chunk_list_is_bounded_by_the_input_and_the_limit() { + let many = format!("{}", "".repeat(10_000)); + let d = index_aepx(many.as_bytes(), Limits::default()).unwrap(); + assert_eq!(d.nodes.len(), 10_000); + assert!(d.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", d.nodes.capacity()); + // The buffer is reserved once, at half the input, and holds nothing here. + assert!(d.buffer.is_empty() && d.buffer.capacity() <= many.len() / 2); + // At the limit the list never holds more slots than the limit. + let d = index_aepx(many.as_bytes(), Limits { max_depth: 64, max_chunks: 10_000 }).unwrap(); + assert!(d.nodes.capacity() <= 10_000); + let few = format!("{}", "".repeat(100)); + let d = index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 100 }).unwrap(); + assert!(d.nodes.capacity() <= 100, "{} slots for at most 100 chunks", d.nodes.capacity()); + assert_eq!(index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 99 }).err(), Some(AepError::TooManyChunks)); + // A tiny input cannot hold more than a third of its bytes in elements. + let tiny = doc(""); + assert!(index_aepx(tiny.as_bytes(), Limits::default()).unwrap().nodes.capacity() <= tiny.len() / 3); + } + + /// What a closed element drops is gone from the buffer and the chunk list, but was counted. + #[test] + fn dropped_content_leaves_nothing_behind_but_still_counts() { + // `bdata` wins over the children: they are read, counted and dropped. + let x = doc("xyzq"); + let d = index_aepx(x.as_bytes(), Limits::default()).unwrap(); + assert_eq!((d.buffer.as_slice(), d.nodes.len()), (&[1u8][..], 1)); + assert_eq!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 4 }).err(), Some(AepError::TooManyChunks)); + assert!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 5 }).is_ok()); + // A child's error is raised although the child would be dropped. + assert!(matches!(index_aepx(doc("").as_bytes(), Limits::default()), Err(AepError::Xml { .. }))); + // Only the first XMP packet stays in the buffer. + let packets = doc(&"packet".repeat(1000)); + let d = index_aepx(packets.as_bytes(), Limits::default()).unwrap(); + assert_eq!((d.buffer.as_slice(), d.trailer), (&b"packet"[..], (0, 6))); + // A number keeps its 4 or 8 bytes, not its text. + let d = index_aepx(doc(" 7 1.5").as_bytes(), Limits::default()).unwrap(); + assert_eq!(d.buffer, [&[0, 0, 0, 7][..], &1.5f64.to_be_bytes()[..]].concat()); + assert_eq!(d.nodes.len(), 2); + } + + /// Bodies keep what the element says they are, even where reading the same bytes as an `.aep` + /// would decide otherwise: a nested id whose `bdata` is a chunk sequence stays data, an empty + /// `btdk` stays an opaque list, an empty list at the deepest level stays a list. + #[test] + fn bodies_keep_the_kind_the_element_gives_them() { + let x = doc(""); + let AepxDrafts { buffer, nodes, trailer } = index_aepx(x.as_bytes(), Limits::default()).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + let top: Vec<_> = index.chunks().collect(); + assert_eq!(top.len(), 3); + assert_eq!(top[0].bytes().map(<[u8]>::len), Some(11)); + assert!(top[0].children().next().is_none()); + assert!(top[1].is_opaque_list() && top[1].bytes() == Some(&[][..])); + assert!(top[2].is_opaque_list() && top[2].bytes().map(<[u8]>::len) == Some(12)); + // 64 levels: the innermost (empty) list is a list, not opaque. + let deep = format!("{}{}", "".repeat(64), "".repeat(64)); + let AepxDrafts { buffer, nodes, trailer } = index_aepx(deep.as_bytes(), Limits::default()).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64)); + assert!(index.walk().all(|(_, n)| n.is_list(b"Fold") && !n.is_opaque_list())); + same(x.as_bytes(), Limits::default()); + same(deep.as_bytes(), Limits::default()); + } + + /// Hostile input is refused by both readers with the same errors. + #[test] + fn hostile_input_is_refused_alike_by_both_readers() { + let hostile = [ + format!("{}{}", "".repeat(100_000), "".repeat(100_000)), + format!("{}", "".repeat(1000)), + doc(&format!("", "00".repeat(100_000))), + doc(&format!("", "00".repeat(100_000))), + doc("4294967296"), + doc("1e400x"), + doc("�"), + doc("�"), + doc(&format!("{}", "".repeat(1000))), + format!("{}", "".repeat(10)), + " = + entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aepx")).collect(); + paths.sort(); + for path in paths { + let bytes = std::fs::read(&path).unwrap(); + for l in limit_sets() { + let index = index_aepx(&bytes, l).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned()); + assert_eq!(index, read_aepx_with_limits(&bytes, l), "{}", path.display()); + } + } + } } diff --git a/crates/aep/src/container/aepx/tree.rs b/crates/aep/src/container/aepx/tree.rs new file mode 100644 index 000000000..ba7e20ed5 --- /dev/null +++ b/crates/aep/src/container/aepx/tree.rs @@ -0,0 +1,361 @@ +//! The `.aepx` reader of the owned chunk tree ([`super::read_aepx`]): it builds the tree element by +//! element. The importer uses the index reader in `aepx.rs` instead; the tests compare the two on +//! generated, damaged and corpus input (the same tree and the same errors). + +use std::borrow::Cow; + +use super::{FILE_REFERENCE, OPAQUE_LIST_TYPES, ROOT, STRING, XMP, alias_json, chunk_id, decode_entities, decode_hex, xml}; +use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::container::error::AepError; +use crate::container::read::Limits; + +/// [`super::read_aepx_with_limits`]. +pub(super) fn read(data: &[u8], limits: Limits) -> Result, AepError> { + let text = match std::str::from_utf8(data) { + Ok(t) => t.as_bytes(), + Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), + }; + Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() +} + +/// What an open element becomes when it closes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + /// The root element: its children are the top-level chunks. + Root, + /// A chunk with this id (or a `LIST` with this list type). + Chunk([u8; 4]), + /// A `Utf8` chunk. + Utf8, + /// An `alas` chunk built from the attributes. + Alias, + /// The XMP packet (only directly below the root). + Xmp, + /// An element without a chunk counterpart (a name that is no chunk id); skipped with its content. + Skip, +} + +struct Frame { + kind: Kind, + /// Offset of the start tag (for errors). + at: usize, + /// The name as written, to match the end tag. + name: (usize, usize), + /// The decoded `bdata` attribute. + bdata: Option>, + /// The attributes of a `fileReference`, in document order. + attrs: Vec<(String, String)>, + /// Collected text (only for `string` and the XMP packet). + text: Vec, + children: Vec>, +} + +struct Reader<'d> { + data: &'d [u8], + pos: usize, + limits: Limits, + count: usize, + stack: Vec, + /// The top-level chunks once the root element has closed. + top: Option>>, + trailer: Option>, +} + +impl Reader<'_> { + fn run(mut self) -> Result, AepError> { + if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { + self.pos = 3; + } + while self.pos < self.data.len() { + if self.rest().starts_with(b"<") { + self.markup()?; + } else { + self.text()?; + } + } + if let Some(open) = self.stack.last() { + return Err(xml(open.at, "an element is not closed before the end of the file")); + } + let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; + Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + } + + fn rest(&self) -> &[u8] { + self.data.get(self.pos..).unwrap_or(&[]) + } + + /// The offset of `needle` at or after `from`. + fn find(&self, from: usize, needle: &[u8]) -> Option { + let hay = self.data.get(from..)?; + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + let first = *needle.first()?; + let mut i = 0; + while let Some(p) = hay.get(i..)?.iter().position(|b| *b == first) { + let at = i.checked_add(p)?; + if hay.get(at..)?.starts_with(needle) { + return from.checked_add(at); + } + i = at.checked_add(1)?; + } + None + } + + /// Skips to just after `close`, which must follow. + fn skip_past(&mut self, close: &[u8], what: &'static str) -> Result { + let at = self.find(self.pos, close).ok_or(xml(self.pos, what))?; + self.pos = at.saturating_add(close.len()); + Ok(at) + } + + fn markup(&mut self) -> Result<(), AepError> { + let rest = self.rest(); + if rest.starts_with(b"", "a processing instruction is not closed")?; + } else if rest.starts_with(b"", "a comment is not closed")?; + } else if rest.starts_with(b"", "a CDATA section is not closed")?; + let content = self.data.get(start..end).unwrap_or(&[]); + let at = start; + match self.stack.last_mut() { + Some(frame) => { + if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { + frame.text.extend_from_slice(content); + } + } + None => return Err(xml(at, "CDATA outside the root element")), + } + } else if rest.starts_with(b" bool { + !(b.is_ascii_whitespace() || matches!(b, b'<' | b'>' | b'/' | b'=' | b'"' | b'\'' | b'&')) + } + + fn name(&mut self) -> Result<(usize, usize), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| !Self::is_name_byte(*b)).unwrap_or(self.rest().len()); + if len == 0 { + return Err(xml(start, "a tag or attribute has no name")); + } + self.pos = start.saturating_add(len); + Ok((start, self.pos)) + } + + fn skip_ws(&mut self) { + let n = self.rest().iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(self.rest().len()); + self.pos = self.pos.saturating_add(n); + } + + fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + self.data.get(a..b).unwrap_or(&[]) + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs = Vec::new(); + let self_closing = loop { + self.skip_ws(); + let rest = self.rest(); + if rest.starts_with(b"/>") { + self.pos = self.pos.saturating_add(2); + break true; + } + if rest.starts_with(b">") { + self.pos = self.pos.saturating_add(1); + break false; + } + if rest.is_empty() { + return Err(xml(at, "a start tag is not closed")); + } + let key = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b"=") { + return Err(xml(self.pos, "an attribute has no value")); + } + self.pos = self.pos.saturating_add(1); + self.skip_ws(); + let quote = *self.rest().first().ok_or(xml(self.pos, "a start tag is not closed"))?; + if quote != b'"' && quote != b'\'' { + return Err(xml(self.pos, "an attribute value is not quoted")); + } + let vstart = self.pos.saturating_add(1); + let vend = self.find(vstart, &[quote]).ok_or(xml(vstart, "an attribute value is not closed"))?; + let raw = self.data.get(vstart..vend).unwrap_or(&[]); + if raw.contains(&b'<') { + return Err(xml(vstart, "`<` inside an attribute value")); + } + // A value without `&` is its own decoding (most of them, every `bdata` After Effects + // writes): borrow it instead of copying it. + let value = if raw.contains(&b'&') { + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + Cow::Owned(value) + } else { + Cow::Borrowed(raw) + }; + self.pos = vend.saturating_add(1); + attrs.push((key, value)); + }; + let kind = self.kind_of(name, at)?; + if kind != Kind::Root { + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + // The root's children are depth 1, like the top-level chunks of an `.aep`. + if self.stack.len() > self.limits.max_depth { + return Err(AepError::TooDeep); + } + } + let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; + if matches!(kind, Kind::Chunk(_)) { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + frame.bdata = Some(decode_hex(value, key.1)?); + } + } else if kind == Kind::Alias { + frame.attrs = attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + } + self.stack.push(frame); + if self_closing { + self.close()?; + } + Ok(()) + } + + /// Classifies an element by its name and where it is. + fn kind_of(&self, name: (usize, usize), at: usize) -> Result { + let n = self.slice(name); + let parent = match self.stack.last() { + None => { + if self.top.is_some() { + return Err(xml(at, "a second root element")); + } + if n != ROOT { + return Err(xml(at, "the root element is not AfterEffectsProject")); + } + return Ok(Kind::Root); + } + Some(p) => p.kind, + }; + Ok(match parent { + Kind::Skip | Kind::Utf8 | Kind::Xmp | Kind::Alias => Kind::Skip, + Kind::Root | Kind::Chunk(_) => { + if n == STRING { + Kind::Utf8 + } else if n == FILE_REFERENCE { + Kind::Alias + } else if n == XMP && parent == Kind::Root { + Kind::Xmp + } else if let Some(id) = chunk_id(n) { + Kind::Chunk(id) + } else { + Kind::Skip + } + } + }) + } + + fn end_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(2); + let name = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b">") { + return Err(xml(at, "an end tag is not closed")); + } + self.pos = self.pos.saturating_add(1); + let open = self.stack.last().ok_or(xml(at, "an end tag without a start tag"))?; + if self.slice(open.name) != self.slice(name) { + return Err(xml(at, "an end tag does not match its start tag")); + } + self.close() + } + + /// Closes the innermost element and hands its chunk to the parent. + fn close(&mut self) -> Result<(), AepError> { + let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; + let chunk = match frame.kind { + Kind::Root => { + self.top = Some(frame.children); + return Ok(()); + } + Kind::Skip => return Ok(()), + Kind::Xmp => { + if self.trailer.is_none() { + self.trailer = Some(frame.text); + } + return Ok(()); + } + Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), + Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), + Kind::Chunk(id) => chunk_of(id, frame)?, + }; + if let Some(parent) = self.stack.last_mut() { + parent.children.push(chunk); + } + Ok(()) + } + + fn text(&mut self) -> Result<(), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); + let end = start.saturating_add(len); + self.pos = end; + let raw = self.data.get(start..end).unwrap_or(&[]); + match self.stack.last_mut() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + Some(_) => Ok(()), + None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), + None => Err(xml(start, "text outside the root element")), + } + } +} + +/// The chunk of a closed element named by a chunk id. +fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { + if let Some(bytes) = frame.bdata { + if OPAQUE_LIST_TYPES.contains(&id) { + let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; + return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); + } + return Ok(Chunk::data(id, bytes)); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let text = match frame.children.as_slice() { + [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), + _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + (n as u32).to_be_bytes().to_vec() + } else { + text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() + }; + return Ok(Chunk::data(id, body)); + } + if NESTED_IDS.contains(&id) { + if frame.children.is_empty() { + return Ok(Chunk::data(id, Vec::new())); + } + return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); + } + Ok(Chunk::list(id, frame.children)) +} diff --git a/crates/aep/src/container/index.rs b/crates/aep/src/container/index.rs index 30d7fcb9d..695d1e9f3 100644 --- a/crates/aep/src/container/index.rs +++ b/crates/aep/src/container/index.rs @@ -20,6 +20,10 @@ //! [`AepError::TooLarge`] when the array cannot grow (a failed `try_reserve`), and //! [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. //! +//! An `.aepx` is indexed by its own reader (`container/aepx.rs`), which decodes every body into +//! one buffer and fills the entries as it reads; the index then borrows that buffer +//! (`TreeIndex::from_drafts`). It gives the tree [`crate::container::read_aepx`] gives. +//! //! [`Node`] is the handle the typed view ([`crate::container::Project`] and the types below it) //! reads through: a chunk of either tree. The view works the same on both. @@ -33,7 +37,7 @@ use crate::container::read::{Limits, container_end, id4}; /// What an entry's body is (the [`Body`] variants). #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Kind { +pub(crate) enum Kind { Data, /// [`Body::Chunks`]. Nested, @@ -63,6 +67,28 @@ impl<'a> Entry<'a> { } } +/// An entry of an index whose bytes are still being written (the `.aepx` reader decodes into a +/// buffer that grows, so it keeps positions, not slices): the body is `start..start + len` of +/// that buffer. The fields are those of [`Entry`]; [`TreeIndex::from_drafts`] turns the list +/// into an index once the buffer is final. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Draft { + pub(crate) start: usize, + pub(crate) len: usize, + pub(crate) id: [u8; 4], + pub(crate) list_type: [u8; 4], + pub(crate) end: u32, + pub(crate) kind: Kind, + pub(crate) pad: Option, +} + +impl Draft { + /// A data chunk with no body yet. + pub(crate) fn placeholder(id: [u8; 4]) -> Self { + Self { start: 0, len: 0, id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None } + } +} + /// The chunk tree of a file as one flat, read-only array (see the module documentation). #[derive(Clone)] pub struct TreeIndex<'a> { @@ -112,8 +138,19 @@ impl<'a> TreeIndex<'a> { Ok(Self { nodes: builder.nodes, trailer }) } - /// Indexes an already parsed (or built, or `.aepx`) chunk tree. The index borrows the file's - /// bytes. + /// The index of the entries `drafts` (in depth-first order, see [`Draft`]) whose bodies and + /// trailer are ranges of `buffer`. A range outside `buffer` reads as empty. The list is turned + /// into entries; a [`Draft`] and an [`Entry`] have the same size and alignment, so the standard + /// library reuses the list's allocation for them. + pub(crate) fn from_drafts(buffer: &'a [u8], drafts: Vec, trailer: (usize, usize)) -> Self { + let slice = |start: usize, len: usize| start.checked_add(len).and_then(|end| buffer.get(start..end)).unwrap_or(&[]); + let nodes = + drafts.into_iter().map(|d| Entry { data: slice(d.start, d.len), id: d.id, list_type: d.list_type, end: d.end, kind: d.kind, pad: d.pad }).collect(); + let trailer = buffer.get(trailer.0..trailer.1).unwrap_or(&[]); + Self { nodes, trailer } + } + + /// Indexes an already parsed (or built) chunk tree. The index borrows the file's bytes. /// /// # Errors /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. diff --git a/crates/aep/src/import.rs b/crates/aep/src/import.rs index 4c7767daa..be93920ac 100644 --- a/crates/aep/src/import.rs +++ b/crates/aep/src/import.rs @@ -1,6 +1,6 @@ //! After Effects project (`.aep`) import: the typed view of [`effectcraft_aep`] mapped into an -//! EffectCraft [`Project`]. The XML form (`.aepx`) is accepted as well: it is read into the same -//! chunk tree ([`crate::parse_any`] picks the reader by content). +//! EffectCraft [`Project`]. The XML form (`.aepx`) is accepted as well: both are read into the same +//! flat chunk index (an `.aep` in place, an `.aepx` decoded into one buffer first). //! //! What arrives: the project item tree (folders), project settings (depth, working space, time //! display), compositions (size, exact frame rate, duration, work area, background, pixel @@ -43,6 +43,7 @@ mod time; use effectcraft_project::{ItemId, Project, ProjectSettings}; use crate::container::index::{Root, TreeIndex}; +use crate::container::{AepxDrafts, Limits, index_aepx}; pub use effects::supported_effects; pub use report::{AeVersionInfo, ImportReport, Imported, LIMITATIONS, Loss, LossCategory, MAX_DETAILS, TESTED_MAJORS, UNKNOWN}; @@ -118,10 +119,11 @@ pub struct ImportResult { /// error `project` is unchanged. pub fn import(project: &mut Project, bytes: &[u8], options: &ImportOptions<'_>) -> Result { // The importer only reads: an `.aep` is indexed in place (one flat array that borrows the - // bytes); an `.aepx` is read into the chunk tree, which the view reads directly. + // bytes); an `.aepx` is decoded into one buffer, which the index borrows. if crate::is_aepx(bytes) { - let file = crate::read_aepx(bytes)?; - import_root(project, Root::from(&file), options) + let AepxDrafts { buffer, nodes, trailer } = index_aepx(bytes, Limits::default())?; + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + import_root(project, Root::from(&index), options) } else { let index = TreeIndex::parse(bytes)?; import_root(project, Root::from(&index), options) @@ -243,4 +245,45 @@ mod tests { both(&bytes, &path.display().to_string()); } } + + /// Imports an `.aepx` through the owned tree ([`crate::read_aepx`]) and through the index + /// ([`import`]): the same project, report and settings, or the same error. + fn both_aepx(bytes: &[u8], name: &str) -> Result { + let exists = |_: &str| true; + let options = ImportOptions { allow_losses: true, ..ImportOptions::new(&exists) }; + let a = crate::read_aepx(bytes).map_err(AepImportError::from).and_then(|file| import_root(&mut Project::default(), Root::from(&file), &options)); + let b = import(&mut Project::default(), bytes, &options); + assert!(a == b, "{name}: the .aepx import differs through the index"); + b + } + + /// A synthetic `.aepx` imports the same through the index as through the owned tree. + #[test] + fn a_synthetic_aepx_imports_the_same_through_the_index() { + // The comp of the owned-tree test above (`idta` type 4, `cdta` 1/30 s frame duration). + let idta = format!("0004{}", "00".repeat(62)); + let cdta = format!("00000000000000010000001e{}", "00".repeat(188)); + let x = format!( + "Compx", + "00".repeat(20) + ); + let imported = both_aepx(x.as_bytes(), "synthetic").expect("the synthetic project imports"); + assert_eq!(imported.comps.len(), 1); + assert_eq!(both_aepx(b"", "empty").err(), Some(AepImportError::NoProject)); + assert!(matches!(both_aepx(b"", "broken"), Err(AepImportError::Container(crate::AepError::Xml { .. })))); + } + + /// Every `.aepx` of the fixture directory (skipped without `EFFECTCRAFT_AEP_FIXTURES`). + #[test] + fn every_fixture_aepx_imports_the_same_through_the_index() { + let Some(dir) = std::env::var_os("EFFECTCRAFT_AEP_FIXTURES").filter(|d| !d.is_empty()) else { return }; + let Ok(entries) = std::fs::read_dir(dir) else { return }; + let mut paths: Vec = + entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aepx")).collect(); + paths.sort(); + for path in paths { + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + let _ = both_aepx(&bytes, &path.display().to_string()); + } + } } diff --git a/docs/aep-format.md b/docs/aep-format.md index 13971c1d5..21ba31ae1 100644 --- a/docs/aep-format.md +++ b/docs/aep-format.md @@ -119,6 +119,11 @@ counterpart (a name that is not 1 to 4 characters) are skipped with their conten with the byte offset. It runs without recursion; the nesting depth (`Limits::max_depth`) and the element count (`Limits::max_chunks`) are bounded, and everything it allocates is bounded by the size of the input. +- There are two readers. `read_aepx` builds the owned tree element by element. The importer, which + only reads, uses a second reader that decodes every body (hex, text, the alias JSON, the + numbers) into one buffer and records the chunks as a flat depth-first list whose bodies are + ranges of that buffer; it indexes that buffer (`TreeIndex`) as it indexes an `.aep`. The tests + hold both readers to the same tree and the same errors. - Tests: `tests/aepx.rs` compares the five fixture pairs chunk by chunk and byte by byte, compares the typed view of both forms, and runs truncated, bit-flipped, deeply nested and oversized inputs; `crates/aep/tests/it/import/aepx.rs` imports both forms of each pair and expects the same