Merge remote-tracking branch 'origin/aep-audit-3300' into aep-import-bounds-3299

crates/aep/README.md

@@ -21,7 +21,9 @@ tree; pad bytes, odd bodies and the bytes after the container, an XMP packet, ar
 
 The XML form of a project (`.aepx`) is read into the same tree by `read_aepx` (a small bounded
 XML tokenizer, no dependency); `parse_any` picks the reader by content, so the typed view and the
-importer take either form. The mapping is in `docs/aep-format.md`, section 1.
+importer take either form. The mapping is in `docs/aep-format.md`, section 1. The importer, which
+only reads, does not build the owned tree: it reads either form through the flat index
+`TreeIndex`, an `.aep` in place and an `.aepx` decoded into one buffer first.
 
 Bodies that look like lists but are not chunk sequences (for example `LIST btdk`, raw text) stay
 opaque rather than failing the parse. Input is untrusted: sizes are checked against the remaining

crates/aep/src/container.rs

@@ -37,6 +37,7 @@ mod values;
 mod version;
 mod write;
 
+pub(crate) use aepx::{AepxDrafts, index_aepx};
 pub use aepx::{is_aepx, parse_any, read_aepx, read_aepx_with_limits};
 pub use arbitrary::{
     CURVE_IDENTITY, CryptomatteData, CurvesData, ExtractorChannels, HueSaturationData, LevelsData, MeshData, curve_lut, identifier_channel_type,

crates/aep/src/container/aepx.rs

@@ -12,13 +12,23 @@
 //! instructions. A document type declaration is refused (no entity expansion). It runs without
 //! recursion; the nesting depth and the element count are bounded by [`Limits`], and everything
 //! it allocates is bounded by the size of the input.
+//!
+//! There are two readers. [`read_aepx`] builds the owned tree element by element
+//! (`aepx/tree.rs`). The importer, which only reads, uses `index_aepx`: it decodes every body
+//! (hex, text, the alias JSON, the numbers) into one buffer and records the chunks as entries of
+//! the flat index ([`Draft`]s, in depth-first order, bodies as ranges of the buffer), which
+//! [`crate::container::TreeIndex::from_drafts`] turns into an index that borrows the buffer.
+//! The tests hold both to the same tree and the same errors.
 
 use std::borrow::Cow;
 
-use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS};
+use crate::container::chunk::{AepFile, LIST, NESTED_IDS};
 use crate::container::error::AepError;
+use crate::container::index::{Draft, Kind as EntryKind};
 use crate::container::read::{Limits, parse};
 
+mod tree;
+
 /// The root element of an `.aepx`.
 const ROOT: &[u8] = b"AfterEffectsProject";
 /// The element that holds the XMP packet (the trailer of an `.aep`).
@@ -68,17 +78,59 @@ pub fn read_aepx(data: &[u8]) -> Result<AepFile<'static>, AepError> {
 /// # Errors
 /// See [`read_aepx`].
 pub fn read_aepx_with_limits(data: &[u8], limits: Limits) -> Result<AepFile<'static>, AepError> {
+    tree::read(data, limits)
+}
+
+/// An `.aepx` read for the flat index: every body decoded into one buffer, and the chunks as
+/// [`Draft`]s in depth-first order with their bodies as ranges of that buffer.
+/// [`crate::container::TreeIndex::from_drafts`] makes the index, which borrows the buffer.
+pub(crate) struct AepxDrafts {
+    /// The decoded bodies (hex, text, alias JSON, numbers) and the XMP packet.
+    pub(crate) buffer: Vec<u8>,
+    pub(crate) nodes: Vec<Draft>,
+    /// The XMP packet: a range of `buffer` (empty without one).
+    pub(crate) trailer: (usize, usize),
+}
+
+/// Reads an `.aepx` into [`AepxDrafts`]: the tree [`read_aepx_with_limits`] gives, with the same
+/// errors, without one allocation per chunk.
+///
+/// # Errors
+/// See [`read_aepx_with_limits`]. When memory runs out, two errors come up in more cases than
+/// there: [`AepError::TooLarge`] when the chunk list cannot grow (the owned-tree reader would
+/// abort), and [`AepError::Xml`] "bdata is too large to hold in memory" also when the shared body
+/// buffer cannot grow (the owned-tree reader reserves each value on its own).
+pub(crate) fn index_aepx(data: &[u8], limits: Limits) -> Result<AepxDrafts, AepError> {
     let text = match std::str::from_utf8(data) {
         Ok(t) => t.as_bytes(),
         Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")),
     };
-    Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run()
+    // Every counted element takes at least 3 bytes (`<a>` in a document cut short), so there are
+    // never more chunks than this.
+    let bound = (text.len() / 3).min(limits.max_chunks);
+    let mut nodes = Vec::new();
+    let _ = nodes.try_reserve_exact(bound.min(INITIAL_SLOTS));
+    // Hex is most of an `.aepx` and decodes to half its length: one buffer of that size usually
+    // holds everything. A failed reservation is not an error here; the buffer grows as it goes.
+    let mut buffer = Vec::new();
+    let _ = buffer.try_reserve_exact(text.len() / 2);
+    let reader = Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), root_closed: false, trailer: None, buffer, nodes, bound, attrs: Vec::new() };
+    reader.run()
 }
 
+/// The slots the chunk list starts with; it doubles from there (as the array of
+/// [`crate::container::TreeIndex`] does).
+const INITIAL_SLOTS: usize = 1024;
+
 fn xml(offset: usize, what: &'static str) -> AepError {
     AepError::Xml { offset, what }
 }
 
+/// The pad byte of a body of `len` bytes, as [`crate::container::Chunk::data`] sets it.
+fn pad_of(len: usize) -> Option<u8> {
+    if len % 2 == 1 { Some(0) } else { None }
+}
+
 /// What an open element becomes when it closes.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum Kind {
@@ -102,13 +154,14 @@ struct Frame {
     at: usize,
     /// The name as written, to match the end tag.
     name: (usize, usize),
-    /// The decoded `bdata` attribute.
-    bdata: Option<Vec<u8>>,
+    /// The chunk's entry in the list (for `Chunk`, `Utf8` and `Alias`).
+    entry: Option<usize>,
+    /// The buffer length when the element opened: its body starts here.
+    mark: usize,
+    /// The end of the decoded `bdata` attribute in the buffer.
+    bdata_end: Option<usize>,
     /// The attributes of a `fileReference`, in document order.
     attrs: Vec<(String, String)>,
-    /// Collected text (only for `string` and the XMP packet).
-    text: Vec<u8>,
-    children: Vec<Chunk<'static>>,
 }
 
 struct Reader<'d> {
@@ -117,13 +170,19 @@ struct Reader<'d> {
     limits: Limits,
     count: usize,
     stack: Vec<Frame>,
-    /// The top-level chunks once the root element has closed.
-    top: Option<Vec<Chunk<'static>>>,
-    trailer: Option<Vec<u8>>,
+    /// True once the root element has closed.
+    root_closed: bool,
+    trailer: Option<(usize, usize)>,
+    buffer: Vec<u8>,
+    nodes: Vec<Draft>,
+    /// The most chunks the input can hold under the limits: `nodes` never reserves past it.
+    bound: usize,
+    /// The attributes of the current start tag; kept to reuse its allocation.
+    attrs: Vec<((usize, usize), Cow<'d, [u8]>)>,
 }
 
-impl Reader<'_> {
-    fn run(mut self) -> Result<AepFile<'static>, AepError> {
+impl<'d> Reader<'d> {
+    fn run(mut self) -> Result<AepxDrafts, AepError> {
         if self.data.starts_with(&[0xef, 0xbb, 0xbf]) {
             self.pos = 3;
         }
@@ -137,11 +196,13 @@ impl Reader<'_> {
         if let Some(open) = self.stack.last() {
             return Err(xml(open.at, "an element is not closed before the end of the file"));
         }
-        let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?;
-        Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) })
+        if !self.root_closed {
+            return Err(xml(self.data.len(), "no AfterEffectsProject root element"));
+        }
+        Ok(AepxDrafts { buffer: self.buffer, nodes: self.nodes, trailer: self.trailer.unwrap_or((0, 0)) })
     }
 
-    fn rest(&self) -> &[u8] {
+    fn rest(&self) -> &'d [u8] {
         self.data.get(self.pos..).unwrap_or(&[])
     }
 
@@ -181,14 +242,13 @@ impl Reader<'_> {
             self.pos = start;
             let end = self.skip_past(b"]]>", "a CDATA section is not closed")?;
             let content = self.data.get(start..end).unwrap_or(&[]);
-            let at = start;
-            match self.stack.last_mut() {
+            match self.stack.last() {
                 Some(frame) => {
                     if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) {
-                        frame.text.extend_from_slice(content);
+                        self.buffer.extend_from_slice(content);
                     }
                 }
-                None => return Err(xml(at, "CDATA outside the root element")),
+                None => return Err(xml(start, "CDATA outside the root element")),
             }
         } else if rest.starts_with(b"<!") {
             return Err(xml(self.pos, "document type declarations are not supported"));
@@ -219,25 +279,22 @@ impl Reader<'_> {
         self.pos = self.pos.saturating_add(n);
     }
 
-    fn slice(&self, (a, b): (usize, usize)) -> &[u8] {
+    fn slice(&self, (a, b): (usize, usize)) -> &'d [u8] {
         self.data.get(a..b).unwrap_or(&[])
     }
 
-    fn start_tag(&mut self) -> Result<(), AepError> {
-        let at = self.pos;
-        self.pos = self.pos.saturating_add(1);
-        let name = self.name()?;
-        let mut attrs = Vec::new();
-        let self_closing = loop {
+    /// Reads the attributes of a start tag into `attrs`; true when the tag closes itself.
+    fn attributes(&mut self, at: usize, attrs: &mut Vec<((usize, usize), Cow<'d, [u8]>)>) -> Result<bool, AepError> {
+        loop {
             self.skip_ws();
             let rest = self.rest();
             if rest.starts_with(b"/>") {
                 self.pos = self.pos.saturating_add(2);
-                break true;
+                return Ok(true);
             }
             if rest.starts_with(b">") {
                 self.pos = self.pos.saturating_add(1);
-                break false;
+                return Ok(false);
             }
             if rest.is_empty() {
                 return Err(xml(at, "a start tag is not closed"));
@@ -270,7 +327,16 @@ impl Reader<'_> {
             };
             self.pos = vend.saturating_add(1);
             attrs.push((key, value));
-        };
+        }
+    }
+
+    fn start_tag(&mut self) -> Result<(), AepError> {
+        let at = self.pos;
+        self.pos = self.pos.saturating_add(1);
+        let name = self.name()?;
+        let mut attrs = std::mem::take(&mut self.attrs);
+        attrs.clear();
+        let self_closing = self.attributes(at, &mut attrs)?;
         let kind = self.kind_of(name, at)?;
         if kind != Kind::Root {
             self.count = self.count.saturating_add(1);
@@ -282,14 +348,25 @@ impl Reader<'_> {
                 return Err(AepError::TooDeep);
             }
         }
-        let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() };
-        if matches!(kind, Kind::Chunk(_)) {
-            if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") {
-                frame.bdata = Some(decode_hex(value, key.1)?);
+        let mark = self.buffer.len();
+        let mut frame = Frame { kind, at, name, entry: None, mark, bdata_end: None, attrs: Vec::new() };
+        match kind {
+            Kind::Chunk(id) => {
+                if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") {
+                    decode_hex_into(value, key.1, MAX_BDATA, &mut self.buffer)?;
+                    frame.bdata_end = Some(self.buffer.len());
+                }
+                frame.entry = Some(self.open_entry(id)?);
             }
-        } else if kind == Kind::Alias {
-            frame.attrs = attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect();
+            Kind::Utf8 => frame.entry = Some(self.open_entry(*b"Utf8")?),
+            Kind::Alias => {
+                frame.attrs =
+                    attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect();
+                frame.entry = Some(self.open_entry(*b"alas")?);
+            }
+            Kind::Root | Kind::Xmp | Kind::Skip => {}
         }
+        self.attrs = attrs;
         self.stack.push(frame);
         if self_closing {
             self.close()?;
@@ -297,12 +374,38 @@ impl Reader<'_> {
         Ok(())
     }
 
+    /// Appends the entry of a chunk whose element just opened; its body is filled in when it
+    /// closes. The list grows by its length (at least 16) but never past [`Self::bound`]; an
+    /// allocation the target cannot make is an error, not an abort.
+    fn open_entry(&mut self, id: [u8; 4]) -> Result<usize, AepError> {
+        let at = self.nodes.len();
+        // This entry and the one after it must have `u32` indices.
+        u32::try_from(at.checked_add(1).ok_or(AepError::TooManyChunks)?).map_err(|_| AepError::TooManyChunks)?;
+        if at == self.nodes.capacity() {
+            // Past the bound (which only a wrong guess could reach) it keeps doubling.
+            let room = self.bound.saturating_sub(at);
+            let more = if room == 0 { at.max(16) } else { at.max(16).min(room) };
+            self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?;
+        }
+        self.nodes.push(Draft::placeholder(id));
+        Ok(at)
+    }
+
+    /// Fills in the entry at `at` once its element has closed; its subtree is what follows it.
+    fn finish(&mut self, at: usize, id: [u8; 4], kind: EntryKind, list_type: [u8; 4], body: (usize, usize), pad: Option<u8>) -> Result<(), AepError> {
+        let end = u32::try_from(self.nodes.len()).map_err(|_| AepError::TooManyChunks)?;
+        if let Some(d) = self.nodes.get_mut(at) {
+            *d = Draft { start: body.0, len: body.1.saturating_sub(body.0), id, list_type, end, kind, pad };
+        }
+        Ok(())
+    }
+
     /// Classifies an element by its name and where it is.
     fn kind_of(&self, name: (usize, usize), at: usize) -> Result<Kind, AepError> {
         let n = self.slice(name);
         let parent = match self.stack.last() {
             None => {
-                if self.top.is_some() {
+                if self.root_closed {
                     return Err(xml(at, "a second root element"));
                 }
                 if n != ROOT {
@@ -346,39 +449,102 @@ impl Reader<'_> {
         self.close()
     }
 
-    /// Closes the innermost element and hands its chunk to the parent.
+    /// Closes the innermost element and fills in its chunk's entry.
     fn close(&mut self) -> Result<(), AepError> {
         let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?;
-        let chunk = match frame.kind {
-            Kind::Root => {
-                self.top = Some(frame.children);
-                return Ok(());
-            }
-            Kind::Skip => return Ok(()),
+        match frame.kind {
+            Kind::Root => self.root_closed = true,
+            Kind::Skip => {}
             Kind::Xmp => {
+                // Only the first packet is kept; a later one is dropped from the buffer (its
+                // children are skipped, so nothing else was written after it opened).
                 if self.trailer.is_none() {
-                    self.trailer = Some(frame.text);
+                    self.trailer = Some((frame.mark, self.buffer.len()));
+                } else {
+                    self.buffer.truncate(frame.mark);
                 }
-                return Ok(());
             }
-            Kind::Utf8 => Chunk::data(*b"Utf8", frame.text),
-            Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()),
-            Kind::Chunk(id) => chunk_of(id, frame)?,
-        };
-        if let Some(parent) = self.stack.last_mut() {
-            parent.children.push(chunk);
+            Kind::Utf8 => {
+                let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?;
+                let end = self.buffer.len();
+                self.finish(at, *b"Utf8", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?;
+            }
+            Kind::Alias => {
+                let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?;
+                self.buffer.extend_from_slice(alias_json(&frame.attrs).as_bytes());
+                let end = self.buffer.len();
+                self.finish(at, *b"alas", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?;
+            }
+            Kind::Chunk(id) => self.close_chunk(id, &frame)?,
         }
         Ok(())
     }
 
+    /// The chunk of a closed element named by a chunk id (what the owned reader's `chunk_of`
+    /// built): `bdata` wins over children, `numS` / `ppSn` hold their number, a nested id holds its
+    /// children (or nothing), any other name is a `LIST` of that type.
+    fn close_chunk(&mut self, id: [u8; 4], frame: &Frame) -> Result<(), AepError> {
+        let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?;
+        let after = at.checked_add(1).ok_or(AepError::TooManyChunks)?;
+        if let Some(bdata_end) = frame.bdata_end {
+            // The children were read (and counted, and checked) but are not part of the chunk.
+            self.nodes.truncate(after);
+            self.buffer.truncate(bdata_end);
+            let body = (frame.mark, bdata_end);
+            let pad = pad_of(bdata_end.saturating_sub(frame.mark));
+            if OPAQUE_LIST_TYPES.contains(&id) {
+                return self.finish(at, LIST, EntryKind::OpaqueList, id, body, pad);
+            }
+            return self.finish(at, id, EntryKind::Data, [0; 4], body, pad);
+        }
+        // Numbers written as text: `<numS><string>0</string></numS>` and `<ppSn><string>150.000000000</string></ppSn>`.
+        if &id == b"numS" || &id == b"ppSn" {
+            let missing = || xml(frame.at, "a numS or ppSn element without its number");
+            // Exactly one child chunk, a `Utf8`: the entry after this one, whose subtree runs to the end.
+            let only = self.nodes.get(after).copied().filter(|c| usize::try_from(c.end).ok() == Some(self.nodes.len())).ok_or_else(missing)?;
+            if only.id != *b"Utf8" {
+                return Err(missing());
+            }
+            let text = match only.kind {
+                EntryKind::Data | EntryKind::OpaqueList => only.start.checked_add(only.len).and_then(|end| self.buffer.get(only.start..end)).unwrap_or(&[]),
+                EntryKind::List | EntryKind::Nested => &[],
+            };
+            let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim();
+            let body: [u8; 8];
+            let bytes: &[u8] = if &id == b"numS" {
+                // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes.
+                let n = text.parse::<i64>().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n));
+                let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?;
+                body = u64::from(n as u32).to_be_bytes();
+                body.get(4..).unwrap_or(&[])
+            } else {
+                body = text.parse::<f64>().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes();
+                &body
+            };
+            self.nodes.truncate(after);
+            self.buffer.truncate(frame.mark);
+            self.buffer.extend_from_slice(bytes);
+            let end = self.buffer.len();
+            return self.finish(at, id, EntryKind::Data, [0; 4], (frame.mark, end), pad_of(bytes.len()));
+        }
+        let empty = (frame.mark, frame.mark);
+        if NESTED_IDS.contains(&id) {
+            if self.nodes.len() == after {
+                return self.finish(at, id, EntryKind::Data, [0; 4], empty, None);
+            }
+            return self.finish(at, id, EntryKind::Nested, [0; 4], empty, None);
+        }
+        self.finish(at, LIST, EntryKind::List, id, empty, None)
+    }
+
     fn text(&mut self) -> Result<(), AepError> {
         let start = self.pos;
         let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len());
         let end = start.saturating_add(len);
         self.pos = end;
         let raw = self.data.get(start..end).unwrap_or(&[]);
-        match self.stack.last_mut() {
-            Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text),
+        match self.stack.last() {
+            Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut self.buffer),
             Some(_) => Ok(()),
             None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()),
             None => Err(xml(start, "text outside the root element")),
@@ -399,41 +565,6 @@ fn chunk_id(name: &[u8]) -> Option<[u8; 4]> {
     Some(id)
 }
 
-/// The chunk of a closed element named by a chunk id.
-fn chunk_of(id: [u8; 4], frame: Frame) -> Result<Chunk<'static>, AepError> {
-    if let Some(bytes) = frame.bdata {
-        if OPAQUE_LIST_TYPES.contains(&id) {
-            let pad = if bytes.len() % 2 == 1 { Some(0) } else { None };
-            return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad });
-        }
-        return Ok(Chunk::data(id, bytes));
-    }
-    // Numbers written as text: `<numS><string>0</string></numS>` and `<ppSn><string>150.000000000</string></ppSn>`.
-    if &id == b"numS" || &id == b"ppSn" {
-        let text = match frame.children.as_slice() {
-            [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]),
-            _ => return Err(xml(frame.at, "a numS or ppSn element without its number")),
-        };
-        let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim();
-        let body = if &id == b"numS" {
-            // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes.
-            let n = text.parse::<i64>().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n));
-            let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?;
-            (n as u32).to_be_bytes().to_vec()
-        } else {
-            text.parse::<f64>().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec()
-        };
-        return Ok(Chunk::data(id, body));
-    }
-    if NESTED_IDS.contains(&id) {
-        if frame.children.is_empty() {
-            return Ok(Chunk::data(id, Vec::new()));
-        }
-        return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None });
-    }
-    Ok(Chunk::list(id, frame.children))
-}
-
 /// Appends `raw` with the predefined entities and character references decoded.
 fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec<u8>) -> Result<(), AepError> {
     let mut i = 0;
@@ -471,7 +602,7 @@ fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec<u8>) -> Result<(), A
     Ok(())
 }
 
-/// Decodes a `bdata` value (hex digits, two per byte).
+/// Decodes a `bdata` value (hex digits, two per byte) into a new `Vec` of exactly its size.
 fn decode_hex(text: &[u8], offset: usize) -> Result<Vec<u8>, AepError> {
     decode_hex_at_most(text, offset, MAX_BDATA)
 }
@@ -482,35 +613,63 @@ const MAX_BDATA: usize = u32::MAX as usize;
 
 /// [`decode_hex`] with the decoded-size limit as a parameter (so tests can reach it).
 fn decode_hex_at_most(text: &[u8], offset: usize, max: usize) -> Result<Vec<u8>, AepError> {
-    if text.len() % 2 == 1 {
-        return Err(xml(offset, "bdata has an odd number of hex digits"));
-    }
+    // Checked before anything is reserved, then sized once (collecting into `Option<Vec>` has no
+    // size hint and regrows step by step), and without aborting when the memory is not there.
+    let pairs = check_hex(text, offset, max)?;
+    let mut out = Vec::new();
+    out.try_reserve_exact(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?;
+    decode_checked(text, offset, &mut out)?;
+    Ok(out)
+}
+
+/// Appends the bytes of a `bdata` value (hex digits, two per byte, at most `max` bytes) to `out`.
+/// On an error `out` is unchanged.
+fn decode_hex_into(text: &[u8], offset: usize, max: usize, out: &mut Vec<u8>) -> Result<(), AepError> {
+    let pairs = check_hex(text, offset, max)?;
+    // Grown at most once per value (amortised over a shared buffer), and without aborting when the
+    // memory is not there.
+    out.try_reserve(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?;
+    decode_checked(text, offset, out)
+}
+
+/// Appends the bytes of a `bdata` value that [`check_hex`] accepted. On an error (a digit the
+/// check let through, which it does not) `out` is unchanged.
+fn decode_checked(text: &[u8], offset: usize, out: &mut Vec<u8>) -> Result<(), AepError> {
     let digit = |b: u8| match b {
         b'0'..=b'9' => Some(b - b'0'),
         b'a'..=b'f' => Some(b - b'a' + 10),
         b'A'..=b'F' => Some(b - b'A' + 10),
         _ => None,
     };
-    let (pairs, _) = text.as_chunks::<2>();
-    if pairs.len() > max {
+    let mark = out.len();
+    for [hi, lo] in text.as_chunks::<2>().0 {
+        match (digit(*hi), digit(*lo)) {
+            (Some(h), Some(l)) => out.push((h << 4) | l),
+            _ => {
+                out.truncate(mark);
+                return Err(xml(offset, "bdata holds a character that is not a hex digit"));
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Checks a `bdata` value before anything is reserved for it: an even number of hex digits, at
+/// most `max` bytes. Returns the number of bytes it decodes to.
+fn check_hex(text: &[u8], offset: usize, max: usize) -> Result<usize, AepError> {
+    if text.len() % 2 == 1 {
+        return Err(xml(offset, "bdata has an odd number of hex digits"));
+    }
+    let pairs = text.len() / 2;
+    if pairs > max {
         return Err(AepError::TooLarge);
     }
-    // Nothing is reserved for a value that does not decode: one cheap pass checks every digit first.
-    // Blocks without an early exit inside, so the check vectorises; it stops at the first bad block.
+    // One cheap pass checks every digit first. Blocks without an early exit inside, so the check
+    // vectorises; it stops at the first bad block.
     if !text.chunks(4096).all(|block| block.iter().fold(true, |ok, b| ok & b.is_ascii_hexdigit())) {
         return Err(xml(offset, "bdata holds a character that is not a hex digit"));
     }
-    // Sized once (collecting into `Option<Vec>` has no size hint and regrows step by step), and
-    // without aborting when the memory is not there.
-    let mut out = Vec::new();
-    out.try_reserve_exact(pairs.len()).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?;
-    for [hi, lo] in pairs {
-        match (digit(*hi), digit(*lo)) {
-            (Some(h), Some(l)) => out.push((h << 4) | l),
-            _ => return Err(xml(offset, "bdata holds a character that is not a hex digit")),
-        }
-    }
-    Ok(out)
+    Ok(pairs)
 }
 
 /// The JSON body of an `alas` chunk from the attributes of a `fileReference`, keys in document
@@ -558,6 +717,7 @@ fn json_str(out: &mut String, s: &str) {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::container::index::TreeIndex;
 
     fn doc(body: &str) -> String {
         format!("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<AfterEffectsProject xmlns=\"x\">{body}</AfterEffectsProject>\n")
@@ -753,4 +913,292 @@ mod tests {
         let many = format!("<AfterEffectsProject>{}</AfterEffectsProject>", "<a/>".repeat(10));
         assert_eq!(read_aepx_with_limits(many.as_bytes(), Limits { max_depth: 64, max_chunks: 9 }), Err(AepError::TooManyChunks));
     }
+
+    /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate.
+    struct Rng(u64);
+
+    impl Rng {
+        /// The next pseudo-random number.
+        fn next(&mut self) -> u64 {
+            let mut x = self.0;
+            x ^= x >> 12;
+            x ^= x << 25;
+            x ^= x >> 27;
+            self.0 = x;
+            x.wrapping_mul(0x2545_f491_4f6c_dd1d)
+        }
+
+        /// A number below `n` (0 for `n == 0`).
+        fn below(&mut self, n: usize) -> usize {
+            if n == 0 { 0 } else { usize::try_from(self.next() % n as u64).unwrap() }
+        }
+
+        /// One of `items`.
+        fn pick<'s>(&mut self, items: &[&'s str]) -> &'s str {
+            items[self.below(items.len())]
+        }
+    }
+
+    /// `n` random bytes as hex, upper or lower case.
+    fn hex(rng: &mut Rng, n: usize) -> String {
+        let upper = rng.below(4) == 0;
+        (0..n).map(|_| if upper { format!("{:02X}", rng.below(256)) } else { format!("{:02x}", rng.below(256)) }).collect()
+    }
+
+    /// Random `.aepx` content: every element form the reader knows (data, lists, nested ids,
+    /// strings, numbers, `btdk`, file references, skipped elements, XMP packets), `bdata` next to
+    /// children, bodies that read as chunk sequences, entities, CDATA, comments and stray text.
+    fn content(rng: &mut Rng, depth: usize, budget: &mut usize, out: &mut String) {
+        for _ in 0..rng.below(6) {
+            if *budget == 0 {
+                return;
+            }
+            *budget -= 1;
+            match rng.below(20) {
+                0..=3 => {
+                    let id = rng.pick(&["cdat", "tdmn", "head", "Utf8", "idta", "LIST", "a"]);
+                    let n = rng.below(9);
+                    out.push_str(&format!("<{id} bdata=\"{}\"/>", hex(rng, n)));
+                }
+                4..=6 if depth < 10 => {
+                    let id = rng.pick(&["Fold", "tdgp", "Layr", "Pin", "LIST", "btdk", "sspc"]);
+                    let attr = if rng.below(5) == 0 { format!(" bdata=\"{}\"", hex(rng, 3)) } else { String::new() };
+                    out.push_str(&format!("<{id}{attr}>"));
+                    content(rng, depth + 1, budget, out);
+                    out.push_str(&format!("</{id}>"));
+                }
+                7 => {
+                    let id = rng.pick(&["tdsn", "pdnm", "fnam"]);
+                    match rng.below(4) {
+                        0 => out.push_str(&format!("<{id}/>")),
+                        1 => out.push_str(&format!("<{id}><string>name &amp; &#x41;</string></{id}>")),
+                        // A body that is itself a chunk sequence (`Utf8`, 1 byte, pad): still data.
+                        2 => out.push_str(&format!("<{id} bdata=\"557466380000000161{}\"/>", if rng.below(2) == 0 { "00" } else { "" })),
+                        _ => out.push_str(&format!("<{id}><string>a</string><string>b</string></{id}>")),
+                    }
+                }
+                8 => {
+                    let text = rng.pick(&["plain", "A &lt; B", "<![CDATA[x<y]]>", "a<!-- c -->b", "", "&#233;t&#xE9;", "x<Skip>gone</Skip>y", "\r\n"]);
+                    out.push_str(&format!("<string>{text}</string>"));
+                }
+                9 => {
+                    let id = rng.pick(&["numS", "ppSn"]);
+                    let inner = rng.pick(&[
+                        "<string>0</string>",
+                        "<string> 7 </string>",
+                        "<string>-2147483648</string>",
+                        "<string>150.000000000</string>",
+                        "<string>x</string>",
+                        "",
+                        "<string>1</string><string>2</string>",
+                        "<Utf8 bdata=\"31\"/>",
+                        "<cdat bdata=\"31\"/>",
+                        "<Comment>3</Comment><string>4</string>",
+                    ]);
+                    let attr = if rng.below(6) == 0 { " bdata=\"00000002\"" } else { "" };
+                    out.push_str(&format!("<{id}{attr}>{inner}</{id}>"));
+                }
+                10 => {
+                    let n = rng.pick(&["", "41", "414243", "4c495354000000000000"]);
+                    out.push_str(&format!("<btdk bdata=\"{n}\"/>"));
+                }
+                11 => {
+                    let platform = rng.pick(&["Win", "1", "Mac"]);
+                    out.push_str(&format!(
+                        "<Als2><fileReference ascendcount_base=\"1\" fullpath=\"C:\\a\\&quot;b&#x9;.png\" platform=\"{platform}\" target_is_folder=\"{}\"/></Als2>",
+                        rng.below(3)
+                    ));
+                }
+                12 if depth < 10 => {
+                    out.push_str("<Comment note=\"&amp;\">text<a bdata=\"00\"/>");
+                    content(rng, depth + 1, budget, out);
+                    out.push_str("</Comment>");
+                }
+                13 => {
+                    let packet = rng.pick(&["<![CDATA[<?xpacket begin?>x<?xpacket end?>]]>", "plain &amp; text", ""]);
+                    out.push_str(&format!("<ProjectXMPMetadata>{packet}</ProjectXMPMetadata>"));
+                }
+                14 => out.push_str(rng.pick(&["<!-- comment -->", "<?pi x?>", "\n  ", "stray"])),
+                _ => {
+                    let n = rng.below(5);
+                    out.push_str(&format!("<tdb4 bdata=\"{}\" x='y'></tdb4>", hex(rng, n)));
+                }
+            }
+        }
+    }
+
+    /// The default limits and six tight ones (depth 0, 1 and 3; 0, 7 and 40 chunks).
+    fn limit_sets() -> [Limits; 7] {
+        [
+            Limits { max_depth: 0, max_chunks: 10_000_000 },
+            Limits { max_depth: 64, max_chunks: 0 },
+            Limits::default(),
+            Limits { max_depth: 1, max_chunks: 10_000_000 },
+            Limits { max_depth: 3, max_chunks: 10_000_000 },
+            Limits { max_depth: 64, max_chunks: 7 },
+            Limits { max_depth: 64, max_chunks: 40 },
+        ]
+    }
+
+    /// The index reader and the owned-tree reader give the same `Result` (the tree, or the error),
+    /// and the index walks like the index of the owned tree.
+    fn same(bytes: &[u8], limits: Limits) {
+        let old = read_aepx_with_limits(bytes, limits);
+        let new = index_aepx(bytes, limits).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned());
+        assert_eq!(new, old, "{}", String::from_utf8_lossy(bytes));
+        if let Ok(file) = &old {
+            let AepxDrafts { buffer, nodes, trailer } = index_aepx(bytes, limits).unwrap();
+            let index = TreeIndex::from_drafts(&buffer, nodes, trailer);
+            let expected = TreeIndex::from_file(file).unwrap();
+            assert_eq!(index.len(), expected.len());
+            assert_eq!(index.trailer(), expected.trailer());
+            let a: Vec<String> = index.walk().map(|(d, n)| format!("{d} {n:?}")).collect();
+            let b: Vec<String> = expected.walk().map(|(d, n)| format!("{d} {n:?}")).collect();
+            assert_eq!(a, b);
+        }
+    }
+
+    /// The two readers give the same tree and errors on generated and damaged documents.
+    #[test]
+    fn the_two_readers_agree_on_generated_and_damaged_documents() {
+        let mut rng = Rng(0x9e37_79b9_7f4a_7c15);
+        let noise = b"<>/=\"'&;#x0aZ \n![]-?";
+        for _ in 0..600 {
+            let mut budget = 80;
+            let mut body = String::new();
+            content(&mut rng, 0, &mut budget, &mut body);
+            let x = doc(&body);
+            for l in limit_sets() {
+                same(x.as_bytes(), l);
+            }
+            for _ in 0..4 {
+                let mut bad = x.clone().into_bytes();
+                for _ in 0..1 + rng.below(3) {
+                    let at = rng.below(bad.len());
+                    bad[at] = noise[rng.below(noise.len())];
+                }
+                for l in limit_sets() {
+                    same(&bad, l);
+                }
+            }
+            let cut = rng.below(x.len());
+            same(&x.as_bytes()[..cut], Limits::default());
+            // Bytes that are not UTF-8, and a multi-byte character cut in half.
+            let mut bad = x.clone().into_bytes();
+            let at = rng.below(bad.len());
+            bad.insert(at, 0xff);
+            same(&bad, Limits::default());
+            let open = format!("{}<string>é", x.trim_end().trim_end_matches("</AfterEffectsProject>")).into_bytes();
+            same(&open[..open.len() - 1], Limits::default());
+        }
+    }
+
+    /// The flat chunk list never reserves more slots than the input or the chunk limit allows.
+    #[test]
+    fn the_chunk_list_is_bounded_by_the_input_and_the_limit() {
+        let many = format!("<AfterEffectsProject>{}</AfterEffectsProject>", "<a/>".repeat(10_000));
+        let d = index_aepx(many.as_bytes(), Limits::default()).unwrap();
+        assert_eq!(d.nodes.len(), 10_000);
+        assert!(d.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", d.nodes.capacity());
+        // The buffer is reserved once, at half the input, and holds nothing here.
+        assert!(d.buffer.is_empty() && d.buffer.capacity() <= many.len() / 2);
+        // At the limit the list never holds more slots than the limit.
+        let d = index_aepx(many.as_bytes(), Limits { max_depth: 64, max_chunks: 10_000 }).unwrap();
+        assert!(d.nodes.capacity() <= 10_000);
+        let few = format!("<AfterEffectsProject>{}</AfterEffectsProject>", "<a/>".repeat(100));
+        let d = index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 100 }).unwrap();
+        assert!(d.nodes.capacity() <= 100, "{} slots for at most 100 chunks", d.nodes.capacity());
+        assert_eq!(index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 99 }).err(), Some(AepError::TooManyChunks));
+        // A tiny input cannot hold more than a third of its bytes in elements.
+        let tiny = doc("<a/>");
+        assert!(index_aepx(tiny.as_bytes(), Limits::default()).unwrap().nodes.capacity() <= tiny.len() / 3);
+    }
+
+    /// What a closed element drops is gone from the buffer and the chunk list, but was counted.
+    #[test]
+    fn dropped_content_leaves_nothing_behind_but_still_counts() {
+        // `bdata` wins over the children: they are read, counted and dropped.
+        let x = doc("<a bdata=\"01\"><b bdata=\"0203\"/><string>xyz</string><tdsn><string>q</string></tdsn></a>");
+        let d = index_aepx(x.as_bytes(), Limits::default()).unwrap();
+        assert_eq!((d.buffer.as_slice(), d.nodes.len()), (&[1u8][..], 1));
+        assert_eq!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 4 }).err(), Some(AepError::TooManyChunks));
+        assert!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 5 }).is_ok());
+        // A child's error is raised although the child would be dropped.
+        assert!(matches!(index_aepx(doc("<a bdata=\"01\"><numS></numS></a>").as_bytes(), Limits::default()), Err(AepError::Xml { .. })));
+        // Only the first XMP packet stays in the buffer.
+        let packets = doc(&"<ProjectXMPMetadata>packet</ProjectXMPMetadata>".repeat(1000));
+        let d = index_aepx(packets.as_bytes(), Limits::default()).unwrap();
+        assert_eq!((d.buffer.as_slice(), d.trailer), (&b"packet"[..], (0, 6)));
+        // A number keeps its 4 or 8 bytes, not its text.
+        let d = index_aepx(doc("<numS><string>   7   </string></numS><ppSn><string>1.5</string></ppSn>").as_bytes(), Limits::default()).unwrap();
+        assert_eq!(d.buffer, [&[0, 0, 0, 7][..], &1.5f64.to_be_bytes()[..]].concat());
+        assert_eq!(d.nodes.len(), 2);
+    }
+
+    /// Bodies keep what the element says they are, even where reading the same bytes as an `.aep`
+    /// would decide otherwise: a nested id whose `bdata` is a chunk sequence stays data, an empty
+    /// `btdk` stays an opaque list, an empty list at the deepest level stays a list.
+    #[test]
+    fn bodies_keep_the_kind_the_element_gives_them() {
+        let x = doc("<tdsn bdata=\"5574663800000001610000\"/><btdk bdata=\"\"/><btdk bdata=\"4c4953540000000474657374\"/>");
+        let AepxDrafts { buffer, nodes, trailer } = index_aepx(x.as_bytes(), Limits::default()).unwrap();
+        let index = TreeIndex::from_drafts(&buffer, nodes, trailer);
+        let top: Vec<_> = index.chunks().collect();
+        assert_eq!(top.len(), 3);
+        assert_eq!(top[0].bytes().map(<[u8]>::len), Some(11));
+        assert!(top[0].children().next().is_none());
+        assert!(top[1].is_opaque_list() && top[1].bytes() == Some(&[][..]));
+        assert!(top[2].is_opaque_list() && top[2].bytes().map(<[u8]>::len) == Some(12));
+        // 64 levels: the innermost (empty) list is a list, not opaque.
+        let deep = format!("<AfterEffectsProject>{}{}</AfterEffectsProject>", "<Fold>".repeat(64), "</Fold>".repeat(64));
+        let AepxDrafts { buffer, nodes, trailer } = index_aepx(deep.as_bytes(), Limits::default()).unwrap();
+        let index = TreeIndex::from_drafts(&buffer, nodes, trailer);
+        assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64));
+        assert!(index.walk().all(|(_, n)| n.is_list(b"Fold") && !n.is_opaque_list()));
+        same(x.as_bytes(), Limits::default());
+        same(deep.as_bytes(), Limits::default());
+    }
+
+    /// Hostile input is refused by both readers with the same errors.
+    #[test]
+    fn hostile_input_is_refused_alike_by_both_readers() {
+        let hostile = [
+            format!("<AfterEffectsProject>{}{}</AfterEffectsProject>", "<a>".repeat(100_000), "</a>".repeat(100_000)),
+            format!("<AfterEffectsProject>{}", "<string>".repeat(1000)),
+            doc(&format!("<a bdata=\"{}0\"/>", "00".repeat(100_000))),
+            doc(&format!("<a bdata=\"{}zz\"/>", "00".repeat(100_000))),
+            doc("<numS><string>4294967296</string></numS>"),
+            doc("<ppSn><string>1e400x</string></ppSn>"),
+            doc("<string>&#xD800;</string>"),
+            doc("<string>&#99999999999;</string>"),
+            doc(&format!("<Fold>{}", "<b/>".repeat(1000))),
+            format!("<AfterEffectsProject>{}</AfterEffectsProject><x/>", "<a/>".repeat(10)),
+            "<AfterEffectsProject><a bdata=\"00".to_string(),
+            "<AfterEffectsProject><![CDATA[x".to_string(),
+        ];
+        for x in &hostile {
+            assert!(read_aepx(x.as_bytes()).is_err(), "{}", &x[..x.len().min(80)]);
+            for l in limit_sets() {
+                same(x.as_bytes(), l);
+            }
+        }
+    }
+
+    /// Every `.aepx` of the fixture directory gives the same tree through both readers (skipped
+    /// without `EFFECTCRAFT_AEP_FIXTURES`).
+    #[test]
+    fn every_fixture_aepx_gives_the_same_tree_through_both_readers() {
+        let Some(dir) = std::env::var_os("EFFECTCRAFT_AEP_FIXTURES").filter(|d| !d.is_empty()) else { return };
+        let Ok(entries) = std::fs::read_dir(dir) else { return };
+        let mut paths: Vec<std::path::PathBuf> =
+            entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aepx")).collect();
+        paths.sort();
+        for path in paths {
+            let bytes = std::fs::read(&path).unwrap();
+            for l in limit_sets() {
+                let index = index_aepx(&bytes, l).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned());
+                assert_eq!(index, read_aepx_with_limits(&bytes, l), "{}", path.display());
+            }
+        }
+    }
 }

crates/aep/src/container/aepx/tree.rs

@@ -0,0 +1,361 @@
+//! The `.aepx` reader of the owned chunk tree ([`super::read_aepx`]): it builds the tree element by
+//! element. The importer uses the index reader in `aepx.rs` instead; the tests compare the two on
+//! generated, damaged and corpus input (the same tree and the same errors).
+
+use std::borrow::Cow;
+
+use super::{FILE_REFERENCE, OPAQUE_LIST_TYPES, ROOT, STRING, XMP, alias_json, chunk_id, decode_entities, decode_hex, xml};
+use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS};
+use crate::container::error::AepError;
+use crate::container::read::Limits;
+
+/// [`super::read_aepx_with_limits`].
+pub(super) fn read(data: &[u8], limits: Limits) -> Result<AepFile<'static>, AepError> {
+    let text = match std::str::from_utf8(data) {
+        Ok(t) => t.as_bytes(),
+        Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")),
+    };
+    Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run()
+}
+
+/// What an open element becomes when it closes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Kind {
+    /// The root element: its children are the top-level chunks.
+    Root,
+    /// A chunk with this id (or a `LIST` with this list type).
+    Chunk([u8; 4]),
+    /// A `Utf8` chunk.
+    Utf8,
+    /// An `alas` chunk built from the attributes.
+    Alias,
+    /// The XMP packet (only directly below the root).
+    Xmp,
+    /// An element without a chunk counterpart (a name that is no chunk id); skipped with its content.
+    Skip,
+}
+
+struct Frame {
+    kind: Kind,
+    /// Offset of the start tag (for errors).
+    at: usize,
+    /// The name as written, to match the end tag.
+    name: (usize, usize),
+    /// The decoded `bdata` attribute.
+    bdata: Option<Vec<u8>>,
+    /// The attributes of a `fileReference`, in document order.
+    attrs: Vec<(String, String)>,
+    /// Collected text (only for `string` and the XMP packet).
+    text: Vec<u8>,
+    children: Vec<Chunk<'static>>,
+}
+
+struct Reader<'d> {
+    data: &'d [u8],
+    pos: usize,
+    limits: Limits,
+    count: usize,
+    stack: Vec<Frame>,
+    /// The top-level chunks once the root element has closed.
+    top: Option<Vec<Chunk<'static>>>,
+    trailer: Option<Vec<u8>>,
+}
+
+impl Reader<'_> {
+    fn run(mut self) -> Result<AepFile<'static>, AepError> {
+        if self.data.starts_with(&[0xef, 0xbb, 0xbf]) {
+            self.pos = 3;
+        }
+        while self.pos < self.data.len() {
+            if self.rest().starts_with(b"<") {
+                self.markup()?;
+            } else {
+                self.text()?;
+            }
+        }
+        if let Some(open) = self.stack.last() {
+            return Err(xml(open.at, "an element is not closed before the end of the file"));
+        }
+        let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?;
+        Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) })
+    }
+
+    fn rest(&self) -> &[u8] {
+        self.data.get(self.pos..).unwrap_or(&[])
+    }
+
+    /// The offset of `needle` at or after `from`.
+    fn find(&self, from: usize, needle: &[u8]) -> Option<usize> {
+        let hay = self.data.get(from..)?;
+        if needle.is_empty() || hay.len() < needle.len() {
+            return None;
+        }
+        let first = *needle.first()?;
+        let mut i = 0;
+        while let Some(p) = hay.get(i..)?.iter().position(|b| *b == first) {
+            let at = i.checked_add(p)?;
+            if hay.get(at..)?.starts_with(needle) {
+                return from.checked_add(at);
+            }
+            i = at.checked_add(1)?;
+        }
+        None
+    }
+
+    /// Skips to just after `close`, which must follow.
+    fn skip_past(&mut self, close: &[u8], what: &'static str) -> Result<usize, AepError> {
+        let at = self.find(self.pos, close).ok_or(xml(self.pos, what))?;
+        self.pos = at.saturating_add(close.len());
+        Ok(at)
+    }
+
+    fn markup(&mut self) -> Result<(), AepError> {
+        let rest = self.rest();
+        if rest.starts_with(b"<?") {
+            self.skip_past(b"?>", "a processing instruction is not closed")?;
+        } else if rest.starts_with(b"<!--") {
+            self.skip_past(b"-->", "a comment is not closed")?;
+        } else if rest.starts_with(b"<![CDATA[") {
+            let start = self.pos.saturating_add(9);
+            self.pos = start;
+            let end = self.skip_past(b"]]>", "a CDATA section is not closed")?;
+            let content = self.data.get(start..end).unwrap_or(&[]);
+            let at = start;
+            match self.stack.last_mut() {
+                Some(frame) => {
+                    if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) {
+                        frame.text.extend_from_slice(content);
+                    }
+                }
+                None => return Err(xml(at, "CDATA outside the root element")),
+            }
+        } else if rest.starts_with(b"<!") {
+            return Err(xml(self.pos, "document type declarations are not supported"));
+        } else if rest.starts_with(b"</") {
+            self.end_tag()?;
+        } else {
+            self.start_tag()?;
+        }
+        Ok(())
+    }
+
+    fn is_name_byte(b: u8) -> bool {
+        !(b.is_ascii_whitespace() || matches!(b, b'<' | b'>' | b'/' | b'=' | b'"' | b'\'' | b'&'))
+    }
+
+    fn name(&mut self) -> Result<(usize, usize), AepError> {
+        let start = self.pos;
+        let len = self.rest().iter().position(|b| !Self::is_name_byte(*b)).unwrap_or(self.rest().len());
+        if len == 0 {
+            return Err(xml(start, "a tag or attribute has no name"));
+        }
+        self.pos = start.saturating_add(len);
+        Ok((start, self.pos))
+    }
+
+    fn skip_ws(&mut self) {
+        let n = self.rest().iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(self.rest().len());
+        self.pos = self.pos.saturating_add(n);
+    }
+
+    fn slice(&self, (a, b): (usize, usize)) -> &[u8] {
+        self.data.get(a..b).unwrap_or(&[])
+    }
+
+    fn start_tag(&mut self) -> Result<(), AepError> {
+        let at = self.pos;
+        self.pos = self.pos.saturating_add(1);
+        let name = self.name()?;
+        let mut attrs = Vec::new();
+        let self_closing = loop {
+            self.skip_ws();
+            let rest = self.rest();
+            if rest.starts_with(b"/>") {
+                self.pos = self.pos.saturating_add(2);
+                break true;
+            }
+            if rest.starts_with(b">") {
+                self.pos = self.pos.saturating_add(1);
+                break false;
+            }
+            if rest.is_empty() {
+                return Err(xml(at, "a start tag is not closed"));
+            }
+            let key = self.name()?;
+            self.skip_ws();
+            if !self.rest().starts_with(b"=") {
+                return Err(xml(self.pos, "an attribute has no value"));
+            }
+            self.pos = self.pos.saturating_add(1);
+            self.skip_ws();
+            let quote = *self.rest().first().ok_or(xml(self.pos, "a start tag is not closed"))?;
+            if quote != b'"' && quote != b'\'' {
+                return Err(xml(self.pos, "an attribute value is not quoted"));
+            }
+            let vstart = self.pos.saturating_add(1);
+            let vend = self.find(vstart, &[quote]).ok_or(xml(vstart, "an attribute value is not closed"))?;
+            let raw = self.data.get(vstart..vend).unwrap_or(&[]);
+            if raw.contains(&b'<') {
+                return Err(xml(vstart, "`<` inside an attribute value"));
+            }
+            // A value without `&` is its own decoding (most of them, every `bdata` After Effects
+            // writes): borrow it instead of copying it.
+            let value = if raw.contains(&b'&') {
+                let mut value = Vec::new();
+                decode_entities(raw, vstart, &mut value)?;
+                Cow::Owned(value)
+            } else {
+                Cow::Borrowed(raw)
+            };
+            self.pos = vend.saturating_add(1);
+            attrs.push((key, value));
+        };
+        let kind = self.kind_of(name, at)?;
+        if kind != Kind::Root {
+            self.count = self.count.saturating_add(1);
+            if self.count > self.limits.max_chunks {
+                return Err(AepError::TooManyChunks);
+            }
+            // The root's children are depth 1, like the top-level chunks of an `.aep`.
+            if self.stack.len() > self.limits.max_depth {
+                return Err(AepError::TooDeep);
+            }
+        }
+        let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() };
+        if matches!(kind, Kind::Chunk(_)) {
+            if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") {
+                frame.bdata = Some(decode_hex(value, key.1)?);
+            }
+        } else if kind == Kind::Alias {
+            frame.attrs = attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect();
+        }
+        self.stack.push(frame);
+        if self_closing {
+            self.close()?;
+        }
+        Ok(())
+    }
+
+    /// Classifies an element by its name and where it is.
+    fn kind_of(&self, name: (usize, usize), at: usize) -> Result<Kind, AepError> {
+        let n = self.slice(name);
+        let parent = match self.stack.last() {
+            None => {
+                if self.top.is_some() {
+                    return Err(xml(at, "a second root element"));
+                }
+                if n != ROOT {
+                    return Err(xml(at, "the root element is not AfterEffectsProject"));
+                }
+                return Ok(Kind::Root);
+            }
+            Some(p) => p.kind,
+        };
+        Ok(match parent {
+            Kind::Skip | Kind::Utf8 | Kind::Xmp | Kind::Alias => Kind::Skip,
+            Kind::Root | Kind::Chunk(_) => {
+                if n == STRING {
+                    Kind::Utf8
+                } else if n == FILE_REFERENCE {
+                    Kind::Alias
+                } else if n == XMP && parent == Kind::Root {
+                    Kind::Xmp
+                } else if let Some(id) = chunk_id(n) {
+                    Kind::Chunk(id)
+                } else {
+                    Kind::Skip
+                }
+            }
+        })
+    }
+
+    fn end_tag(&mut self) -> Result<(), AepError> {
+        let at = self.pos;
+        self.pos = self.pos.saturating_add(2);
+        let name = self.name()?;
+        self.skip_ws();
+        if !self.rest().starts_with(b">") {
+            return Err(xml(at, "an end tag is not closed"));
+        }
+        self.pos = self.pos.saturating_add(1);
+        let open = self.stack.last().ok_or(xml(at, "an end tag without a start tag"))?;
+        if self.slice(open.name) != self.slice(name) {
+            return Err(xml(at, "an end tag does not match its start tag"));
+        }
+        self.close()
+    }
+
+    /// Closes the innermost element and hands its chunk to the parent.
+    fn close(&mut self) -> Result<(), AepError> {
+        let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?;
+        let chunk = match frame.kind {
+            Kind::Root => {
+                self.top = Some(frame.children);
+                return Ok(());
+            }
+            Kind::Skip => return Ok(()),
+            Kind::Xmp => {
+                if self.trailer.is_none() {
+                    self.trailer = Some(frame.text);
+                }
+                return Ok(());
+            }
+            Kind::Utf8 => Chunk::data(*b"Utf8", frame.text),
+            Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()),
+            Kind::Chunk(id) => chunk_of(id, frame)?,
+        };
+        if let Some(parent) = self.stack.last_mut() {
+            parent.children.push(chunk);
+        }
+        Ok(())
+    }
+
+    fn text(&mut self) -> Result<(), AepError> {
+        let start = self.pos;
+        let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len());
+        let end = start.saturating_add(len);
+        self.pos = end;
+        let raw = self.data.get(start..end).unwrap_or(&[]);
+        match self.stack.last_mut() {
+            Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text),
+            Some(_) => Ok(()),
+            None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()),
+            None => Err(xml(start, "text outside the root element")),
+        }
+    }
+}
+
+/// The chunk of a closed element named by a chunk id.
+fn chunk_of(id: [u8; 4], frame: Frame) -> Result<Chunk<'static>, AepError> {
+    if let Some(bytes) = frame.bdata {
+        if OPAQUE_LIST_TYPES.contains(&id) {
+            let pad = if bytes.len() % 2 == 1 { Some(0) } else { None };
+            return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad });
+        }
+        return Ok(Chunk::data(id, bytes));
+    }
+    // Numbers written as text: `<numS><string>0</string></numS>` and `<ppSn><string>150.000000000</string></ppSn>`.
+    if &id == b"numS" || &id == b"ppSn" {
+        let text = match frame.children.as_slice() {
+            [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]),
+            _ => return Err(xml(frame.at, "a numS or ppSn element without its number")),
+        };
+        let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim();
+        let body = if &id == b"numS" {
+            // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes.
+            let n = text.parse::<i64>().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n));
+            let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?;
+            (n as u32).to_be_bytes().to_vec()
+        } else {
+            text.parse::<f64>().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec()
+        };
+        return Ok(Chunk::data(id, body));
+    }
+    if NESTED_IDS.contains(&id) {
+        if frame.children.is_empty() {
+            return Ok(Chunk::data(id, Vec::new()));
+        }
+        return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None });
+    }
+    Ok(Chunk::list(id, frame.children))
+}

crates/aep/src/container/index.rs

@@ -20,6 +20,10 @@
 //! [`AepError::TooLarge`] when the array cannot grow (a failed `try_reserve`), and
 //! [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks.
 //!
+//! An `.aepx` is indexed by its own reader (`container/aepx.rs`), which decodes every body into
+//! one buffer and fills the entries as it reads; the index then borrows that buffer
+//! (`TreeIndex::from_drafts`). It gives the tree [`crate::container::read_aepx`] gives.
+//!
 //! [`Node`] is the handle the typed view ([`crate::container::Project`] and the types below it)
 //! reads through: a chunk of either tree. The view works the same on both.
 
@@ -33,7 +37,7 @@ use crate::container::read::{Limits, container_end, id4};
 
 /// What an entry's body is (the [`Body`] variants).
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum Kind {
+pub(crate) enum Kind {
     Data,
     /// [`Body::Chunks`].
     Nested,
@@ -63,6 +67,28 @@ impl<'a> Entry<'a> {
     }
 }
 
+/// An entry of an index whose bytes are still being written (the `.aepx` reader decodes into a
+/// buffer that grows, so it keeps positions, not slices): the body is `start..start + len` of
+/// that buffer. The fields are those of [`Entry`]; [`TreeIndex::from_drafts`] turns the list
+/// into an index once the buffer is final.
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct Draft {
+    pub(crate) start: usize,
+    pub(crate) len: usize,
+    pub(crate) id: [u8; 4],
+    pub(crate) list_type: [u8; 4],
+    pub(crate) end: u32,
+    pub(crate) kind: Kind,
+    pub(crate) pad: Option<u8>,
+}
+
+impl Draft {
+    /// A data chunk with no body yet.
+    pub(crate) fn placeholder(id: [u8; 4]) -> Self {
+        Self { start: 0, len: 0, id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None }
+    }
+}
+
 /// The chunk tree of a file as one flat, read-only array (see the module documentation).
 #[derive(Clone)]
 pub struct TreeIndex<'a> {
@@ -112,8 +138,19 @@ impl<'a> TreeIndex<'a> {
         Ok(Self { nodes: builder.nodes, trailer })
     }
 
-    /// Indexes an already parsed (or built, or `.aepx`) chunk tree. The index borrows the file's
-    /// bytes.
+    /// The index of the entries `drafts` (in depth-first order, see [`Draft`]) whose bodies and
+    /// trailer are ranges of `buffer`. A range outside `buffer` reads as empty. The list is turned
+    /// into entries; a [`Draft`] and an [`Entry`] have the same size and alignment, so the standard
+    /// library reuses the list's allocation for them.
+    pub(crate) fn from_drafts(buffer: &'a [u8], drafts: Vec<Draft>, trailer: (usize, usize)) -> Self {
+        let slice = |start: usize, len: usize| start.checked_add(len).and_then(|end| buffer.get(start..end)).unwrap_or(&[]);
+        let nodes =
+            drafts.into_iter().map(|d| Entry { data: slice(d.start, d.len), id: d.id, list_type: d.list_type, end: d.end, kind: d.kind, pad: d.pad }).collect();
+        let trailer = buffer.get(trailer.0..trailer.1).unwrap_or(&[]);
+        Self { nodes, trailer }
+    }
+
+    /// Indexes an already parsed (or built) chunk tree. The index borrows the file's bytes.
     ///
     /// # Errors
     /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks.

crates/aep/src/import.rs

@@ -1,6 +1,6 @@
 //! After Effects project (`.aep`) import: the typed view of [`effectcraft_aep`] mapped into an
-//! EffectCraft [`Project`]. The XML form (`.aepx`) is accepted as well: it is read into the same
-//! chunk tree ([`crate::parse_any`] picks the reader by content).
+//! EffectCraft [`Project`]. The XML form (`.aepx`) is accepted as well: both are read into the same
+//! flat chunk index (an `.aep` in place, an `.aepx` decoded into one buffer first).
 //!
 //! What arrives: the project item tree (folders), project settings (depth, working space, time
 //! display), compositions (size, exact frame rate, duration, work area, background, pixel
@@ -43,6 +43,7 @@ mod time;
 use effectcraft_project::{ItemId, Project, ProjectSettings};
 
 use crate::container::index::{Root, TreeIndex};
+use crate::container::{AepxDrafts, Limits, index_aepx};
 
 pub use effects::supported_effects;
 pub use report::{AeVersionInfo, ImportReport, Imported, LIMITATIONS, Loss, LossCategory, MAX_DETAILS, TESTED_MAJORS, UNKNOWN};
@@ -118,10 +119,11 @@ pub struct ImportResult {
 /// error `project` is unchanged.
 pub fn import(project: &mut Project, bytes: &[u8], options: &ImportOptions<'_>) -> Result<ImportResult, AepImportError> {
     // The importer only reads: an `.aep` is indexed in place (one flat array that borrows the
-    // bytes); an `.aepx` is read into the chunk tree, which the view reads directly.
+    // bytes); an `.aepx` is decoded into one buffer, which the index borrows.
     if crate::is_aepx(bytes) {
-        let file = crate::read_aepx(bytes)?;
-        import_root(project, Root::from(&file), options)
+        let AepxDrafts { buffer, nodes, trailer } = index_aepx(bytes, Limits::default())?;
+        let index = TreeIndex::from_drafts(&buffer, nodes, trailer);
+        import_root(project, Root::from(&index), options)
     } else {
         let index = TreeIndex::parse(bytes)?;
         import_root(project, Root::from(&index), options)
@@ -243,4 +245,45 @@ mod tests {
             both(&bytes, &path.display().to_string());
         }
     }
+
+    /// Imports an `.aepx` through the owned tree ([`crate::read_aepx`]) and through the index
+    /// ([`import`]): the same project, report and settings, or the same error.
+    fn both_aepx(bytes: &[u8], name: &str) -> Result<ImportResult, AepImportError> {
+        let exists = |_: &str| true;
+        let options = ImportOptions { allow_losses: true, ..ImportOptions::new(&exists) };
+        let a = crate::read_aepx(bytes).map_err(AepImportError::from).and_then(|file| import_root(&mut Project::default(), Root::from(&file), &options));
+        let b = import(&mut Project::default(), bytes, &options);
+        assert!(a == b, "{name}: the .aepx import differs through the index");
+        b
+    }
+
+    /// A synthetic `.aepx` imports the same through the index as through the owned tree.
+    #[test]
+    fn a_synthetic_aepx_imports_the_same_through_the_index() {
+        // The comp of the owned-tree test above (`idta` type 4, `cdta` 1/30 s frame duration).
+        let idta = format!("0004{}", "00".repeat(62));
+        let cdta = format!("00000000000000010000001e{}", "00".repeat(188));
+        let x = format!(
+            "<?xml version=\"1.0\"?><AfterEffectsProject><head bdata=\"{}\"/><Fold><Item><idta bdata=\"{idta}\"/><string>Comp</string><cdta bdata=\"{cdta}\"/></Item></Fold><ProjectXMPMetadata>x</ProjectXMPMetadata></AfterEffectsProject>",
+            "00".repeat(20)
+        );
+        let imported = both_aepx(x.as_bytes(), "synthetic").expect("the synthetic project imports");
+        assert_eq!(imported.comps.len(), 1);
+        assert_eq!(both_aepx(b"<AfterEffectsProject/>", "empty").err(), Some(AepImportError::NoProject));
+        assert!(matches!(both_aepx(b"<AfterEffectsProject><a>", "broken"), Err(AepImportError::Container(crate::AepError::Xml { .. }))));
+    }
+
+    /// Every `.aepx` of the fixture directory (skipped without `EFFECTCRAFT_AEP_FIXTURES`).
+    #[test]
+    fn every_fixture_aepx_imports_the_same_through_the_index() {
+        let Some(dir) = std::env::var_os("EFFECTCRAFT_AEP_FIXTURES").filter(|d| !d.is_empty()) else { return };
+        let Ok(entries) = std::fs::read_dir(dir) else { return };
+        let mut paths: Vec<std::path::PathBuf> =
+            entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aepx")).collect();
+        paths.sort();
+        for path in paths {
+            let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
+            let _ = both_aepx(&bytes, &path.display().to_string());
+        }
+    }
 }

docs/aep-format.md

@@ -119,6 +119,11 @@ counterpart (a name that is not 1 to 4 characters) are skipped with their conten
   with the byte offset. It runs without recursion; the nesting depth (`Limits::max_depth`) and the
   element count (`Limits::max_chunks`) are bounded, and everything it allocates is bounded by the
   size of the input.
+- There are two readers. `read_aepx` builds the owned tree element by element. The importer, which
+  only reads, uses a second reader that decodes every body (hex, text, the alias JSON, the
+  numbers) into one buffer and records the chunks as a flat depth-first list whose bodies are
+  ranges of that buffer; it indexes that buffer (`TreeIndex`) as it indexes an `.aep`. The tests
+  hold both readers to the same tree and the same errors.
 - Tests: `tests/aepx.rs` compares the five fixture pairs chunk by chunk and byte by byte, compares
   the typed view of both forms, and runs truncated, bit-flipped, deeply nested and oversized
   inputs; `crates/aep/tests/it/import/aepx.rs` imports both forms of each pair and expects the same