From 06af3f2d46a96b1f935592fabf4c30653b1d5825 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 12:49:55 -0500 Subject: [PATCH] Add craft-cfb: Compound File Binary reader and writer (#818) The base is FilmCraft's filmcraft-cfb (reader, writer, tests) at filmcraft 6d5269ed. Ported on top of it, each with a test: From GridCraft's and DeckCraft's readers: only sector shifts 9 and 12; a file that ends inside its last sector still reads; a chain loop shorter than the stream is an error instead of repeated data; a damaged mini stream no longer fails the open; links to missing directory entries are ignored; a stream size cap (Limits, default 1 GiB, and a per-call cap); a lenient directory scan (directory_names, has_entry_named) for files too damaged to open. From WordCraft's guard: a FAT sector listed twice is refused; spare FAT sectors and zero-padded DIFAT sectors are accepted. New: Limits::max_dir_entries, walk() and path() to list entries, has_stream, Writer::stream_at and write_streams, and only storages the root reaches are walked. --- Cargo.lock | 4 + Cargo.toml | 1 + README.md | 3 + crates/cfb/Cargo.toml | 13 + crates/cfb/README.md | 190 +++++++++ crates/cfb/src/lib.rs | 131 ++++++ crates/cfb/src/read.rs | 571 +++++++++++++++++++++++++ crates/cfb/src/scan.rs | 91 ++++ crates/cfb/src/tests.rs | 442 +++++++++++++++++++ crates/cfb/src/tests/ported.rs | 704 +++++++++++++++++++++++++++++++ crates/cfb/src/write.rs | 478 +++++++++++++++++++++ crates/cfb/tests/writer_scale.rs | 32 ++ 12 files changed, 2660 insertions(+) create mode 100644 crates/cfb/Cargo.toml create mode 100644 crates/cfb/README.md create mode 100644 crates/cfb/src/lib.rs create mode 100644 crates/cfb/src/read.rs create mode 100644 crates/cfb/src/scan.rs create mode 100644 crates/cfb/src/tests.rs create mode 100644 crates/cfb/src/tests/ported.rs create mode 100644 crates/cfb/src/write.rs create mode 100644 crates/cfb/tests/writer_scale.rs diff --git a/Cargo.lock b/Cargo.lock index 7643420ed..213a2ce39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -825,6 +825,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-cfb" +version = "0.1.0" + [[package]] name = "craft-cms" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 01dfe9234..08e45a86d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ bytemuck = "1" kurbo = "0.13" thiserror = "2" craft-atomic = { path = "crates/atomic" } +craft-cfb = { path = "crates/cfb" } craft-cms = { path = "crates/cms" } craft-crw = { path = "crates/crw" } craft-dialogs = { path = "crates/dialogs" } diff --git a/README.md b/README.md index 351596d65..012d119b6 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,9 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " = w.finish(); +``` + +Layout: stream sectors (streams of 4096 bytes or more), mini stream, mini FAT, directory, FAT, +DIFAT. Each chain is contiguous. Sibling trees are balanced binary search trees in [MS-CFB] name +order (length first, then upper-cased code units) coloured as valid red-black trees. Names are +checked (1 to 31 UTF-16 code units, none of `/ \ : !` or NUL, unique per storage, +case-insensitively); the duplicate check is a hash lookup, so adding n children is O(n). + +## What it does not do + +- No `Read + Seek` input and no streaming output: files are read from and written to memory. +- No editing in place. To change a file, read its streams and write a new one. +- No timestamps or state bits on write (they are read). +- No transactions, no sector-level access, no free-sector reuse. +- Version 3 files are not limited to the 2 GB the specification allows them. +- Nothing about stream contents (property sets, `\1CompObj`, encryption). + +## Where it came from + +The base is FilmCraft's `filmcraft-cfb` (reader, writer and tests, moved as they were). Three +other Craft apps had their own readers; what they did and the base did not was ported, each with +a test in `src/tests/ported.rs`. Where the copies disagreed: + +| Question | FilmCraft | GridCraft / DeckCraft | WordCraft (guard in front of the `cfb` crate) | Here | +|---|---|---|---|---| +| Sector shift | 7 to 16 | 9 or 12 | by version: 3 → 512, 4 → 4096 | 9 or 12 (the specification; a smaller sector overlaps the header) | +| Byte order mark | required | not checked | checked by `cfb` | required by `open`; not checked by `directory_names` | +| Where the FAT sector list ends | the header's count | every listed number | the header's count, trailing zeros and free slots trimmed | the header's count: it is what makes zero-padded DIFAT sectors readable | +| A FAT sector listed twice | allowed (FAT bounded by the header's count) | allowed | refused | refused | +| More FAT sectors than the file has sectors | refused | refused | refused; a tighter bound had refused real files with spare FAT sectors | refused; spare sectors accepted | +| File ends inside its last sector | error | reads short, if it is the chain's last sector | (left to `cfb`) | GridCraft's rule, for streams, the directory, the mini FAT and the mini stream; FAT and DIFAT sectors must be whole | +| Chain loop | step count bounded by the file's sectors; a loop inside the stream's size read back as repeated data | a visited flag per FAT entry, allocated per read | (left to `cfb`) | step bound, plus a check for a repeated sector in the chain (memory proportional to the stream, not to the FAT) | +| Damaged or short mini stream | `open` fails | `open` succeeds, the stream read fails | (left to `cfb`) | `open` succeeds | +| Stream size cap | the file's length | 1 GiB | a per-call cap at each caller | `Limits::max_stream_size` (default 1 GiB) and a per-call cap | +| Size before version 4 | high half ignored when the version is 3 | high half ignored when the version is below 4 | (left to `cfb`) | below 4 | +| Link to a missing directory entry | `open` fails | ignored | (left to `cfb`) | ignored | +| Entry reached twice | `open` fails | the lookup fails if it gets that far | (left to `cfb`) | `open` fails | +| Which storages are walked | every storage entry, in directory order (an orphan pointing into the live tree made `open` fail) | the root only | (left to `cfb`) | every storage the root reaches | +| Name lookup | [MS-CFB] upper-casing, sorted index | ASCII case-insensitive, linear, streams only | `cfb` | FilmCraft's; `has_stream` for "a stream of that name" | +| Name of an entry | the code units the length covers, up to a NUL | the code units the length covers | `cfb` | FilmCraft's | +| Directory size | the file's length | 1024 sectors in the scan | (left to `cfb`) | `Limits::max_dir_entries` | +| Largest regular sector number | `<= 0xFFFFFFFA` | `< 0xFFFFFFFA` | `cfb` | `<= 0xFFFFFFFA` (no file can hold either) | + +## Migration + +FilmCraft (`filmcraft_cfb`): + +| Before | After | +|---|---| +| `filmcraft_cfb::{CompoundFile, Entry, EntryKind, Writer, NodeId, Version, Error, sniff, compare_names, validate_name}` | the same names in `craft_cfb` | +| `CompoundFile::open(bytes)` with no stream cap | `CompoundFile::open_with(bytes, Limits { max_stream_size: u64::MAX, ..Limits::default() })` if streams over 1 GiB must read | +| matching on `Error` | `Error` is `#[non_exhaustive]` and has `LimitExceeded` | + +GridCraft (`crates/xlsx/src/cfb.rs`) and DeckCraft (`crates/pptx/src/cfb.rs`): + +| Before | After | +|---|---| +| `cfb::SIGNATURE`, `bytes.starts_with(&SIGNATURE)` | `craft_cfb::SIGNATURE`, `craft_cfb::sniff(bytes)` | +| `cfb::is_encrypted(bytes)` | `["EncryptedPackage", "EncryptionInfo"].iter().any(\|n\| craft_cfb::has_entry_named(bytes, n, &limits))` | +| `directory_names(bytes)` | `craft_cfb::directory_names(bytes, &limits)` | +| `Compound::open(bytes)` | `CompoundFile::open(bytes)` | +| `cf.has_root_stream(name)` | `cf.has_stream(name)` | +| `cf.root_stream(name)` → `Ok(None)` when absent | `if cf.has_stream(name) { Some(cf.read_path(name)?) } else { None }` | +| `MAX_STREAM` (1 GiB) | the default `Limits::max_stream_size` | +| `testing::compound_file(streams)` | `craft_cfb::write_streams(Version::V3, streams)` (a different sector layout: tests that cut or patch the file at fixed offsets need new offsets) | + +WordCraft (`wordcraft_doc::cfb_guard` and the `cfb` crate): + +| Before | After | +|---|---| +| `cfb_guard::check(bytes)?` then `cfb::CompoundFile::open(Cursor::new(bytes))` | `CompoundFile::open(bytes)` (the guard's checks are part of `open`) | +| `comp.is_stream(path)` | `cf.has_stream(path)` | +| `comp.open_stream(path)` then `take(cap + 1).read_to_end(..)` | `cf.read_path_capped(path, cap)` (`Error::LimitExceeded` when larger) | +| `CompoundFile::create_with_version(v, Cursor::new(Vec::new()))`, `create_storage_all`, `create_stream(..).write_all(..)`, `flush`, `into_inner().into_inner()` | `Writer::new(v)`, `stream_at(path, bytes)`, `finish()`, or `write_streams(v, &[(path, bytes)])` | +| `cfb::Version::{V3, V4}` | `craft_cfb::Version::{V3, V4}` | +| `cfb_guard::repeating_difat_sample()` (test input) | not exported; the regression is `a_repeating_difat_is_refused` here | + +## Tests + +`src/tests.rs` (from FilmCraft): round trips for both versions with empty, mini, boundary +(63/64/65, 4095/4096/4097 bytes) and multi-sector streams and up to 40 siblings; red-black +invariants of every sibling tree; a 7.5 MB stream that needs DIFAT sectors; Unicode and +case-insensitive names; random mutations; DIFAT and directory chain loops. + +`src/tests/ported.rs`: files from a second, independent test writer with another layout and an +unsorted sibling list; every behaviour in the table above; each limit; truncation at every length +and every single-bit flip of two small files, and every special sector value in every header, FAT +and directory field, none of which may panic or hang. + +`tests/writer_scale.rs`: 40 000 children in one storage in under 3 s; duplicate-name rules. diff --git a/crates/cfb/src/lib.rs b/crates/cfb/src/lib.rs new file mode 100644 index 000000000..271bb08b5 --- /dev/null +++ b/crates/cfb/src/lib.rs @@ -0,0 +1,131 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Clean-room Compound File Binary format reader and writer ("OLE structured storage": the +//! container of legacy Office files, Outlook `.msg` messages, encrypted OOXML packages and AAF), +//! written from the public Microsoft Open Specification **[MS-CFB]: Compound File Binary File +//! Format** (protocol revision 10.0 and earlier editions; the on-disk format has not changed +//! since 1.0). +//! +//! A compound file is a small FAT file system inside one file: fixed-size sectors (512 bytes in +//! version 3, 4096 bytes in version 4) chained by a file allocation table, a directory of 128-byte +//! entries arranged as one red-black tree of siblings per storage, and a "mini stream" with a mini +//! FAT for streams smaller than 4096 bytes. +//! +//! - [`CompoundFile::open`] parses a file held in memory (no copies of large streams; every chain +//! is bounds- and cycle-checked, so damaged files give errors, never panics or endless loops). +//! [`Limits`] caps the stream size and the number of directory entries. +//! - [`Writer`] builds a tree of storages and streams and lays it out in one pass +//! ([`Writer::finish`]): stream sectors, mini stream, mini FAT, directory, FAT and DIFAT. +//! [`write_streams`] is the one-call form for a list of `(path, bytes)`. +//! - [`directory_names`] and [`has_entry_named`] answer "does this file hold an entry called X" +//! for files too damaged to open. +//! +//! No dependencies beyond `std`, no `unsafe`, nothing platform-specific. + +mod read; +mod scan; +mod write; + +pub use read::{CompoundFile, Entry, EntryKind, Limits}; +pub use scan::{directory_names, has_entry_named}; +pub use write::{NodeId, Version, Writer, write_streams}; + +use std::cmp::Ordering; +use std::fmt; + +/// The 8-byte header signature (§2.2). +pub const SIGNATURE: [u8; 8] = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; +/// Largest regular sector number (§2.1). +pub const MAXREGSECT: u32 = 0xFFFF_FFFA; +/// Sector used by the DIFAT. +pub const DIFSECT: u32 = 0xFFFF_FFFC; +/// Sector used by the FAT. +pub const FATSECT: u32 = 0xFFFF_FFFD; +/// End of a sector chain. +pub const ENDOFCHAIN: u32 = 0xFFFF_FFFE; +/// Unallocated sector. +pub const FREESECT: u32 = 0xFFFF_FFFF; +/// No sibling / child directory entry. +pub const NOSTREAM: u32 = 0xFFFF_FFFF; +/// Streams shorter than this live in the mini stream (§2.2 `Mini Stream Cutoff Size`). +pub const MINI_STREAM_CUTOFF: u32 = 4096; +/// Mini sector size (§2.2: mini sector shift 6). +pub const MINI_SECTOR_SIZE: usize = 64; +/// Longest directory entry name in UTF-16 code units, without the terminating NUL (§2.6.1). +pub const MAX_NAME_LEN: usize = 31; + +/// Errors from the reader and writer. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// Not a compound file (bad signature or byte order mark). + NotCfb, + /// Structurally invalid or truncated. + Invalid(String), + /// No entry with that name / path. + NotFound(String), + /// A name the format cannot store (too long, illegal characters, duplicate). + BadName(String), + /// The file is well-formed so far but asks for more than a [`Limits`] value or a per-call cap. + LimitExceeded(String), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::NotCfb => f.write_str("not a compound file"), + Error::Invalid(s) => write!(f, "invalid compound file: {s}"), + Error::NotFound(s) => write!(f, "no entry {s:?}"), + Error::BadName(s) => write!(f, "invalid entry name {s:?}"), + Error::LimitExceeded(s) => write!(f, "compound file limit exceeded: {s}"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// Whether `head` starts with the compound file signature. +pub fn sniff(head: &[u8]) -> bool { + head.starts_with(&SIGNATURE) +} + +fn upper(u: u16) -> u16 { + if (b'a' as u16..=b'z' as u16).contains(&u) { + return u - 32; + } + if u < 0x80 { + return u; + } + match char::from_u32(u as u32) { + Some(c) => { + let mut up = c.to_uppercase(); + match (up.next(), up.next()) { + (Some(x), None) if (x as u32) <= 0xFFFF => x as u32 as u16, + _ => u, + } + } + None => u, + } +} + +/// The sibling order of directory entries (§2.6.4): shorter names first, then a code-unit +/// comparison of the upper-cased names. +pub fn compare_names(a: &str, b: &str) -> Ordering { + // No allocation: the iterators are walked in place (`child` calls this once per sibling). + let (la, lb) = (a.encode_utf16().count(), b.encode_utf16().count()); + la.cmp(&lb).then_with(|| a.encode_utf16().map(upper).cmp(b.encode_utf16().map(upper))) +} + +/// Check a directory entry name (§2.6.1): 1–31 UTF-16 code units, none of `/ \ : !`, and no NUL +/// (the stored name is NUL-terminated, so a NUL would cut the name short when read back). +pub fn validate_name(name: &str) -> Result<()> { + let n = name.encode_utf16().count(); + if n == 0 || n > MAX_NAME_LEN || name.contains(['/', '\\', ':', '!', '\0']) { + return Err(Error::BadName(name.to_string())); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/cfb/src/read.rs b/crates/cfb/src/read.rs new file mode 100644 index 000000000..decd895da --- /dev/null +++ b/crates/cfb/src/read.rs @@ -0,0 +1,571 @@ +//! Reading compound files ([MS-CFB] §2.2–2.6). + +use crate::{ENDOFCHAIN, Error, MAXREGSECT, MINI_SECTOR_SIZE, NOSTREAM, Result, compare_names, sniff}; + +/// Caps on what a file may ask the reader to allocate. +/// +/// Everything else `open` allocates (FAT, mini FAT, mini stream) is bounded by the length of the +/// input itself: no chain can hold more sectors than the file has. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Limits { + /// Largest stream [`CompoundFile::read`] returns. Default 1 GiB. + pub max_stream_size: u64, + /// Most directory entries (128 bytes each in the file) `open` accepts. Default 1 048 576. + pub max_dir_entries: usize, +} + +impl Default for Limits { + fn default() -> Self { + Limits { max_stream_size: 1 << 30, max_dir_entries: 1 << 20 } + } +} + +/// Directory entry object type (§2.6.1). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EntryKind { + Root, + Storage, + Stream, + /// Unallocated (type 0) or an unknown type. + Unknown, +} + +/// One directory entry. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Entry { + pub name: String, + pub kind: EntryKind, + /// Class id of a storage. + pub clsid: [u8; 16], + pub state_bits: u32, + /// FILETIME stamps (0 when not set). + pub created: u64, + pub modified: u64, + /// First sector (regular or mini) of a stream; the mini stream's for the root. + pub start: u32, + pub size: u64, + /// The storage this entry is a child of. `None` for the root and for entries the root's tree + /// does not reach (unallocated entries, orphans). + pub parent: Option, + /// Child entries of a storage, in the in-order position of its sibling tree. For a well-formed + /// file that is sorted by [`compare_names`]; the reader does not validate the tree's order, so + /// a hostile file may give any order (lookups by name do not depend on it). Read-only after + /// open: [`CompoundFile::child`] answers from an index built from this list and the entry names + /// when the file is parsed, so changing either afterwards leaves the lookup stale. + pub children: Vec, + /// `children` sorted by [`compare_names`] (stable, so equal names keep their in-order + /// position): [`CompoundFile::child`] binary-searches it instead of scanning every sibling, + /// which made resolving every member of a wide storage quadratic. + by_name: Vec, + left: u32, + right: u32, + child: u32, +} + +/// A parsed compound file over borrowed bytes. +#[derive(Clone, Debug)] +pub struct CompoundFile<'a> { + data: &'a [u8], + limits: Limits, + /// Major version: 3 (512-byte sectors) or 4 (4096-byte sectors). + pub version: u16, + /// Header CLSID (normally zero). + pub header_clsid: [u8; 16], + sector_size: usize, + mini_cutoff: u64, + fat: Vec, + minifat: Vec, + mini_stream: Vec, + /// Why the mini stream could not be read. The file still opens (streams in regular sectors + /// stay readable); reading a stream held in the mini stream returns this error. + mini_error: Option, + /// The directory entries. Read-only after open (see [`Entry::children`]): the by-name index + /// behind [`CompoundFile::child`] is built once when the file is parsed. + pub entries: Vec, +} + +// Little-endian reads that cannot panic: a read past the end gives 0. Callers pass slices they +// have already sized (the 512-byte header, a whole sector, a 128-byte directory entry). +pub(crate) fn u16_at(b: &[u8], at: usize) -> u16 { + b.get(at..).and_then(|s| s.first_chunk::<2>()).map_or(0, |c| u16::from_le_bytes(*c)) +} +pub(crate) fn u32_at(b: &[u8], at: usize) -> u32 { + b.get(at..).and_then(|s| s.first_chunk::<4>()).map_or(0, |c| u32::from_le_bytes(*c)) +} +fn u64_at(b: &[u8], at: usize) -> u64 { + b.get(at..).and_then(|s| s.first_chunk::<8>()).map_or(0, |c| u64::from_le_bytes(*c)) +} +fn clsid_at(b: &[u8], at: usize) -> [u8; 16] { + b.get(at..).and_then(|s| s.first_chunk::<16>()).copied().unwrap_or([0; 16]) +} + +/// Byte range `[start, end)` of unit `index + skip` of `unit` bytes each, or `None` when the +/// arithmetic overflows `usize` (reachable on 32-bit targets such as wasm32). +pub(crate) fn span(index: usize, skip: usize, unit: usize) -> Option<(usize, usize)> { + let start = index.checked_add(skip)?.checked_mul(unit)?; + Some((start, start.checked_add(unit)?)) +} + +/// The first value that occurs twice in `ids`. +fn repeated(ids: &[u32]) -> Option { + let mut sorted = ids.to_vec(); + sorted.sort_unstable(); + sorted.windows(2).find(|w| w.first() == w.get(1)).and_then(|w| w.first().copied()) +} + +fn invalid(s: impl Into) -> Error { + Error::Invalid(s.into()) +} + +impl<'a> CompoundFile<'a> { + /// Parse the header, FAT, mini FAT, directory and mini stream of `data` with the default + /// [`Limits`]. + pub fn open(data: &'a [u8]) -> Result> { + Self::open_with(data, Limits::default()) + } + + /// [`CompoundFile::open`] with caller-set limits. + pub fn open_with(data: &'a [u8], limits: Limits) -> Result> { + if data.len() < 512 || !sniff(data) || u16_at(data, 28) != 0xFFFE { + return Err(Error::NotCfb); + } + let version = u16_at(data, 26); + let shift = u16_at(data, 30); + // §2.2: 9 in version 3 and 12 in version 4, nothing else. The version field itself is not + // checked: the shift is what decides where every sector lies. + if shift != 9 && shift != 12 { + return Err(invalid(format!("sector shift {shift}"))); + } + let mini_shift = u16_at(data, 32); + if 1usize.checked_shl(mini_shift as u32) != Some(MINI_SECTOR_SIZE) { + return Err(invalid(format!("mini sector shift {mini_shift}"))); + } + let sector_size = 1usize << shift; + let n_fat = u32_at(data, 44) as usize; + let first_dir = u32_at(data, 48); + let first_minifat = u32_at(data, 60); + let first_difat = u32_at(data, 68); + let mut cf = CompoundFile { + data, + limits, + version, + header_clsid: clsid_at(data, 8), + sector_size, + mini_cutoff: u32_at(data, 56) as u64, + fat: Vec::new(), + minifat: Vec::new(), + mini_stream: Vec::new(), + mini_error: None, + entries: Vec::new(), + }; + cf.load_fat(n_fat, first_difat)?; + // Directory. + let dir = cf.read_chain(first_dir, None)?; + let (raw_entries, _) = dir.as_chunks::<128>(); + if raw_entries.len() > limits.max_dir_entries { + return Err(Error::LimitExceeded(format!("{} directory entries, the limit is {}", raw_entries.len(), limits.max_dir_entries))); + } + cf.entries = raw_entries.iter().map(|e| parse_entry(e, version)).collect(); + let (root_start, root_size) = match cf.entries.first() { + None => return Err(invalid("empty directory")), + Some(root) if root.kind != EntryKind::Root => return Err(invalid("first directory entry is not the root")), + Some(root) => (root.start, root.size), + }; + // Mini FAT and mini stream. + if first_minifat <= MAXREGSECT { + let mf = cf.read_chain(first_minifat, None)?; + cf.minifat = mf.as_chunks::<4>().0.iter().map(|c| u32::from_le_bytes(*c)).collect(); + } + if root_start <= MAXREGSECT && root_size > 0 { + // A mini stream that is damaged or shorter than the root entry says does not stop the + // file from opening: only reads that need the missing part fail. + let size = root_size.min(data.len() as u64) as usize; + match cf.read_chain(root_start, Some(size)) { + Ok(m) => cf.mini_stream = m, + Err(e) => cf.mini_error = Some(e), + } + } + cf.build_tree()?; + Ok(cf) + } + + /// An upper bound on the sectors any one chain can hold: the sectors of the file (counting the + /// header and a cut last sector). + fn max_sectors(&self) -> usize { + self.data.len() / self.sector_size + 1 + } + + /// Read the FAT: its sector numbers are the first `n_fat` of the 109 header slots followed by + /// the DIFAT chain. + fn load_fat(&mut self, n_fat: usize, first_difat: u32) -> Result<()> { + let (data, sector_size) = (self.data, self.sector_size); + let max_sectors = self.max_sectors(); + if n_fat > max_sectors { + return Err(invalid(format!("{n_fat} FAT sectors in a {}-byte file", data.len()))); + } + let mut fat_sectors: Vec = (0..109).map(|i| u32_at(data, 76 + i * 4)).filter(|&s| s <= MAXREGSECT).collect(); + let mut d = first_difat; + let mut seen = 0; + while d <= MAXREGSECT && fat_sectors.len() < n_fat { + seen += 1; + // a DIFAT chain cannot have more sectors than the file holds; the header's DIFAT sector + // count (up to 2^32 - 1) is not trusted, or a self-pointing sector loops that often + if seen > max_sectors { + return Err(invalid(format!("DIFAT chain loops: more than {max_sectors} sectors in a {}-byte file", data.len()))); + } + let s = self.sector(d)?; + let per = sector_size / 4 - 1; + fat_sectors.extend((0..per).map(|i| u32_at(s, i * 4)).filter(|&v| v <= MAXREGSECT)); + d = u32_at(s, per * 4); + } + // Slots past the header's count are padding (free, or zero from some writers). + fat_sectors.truncate(n_fat); + // Every FAT sector is a sector of the file, so none is listed twice. A file that names one + // sector over and over would otherwise build a FAT many times its own size. + if let Some(dup) = repeated(&fat_sectors) { + return Err(invalid(format!("FAT sector {dup} is listed twice"))); + } + let mut fat = Vec::with_capacity(fat_sectors.len().saturating_mul(sector_size / 4)); + for s in fat_sectors { + let b = self.sector(s)?; + fat.extend(b.as_chunks::<4>().0.iter().map(|c| u32::from_le_bytes(*c))); + } + self.fat = fat; + Ok(()) + } + + /// Sector `n`, whole (FAT and DIFAT sectors: a truncated one is an error). + fn sector(&self, n: u32) -> Result<&'a [u8]> { + let s = self.sector_tail(n)?; + if s.len() < self.sector_size { + return Err(invalid(format!("sector {n} is outside the file"))); + } + Ok(s) + } + + /// Sector `n`, or the part of it the file holds when the file ends inside it (some writers do + /// not pad the last sector). Never empty. + fn sector_tail(&self, n: u32) -> Result<&'a [u8]> { + let outside = || invalid(format!("sector {n} is outside the file")); + if n > MAXREGSECT { + return Err(outside()); + } + // sector 0 follows the header sector + let (start, end) = usize::try_from(n).ok().and_then(|i| span(i, 1, self.sector_size)).ok_or_else(|| invalid("sector number overflow"))?; + self.data.get(start..end.min(self.data.len())).filter(|s| !s.is_empty()).ok_or_else(outside) + } + + /// The sector numbers of the chain from `start`: all of it, or its first `want` sectors. + fn chain_ids(&self, start: u32, want: Option) -> Result> { + // A chain cannot visit more sectors than the file holds. Not `self.fat.len()`: the FAT can + // be longer than the file has sectors. + let max_steps = self.max_sectors(); + let want = want.unwrap_or(usize::MAX); + let mut ids = Vec::new(); + let mut s = start; + while s != ENDOFCHAIN && ids.len() < want { + if s > MAXREGSECT { + return Err(invalid(format!("bad sector {s:#x} in a chain"))); + } + if ids.len() >= max_steps { + return Err(invalid(format!("sector chain loops: more than {max_steps} sectors in a {}-byte file", self.data.len()))); + } + ids.push(s); + if ids.len() == want { + break; + } + // a free entry (`FREESECT`) is above `MAXREGSECT` and is rejected at the top of the loop + s = *self.fat.get(s as usize).ok_or_else(|| invalid(format!("sector {s} has no FAT entry")))?; + } + // A loop shorter than the stream would otherwise read back as repeated data. + if let Some(dup) = repeated(&ids) { + return Err(invalid(format!("sector chain loops: sector {dup} is visited twice"))); + } + Ok(ids) + } + + /// Concatenate a regular sector chain, up to `limit` bytes. The result is shorter than `limit` + /// when the chain ends first or the file ends inside the chain's last sector; a sector cut + /// short anywhere else in the chain is an error. + fn read_chain(&self, start: u32, limit: Option) -> Result> { + let ids = self.chain_ids(start, limit.map(|l| l.div_ceil(self.sector_size)))?; + // `limit` is at most the file length (callers check) and the chain holds distinct sectors + // of the file, so this capacity is bounded by the file's own size. + let mut out = Vec::with_capacity(limit.unwrap_or(ids.len().saturating_mul(self.sector_size)).min(self.data.len())); + for (i, &s) in ids.iter().enumerate() { + let b = self.sector_tail(s)?; + if b.len() < self.sector_size && i + 1 != ids.len() { + return Err(invalid(format!("sector {s} is cut short inside a chain (the file may be truncated)"))); + } + out.extend_from_slice(b); + } + if let Some(l) = limit { + out.truncate(l); + } + Ok(out) + } + + /// The `size` bytes of a stream held in the mini stream. As in [`Self::read_chain`], the mini + /// stream may end inside the chain's last mini sector. + fn read_mini_chain(&self, start: u32, size: usize) -> Result> { + if let Some(e) = &self.mini_error { + return Err(e.clone()); + } + // `size` is at most the file length (the caller checks), which bounds the walk. + let want = size.div_ceil(MINI_SECTOR_SIZE); + let mut ids = Vec::new(); + let mut s = start; + while ids.len() < want { + if s > MAXREGSECT { + return Err(invalid("mini stream chain ends early")); + } + ids.push(s); + s = *self.minifat.get(s as usize).ok_or_else(|| invalid("mini sector has no mini FAT entry"))?; + } + if repeated(&ids).is_some() { + return Err(invalid("mini sector chain loops")); + } + let mut out = Vec::with_capacity(size); + for (i, &s) in ids.iter().enumerate() { + let b = usize::try_from(s) + .ok() + .and_then(|i| span(i, 0, MINI_SECTOR_SIZE)) + .and_then(|(start, end)| self.mini_stream.get(start..end.min(self.mini_stream.len()))) + .filter(|b| !b.is_empty()) + .ok_or_else(|| invalid("mini sector outside the mini stream"))?; + if b.len() < MINI_SECTOR_SIZE && i + 1 != ids.len() { + return Err(invalid("mini sector is cut short inside a chain (the file may be truncated)")); + } + out.extend_from_slice(b); + } + if out.len() < size { + return Err(invalid("stream is truncated")); + } + out.truncate(size); + Ok(out) + } + + /// Fill in `children`, `parent` and the by-name index of every storage the root reaches. + /// Entries nothing reaches (orphans) are left alone, whatever they point at. + fn build_tree(&mut self) -> Result<()> { + let n = self.entries.len(); + let mut owner = vec![false; n]; + if let Some(root) = owner.first_mut() { + *root = true; + } + // Storages whose children are still to be walked. Every entry is pushed at most once (it + // is claimed in `owner` first), so this ends after at most `n` storages. + let mut pending = vec![0usize]; + while let Some(parent) = pending.pop() { + let mut kids = Vec::new(); + // in-order walk of the sibling tree, iterative, cycle-safe + let mut stack: Vec = Vec::new(); + let mut cur = self.entries.get(parent).map_or(NOSTREAM, |e| e.child); + loop { + // A link to an entry that does not exist is treated as no link (the rest of the + // tree stays readable); a link to an entry already in the tree is a cycle or a + // shared subtree, which has no sensible reading. + while let Some(e) = usize::try_from(cur).ok().and_then(|i| self.entries.get(i).map(|e| (i, e.left))) { + let (i, left) = e; + if owner.get(i).copied().unwrap_or(true) { + return Err(invalid(format!("directory entry {i} is referenced twice"))); + } + if let Some(o) = owner.get_mut(i) { + *o = true; + } + stack.push(i); + cur = left; + } + let Some(top) = stack.pop() else { break }; + kids.push(top); + cur = self.entries.get(top).map_or(NOSTREAM, |e| e.right); + } + for &k in &kids { + if let Some(e) = self.entries.get_mut(k) { + e.parent = Some(parent); + if matches!(e.kind, EntryKind::Root | EntryKind::Storage) { + pending.push(k); + } + } + } + let mut by_name = kids.clone(); + by_name.sort_by(|&a, &b| compare_names(self.name(a), self.name(b))); + if let Some(p) = self.entries.get_mut(parent) { + p.children = kids; + p.by_name = by_name; + } + } + Ok(()) + } + + fn name(&self, id: usize) -> &str { + self.entries.get(id).map_or("", |e| e.name.as_str()) + } + + /// The root storage entry (id 0). + pub fn root(&self) -> &Entry { + // `open` refuses an empty directory, and `entries` is documented read-only after it. + static EMPTY: Entry = Entry { + name: String::new(), + kind: EntryKind::Unknown, + clsid: [0; 16], + state_bits: 0, + created: 0, + modified: 0, + start: ENDOFCHAIN, + size: 0, + parent: None, + children: Vec::new(), + by_name: Vec::new(), + left: NOSTREAM, + right: NOSTREAM, + child: NOSTREAM, + }; + self.entries.first().unwrap_or(&EMPTY) + } + + pub fn entry(&self, id: usize) -> Option<&Entry> { + self.entries.get(id) + } + + /// Child of storage `parent` named `name` (case-insensitive as in §2.6.4). When a damaged + /// file holds several children of the same name, the first in sibling-tree order wins. + pub fn child(&self, parent: usize, name: &str) -> Option { + let by_name = &self.entries.get(parent)?.by_name; + let i = by_name.partition_point(|&c| compare_names(self.name(c), name).is_lt()); + by_name.get(i).copied().filter(|&c| compare_names(self.name(c), name).is_eq()) + } + + /// Entry at a `/`-separated path below the root (`""` is the root; empty components, so a + /// leading `/` too, are ignored). + pub fn find(&self, path: &str) -> Result { + let mut cur = 0; + for part in path.split('/').filter(|p| !p.is_empty()) { + cur = self.child(cur, part).ok_or_else(|| Error::NotFound(path.to_string()))?; + } + Ok(cur) + } + + /// Whether `path` names a stream. + pub fn has_stream(&self, path: &str) -> bool { + self.find(path).ok().and_then(|id| self.entries.get(id)).is_some_and(|e| e.kind == EntryKind::Stream) + } + + /// Every storage and stream below the root, parents before their children and siblings in + /// sibling-tree order. [`CompoundFile::path`] gives the path of each. + pub fn walk(&self) -> Vec { + let mut out = Vec::new(); + // ends: `build_tree` gave every entry at most one parent, so each is pushed at most once + let mut stack: Vec = self.root().children.iter().rev().copied().collect(); + while let Some(id) = stack.pop() { + out.push(id); + if let Some(e) = self.entries.get(id) { + stack.extend(e.children.iter().rev()); + } + } + out + } + + /// The `/`-separated path of entry `id` below the root (`""` for the root itself), or `None` + /// for an id that does not exist or an entry the root does not reach. Names are joined as + /// they are: a damaged file may hold a name with a `/` in it, which `find` will not resolve. + pub fn path(&self, id: usize) -> Option { + let mut parts = Vec::new(); + let mut cur = id; + // a parent chain is never longer than the directory + for _ in 0..=self.entries.len() { + if cur == 0 { + parts.reverse(); + return Some(parts.join("/")); + } + let e = self.entries.get(cur)?; + parts.push(e.name.as_str()); + cur = e.parent?; + } + None + } + + /// Contents of stream `id`, refused above [`Limits::max_stream_size`]. + pub fn read(&self, id: usize) -> Result> { + self.read_capped(id, u64::MAX) + } + + /// Contents of stream `id`, refused when it is larger than `cap` bytes (or than + /// [`Limits::max_stream_size`], whichever is smaller). Nothing is allocated for a refused + /// stream. + pub fn read_capped(&self, id: usize, cap: u64) -> Result> { + let e = self.entries.get(id).ok_or_else(|| Error::NotFound(format!("entry {id}")))?; + if e.kind != EntryKind::Stream { + return Err(invalid(format!("{:?} is not a stream", e.name))); + } + let cap = cap.min(self.limits.max_stream_size); + if e.size > cap { + return Err(Error::LimitExceeded(format!("stream {:?} is {} bytes, the limit is {cap}", e.name, e.size))); + } + if e.size == 0 { + return Ok(Vec::new()); + } + let size = usize::try_from(e.size) + .ok() + .filter(|&s| s <= self.data.len()) + .ok_or_else(|| invalid(format!("stream {:?} is larger than the file", e.name)))?; + if e.size < self.mini_cutoff { + return self.read_mini_chain(e.start, size); + } + let out = self.read_chain(e.start, Some(size))?; + if out.len() < size { + return Err(invalid("stream is truncated")); + } + Ok(out) + } + + /// Contents of the stream at `path`. + pub fn read_path(&self, path: &str) -> Result> { + self.read(self.find(path)?) + } + + /// Contents of the stream at `path`, refused when larger than `cap` bytes. + pub fn read_path_capped(&self, path: &str, cap: u64) -> Result> { + self.read_capped(self.find(path)?, cap) + } + + /// Sector size in bytes (512 or 4096). + pub fn sector_size(&self) -> usize { + self.sector_size + } +} + +/// The name of a 128-byte directory entry: the UTF-16 code units its length field covers, up to +/// the first NUL. +pub(crate) fn entry_name(b: &[u8; 128]) -> String { + let name_len = (u16_at(b, 64) as usize).min(64); + let units: Vec = (0..name_len / 2).map(|i| u16_at(b, i * 2)).take_while(|&u| u != 0).collect(); + String::from_utf16_lossy(&units) +} + +fn parse_entry(b: &[u8; 128], version: u16) -> Entry { + let kind = match b.get(66) { + Some(5) => EntryKind::Root, + Some(1) => EntryKind::Storage, + Some(2) => EntryKind::Stream, + _ => EntryKind::Unknown, + }; + let size = u64_at(b, 120); + Entry { + name: entry_name(b), + kind, + clsid: clsid_at(b, 80), + state_bits: u32_at(b, 96), + created: u64_at(b, 100), + modified: u64_at(b, 108), + start: u32_at(b, 116), + // only version 4 has 64-bit sizes; earlier files may leave garbage in the high half (§2.6.3) + size: if version >= 4 { size } else { size & 0xFFFF_FFFF }, + parent: None, + children: Vec::new(), + by_name: Vec::new(), + left: u32_at(b, 68), + right: u32_at(b, 72), + child: u32_at(b, 76), + } +} diff --git a/crates/cfb/src/scan.rs b/crates/cfb/src/scan.rs new file mode 100644 index 000000000..e16abee39 --- /dev/null +++ b/crates/cfb/src/scan.rs @@ -0,0 +1,91 @@ +//! A lenient look at the directory, for telling what a file is when it may be too damaged to +//! open: only the header, the DIFAT and the directory chain are followed. No FAT is built and +//! nothing is allocated beyond the names themselves. + +use crate::read::{Limits, entry_name, span, u16_at, u32_at}; +use crate::{ENDOFCHAIN, MAXREGSECT, compare_names, sniff}; + +/// The names of the directory entries of a compound file, in directory order (the root entry +/// first; unallocated entries, which have no name, are left out), or `None` when `bytes` is not a +/// compound file or its directory cannot be followed. +/// +/// More forgiving than [`crate::CompoundFile::open`]: the byte order mark, the FAT sector count, +/// the mini FAT, the mini stream and the sibling trees are not looked at, so entries no storage +/// links to are listed too. A directory chain that loops, or that holds more than +/// [`Limits::max_dir_entries`] entries, gives the names read up to that point. +pub fn directory_names(bytes: &[u8], limits: &Limits) -> Option> { + if bytes.len() < 512 || !sniff(bytes) { + return None; + } + let shift = u16_at(bytes, 30); + if shift != 9 && shift != 12 { + return None; + } + let size = 1usize << shift; + let per = size / 4; + // No chain is longer than the file has sectors: the bound on both walks below. + let max_sectors = bytes.len() / size + 1; + let sector = |id: u32| -> Option<&[u8]> { + let (start, end) = span(usize::try_from(id).ok()?, 1, size)?; + bytes.get(start..end) + }; + // The DIFAT: the FAT's sector ids, 109 in the header then chained DIFAT sectors. + let mut difat: Vec = (0..109).map(|i| u32_at(bytes, 76 + i * 4)).filter(|&s| s <= MAXREGSECT).collect(); + let mut next = u32_at(bytes, 68); + for _ in 0..max_sectors { + if next > MAXREGSECT { + break; + } + let s = sector(next)?; + difat.extend((0..per - 1).map(|i| u32_at(s, i * 4)).filter(|&x| x <= MAXREGSECT)); + next = u32_at(s, (per - 1) * 4); + } + let fat_next = |id: u32| -> Option { + let idx = usize::try_from(id).ok()?; + Some(u32_at(sector(*difat.get(idx / per)?)?, (idx % per) * 4)) + }; + let mut names = Vec::new(); + let mut id = u32_at(bytes, 48); + for _ in 0..max_sectors { + if id == ENDOFCHAIN { + return Some(names); + } + if id > MAXREGSECT { + return None; + } + for e in sector(id)?.as_chunks::<128>().0 { + // an allocated entry's name length counts its terminating NUL: 2 to 64 bytes + if !(2..=64).contains(&u16_at(e, 64)) { + continue; + } + if names.len() >= limits.max_dir_entries { + return Some(names); + } + names.push(entry_name(e)); + } + id = fat_next(id)?; + } + // A cyclic chain: report what was read. + Some(names) +} + +/// Whether the compound file `bytes` holds a directory entry called `name` (compared as sibling +/// names are, so case-insensitively), at any depth. `false` when `bytes` does not start with the +/// compound file signature. +/// +/// When the directory cannot be followed (a truncated or damaged file), falls back to searching +/// the bytes for the name in UTF-16LE, so a damaged file still answers. That fallback is +/// case-sensitive and can match stream contents: use it to choose an error message, not to make a +/// security decision. +pub fn has_entry_named(bytes: &[u8], name: &str, limits: &Limits) -> bool { + if !sniff(bytes) { + return false; + } + match directory_names(bytes, limits) { + Some(names) => names.iter().any(|n| compare_names(n, name).is_eq()), + None => { + let needle: Vec = name.encode_utf16().flat_map(u16::to_le_bytes).collect(); + !needle.is_empty() && bytes.windows(needle.len()).any(|w| w == needle.as_slice()) + } + } +} diff --git a/crates/cfb/src/tests.rs b/crates/cfb/src/tests.rs new file mode 100644 index 000000000..7625afb52 --- /dev/null +++ b/crates/cfb/src/tests.rs @@ -0,0 +1,442 @@ +//! Round trips, the writer's layout and the reader's loop bounds. `tests/ported.rs` holds the +//! tests for the behaviour taken from the other readers this crate replaces. + +use super::*; + +mod ported; + +fn pattern(n: usize, seed: u8) -> Vec { + (0..n).map(|i| (i as u32).wrapping_mul(31).wrapping_add(seed as u32) as u8).collect() +} + +/// A tree with mini and regular streams, nested storages and many siblings. +fn sample(version: Version, siblings: usize) -> (Vec, Vec<(String, Vec)>) { + let mut w = Writer::new(version); + w.set_clsid(Writer::ROOT, [7; 16]); + let mut expect = Vec::new(); + let a = w.storage(Writer::ROOT, "Header-2", [1; 16]).unwrap(); + let b = w.storage(a, "Content-3b03", [2; 16]).unwrap(); + for (i, size) in [0usize, 1, 63, 64, 65, 4095, 4096, 4097, 20_000].iter().enumerate() { + let name = format!("s{i}"); + let data = pattern(*size, i as u8); + w.stream(b, &name, data.clone()).unwrap(); + expect.push((format!("Header-2/Content-3b03/{name}"), data)); + } + for i in 0..siblings { + let name = format!("Mobs-1901{{{i:x}}}"); + let s = w.storage(Writer::ROOT, &name, [3; 16]).unwrap(); + let data = pattern(10 + i * 7, 99); + w.stream(s, "properties", data.clone()).unwrap(); + expect.push((format!("{name}/properties"), data)); + } + (w.finish(), expect) +} + +fn check(bytes: &[u8], expect: &[(String, Vec)]) { + let cf = CompoundFile::open(bytes).unwrap(); + assert_eq!(cf.root().clsid, [7; 16]); + for (path, data) in expect { + assert_eq!(&cf.read_path(path).unwrap(), data, "{path}"); + } + let h = cf.find("Header-2").unwrap(); + assert_eq!(cf.entries[h].clsid, [1; 16]); + assert_eq!(cf.entries[h].kind, EntryKind::Storage); + // children come back sorted in CFB order + for e in &cf.entries { + for w in e.children.windows(2) { + assert!(compare_names(&cf.entries[w[0]].name, &cf.entries[w[1]].name).is_lt()); + } + } +} + +/// Every path from a node to a nil leaf has the same number of black nodes, and no red node has +/// a red child. +fn check_red_black(bytes: &[u8]) { + let cf = CompoundFile::open(bytes).unwrap(); + let raw = |i: u32| -> (u32, u32, u8) { + let ss = cf.sector_size(); + let dir_start = u32::from_le_bytes(bytes[48..52].try_into().unwrap()); + // the writer stores the directory contiguously + let at = (dir_start as usize + 1) * ss + i as usize * 128; + let e = &bytes[at..at + 128]; + (u32::from_le_bytes(e[68..72].try_into().unwrap()), u32::from_le_bytes(e[72..76].try_into().unwrap()), e[67]) + }; + fn black_height(raw: &dyn Fn(u32) -> (u32, u32, u8), n: u32, parent_red: bool) -> usize { + if n == NOSTREAM { + return 1; + } + let (l, r, c) = raw(n); + let red = c == 0; + assert!(!(red && parent_red), "red node with a red parent"); + let hl = black_height(raw, l, red); + let hr = black_height(raw, r, red); + assert_eq!(hl, hr, "unbalanced black height"); + hl + usize::from(!red) + } + let dir_start = u32::from_le_bytes(bytes[48..52].try_into().unwrap()); + let ss = cf.sector_size(); + for i in 0..cf.entries.len() { + let at = (dir_start as usize + 1) * ss + i * 128; + let child = u32::from_le_bytes(bytes[at + 76..at + 80].try_into().unwrap()); + if child != NOSTREAM { + let (_, _, c) = raw(child); + assert_eq!(c, 1, "tree roots are black"); + black_height(&raw, child, false); + } + } +} + +#[test] +fn round_trip_v3_and_v4() { + for v in [Version::V3, Version::V4] { + for n in [0, 1, 2, 3, 7, 8, 40] { + let (bytes, expect) = sample(v, n); + assert_eq!(bytes.len() % v_sector(v), 0); + check(&bytes, &expect); + check_red_black(&bytes); + } + } +} + +fn v_sector(v: Version) -> usize { + if v == Version::V3 { 512 } else { 4096 } +} + +#[test] +fn header_fields() { + let (b, _) = sample(Version::V4, 3); + assert!(sniff(&b)); + assert_eq!(&b[24..26], &0x3Eu16.to_le_bytes()); + assert_eq!(&b[26..28], &4u16.to_le_bytes()); + assert_eq!(&b[30..32], &12u16.to_le_bytes()); + let (b, _) = sample(Version::V3, 3); + assert_eq!(&b[26..28], &3u16.to_le_bytes()); + assert_eq!(&b[30..32], &9u16.to_le_bytes()); + assert_eq!(&b[40..44], &0u32.to_le_bytes(), "version 3 leaves the directory sector count 0"); + assert_eq!(CompoundFile::open(&b).unwrap().version, 3); +} + +#[test] +fn large_file_needs_difat() { + // > 109 FAT sectors with 512-byte sectors: 109 * 128 sectors * 512 bytes ≈ 7.1 MB + let mut w = Writer::new(Version::V3); + let big = pattern(7_500_000, 5); + w.stream(Writer::ROOT, "big", big.clone()).unwrap(); + w.stream(Writer::ROOT, "small", vec![1, 2, 3]).unwrap(); + let b = w.finish(); + let n_difat = u32::from_le_bytes(b[72..76].try_into().unwrap()); + assert!(n_difat >= 1); + let cf = CompoundFile::open(&b).unwrap(); + assert_eq!(cf.read_path("big").unwrap(), big); + assert_eq!(cf.read_path("small").unwrap(), vec![1, 2, 3]); +} + +#[test] +fn names() { + assert!(compare_names("b", "aa").is_lt(), "shorter first"); + assert!(compare_names("abc", "ABD").is_lt()); + assert!(compare_names("Abc", "aBC").is_eq()); + assert!(validate_name("a/b").is_err()); + assert!(validate_name(&"x".repeat(32)).is_err()); + assert!(validate_name(&"x".repeat(31)).is_ok()); + let mut w = Writer::default(); + w.stream(Writer::ROOT, "Name", vec![]).unwrap(); + assert!(matches!(w.stream(Writer::ROOT, "NAME", vec![]), Err(Error::BadName(_)))); + let s = w.stream(Writer::ROOT, "x", vec![]).unwrap(); + assert!(w.storage(s, "y", [0; 16]).is_err(), "streams have no children"); + let cf_bytes = w.finish(); + let cf = CompoundFile::open(&cf_bytes).unwrap(); + assert!(cf.find("name").is_ok(), "lookups are case-insensitive"); + assert!(matches!(cf.find("nope"), Err(Error::NotFound(_)))); + assert!(cf.read(0).is_err(), "the root is not a stream"); +} + +#[test] +fn unicode_names() { + let mut w = Writer::default(); + w.stream(Writer::ROOT, "Ünïcødé ストリーム", vec![9; 10]).unwrap(); + let b = w.finish(); + let cf = CompoundFile::open(&b).unwrap(); + assert_eq!(cf.read_path("üNÏCØDÉ ストリーム").unwrap(), vec![9; 10]); +} + +#[test] +fn not_cfb() { + assert_eq!(CompoundFile::open(b"hello").unwrap_err(), Error::NotCfb); + assert_eq!(CompoundFile::open(&[0u8; 600]).unwrap_err(), Error::NotCfb); +} + +#[test] +fn truncation_never_panics() { + let (b, expect) = sample(Version::V3, 20); + let mut ok = 0; + for cut in (0..b.len()).step_by(29) { + if let Ok(cf) = CompoundFile::open(&b[..cut]) { + ok += 1; + for (path, data) in &expect { + if let Ok(d) = cf.read_path(path) { + assert_eq!(&d, data, "a stream that reads back is complete"); + } + } + } + } + let _ = ok; +} + +#[test] +fn mutation_never_panics_or_hangs() { + let (b, _) = sample(Version::V3, 12); + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + for _ in 0..1500 { + let mut g = b.clone(); + for _ in 0..6 { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + // bias towards the header, FAT and directory (the end of the file) + let at = if seed & 1 == 0 { (seed >> 8) as usize % 512 } else { g.len() - 1 - (seed >> 8) as usize % 2048.min(g.len()) }; + g[at] = (seed >> 40) as u8; + } + if let Ok(cf) = CompoundFile::open(&g) { + for i in 0..cf.entries.len() { + let _ = cf.read(i); + } + } + } +} + +fn put_u32(b: &mut [u8], at: usize, v: u32) { + b[at..at + 4].copy_from_slice(&v.to_le_bytes()); +} + +/// A 512-byte header (§2.2) with sector shift `shift`: no directory, mini FAT or DIFAT chain, and +/// all 109 FAT slots free. Tests fill in the fields they attack. +fn bare_header(shift: u16) -> Vec { + let mut h = vec![0u8; 512]; + h[..8].copy_from_slice(&SIGNATURE); + h[24..26].copy_from_slice(&0x3Eu16.to_le_bytes()); + h[26..28].copy_from_slice(&(if shift == 9 { 3u16 } else { 4 }).to_le_bytes()); + h[28..30].copy_from_slice(&0xFFFEu16.to_le_bytes()); + h[30..32].copy_from_slice(&shift.to_le_bytes()); + h[32..34].copy_from_slice(&6u16.to_le_bytes()); + for at in [48, 60, 68] { + put_u32(&mut h, at, ENDOFCHAIN); + } + put_u32(&mut h, 56, MINI_STREAM_CUTOFF); + h[76..].fill(0xFF); + h +} + +/// A DIFAT sector that names no FAT sector and points to itself. The loop guard used to be the +/// header's DIFAT sector count (up to 2^32 - 1), so this 1 KB file kept `open` busy for 14-16 s here (minutes with `u32::MAX`), reading a sector each pass. A DIFAT +/// chain cannot be longer than the file, so the walk now stops after as many passes as the file +/// has sectors (1024 / 512 + 1 = 3 here) and fails on the 4th. +#[test] +fn difat_self_loop_fails_fast() { + let mut b = bare_header(9); + put_u32(&mut b, 44, 1); // one FAT sector, to be found through the DIFAT + put_u32(&mut b, 68, 0); // first DIFAT sector + put_u32(&mut b, 72, 100_000_000); // claimed DIFAT sector count (inflates the old loop bound) + let mut difat = vec![0xFF; 512]; // no FAT sector numbers... + put_u32(&mut difat, 508, 0); // ...and the next DIFAT sector is this one + b.extend_from_slice(&difat); + // The message names the bound the walk stopped at: the file's 3 sectors, not the header's + // 100,000,000 (the old guard allowed 100,000,003 passes). + let expect = format!("DIFAT chain loops: more than 3 sectors in a {}-byte file", b.len()); + assert_eq!(CompoundFile::open(&b).unwrap_err(), Error::Invalid(expect)); +} + +/// All 109 header slots name the same FAT sector, so the FAT would be 109 copies of it. The +/// reader used to build that FAT and then follow a looping directory chain through it, appending +/// a sector per FAT entry before failing: a 457 MB buffer per open of this 450 KB file. No file +/// lists a FAT sector twice, so the list is refused before any of it is read. +#[test] +fn duplicated_fat_sector_is_refused() { + let ss = 4096; + let mut b = bare_header(12); + put_u32(&mut b, 44, 109); // FAT sectors: the 109 header slots... + for i in 0..109 { + put_u32(&mut b, 76 + i * 4, 0); // ...all naming sector 0 + } + put_u32(&mut b, 48, 0); // the directory starts at sector 0 + b.resize(ss, 0); + let mut fat = vec![0xFF; ss]; + put_u32(&mut fat, 0, 0); // FAT[0] = 0: sector 0 is followed by itself + b.extend_from_slice(&fat); + b.resize(110 * ss, 0); // big enough for 109 FAT sectors to pass the size check + assert_eq!(CompoundFile::open(&b).unwrap_err(), Error::Invalid("FAT sector 0 is listed twice".into())); +} + +/// A directory chain that loops (sector 1 → 1) stops after as many sectors as the file has, not +/// after as many as the FAT has entries. +#[test] +fn looping_directory_chain_fails_fast() { + let mut b = bare_header(9); + put_u32(&mut b, 44, 1); + put_u32(&mut b, 76, 0); // the FAT is sector 0 + put_u32(&mut b, 48, 1); // the directory starts at sector 1 + let mut fat = vec![0xFF; 512]; + put_u32(&mut fat, 0, FATSECT); + put_u32(&mut fat, 4, 1); // FAT[1] = 1 + b.extend_from_slice(&fat); + b.resize(3 * 512, 0); + let expect = format!("sector chain loops: more than 4 sectors in a {}-byte file", b.len()); + assert_eq!(CompoundFile::open(&b).unwrap_err(), Error::Invalid(expect)); +} + +/// Positive control for the chain bound: the longest chains a file can hold still read back, in +/// both versions: a directory (read without a size limit) and a stream, each filling every sector +/// but the header and the FAT (and the directory, for the stream). +#[test] +fn chains_filling_the_file_still_read() { + for v in [Version::V3, Version::V4] { + let ss = v_sector(v); + let mut w = Writer::new(v); + for i in 0..1000 { + w.stream(Writer::ROOT, &format!("e{i}"), Vec::new()).unwrap(); + } + let b = w.finish(); + assert!(b.len() / ss <= (1001 * 128usize).div_ceil(ss) + 3, "header, directory, FAT"); + let cf = CompoundFile::open(&b).unwrap(); + assert_eq!(cf.root().children.len(), 1000); + assert_eq!(cf.read_path("e999").unwrap(), Vec::::new()); + + let mut w = Writer::new(v); + let data = pattern(64 * ss, 3); + w.stream(Writer::ROOT, "big", data.clone()).unwrap(); + let b = w.finish(); + assert_eq!(b.len(), (1 + 64 + 2) * ss, "header, stream, directory, FAT"); + assert_eq!(CompoundFile::open(&b).unwrap().read_path("big").unwrap(), data); + } +} + +#[test] +fn nul_in_a_name_is_rejected() { + assert!(validate_name("a\0b").is_err()); + assert!(validate_name("\0").is_err()); + let mut w = Writer::default(); + assert!(matches!(w.stream(Writer::ROOT, "a\0b", vec![1]), Err(Error::BadName(_)))); + assert!(matches!(w.storage(Writer::ROOT, "x\0", [0; 16]), Err(Error::BadName(_)))); +} + +#[test] +fn span_is_checked() { + use crate::read::span; + assert_eq!(span(0, 1, 512), Some((512, 1024))); + assert_eq!(span(3, 0, 64), Some((192, 256))); + // On a 32-bit `usize` these are the overflows of `sector` and `read_mini_chain`; the checked + // arithmetic is the same code on 64-bit, so usize::MAX exercises it here. (A real 32-bit + // target is not run.) + assert_eq!(span(usize::MAX, 1, 512), None); + assert_eq!(span(usize::MAX, 0, 64), None); + assert_eq!(span(usize::MAX / 64, 0, 64), None, "start fits, end does not"); + assert_eq!(span(usize::MAX - 1, 1, 1), None); +} + +/// The allocating comparison compare_names used before the iterator rewrite. +fn compare_names_old(a: &str, b: &str) -> std::cmp::Ordering { + let ua: Vec = a.encode_utf16().collect(); + let ub: Vec = b.encode_utf16().collect(); + ua.len().cmp(&ub.len()).then_with(|| ua.iter().map(|&u| upper(u)).cmp(ub.iter().map(|&u| upper(u)))) +} + +#[test] +fn compare_names_matches_the_allocating_version() { + let names = [ + "", + "a", + "A", + "b", + "B", + "ab", + "AB", + "aB", + "abc", + "ABD", + "Abc", + "aBC", + "z", + "Z", + "{", + "_", + "ß", + "SS", + "ss", + "ü", + "Ü", + "é", + "É", + "dž", + "Dž", + "DŽ", + "ストリーム", + "ｓｔ", + "Ｓｔ", + "😀", + "😁", + "a😀", + "A😀", + "\u{ffff}", + "\u{10000}", + "Header-2", + "HEADER-2", + "header-2", + "Mobs-1901{a}", + "mobs-1901{A}", + ]; + for a in names { + for b in names { + assert_eq!(compare_names(a, b), compare_names_old(a, b), "{a:?} vs {b:?}"); + } + } + // every BMP scalar against a few fixed names + for c in (0u32..=0x10FFFF).filter_map(char::from_u32) { + let s = c.to_string(); + for other in ["a", "Z", "ü", "dž"] { + assert_eq!(compare_names(&s, other), compare_names_old(&s, other)); + assert_eq!(compare_names(other, &s), compare_names_old(other, &s)); + } + } +} + +/// Name lookups go through a sorted per-storage index: every child of a wide storage resolves +/// (in any case), and a hostile file whose sibling tree is out of order still resolves by name, +/// with the first child in tree order winning a duplicated name. +#[test] +fn child_lookup_uses_a_sorted_index() { + let mut w = Writer::default(); + for i in 0..2000 { + w.stream(Writer::ROOT, &format!("m{i}"), vec![i as u8]).unwrap(); + } + let b = w.finish(); + let cf = CompoundFile::open(&b).unwrap(); + for i in (0..2000).step_by(37) { + let id = cf.child(0, &format!("M{i}")).unwrap(); + assert_eq!(cf.read(id).unwrap(), vec![i as u8]); + } + assert_eq!(cf.child(0, "m2000"), None); + assert_eq!(cf.child(12345, "m1"), None); + + let mut w = Writer::default(); + for name in ["aa", "bb", "cc"] { + w.stream(Writer::ROOT, name, name.as_bytes().to_vec()).unwrap(); + } + let mut b = w.finish(); + let utf16 = |s: &str| s.encode_utf16().flat_map(u16::to_le_bytes).collect::>(); + let at = |b: &[u8], s: &str| b.windows(4).position(|w| w == utf16(s)).unwrap(); + // swap the names "aa" and "cc": the tree's in-order walk is now cc, bb, aa + let (a, c) = (at(&b, "aa"), at(&b, "cc")); + b[a..a + 4].copy_from_slice(&utf16("cc")); + b[c..c + 4].copy_from_slice(&utf16("aa")); + let cf = CompoundFile::open(&b).unwrap(); + let names: Vec<&str> = cf.root().children.iter().map(|&c| cf.entries[c].name.as_str()).collect(); + assert_eq!(names, ["cc", "bb", "aa"]); + assert_eq!(cf.read_path("AA").unwrap(), b"cc"); + assert_eq!(cf.read_path("cc").unwrap(), b"aa"); + assert_eq!(cf.read_path("bb").unwrap(), b"bb"); + // a duplicated name resolves to the first child in tree order + let bb = at(&b, "bb"); + b[bb..bb + 4].copy_from_slice(&utf16("cc")); + let cf = CompoundFile::open(&b).unwrap(); + assert_eq!(cf.read_path("cc").unwrap(), b"aa"); +} diff --git a/crates/cfb/src/tests/ported.rs b/crates/cfb/src/tests/ported.rs new file mode 100644 index 000000000..f29eb7ea9 --- /dev/null +++ b/crates/cfb/src/tests/ported.rs @@ -0,0 +1,704 @@ +//! Tests for what the other readers this crate replaces did and the base reader did not: a +//! spreadsheet reader (unpadded last sectors, looping stream chains, a stream size cap, a lenient +//! directory scan), a presentation reader (the same scan) and a word processor's guard in front +//! of a third-party reader (repeated FAT sectors, spare FAT sectors, zero-padded DIFAT sectors). +//! Plus the limits, the listing API and exhaustive truncation and bit flips. + +use super::{bare_header, pattern, put_u32}; +use crate::{ + CompoundFile, ENDOFCHAIN, EntryKind, Error, FATSECT, FREESECT, Limits, NOSTREAM, SIGNATURE, Version, Writer, directory_names, has_entry_named, + write_streams, +}; + +/// A version 3 file written by a second, independent writer with a different layout from +/// [`Writer`]: FAT first, then the directory, the mini FAT, the mini stream and each large stream +/// last, and the root's children chained as right siblings in the order given (a degenerate, +/// unsorted, all-black tree, as simple writers produce). Up to 109 FAT sectors. +fn foreign_file(streams: &[(&str, &[u8])]) -> Vec { + const SECTOR: usize = 512; + const MINI: usize = 64; + const CUTOFF: usize = 4096; + let is_mini = |d: &[u8]| d.len() < CUTOFF; + + let mut container = vec![]; + let mut mini_fat: Vec = vec![]; + let mut mini_starts = vec![]; + for (_, d) in streams { + if !is_mini(d) || d.is_empty() { + mini_starts.push(ENDOFCHAIN); + continue; + } + let first = mini_fat.len(); + let n = d.len().div_ceil(MINI); + for k in 0..n { + mini_fat.push(if k + 1 == n { ENDOFCHAIN } else { (first + k + 1) as u32 }); + } + mini_starts.push(first as u32); + container.extend_from_slice(d); + container.resize((first + n) * MINI, 0); + } + + // Regular-sector blobs, in order: directory, mini FAT, mini stream, then each large stream. + let dir_len = (streams.len() + 1).div_ceil(4) * SECTOR; + let mut blobs: Vec> = vec![vec![0; dir_len], mini_fat.iter().flat_map(|v| v.to_le_bytes()).collect(), container.clone()]; + blobs.extend(streams.iter().map(|(_, d)| if is_mini(d) { vec![] } else { d.to_vec() })); + let counts: Vec = blobs.iter().map(|b| b.len().div_ceil(SECTOR)).collect(); + let body: usize = counts.iter().sum(); + let mut fat_sectors = 1; + while fat_sectors * 128 < fat_sectors + body { + fat_sectors += 1; + } + assert!(fat_sectors <= 109, "the test writer has no DIFAT sectors"); + let mut fat = vec![FREESECT; fat_sectors * 128]; + fat[..fat_sectors].fill(FATSECT); + let mut starts = vec![]; + let mut next = fat_sectors; + for &n in &counts { + if n == 0 { + starts.push(ENDOFCHAIN); + continue; + } + starts.push(next as u32); + for k in 0..n { + fat[next + k] = if k + 1 == n { ENDOFCHAIN } else { (next + k + 1) as u32 }; + } + next += n; + } + + let put16 = |b: &mut [u8], at: usize, v: u16| b[at..at + 2].copy_from_slice(&v.to_le_bytes()); + let dir = &mut blobs[0]; + for k in 0..dir_len / 128 { + for at in [68, 72, 76] { + put_u32(dir, k * 128 + at, NOSTREAM); + } + } + let mut entry = |k: usize, name: &str, kind: u8, start: u32, size: usize| { + let e = k * 128; + let units: Vec = name.encode_utf16().flat_map(u16::to_le_bytes).collect(); + dir[e..e + units.len()].copy_from_slice(&units); + put16(dir, e + 64, units.len() as u16 + 2); + dir[e + 66] = kind; + dir[e + 67] = 1; // black + if kind == 5 && !streams.is_empty() { + put_u32(dir, e + 76, 1); + } + if kind == 2 && k < streams.len() { + put_u32(dir, e + 72, k as u32 + 1); + } + put_u32(dir, e + 116, start); + put_u32(dir, e + 120, size as u32); + }; + entry(0, "Root Entry", 5, starts[2], container.len()); + for (k, (name, data)) in streams.iter().enumerate() { + let start = if is_mini(data) { mini_starts[k] } else { starts[3 + k] }; + entry(k + 1, name, 2, start, data.len()); + } + + let mut out = vec![0u8; SECTOR * (1 + fat_sectors + body)]; + out[..8].copy_from_slice(&SIGNATURE); + put16(&mut out, 24, 0x3E); // minor version + put16(&mut out, 26, 3); // major version + put16(&mut out, 28, 0xFFFE); // byte order + put16(&mut out, 30, 9); // 512-byte sectors + put16(&mut out, 32, 6); // 64-byte mini sectors + put_u32(&mut out, 44, fat_sectors as u32); + put_u32(&mut out, 48, starts[0]); + put_u32(&mut out, 56, CUTOFF as u32); + put_u32(&mut out, 60, starts[1]); // first mini FAT sector (END when there is none) + put_u32(&mut out, 64, counts[1] as u32); + put_u32(&mut out, 68, ENDOFCHAIN); // no DIFAT sectors + for i in 0..109 { + put_u32(&mut out, 76 + i * 4, if i < fat_sectors { i as u32 } else { FREESECT }); + } + for (i, v) in fat.iter().enumerate() { + put_u32(&mut out, SECTOR + i * 4, *v); + } + for (blob, start) in blobs.iter().zip(&starts) { + if !blob.is_empty() { + let at = SECTOR * (1 + *start as usize); + out[at..at + blob.len()].copy_from_slice(blob); + } + } + out +} + +/// Byte offset of directory entry `i` in a file whose directory is contiguous (both writers'). +fn dir_at(b: &[u8], i: usize) -> usize { + let ss = 1usize << u16::from_le_bytes([b[30], b[31]]); + (u32::from_le_bytes(b[48..52].try_into().unwrap()) as usize + 1) * ss + i * 128 +} + +fn invalid_containing(r: crate::Result>, what: &str) { + match r { + Err(Error::Invalid(m)) if m.contains(what) => {} + other => panic!("expected an Invalid error containing {what:?}, got {other:?}"), + } +} + +// ---- an independent writer's files ------------------------------------------------------- + +/// The reader does not depend on the sibling tree being sorted or balanced: a right-linked list +/// in insertion order resolves every name, mini and regular streams alike. +#[test] +fn a_foreign_layout_with_an_unsorted_sibling_list_reads() { + let streams: Vec<(String, Vec)> = + [("Workbook", 9000), ("\u{5}SummaryInformation", 300), ("a", 0), ("Zeta", 4096), ("beta", 4095), ("CompObj", 1)] + .iter() + .enumerate() + .map(|(i, (n, len))| (n.to_string(), pattern(*len, i as u8))) + .collect(); + let refs: Vec<(&str, &[u8])> = streams.iter().map(|(n, d)| (n.as_str(), d.as_slice())).collect(); + let file = foreign_file(&refs); + let cf = CompoundFile::open(&file).unwrap(); + for (name, data) in &streams { + assert_eq!(&cf.read_path(&name.to_uppercase()).unwrap(), data, "{name}"); + } + let listed: Vec = cf.walk().into_iter().map(|id| cf.path(id).unwrap()).collect(); + assert_eq!(listed, streams.iter().map(|(n, _)| n.clone()).collect::>(), "sibling-tree order"); +} + +// ---- unpadded last sector ------------------------------------------------------------------ + +/// One regular-sector stream of `len` bytes (FAT in sector 0, directory in sector 1, the stream +/// from sector 2), cut right after the stream's last byte, the way some writers leave the final +/// sector unpadded. +fn unpadded(len: usize) -> (Vec, Vec) { + let data: Vec = (0..len).map(|i| (i % 251) as u8).collect(); + let mut file = foreign_file(&[("Workbook", &data)]); + file.truncate(512 * 3 + len); + (file, data) +} + +#[test] +fn an_unpadded_last_sector_still_reads() { + let (file, data) = unpadded(5000); + assert_ne!(file.len() % 512, 0); + let cf = CompoundFile::open(&file).unwrap(); + assert_eq!(cf.read_path("Workbook").unwrap(), data); + // Bytes the stream needs that the file doesn't hold are still damage. + let cf = CompoundFile::open(&file[..file.len() - 1]).unwrap(); + invalid_containing(cf.read_path("Workbook"), "truncated"); +} + +#[test] +fn an_unpadded_last_mini_sector_still_reads() { + let data: Vec = (0..1000u32).map(|i| (i % 251) as u8).collect(); + let mut file = foreign_file(&[("Workbook", &data)]); + // Header, FAT, directory, mini FAT, then the mini stream holding the stream's bytes. + file.truncate(512 * 4 + data.len()); + let cf = CompoundFile::open(&file).unwrap(); + assert_eq!(cf.read_path("Workbook").unwrap(), data); + let cf = CompoundFile::open(&file[..file.len() - 1]).unwrap(); + invalid_containing(cf.read_path("Workbook"), "truncated"); +} + +#[test] +fn a_cut_sector_inside_a_chain_is_damage() { + let (mut file, _) = unpadded(5000); + // The cut sector (11) points on to the directory sector and the stream claims those bytes + // too: the partial sector would splice 392 bytes into the middle of the stream. + put_u32(&mut file, 512 + 11 * 4, 1); + put_u32(&mut file, 1024 + 128 + 120, 5512); + let cf = CompoundFile::open(&file).unwrap(); + invalid_containing(cf.read_path("Workbook"), "cut short inside a chain"); +} + +#[test] +fn a_cut_mini_sector_inside_a_chain_is_damage() { + let small = [7u8; 100]; + let data: Vec = (0..1000u32).map(|i| (i % 251) as u8).collect(); + let mut file = foreign_file(&[("Small", &small), ("Workbook", &data)]); + // "Small" holds mini sectors 0-1, "Workbook" 2-17; the mini stream starts at sector 3. + file.truncate(512 * 4 + 128 + data.len()); + assert_eq!(CompoundFile::open(&file).unwrap().read_path("Workbook").unwrap(), data); + // The cut mini sector (17) points on to sector 0 and the stream claims those bytes too. + put_u32(&mut file, 1536 + 17 * 4, 0); + put_u32(&mut file, 1024 + 2 * 128 + 120, 1064); + let cf = CompoundFile::open(&file).unwrap(); + invalid_containing(cf.read_path("Workbook"), "cut short inside a chain"); +} + +/// FAT sectors are never read short: a file cut inside its only FAT sector does not open. +#[test] +fn a_cut_fat_sector_is_refused() { + let file = foreign_file(&[("a", &[1; 10])]); + assert!(CompoundFile::open(&file[..1000]).is_err()); +} + +// ---- loops shorter than the stream --------------------------------------------------------- + +/// A chain that loops inside the stream's declared size used to read back as the same sectors +/// over and over. It is damage. +#[test] +fn a_stream_chain_that_loops_within_its_size_is_refused() { + let data = pattern(5000, 1); + let mut file = foreign_file(&[("Workbook", &data)]); + assert_eq!(CompoundFile::open(&file).unwrap().read_path("Workbook").unwrap(), data); + put_u32(&mut file, 512 + 3 * 4, 2); // FAT[3] = 2: sectors 2, 3, 2, 3, ... + invalid_containing(CompoundFile::open(&file).unwrap().read_path("Workbook"), "sector chain loops: sector 2 is visited twice"); +} + +#[test] +fn a_mini_chain_that_loops_within_its_size_is_refused() { + let data = pattern(200, 1); + let mut file = foreign_file(&[("Small", &data)]); + assert_eq!(CompoundFile::open(&file).unwrap().read_path("Small").unwrap(), data); + put_u32(&mut file, 1536 + 4, 0); // mini FAT[1] = 0: mini sectors 0, 1, 0, 1 + invalid_containing(CompoundFile::open(&file).unwrap().read_path("Small"), "mini sector chain loops"); +} + +// ---- a damaged mini stream ----------------------------------------------------------------- + +/// A mini stream that cannot be read, or is shorter than the root entry says, does not stop the +/// file from opening: the streams in regular sectors still read. +#[test] +fn a_damaged_mini_stream_only_fails_the_streams_in_it() { + let (small, big) = (pattern(100, 1), pattern(5000, 2)); + let mut file = foreign_file(&[("Small", &small), ("Workbook", &big)]); + let root = dir_at(&file, 0); + // the root entry claims a mini stream far longer than its chain: what is there still reads + put_u32(&mut file, root + 120, 1 << 20); + let cf = CompoundFile::open(&file).unwrap(); + assert_eq!(cf.read_path("Small").unwrap(), small); + // the mini stream starts at a sector outside the file + put_u32(&mut file, root + 116, 5000); + let cf = CompoundFile::open(&file).unwrap(); + assert_eq!(cf.read_path("Workbook").unwrap(), big); + invalid_containing(cf.read_path("Small"), "sector 5000 has no FAT entry"); +} + +// ---- stream size cap and the other limits -------------------------------------------------- + +#[test] +fn stream_size_limit() { + let data = pattern(5000, 3); + let file = write_streams(Version::V3, &[("s", &data), ("t", &[1, 2, 3])]).unwrap(); + let over = |r: crate::Result>| assert!(matches!(r, Err(Error::LimitExceeded(_))), "{r:?}"); + let cf = CompoundFile::open_with(&file, Limits { max_stream_size: 4999, ..Limits::default() }).unwrap(); + over(cf.read_path("s")); + assert_eq!(cf.read_path("t").unwrap(), [1, 2, 3], "a stream under the limit"); + let cf = CompoundFile::open_with(&file, Limits { max_stream_size: 5000, ..Limits::default() }).unwrap(); + assert_eq!(cf.read_path("s").unwrap(), data); + // the per-call cap + over(cf.read_path_capped("s", 4999)); + over(cf.read_capped(cf.find("s").unwrap(), 0)); + assert_eq!(cf.read_path_capped("s", 5000).unwrap(), data); + assert_eq!(cf.read_path_capped("s", u64::MAX).unwrap(), data, "a cap above the limit does not raise it"); +} + +/// The default limit is 1 GiB, and a size the file cannot hold is refused before any allocation +/// whatever the limit. +#[test] +fn a_huge_declared_size_is_refused_without_allocating() { + assert_eq!(Limits::default(), Limits { max_stream_size: 1 << 30, max_dir_entries: 1 << 20 }); + let mut file = write_streams(Version::V4, &[("s", &pattern(5000, 3))]).unwrap(); + let e = dir_at(&file, 1); + file[e + 120..e + 128].copy_from_slice(&((1u64 << 30) + 1).to_le_bytes()); + assert!(matches!(CompoundFile::open(&file).unwrap().read_path("s"), Err(Error::LimitExceeded(_)))); + let cf = CompoundFile::open_with(&file, Limits { max_stream_size: u64::MAX, ..Limits::default() }).unwrap(); + invalid_containing(cf.read_path("s"), "larger than the file"); + file[e + 120..e + 128].copy_from_slice(&u64::MAX.to_le_bytes()); + let cf = CompoundFile::open_with(&file, Limits { max_stream_size: u64::MAX, ..Limits::default() }).unwrap(); + invalid_containing(cf.read_path("s"), "larger than the file"); +} + +#[test] +fn directory_entry_limit() { + let streams: Vec<(String, Vec)> = (0..10).map(|i| (format!("s{i}"), vec![i as u8])).collect(); + let refs: Vec<(&str, &[u8])> = streams.iter().map(|(n, d)| (n.as_str(), d.as_slice())).collect(); + // 11 entries: three 512-byte directory sectors, so 12 entries with the unused one + let file = write_streams(Version::V3, &refs).unwrap(); + let with = |n| CompoundFile::open_with(&file, Limits { max_dir_entries: n, ..Limits::default() }); + assert!(matches!(with(11), Err(Error::LimitExceeded(_)))); + assert_eq!(with(12).unwrap().walk().len(), 10); + assert_eq!( + directory_names(&file, &Limits { max_dir_entries: 4, ..Limits::default() }).unwrap(), + ["Root Entry", "s0", "s1", "s2"], + "the scan reports what it read" + ); +} + +/// Only 512- and 4096-byte sectors exist. Other shifts used to be accepted (down to 128-byte +/// sectors, which overlap the header). +#[test] +fn only_sector_shifts_9_and_12_are_accepted() { + for (version, good) in [(Version::V3, 9u16), (Version::V4, 12)] { + let mut file = write_streams(version, &[("s", &[1; 10])]).unwrap(); + assert!(CompoundFile::open(&file).is_ok()); + for shift in (0..=20).filter(|s| *s != 9 && *s != 12) { + file[30..32].copy_from_slice(&(shift as u16).to_le_bytes()); + assert_eq!(CompoundFile::open(&file).unwrap_err(), Error::Invalid(format!("sector shift {shift}"))); + assert_eq!(directory_names(&file, &Limits::default()), None); + } + file[30..32].copy_from_slice(&good.to_le_bytes()); + file[32..34].copy_from_slice(&7u16.to_le_bytes()); + assert_eq!(CompoundFile::open(&file).unwrap_err(), Error::Invalid("mini sector shift 7".into())); + } +} + +#[test] +fn a_wrong_byte_order_mark_is_not_a_compound_file() { + let mut file = write_streams(Version::V3, &[("s", &[1; 10])]).unwrap(); + file[28] = 0xFF; + file[29] = 0xFE; + assert_eq!(CompoundFile::open(&file).unwrap_err(), Error::NotCfb); +} + +/// Before version 4 a size is 32 bits and writers may leave anything in the high half (§2.6.3). +#[test] +fn version_3_ignores_the_high_half_of_a_size() { + let data = pattern(5000, 9); + let mut file = foreign_file(&[("Workbook", &data)]); + let e = dir_at(&file, 1); + put_u32(&mut file, e + 124, 0xDEAD_BEEF); + assert_eq!(CompoundFile::open(&file).unwrap().read_path("Workbook").unwrap(), data); + // a version field below 3 is treated the same way + file[26] = 2; + assert_eq!(CompoundFile::open(&file).unwrap().read_path("Workbook").unwrap(), data); +} + +// ---- the directory tree -------------------------------------------------------------------- + +/// Three streams a, b, c: `b` (entry 2) is the root of the sibling tree, `a` and `c` its leaves. +fn three() -> Vec { + write_streams(Version::V3, &[("a", b"A"), ("b", b"B"), ("c", b"C")]).unwrap() +} + +/// A sibling link to an entry that does not exist is ignored; the rest of the tree reads. +#[test] +fn a_dangling_sibling_link_is_ignored() { + let mut file = three(); + let c = dir_at(&file, 3); + put_u32(&mut file, c + 68, 1000); // c.left + put_u32(&mut file, c + 72, 0x7FFF_FFFF); // c.right + let cf = CompoundFile::open(&file).unwrap(); + for (name, data) in [("a", b"A"), ("b", b"B"), ("c", b"C")] { + assert_eq!(cf.read_path(name).unwrap(), data); + } + // a dangling child link leaves an empty storage + let root = dir_at(&file, 0); + put_u32(&mut file, root + 76, 4); + let cf = CompoundFile::open(&file).unwrap(); + assert!(cf.walk().is_empty()); +} + +/// A link to an entry already in the tree (a cycle or a shared subtree) is refused. +#[test] +fn an_entry_referenced_twice_is_refused() { + for target in [0u32, 1, 2, 3] { + let mut file = three(); + let c = dir_at(&file, 3); + put_u32(&mut file, c + 72, target); // c.right: the root, a sibling, the tree root, itself + assert_eq!(CompoundFile::open(&file).unwrap_err(), Error::Invalid(format!("directory entry {target} is referenced twice"))); + } +} + +/// Entries the root does not reach are not walked, so a leftover storage entry that points into +/// the live tree (or at itself) does not make the file unreadable. +#[test] +fn an_orphan_entry_does_not_break_the_tree() { + let mut file = write_streams(Version::V3, &[("a", b"A"), ("b", b"B")]).unwrap(); + let orphan = dir_at(&file, 3); // the unused fourth entry of the directory sector + file[orphan] = b'z'; + file[orphan + 64] = 4; + file[orphan + 66] = 1; // a storage + for child in [1u32, 3] { + put_u32(&mut file, orphan + 76, child); + let cf = CompoundFile::open(&file).unwrap(); + assert_eq!(cf.read_path("a").unwrap(), b"A"); + assert_eq!(cf.walk().len(), 2); + assert_eq!(cf.path(3), None, "not below the root"); + assert_eq!(cf.entries[3].parent, None); + assert!(cf.entries[3].children.is_empty()); + } +} + +// ---- listing ------------------------------------------------------------------------------- + +#[test] +fn walk_lists_storages_and_streams_with_paths() { + let file = write_streams( + Version::V4, + &[("Data", b"d"), ("Macros/VBA/dir", b"x"), ("/Macros/VBA/ThisDocument", b"y"), ("Macros/PROJECT", b"z"), ("\u{1}CompObj", b"c")], + ) + .unwrap(); + let cf = CompoundFile::open(&file).unwrap(); + let listed: Vec<(String, EntryKind)> = cf.walk().into_iter().map(|id| (cf.path(id).unwrap(), cf.entries[id].kind)).collect(); + use EntryKind::{Storage, Stream}; + // siblings in [MS-CFB] order: shorter names first, then case-insensitive code units + let expect = [ + ("Data", Stream), + ("Macros", Storage), + ("Macros/VBA", Storage), + ("Macros/VBA/dir", Stream), + ("Macros/VBA/ThisDocument", Stream), + ("Macros/PROJECT", Stream), + ("\u{1}CompObj", Stream), + ]; + assert_eq!(listed, expect.map(|(p, k)| (p.to_string(), k))); + for id in cf.walk() { + assert_eq!(cf.find(&cf.path(id).unwrap()).unwrap(), id, "find is the inverse of path"); + } + assert_eq!(cf.path(0).as_deref(), Some("")); + assert_eq!(cf.path(cf.entries.len()), None); + assert!(cf.has_stream("macros/vba/DIR")); + assert!(cf.has_stream("/Data")); + assert!(!cf.has_stream("Macros"), "a storage"); + assert!(!cf.has_stream("Macros/nope")); + assert!(!cf.has_stream(""), "the root"); + assert_eq!(cf.read_path("/Macros/VBA/ThisDocument").unwrap(), b"y"); +} + +#[test] +fn stream_at_rules() { + let mut w = Writer::default(); + let s = w.storage(Writer::ROOT, "Existing", [9; 16]).unwrap(); + w.stream_at("existing/inner", vec![1]).unwrap(); + assert!(matches!(w.stream_at("Existing/INNER", vec![2]), Err(Error::BadName(_))), "a duplicate"); + assert!(matches!(w.stream_at("Existing/inner/below", vec![2]), Err(Error::Invalid(_))), "a stream is not a storage"); + assert!(matches!(w.stream_at("", vec![]), Err(Error::BadName(_)))); + assert!(matches!(w.stream_at("a/b:c", vec![]), Err(Error::BadName(_)))); + assert!(matches!(write_streams(Version::V3, &[("x", b"1"), ("X", b"2")]), Err(Error::BadName(_)))); + let _ = s; + let file = w.finish(); + let cf = CompoundFile::open(&file).unwrap(); + assert_eq!(cf.entries[cf.find("Existing").unwrap()].clsid, [9; 16], "the existing storage was reused"); + assert_eq!(cf.read_path("Existing/inner").unwrap(), [1]); +} + +// ---- the FAT sector list ------------------------------------------------------------------- + +/// A file of about 1 MB whose DIFAT chain lists one FAT sector 2048 x 127 times. Built naively, +/// its FAT is about 130 MB. +fn repeating_difat_sample(claimed_fat_sectors: u32) -> Vec { + const S: usize = 512; + const CHAIN: u32 = 2048; // DIFAT sectors 1..=CHAIN, sector 0 is the "FAT" + let mut f = vec![0u8; S * (CHAIN as usize + 2)]; + f[..512].copy_from_slice(&bare_header(9)); + put_u32(&mut f, 0x2C, claimed_fat_sectors); + put_u32(&mut f, 0x44, 1); + put_u32(&mut f, 0x48, CHAIN); + f[0x4C..512].fill(0); // the 109 header entries are all 0; each chain sector's entries too + for d in 1..=CHAIN { + put_u32(&mut f, (d as usize + 2) * S - 4, if d == CHAIN { ENDOFCHAIN } else { d + 1 }); + } + f +} + +#[test] +fn a_repeating_difat_is_refused() { + let err = |n| CompoundFile::open(&repeating_difat_sample(n)).unwrap_err(); + // more FAT sectors than the file has sectors + assert_eq!(err(u32::MAX), Error::Invalid(format!("4294967295 FAT sectors in a {}-byte file", 512 * 2050))); + assert_eq!(err(2052), Error::Invalid(format!("2052 FAT sectors in a {}-byte file", 512 * 2050))); + // a count the file could hold: the list is still one sector over and over + assert_eq!(err(2050), Error::Invalid("FAT sector 0 is listed twice".into())); + assert_eq!(err(2), Error::Invalid("FAT sector 0 is listed twice".into())); + // the lenient scan never builds a FAT: it walks the DIFAT chain once and finds no directory + assert_eq!(directory_names(&repeating_difat_sample(2050), &Limits::default()), Some(vec![])); +} + +/// Append `spare` free FAT sectors to a file with fewer than 109 and list them in the header. +fn with_spare_fat_sectors(mut file: Vec, spare: u32) -> Vec { + let n_fat = u32::from_le_bytes(file[44..48].try_into().unwrap()); + let sectors = (file.len() / 512 - 1) as u32; + for i in 0..spare { + put_u32(&mut file, 76 + (n_fat + i) as usize * 4, sectors + i); + file.extend_from_slice(&[0xFF; 512]); + } + put_u32(&mut file, 44, n_fat + spare); + file +} + +/// Writers reserve FAT sectors for growth: a real encrypted package of 78 sectors lists 4, far +/// more than its size needs. Only a list longer than the file, or a repeat, is refused. +#[test] +fn spare_fat_sectors_are_accepted() { + let data = pattern(38_000, 4); + let file = with_spare_fat_sectors(write_streams(Version::V3, &[("EncryptedPackage", &data)]).unwrap(), 3); + assert_eq!(file.len() / 512 - 1, 80); + assert_eq!(u32::from_le_bytes(file[44..48].try_into().unwrap()), 4); + assert_eq!(CompoundFile::open(&file).unwrap().read_path("EncryptedPackage").unwrap(), data); + // the same sector listed for two of them is a repeat + let mut twice = file.clone(); + let last = u32::from_le_bytes(twice[76 + 12..76 + 16].try_into().unwrap()); + put_u32(&mut twice, 76 + 8, last); + assert_eq!(CompoundFile::open(&twice).unwrap_err(), Error::Invalid(format!("FAT sector {last} is listed twice"))); + // more FAT sectors claimed than the file has sectors + let mut many = file; + put_u32(&mut many, 44, 83); + assert!(matches!(CompoundFile::open(&many), Err(Error::Invalid(m)) if m.starts_with("83 FAT sectors"))); +} + +/// Some writers pad the last DIFAT sector with zeros, not FREESECT. Sector 0 is a real sector +/// (here the first of the stream), so the padding must not be read as FAT sector numbers: the +/// header's FAT sector count says where the list ends. +#[test] +fn a_zero_padded_difat_sector_is_accepted() { + let big = pattern(7_500_000, 5); + let mut file = write_streams(Version::V3, &[("big", &big)]).unwrap(); + assert_eq!(u32::from_le_bytes(file[72..76].try_into().unwrap()), 1, "one DIFAT sector"); + let difat = (u32::from_le_bytes(file[68..72].try_into().unwrap()) as usize + 1) * 512; + let mut padded = 0; + for i in 0..127 { + if file[difat + i * 4..difat + i * 4 + 4] == [0xFF; 4] { + put_u32(&mut file, difat + i * 4, 0); + padded += 1; + } + } + assert!(padded > 0); + assert_eq!(CompoundFile::open(&file).unwrap().read_path("big").unwrap(), big); +} + +// ---- the lenient directory scan ------------------------------------------------------------ + +/// A minimal file (512-byte sectors: FAT at sector 0, directory at sector 1) whose directory +/// holds a root entry plus the given stream names, linked to nothing: no storage's tree reaches +/// them, and they have no contents. +fn unlinked(streams: &[&str]) -> Vec { + let refs: Vec<(&str, &[u8])> = streams.iter().map(|n| (*n, &[][..])).collect(); + let mut b = foreign_file(&refs); + let root = dir_at(&b, 0); + put_u32(&mut b, root + 76, NOSTREAM); + b +} + +#[test] +fn scan_reads_directory_names() { + let limits = Limits::default(); + let b = unlinked(&["EncryptionInfo", "EncryptedPackage"]); + assert_eq!(directory_names(&b, &limits).unwrap(), ["Root Entry", "EncryptionInfo", "EncryptedPackage"]); + assert!(CompoundFile::open(&b).unwrap().walk().is_empty(), "the tree reaches none of them"); + assert!(has_entry_named(&b, "EncryptedPackage", &limits)); + assert!(has_entry_named(&b, "encryptedpackage", &limits), "compared as sibling names are"); + assert!(!has_entry_named(&b, "Encrypted", &limits), "a whole name, not a prefix"); + let plain = unlinked(&["Workbook", "\u{5}SummaryInformation"]); + assert!(!has_entry_named(&plain, "EncryptedPackage", &limits)); + assert!(has_entry_named(&plain, "\u{5}SummaryInformation", &limits)); + // not a compound file: no answer, and no byte search either + assert_eq!(directory_names(b"PK\x03\x04", &limits), None); + assert!(!has_entry_named(b"PK\x03\x04", "EncryptedPackage", &limits)); + let mut zip_with_the_name = b"PK\x03\x04".to_vec(); + zip_with_the_name.extend("EncryptedPackage".encode_utf16().flat_map(u16::to_le_bytes)); + assert!(!has_entry_named(&zip_with_the_name, "EncryptedPackage", &limits)); + // nested entries are listed too + let nested = write_streams(Version::V4, &[("a/b/c", b"1")]).unwrap(); + assert_eq!(directory_names(&nested, &limits).unwrap(), ["Root Entry", "a", "b", "c"]); +} + +/// A file whose directory cannot be followed still says whether it mentions the name. +#[test] +fn scan_falls_back_to_a_byte_search() { + let limits = Limits::default(); + let b = unlinked(&["EncryptionInfo", "EncryptedPackage"]); + for damaged in [&b[..1024 + 300], &b[..1024 + 128 + 32]] { + assert_eq!(directory_names(damaged, &limits), None); + assert!(has_entry_named(damaged, "EncryptionInfo", &limits)); + assert!(!has_entry_named(damaged, "Workbook", &limits)); + } + assert!(!has_entry_named(&b[..1024], "EncryptionInfo", &limits), "cut before the directory"); + assert!(!has_entry_named(&b[..300], "", &limits)); +} + +#[test] +fn scan_survives_hostile_headers() { + let limits = Limits::default(); + let mut b = unlinked(&["Workbook"]); + for at in [30usize, 48, 68, 76, 512, 516, 1024 + 64] { + for v in [0u32, 1, 7, 0xFFFF_FFFA, 0xFFFF_FFFF, 0x7FFF_FFFF] { + let mut c = b.clone(); + put_u32(&mut c, at, v); + let _ = has_entry_named(&c, "EncryptedPackage", &limits); + for cut in [0usize, 8, 100, 600, 1100] { + let _ = has_entry_named(&c[..cut.min(c.len())], "EncryptedPackage", &limits); + } + } + } + // A directory chain that loops back on itself stops at the bound and reports what it read. + put_u32(&mut b, 516, 1); + assert!(!has_entry_named(&b, "EncryptedPackage", &limits)); + assert!(has_entry_named(&b, "Workbook", &limits)); + // A DIFAT sector that points at itself stops at the bound too. + let mut b = unlinked(&["Workbook"]); + put_u32(&mut b, 68, 1); + assert!(directory_names(&b, &limits).is_some()); +} + +// ---- exhaustive damage --------------------------------------------------------------------- + +/// Everything the crate offers on bytes that may be anything. Streams that read back from a file +/// that is only cut short must be complete. +fn exercise(bytes: &[u8], intact: Option<&[(&str, &[u8])]>) { + let limits = Limits::default(); + let _ = directory_names(bytes, &limits); + let _ = has_entry_named(bytes, "big", &limits); + let Ok(cf) = CompoundFile::open(bytes) else { return }; + for id in cf.walk() { + let _ = cf.path(id); + } + for id in 0..cf.entries.len() { + let _ = cf.read(id); + let _ = cf.path(id); + } + for (path, data) in intact.unwrap_or(&[]) { + if let Ok(d) = cf.read_path(path) { + assert_eq!(&d, data, "{path} read back from a cut file"); + } + } +} + +type Streams = Vec<(&'static str, Vec)>; + +fn small_samples() -> Vec<(Vec, Streams)> { + let streams = vec![("a/b", pattern(100, 1)), ("big", pattern(5000, 2)), ("c", pattern(10, 3)), ("a/empty", vec![])]; + let refs: Vec<(&str, &[u8])> = streams.iter().map(|(n, d)| (*n, d.as_slice())).collect(); + let roots: Vec<(&str, &[u8])> = refs.iter().filter(|(n, _)| !n.contains('/')).copied().collect(); + vec![ + (write_streams(Version::V3, &refs).unwrap(), streams.clone()), + (foreign_file(&roots), streams.into_iter().filter(|(n, _)| !n.contains('/')).collect()), + ] +} + +#[test] +fn truncation_at_every_length_never_panics() { + for (file, streams) in small_samples() { + let refs: Vec<(&str, &[u8])> = streams.iter().map(|(n, d)| (*n, d.as_slice())).collect(); + for cut in 0..=file.len() { + exercise(&file[..cut], Some(&refs)); + } + } + // version 4, where one sector holds what several did + let file = write_streams(Version::V4, &[("a/b", &pattern(100, 1)), ("big", &pattern(5000, 2))]).unwrap(); + for cut in (0..=file.len()).step_by(7) { + exercise(&file[..cut], None); + } +} + +#[test] +fn every_single_bit_flip_never_panics_or_hangs() { + for (mut file, _) in small_samples() { + for bit in 0..file.len() * 8 { + file[bit / 8] ^= 1 << (bit % 8); + exercise(&file, None); + file[bit / 8] ^= 1 << (bit % 8); + } + } +} + +/// Every header and directory field, and the FAT (the first sector in one sample, the last in the +/// other), set to each of the values that mean something special. +#[test] +fn special_values_in_every_field_never_panic() { + for (file, _) in small_samples() { + let dir = dir_at(&file, 0); + let fields = + (0..512).step_by(4).chain((dir..dir + 5 * 128).step_by(4)).chain((512..1024).step_by(4)).chain((file.len() - 512..file.len()).step_by(4)); + for at in fields { + for v in [0u32, 1, 2, 3, 0x7FFF_FFFF, 0x8000_0000, 0xFFFF_FFFA, 0xFFFF_FFFB, 0xFFFF_FFFC, 0xFFFF_FFFD, 0xFFFF_FFFE, 0xFFFF_FFFF] { + let mut g = file.clone(); + put_u32(&mut g, at, v); + exercise(&g, None); + } + } + } +} diff --git a/crates/cfb/src/write.rs b/crates/cfb/src/write.rs new file mode 100644 index 000000000..eb27a4a5a --- /dev/null +++ b/crates/cfb/src/write.rs @@ -0,0 +1,478 @@ +//! Writing compound files ([MS-CFB] §2.2–2.6). +//! +//! Layout written by [`Writer::finish`] (sector numbers ascending): the sectors of every stream of +//! 4096 bytes or more, the mini stream, the mini FAT, the directory, the FAT, then DIFAT sectors +//! when the FAT needs more than the 109 header slots. Every chain is contiguous. Sibling trees are +//! balanced binary search trees coloured as valid red-black trees (§2.6.4). + +use std::collections::HashMap; + +use crate::{ + DIFSECT, ENDOFCHAIN, FATSECT, FREESECT, MINI_SECTOR_SIZE, MINI_STREAM_CUTOFF, NOSTREAM, Result, SIGNATURE, compare_names, validate_name, +}; + +/// File format version. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Version { + /// Version 3: 512-byte sectors. + V3, + /// Version 4: 4096-byte sectors. + #[default] + V4, +} + +impl Version { + fn sector_size(self) -> usize { + match self { + Version::V3 => 512, + Version::V4 => 4096, + } + } +} + +/// A storage or stream added to a [`Writer`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct NodeId(pub usize); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + Root, + Storage, + Stream, +} + +#[derive(Clone, Debug)] +struct Node { + name: String, + kind: Kind, + clsid: [u8; 16], + data: Vec, + children: Vec, + /// The children by name, upper-cased as sibling order compares them: an O(1) duplicate check + /// and lookup. (Scanning every sibling with `compare_names` made adding n children O(n^2): + /// 20 000 children took a minute.) + child_keys: HashMap, usize>, +} + +/// A name as sibling comparison sees it: its UTF-16 code units, upper-cased. +fn name_key(name: &str) -> Vec { + name.encode_utf16().map(crate::upper).collect() +} + +/// Builds a compound file in memory. +#[derive(Clone, Debug)] +pub struct Writer { + version: Version, + nodes: Vec, +} + +impl Default for Writer { + fn default() -> Self { + Writer::new(Version::default()) + } +} + +impl Writer { + /// The root storage. + pub const ROOT: NodeId = NodeId(0); + + pub fn new(version: Version) -> Self { + Writer { + version, + nodes: vec![Node { + name: "Root Entry".into(), + kind: Kind::Root, + clsid: [0; 16], + data: Vec::new(), + children: Vec::new(), + child_keys: HashMap::new(), + }], + } + } + + fn add(&mut self, parent: NodeId, name: &str, kind: Kind, clsid: [u8; 16], data: Vec) -> Result { + validate_name(name)?; + let p = self.nodes.get(parent.0).ok_or_else(|| crate::Error::NotFound(format!("node {}", parent.0)))?; + if p.kind == Kind::Stream { + return Err(crate::Error::Invalid(format!("{:?} is a stream, not a storage", p.name))); + } + let key = name_key(name); + if p.child_keys.contains_key(&key) { + return Err(crate::Error::BadName(format!("duplicate name {name}"))); + } + let id = self.nodes.len(); + self.nodes.push(Node { name: name.to_string(), kind, clsid, data, children: Vec::new(), child_keys: HashMap::new() }); + if let Some(p) = self.nodes.get_mut(parent.0) { + p.children.push(id); + p.child_keys.insert(key, id); + } + Ok(NodeId(id)) + } + + /// Add a storage below `parent`. + pub fn storage(&mut self, parent: NodeId, name: &str, clsid: [u8; 16]) -> Result { + self.add(parent, name, Kind::Storage, clsid, Vec::new()) + } + + /// Add a stream below `parent`. + pub fn stream(&mut self, parent: NodeId, name: &str, data: Vec) -> Result { + self.add(parent, name, Kind::Stream, [0; 16], data) + } + + /// Add a stream at a `/`-separated path below the root, creating the storages on the way that + /// do not exist yet (with a zero class id). Empty path components are ignored. + pub fn stream_at(&mut self, path: &str, data: Vec) -> Result { + let mut parts: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect(); + let name = parts.pop().ok_or_else(|| crate::Error::BadName(path.to_string()))?; + let mut cur = Writer::ROOT; + for part in parts { + let existing = self.nodes.get(cur.0).and_then(|n| n.child_keys.get(&name_key(part)).copied()); + cur = match existing { + Some(id) => NodeId(id), + None => self.storage(cur, part, [0; 16])?, + }; + } + self.stream(cur, name, data) + } + + /// Set the class id of a storage (or the root). + pub fn set_clsid(&mut self, node: NodeId, clsid: [u8; 16]) { + if let Some(n) = self.nodes.get_mut(node.0) { + n.clsid = clsid; + } + } + + /// Lay the file out and return its bytes. + pub fn finish(self) -> Vec { + let ss = self.version.sector_size(); + let layout = self.layout(); + let fat = layout.fat(ss / 4); + let minifat = layout.mini_fat(ss / 4); + let dir = self.directory(&layout); + let mut out = self.header(&layout); + + // Sectors. + let pad = |out: &mut Vec| { + let r = out.len() % ss; + if r != 0 { + out.resize(out.len() + ss - r, 0); + } + }; + for &n in &layout.order { + let node = &self.nodes[n]; + if node.kind == Kind::Stream && node.data.len() as u64 >= MINI_STREAM_CUTOFF as u64 { + out.extend_from_slice(&node.data); + pad(&mut out); + } + } + for &n in &layout.order { + let node = &self.nodes[n]; + if node.kind == Kind::Stream && !node.data.is_empty() && (node.data.len() as u64) < MINI_STREAM_CUTOFF as u64 { + out.extend_from_slice(&node.data); + let r = node.data.len() % MINI_SECTOR_SIZE; + if r != 0 { + out.resize(out.len() + MINI_SECTOR_SIZE - r, 0); + } + } + } + pad(&mut out); + for v in &minifat { + out.extend_from_slice(&v.to_le_bytes()); + } + out.extend_from_slice(&dir); + for v in &fat { + out.extend_from_slice(&v.to_le_bytes()); + } + layout.write_difat(&mut out, ss); + debug_assert_eq!(out.len(), (layout.total + 1) * ss); + out + } + + /// Directory order (depth-first pre-order from the root) and sector placement of every stream. + fn layout(&self) -> Layout { + let ss = self.version.sector_size(); + let per_sector = ss / 4; + let mut order = Vec::with_capacity(self.nodes.len()); + let mut stack = vec![0usize]; + while let Some(n) = stack.pop() { + order.push(n); + for &c in self.nodes[n].children.iter().rev() { + stack.push(c); + } + } + let mut dir_index = vec![0u32; self.nodes.len()]; + for (i, &n) in order.iter().enumerate() { + dir_index[n] = i as u32; + } + + // Stream placement. + let mut start = vec![ENDOFCHAIN; self.nodes.len()]; + let mut next_sector = 0u32; + let mut big_chains: Vec<(u32, u32)> = Vec::new(); // (first, count) + for &n in &order { + let node = &self.nodes[n]; + if node.kind == Kind::Stream && node.data.len() as u64 >= MINI_STREAM_CUTOFF as u64 { + let count = node.data.len().div_ceil(ss) as u32; + start[n] = next_sector; + big_chains.push((next_sector, count)); + next_sector += count; + } + } + let mut next_mini = 0u32; + let mut mini_chains: Vec<(u32, u32)> = Vec::new(); + for &n in &order { + let node = &self.nodes[n]; + if node.kind == Kind::Stream && !node.data.is_empty() && (node.data.len() as u64) < MINI_STREAM_CUTOFF as u64 { + let count = node.data.len().div_ceil(MINI_SECTOR_SIZE) as u32; + start[n] = next_mini; + mini_chains.push((next_mini, count)); + next_mini += count; + } + } + let mini_bytes = next_mini as usize * MINI_SECTOR_SIZE; + let mini_stream_sectors = mini_bytes.div_ceil(ss) as u32; + let mini_stream_start = if mini_stream_sectors > 0 { next_sector } else { ENDOFCHAIN }; + next_sector += mini_stream_sectors; + let minifat_sectors = (next_mini as usize * 4).div_ceil(ss) as u32; + let minifat_start = if minifat_sectors > 0 { next_sector } else { ENDOFCHAIN }; + next_sector += minifat_sectors; + let dir_sectors = (order.len() * 128).div_ceil(ss) as u32; + let dir_start = next_sector; + next_sector += dir_sectors; + let (fat_n, difat_n) = fat_sector_counts(next_sector as usize, per_sector); + let fat_start = next_sector; + let difat_start = fat_start + fat_n as u32; + Layout { + order, + dir_index, + start, + big_chains, + mini_chains, + mini_bytes, + mini_stream_sectors, + mini_stream_start, + minifat_sectors, + minifat_start, + dir_sectors, + dir_start, + fat_n, + difat_n, + fat_start, + difat_start, + total: next_sector as usize + fat_n + difat_n, + } + } + + /// The directory sectors: one 128-byte entry per node in directory order, then unused entries. + fn directory(&self, l: &Layout) -> Vec { + let ss = self.version.sector_size(); + let mut tree = vec![(NOSTREAM, NOSTREAM, NOSTREAM, 1u8); self.nodes.len()]; // left, right, child, colour + for (p, node) in self.nodes.iter().enumerate() { + if node.kind == Kind::Stream || node.children.is_empty() { + continue; + } + let mut kids = node.children.clone(); + kids.sort_by(|&a, &b| compare_names(&self.nodes[a].name, &self.nodes[b].name)); + let height = usize::BITS - (kids.len() + 1).leading_zeros() - 1; // floor(log2(n + 1)) + let perfect = (1usize << height) - 1 == kids.len(); + let root = build_tree(&kids, 0, perfect, height as usize, &mut tree, &l.dir_index); + tree[p].2 = root; + } + let mut dir = Vec::with_capacity(l.dir_sectors as usize * ss); + for &n in &l.order { + dir.extend_from_slice(&self.dir_entry(l, n, tree[n])); + } + while dir.len() < l.dir_sectors as usize * ss { + let mut e = [0u8; 128]; + e[68..80].copy_from_slice(&[0xFF; 12]); + dir.extend_from_slice(&e); + } + dir + } + + /// The directory entry of node `n`; `links` are its (left, right, child, colour). + fn dir_entry(&self, l: &Layout, n: usize, links: (u32, u32, u32, u8)) -> [u8; 128] { + let node = &self.nodes[n]; + let mut e = [0u8; 128]; + let units: Vec = node.name.encode_utf16().collect(); + for (i, u) in units.iter().enumerate() { + e[i * 2..i * 2 + 2].copy_from_slice(&u.to_le_bytes()); + } + e[64..66].copy_from_slice(&(((units.len() + 1) * 2) as u16).to_le_bytes()); + e[66] = match node.kind { + Kind::Root => 5, + Kind::Storage => 1, + Kind::Stream => 2, + }; + let (left, right, child, colour) = links; + e[67] = if node.kind == Kind::Root { 1 } else { colour }; + e[68..72].copy_from_slice(&left.to_le_bytes()); + e[72..76].copy_from_slice(&right.to_le_bytes()); + e[76..80].copy_from_slice(&child.to_le_bytes()); + e[80..96].copy_from_slice(&node.clsid); + let (st, size) = match node.kind { + Kind::Root => (l.mini_stream_start, l.mini_bytes as u64), + Kind::Storage => (0, 0), + Kind::Stream => (if node.data.is_empty() { ENDOFCHAIN } else { l.start[n] }, node.data.len() as u64), + }; + e[116..120].copy_from_slice(&st.to_le_bytes()); + e[120..128].copy_from_slice(&size.to_le_bytes()); + e + } + + /// The header sector. + fn header(&self, l: &Layout) -> Vec { + let ss = self.version.sector_size(); + let v3 = self.version == Version::V3; + let mut out = Vec::with_capacity((l.total + 1) * ss); + out.extend_from_slice(&SIGNATURE); + out.extend_from_slice(&[0; 16]); + out.extend_from_slice(&0x003Eu16.to_le_bytes()); + out.extend_from_slice(&(if v3 { 3u16 } else { 4u16 }).to_le_bytes()); + out.extend_from_slice(&0xFFFEu16.to_le_bytes()); + out.extend_from_slice(&(if v3 { 9u16 } else { 12u16 }).to_le_bytes()); + out.extend_from_slice(&6u16.to_le_bytes()); + out.extend_from_slice(&[0; 6]); + out.extend_from_slice(&(if v3 { 0 } else { l.dir_sectors }).to_le_bytes()); + out.extend_from_slice(&(l.fat_n as u32).to_le_bytes()); + out.extend_from_slice(&l.dir_start.to_le_bytes()); + out.extend_from_slice(&0u32.to_le_bytes()); + out.extend_from_slice(&MINI_STREAM_CUTOFF.to_le_bytes()); + out.extend_from_slice(&l.minifat_start.to_le_bytes()); + out.extend_from_slice(&l.minifat_sectors.to_le_bytes()); + out.extend_from_slice(&(if l.difat_n > 0 { l.difat_start } else { ENDOFCHAIN }).to_le_bytes()); + out.extend_from_slice(&(l.difat_n as u32).to_le_bytes()); + for i in 0..109 { + let v = if i < l.fat_n { l.fat_start + i as u32 } else { FREESECT }; + out.extend_from_slice(&v.to_le_bytes()); + } + out.resize(ss, 0); + out + } +} + +/// A compound file holding `streams`: `(path, contents)` pairs, the path `/`-separated below the +/// root as in [`Writer::stream_at`]. +pub fn write_streams(version: Version, streams: &[(&str, &[u8])]) -> Result> { + let mut w = Writer::new(version); + for (path, data) in streams { + w.stream_at(path, data.to_vec())?; + } + Ok(w.finish()) +} + +/// Where everything lands in the file: directory order, stream starts and the sector counts. +struct Layout { + /// Node indices in directory order. + order: Vec, + /// Directory index of each node. + dir_index: Vec, + /// First (mini) sector of each stream. + start: Vec, + /// (first, count) of each stream held in regular sectors. + big_chains: Vec<(u32, u32)>, + /// (first, count) of each stream held in mini sectors. + mini_chains: Vec<(u32, u32)>, + mini_bytes: usize, + mini_stream_sectors: u32, + mini_stream_start: u32, + minifat_sectors: u32, + minifat_start: u32, + dir_sectors: u32, + dir_start: u32, + fat_n: usize, + difat_n: usize, + fat_start: u32, + difat_start: u32, + /// Sectors after the header. + total: usize, +} + +/// FAT and DIFAT sector counts for `base` other sectors (the FAT must also map itself). +fn fat_sector_counts(base: usize, per_sector: usize) -> (usize, usize) { + let (mut fat_n, mut difat_n) = (1usize, 0usize); + loop { + let total = base + fat_n + difat_n; + let need_fat = total.div_ceil(per_sector); + let need_difat = if need_fat > 109 { (need_fat - 109).div_ceil(per_sector - 1) } else { 0 }; + if need_fat == fat_n && need_difat == difat_n { + return (fat_n, difat_n); + } + fat_n = need_fat.max(fat_n); + difat_n = need_difat.max(difat_n); + } +} + +impl Layout { + fn fat(&self, per_sector: usize) -> Vec { + let mut fat = vec![FREESECT; self.fat_n * per_sector]; + let chain = |first: u32, count: u32, fat: &mut [u32]| { + for i in 0..count { + fat[(first + i) as usize] = if i + 1 == count { ENDOFCHAIN } else { first + i + 1 }; + } + }; + for &(f, c) in &self.big_chains { + chain(f, c, &mut fat); + } + if self.mini_stream_sectors > 0 { + chain(self.mini_stream_start, self.mini_stream_sectors, &mut fat); + } + if self.minifat_sectors > 0 { + chain(self.minifat_start, self.minifat_sectors, &mut fat); + } + chain(self.dir_start, self.dir_sectors, &mut fat); + for i in 0..self.fat_n { + fat[self.fat_start as usize + i] = FATSECT; + } + for i in 0..self.difat_n { + fat[self.difat_start as usize + i] = DIFSECT; + } + debug_assert!(self.total <= fat.len()); + fat + } + + fn mini_fat(&self, per_sector: usize) -> Vec { + let mut minifat = vec![FREESECT; self.minifat_sectors as usize * per_sector]; + for &(f, c) in &self.mini_chains { + for i in 0..c { + minifat[(f + i) as usize] = if i + 1 == c { ENDOFCHAIN } else { f + i + 1 }; + } + } + minifat + } + + /// Append the DIFAT sectors: the FAT sector numbers beyond the header's 109 slots. + fn write_difat(&self, out: &mut Vec, ss: usize) { + let per_sector = ss / 4; + let mut remaining: Vec = (109..self.fat_n).map(|i| self.fat_start + i as u32).collect(); + for i in 0..self.difat_n { + let mut s = Vec::with_capacity(ss); + let take = remaining.len().min(per_sector - 1); + for v in remaining.drain(..take) { + s.extend_from_slice(&v.to_le_bytes()); + } + while s.len() < ss - 4 { + s.extend_from_slice(&FREESECT.to_le_bytes()); + } + let next = if i + 1 < self.difat_n { self.difat_start + i as u32 + 1 } else { ENDOFCHAIN }; + s.extend_from_slice(&next.to_le_bytes()); + out.extend_from_slice(&s); + } + } +} + +/// Balanced BST of `kids[..]` (sorted). Nodes on the bottom level of an imperfect tree are red, +/// so every root-to-leaf path has the same number of black nodes. +fn build_tree(kids: &[usize], depth: usize, perfect: bool, height: usize, tree: &mut [(u32, u32, u32, u8)], dir_index: &[u32]) -> u32 { + if kids.is_empty() { + return NOSTREAM; + } + let mid = kids.len() / 2; + let n = kids[mid]; + let left = build_tree(&kids[..mid], depth + 1, perfect, height, tree, dir_index); + let right = build_tree(&kids[mid + 1..], depth + 1, perfect, height, tree, dir_index); + tree[n].0 = left; + tree[n].1 = right; + tree[n].3 = if !perfect && depth == height { 0 } else { 1 }; + dir_index[n] +} diff --git a/crates/cfb/tests/writer_scale.rs b/crates/cfb/tests/writer_scale.rs new file mode 100644 index 000000000..db60b09ed --- /dev/null +++ b/crates/cfb/tests/writer_scale.rs @@ -0,0 +1,32 @@ +//! The writer scales: adding many children to one storage, and the duplicate-name rules that check keeps. + +use craft_cfb::{Version, Writer}; + +/// Adding a child compared its name with every sibling (allocating twice per comparison), so n children +/// cost O(n^2): 20 000 children in one storage took about a minute. The duplicate check is a hash +/// lookup now. 40 000 plain names take milliseconds now and about 18 s before, so the limit separates them by orders of magnitude on any machine. +#[test] +fn adding_many_children_is_not_quadratic() { + let mut w = Writer::new(Version::V3); + let start = std::time::Instant::now(); + for i in 0..40_000 { + w.stream(Writer::ROOT, &format!("{i}"), vec![1]).expect("unique name"); + } + assert!(start.elapsed() < std::time::Duration::from_secs(3), "{:?}", start.elapsed()); +} + +/// The faster duplicate check still means what sibling comparison means: case-insensitive, also beyond ASCII, +/// per storage (the same name in another storage is fine), and a name that differs only in length is distinct. +#[test] +fn duplicate_names_are_still_rejected() { + let mut w = Writer::new(Version::V3); + let a = w.storage(Writer::ROOT, "Alpha", [0; 16]).unwrap(); + assert!(w.stream(Writer::ROOT, "ALPHA", vec![]).is_err(), "case-insensitive"); + assert!(w.stream(Writer::ROOT, "alpha", vec![]).is_err()); + assert!(w.stream(Writer::ROOT, "Alph", vec![]).is_ok()); + assert!(w.stream(Writer::ROOT, "Alpha2", vec![]).is_ok()); + assert!(w.stream(a, "Alpha", vec![]).is_ok(), "same name in another storage"); + assert!(w.stream(a, "Alpha", vec![]).is_err()); + w.stream(Writer::ROOT, "Ünï", vec![]).unwrap(); + assert!(w.stream(Writer::ROOT, "ÜNÏ", vec![]).is_err(), "non-ASCII case"); +}