diff --git a/Cargo.lock b/Cargo.lock index 347b4470b..e31edd944 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1362,6 +1362,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-pst" +version = "0.1.0" +dependencies = [ + "proptest", +] + [[package]] name = "craft-quit" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 22412afff..fb8ef521a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ craft-print = { path = "crates/print" } craft-proof = { path = "crates/proof" } craft-raster-codecs = { path = "crates/raster-codecs" } craft-psd = { path = "crates/psd" } +craft-pst = { path = "crates/pst" } craft-quit = { path = "crates/quit" } craft-rtf = { path = "crates/rtf" } craft-raw = { path = "crates/raw" } diff --git a/README.md b/README.md index 408af4c2c..c95f5ca55 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,12 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " { + pub key_size: usize, + pub data_size: usize, + /// Leaf records, `key_size + data_size` bytes each, in the order of the tree. + pub records: Vec<&'h [u8]>, +} + +pub(crate) fn read(heap: &Heap, header_hid: u32, max_records: usize, cap: Error) -> Result> { + let h = heap.item(header_hid)?; + if u8_at(h, 0)? != CLIENT_BTH { + return Err(Error::Corrupt("a B-tree-on-heap's signature")); + } + let key_size = usize::from(u8_at(h, 1)?); + let data_size = usize::from(u8_at(h, 2)?); + let levels = u8_at(h, 3)?; + let root = u32_at(h, 4)?; + if !matches!(key_size, 2 | 4 | 8 | 16) || data_size == 0 || data_size > 32 { + return Err(Error::Corrupt("a B-tree-on-heap's record sizes")); + } + if levels > MAX_LEVELS { + return Err(Error::Corrupt("a B-tree-on-heap too deep")); + } + let mut records = Vec::new(); + if root == 0 { + return Ok(Bth { key_size, data_size, records }); + } + let mut seen = HashSet::new(); + let mut left = heap.size(); + let mut stack = vec![(root, levels)]; + while let Some((hid, level)) = stack.pop() { + if !seen.insert(hid) { + return Err(Error::Cycle("a B-tree-on-heap")); + } + let item = heap.item(hid)?; + left = left.checked_sub(item.len()).ok_or(Error::Corrupt("a B-tree-on-heap larger than its heap"))?; + if level == 0 { + let size = key_size.saturating_add(data_size); + if !item.len().is_multiple_of(size) { + return Err(Error::Corrupt("a B-tree-on-heap's records")); + } + if item.len().checked_div(size).unwrap_or(0) > max_records.saturating_sub(records.len()) { + return Err(cap); + } + // A BTH's keys are unique and in order: each must be greater than the one before + // (as a little-endian number), so no record is named twice. + for r in item.chunks_exact(size) { + let key = r.get(..key_size).unwrap_or(&[]); + if let Some(prev) = records.last().and_then(|p: &&[u8]| p.get(..key_size)) + && key.iter().rev().cmp(prev.iter().rev()) != std::cmp::Ordering::Greater + { + return Err(Error::Corrupt("a B-tree-on-heap's keys out of order")); + } + records.push(r); + } + continue; + } + let size = key_size.saturating_add(4); + if !item.len().is_multiple_of(size) { + return Err(Error::Corrupt("a B-tree-on-heap's index")); + } + let below = level.saturating_sub(1); + let n = item.len().checked_div(size).unwrap_or(0); + let mut children = Vec::with_capacity(n); + for i in 0..n { + let r = slice(item, i.saturating_mul(size), size)?; + children.push((u32_at(r, key_size)?, below)); + } + // Last-first, so that the leaves come out in key order. + stack.extend(children.into_iter().rev()); + } + Ok(Bth { key_size, data_size, records }) +} diff --git a/crates/pst/src/bytes.rs b/crates/pst/src/bytes.rs new file mode 100644 index 000000000..153a78be5 --- /dev/null +++ b/crates/pst/src/bytes.rs @@ -0,0 +1,50 @@ +//! Little-endian reads out of a byte slice that fail instead of panicking. + +use crate::error::{Error, Result}; + +const SHORT: Error = Error::Corrupt("a structure runs past its end"); + +pub(crate) fn slice(b: &[u8], at: usize, len: usize) -> Result<&[u8]> { + let end = at.checked_add(len).ok_or(SHORT)?; + b.get(at..end).ok_or(SHORT) +} + +pub(crate) fn u8_at(b: &[u8], at: usize) -> Result { + b.get(at).copied().ok_or(SHORT) +} + +pub(crate) fn u16_at(b: &[u8], at: usize) -> Result { + let s = slice(b, at, 2)?; + Ok(u16::from_le_bytes([s.first().copied().unwrap_or(0), s.get(1).copied().unwrap_or(0)])) +} + +pub(crate) fn u32_at(b: &[u8], at: usize) -> Result { + let s = slice(b, at, 4)?; + let mut a = [0u8; 4]; + a.copy_from_slice(s); + Ok(u32::from_le_bytes(a)) +} + +pub(crate) fn u64_at(b: &[u8], at: usize) -> Result { + let s = slice(b, at, 8)?; + let mut a = [0u8; 8]; + a.copy_from_slice(s); + Ok(u64::from_le_bytes(a)) +} + +pub(crate) fn array16(b: &[u8], at: usize) -> Result<[u8; 16]> { + let s = slice(b, at, 16)?; + let mut a = [0u8; 16]; + a.copy_from_slice(s); + Ok(a) +} + +/// UTF-16LE text, lossy; an odd last byte and trailing NULs are dropped. +pub(crate) fn utf16(b: &[u8]) -> String { + let units: Vec = b.as_chunks::<2>().0.iter().map(|c| u16::from_le_bytes(*c)).collect(); + let mut s = String::from_utf16_lossy(&units); + while s.ends_with('\0') { + s.pop(); + } + s +} diff --git a/crates/pst/src/crc.rs b/crates/pst/src/crc.rs new file mode 100644 index 000000000..d5a226331 --- /dev/null +++ b/crates/pst/src/crc.rs @@ -0,0 +1,39 @@ +//! The CRC of [MS-PST] 5.3: the reflected CRC-32 polynomial (its table starts `0x00000000, +//! 0x77073096, 0xEE0E612C`), seeded with zero and with no final inversion. It guards the header, +//! every page and every block, and is what the file keeps of its password. + +const TABLE: [u32; 256] = table(); + +#[allow(clippy::indexing_slicing, clippy::arithmetic_side_effects)] // a constant, every index below 256 +const fn table() -> [u32; 256] { + let mut t = [0u32; 256]; + let mut i = 0; + while i < 256 { + let mut c = i as u32; + let mut k = 0; + while k < 8 { + c = if c & 1 != 0 { 0xEDB8_8320 ^ (c >> 1) } else { c >> 1 }; + k += 1; + } + t[i] = c; + i += 1; + } + t +} + +/// The CRC of `data`. +pub fn crc32(data: &[u8]) -> u32 { + let mut crc = 0u32; + for &b in data { + let at = ((crc ^ u32::from(b)) & 0xFF) as usize; + crc = TABLE.get(at).copied().unwrap_or(0) ^ (crc >> 8); + } + crc +} + +/// The first entries of the table as [MS-PST] 5.3 prints them (a test checks the rest of the +/// algorithm against the spec's sample header). +#[cfg(test)] +pub(crate) fn table_head() -> [u32; 4] { + [TABLE[0], TABLE[1], TABLE[2], TABLE[255]] +} diff --git a/crates/pst/src/crypt.rs b/crates/pst/src/crypt.rs new file mode 100644 index 000000000..984bacc34 --- /dev/null +++ b/crates/pst/src/crypt.rs @@ -0,0 +1,99 @@ +//! The two keyless encodings of data blocks, [MS-PST] 5.1 (`NDB_CRYPT_PERMUTE`) and 5.2 +//! (`NDB_CRYPT_CYCLIC`). They hide nothing: they keep text from being read straight off the disk. +//! The three tables are the specification's (`mpbbCrypt`, in three parts of 256); `I` undoes `R`. + +/// How a file's data blocks are encoded (`bCryptMethod`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Encryption { + None, + Permute, + Cyclic, +} + +/// Decodes `data` in place, as it was stored in the block with id `bid`. +pub(crate) fn decode(method: Encryption, data: &mut [u8], bid: u64) { + match method { + Encryption::None => {} + Encryption::Permute => { + for b in data.iter_mut() { + *b = at(&I, *b); + } + } + Encryption::Cyclic => cyclic(data, bid as u32), + } +} + +/// Encodes `data` in place (the test writer's side). +#[cfg(any(test, feature = "test-writer"))] +pub(crate) fn encode(method: Encryption, data: &mut [u8], bid: u64) { + match method { + Encryption::None => {} + Encryption::Permute => { + for b in data.iter_mut() { + *b = at(&R, *b); + } + } + Encryption::Cyclic => cyclic(data, bid as u32), + } +} + +/// [MS-PST] 5.2: symmetric, keyed by the low 32 bits of the block id. +fn cyclic(data: &mut [u8], key: u32) { + let mut w = (key ^ (key >> 16)) as u16; + for b in data.iter_mut() { + let lo = w as u8; + let hi = (w >> 8) as u8; + let mut x = b.wrapping_add(lo); + x = at(&R, x); + x = x.wrapping_add(hi); + x = at(&S, x); + x = x.wrapping_sub(hi); + x = at(&I, x); + x = x.wrapping_sub(lo); + *b = x; + w = w.wrapping_add(1); + } +} + +fn at(table: &[u8; 256], b: u8) -> u8 { + table.get(usize::from(b)).copied().unwrap_or(b) +} + +#[cfg(test)] +pub(crate) fn tables() -> (&'static [u8; 256], &'static [u8; 256], &'static [u8; 256]) { + (&R, &S, &I) +} + +const R: [u8; 256] = [ + 65, 54, 19, 98, 168, 33, 110, 187, 244, 22, 204, 4, 127, 100, 232, 93, 30, 242, 203, 42, 116, 197, 94, 53, 210, 149, 71, 158, 150, 45, 154, 136, + 76, 125, 132, 63, 219, 172, 49, 182, 72, 95, 246, 196, 216, 57, 139, 231, 35, 59, 56, 142, 200, 193, 223, 37, 177, 32, 165, 70, 96, 78, 156, 251, + 170, 211, 86, 81, 69, 124, 85, 0, 7, 201, 43, 157, 133, 155, 9, 160, 143, 173, 179, 15, 99, 171, 137, 75, 215, 167, 21, 90, 113, 102, 66, 191, + 38, 74, 107, 152, 250, 234, 119, 83, 178, 112, 5, 44, 253, 89, 58, 134, 126, 206, 6, 235, 130, 120, 87, 199, 141, 67, 175, 180, 28, 212, 91, 205, + 226, 233, 39, 79, 195, 8, 114, 128, 207, 176, 239, 245, 40, 109, 190, 48, 77, 52, 146, 213, 14, 60, 34, 50, 229, 228, 249, 159, 194, 209, 10, + 129, 18, 225, 238, 145, 131, 118, 227, 151, 230, 97, 138, 23, 121, 164, 183, 220, 144, 122, 92, 140, 2, 166, 202, 105, 222, 80, 26, 17, 147, 185, + 82, 135, 88, 252, 237, 29, 55, 73, 27, 106, 224, 41, 51, 153, 189, 108, 217, 148, 243, 64, 84, 111, 240, 198, 115, 184, 214, 62, 101, 24, 68, 31, + 221, 103, 16, 241, 12, 25, 236, 174, 3, 161, 20, 123, 169, 11, 255, 248, 163, 192, 162, 1, 247, 46, 188, 36, 104, 117, 13, 254, 186, 47, 181, + 208, 218, 61, +]; +const S: [u8; 256] = [ + 20, 83, 15, 86, 179, 200, 122, 156, 235, 101, 72, 23, 22, 21, 159, 2, 204, 84, 124, 131, 0, 13, 12, 11, 162, 98, 168, 118, 219, 217, 237, 199, + 197, 164, 220, 172, 133, 116, 214, 208, 167, 155, 174, 154, 150, 113, 102, 195, 99, 153, 184, 221, 115, 146, 142, 132, 125, 165, 94, 209, 93, + 147, 177, 87, 81, 80, 128, 137, 82, 148, 79, 78, 10, 107, 188, 141, 127, 110, 71, 70, 65, 64, 68, 1, 17, 203, 3, 63, 247, 244, 225, 169, 143, 60, + 58, 249, 251, 240, 25, 48, 130, 9, 46, 201, 157, 160, 134, 73, 238, 111, 77, 109, 196, 45, 129, 52, 37, 135, 27, 136, 170, 252, 6, 161, 18, 56, + 253, 76, 66, 114, 100, 19, 55, 36, 106, 117, 119, 67, 255, 230, 180, 75, 54, 92, 228, 216, 53, 61, 69, 185, 44, 236, 183, 49, 43, 41, 7, 104, + 163, 14, 105, 123, 24, 158, 33, 57, 190, 40, 26, 91, 120, 245, 35, 202, 42, 176, 175, 62, 254, 4, 140, 231, 229, 152, 50, 149, 211, 246, 74, 232, + 166, 234, 233, 243, 213, 47, 112, 32, 242, 31, 5, 103, 173, 85, 16, 206, 205, 227, 39, 59, 218, 186, 215, 194, 38, 212, 145, 29, 210, 28, 34, 51, + 248, 250, 241, 90, 239, 207, 144, 182, 139, 181, 189, 192, 191, 8, 151, 30, 108, 226, 97, 224, 198, 193, 89, 171, 187, 88, 222, 95, 223, 96, 121, + 126, 178, 138, +]; +const I: [u8; 256] = [ + 71, 241, 180, 230, 11, 106, 114, 72, 133, 78, 158, 235, 226, 248, 148, 83, 224, 187, 160, 2, 232, 90, 9, 171, 219, 227, 186, 198, 124, 195, 16, + 221, 57, 5, 150, 48, 245, 55, 96, 130, 140, 201, 19, 74, 107, 29, 243, 251, 143, 38, 151, 202, 145, 23, 1, 196, 50, 45, 110, 49, 149, 255, 217, + 35, 209, 0, 94, 121, 220, 68, 59, 26, 40, 197, 97, 87, 32, 144, 61, 131, 185, 67, 190, 103, 210, 70, 66, 118, 192, 109, 91, 126, 178, 15, 22, 41, + 60, 169, 3, 84, 13, 218, 93, 223, 246, 183, 199, 98, 205, 141, 6, 211, 105, 92, 134, 214, 20, 247, 165, 102, 117, 172, 177, 233, 69, 33, 112, 12, + 135, 159, 116, 164, 34, 76, 111, 191, 31, 86, 170, 46, 179, 120, 51, 80, 176, 163, 146, 188, 207, 25, 28, 167, 99, 203, 30, 77, 62, 75, 27, 155, + 79, 231, 240, 238, 173, 58, 181, 89, 4, 234, 64, 85, 37, 81, 229, 122, 137, 56, 104, 82, 123, 252, 39, 174, 215, 189, 250, 7, 244, 204, 142, 95, + 239, 53, 156, 132, 43, 21, 213, 119, 52, 73, 182, 18, 10, 127, 113, 136, 253, 157, 24, 65, 125, 147, 216, 88, 44, 206, 254, 36, 175, 222, 184, + 54, 200, 161, 128, 166, 153, 152, 168, 47, 14, 129, 101, 115, 228, 194, 162, 138, 212, 225, 17, 208, 8, 139, 42, 242, 237, 154, 100, 63, 193, + 108, 249, 236, +]; diff --git a/crates/pst/src/error.rs b/crates/pst/src/error.rs new file mode 100644 index 000000000..069937ae3 --- /dev/null +++ b/crates/pst/src/error.rs @@ -0,0 +1,206 @@ +//! What can go wrong reading a .pst file, and the caps that keep a hostile one bounded. + +use std::fmt; + +/// A failure to read (part of) a .pst file. Nothing in this crate panics: everything a file can +/// get wrong ends here. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// The reader under the file failed. + Io(std::io::ErrorKind), + /// Not a .pst file (its first bytes are not `!BDN` … `SM`). + NotPst, + /// A .pst file of a version this reader doesn't know (`wVer`). + UnsupportedVersion(u16), + /// Data encoded in a way this reader can't undo (`bCryptMethod`, for example Windows + /// Information Protection). + UnsupportedEncryption(u8), + /// A checksum, signature or block id that doesn't match what it guards. + BadChecksum(&'static str), + /// A structure that makes no sense. + Corrupt(&'static str), + /// An offset or length that points past the end of the file. + OutOfBounds, + /// A tree that leads back to where it was (or reads the same part twice). + Cycle(&'static str), + /// Something the file should have and hasn't. + NotFound(&'static str), + /// Past one of the caps in [`Limits`]: left out, not read. + Cap(Cap), + /// The file has a password and it has not been given ([`crate::Pst::unlock`]). + PasswordRequired, + /// The password given is not the file's. + WrongPassword, + /// The operation does not apply to this object (a search folder's items, for example). + Unsupported(&'static str), +} + +/// Which cap was reached. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Cap { + FileSize, + Items, + Folders, + FolderDepth, + ItemSize, + AttachmentSize, + Recipients, + Attachments, + EmbeddedDepth, + BTreePages, + Subnodes, + DataBlocks, + NamedProperties, + /// [`Limits::max_listing_bytes`]: what one folder walk or one listing may read. + ListingSize, + /// [`Limits::max_embedded_messages`]: attached messages read under one item. + EmbeddedMessages, + /// [`Limits::read_multiple`]: more read from the file, in all, than its size allows for. + TotalRead, +} + +impl fmt::Display for Cap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let what = match self { + Cap::FileSize => "the file is larger than the cap", + Cap::Items => "more items than the cap", + Cap::Folders => "more folders than the cap", + Cap::FolderDepth => "folders nested deeper than the cap", + Cap::ItemSize => "an item larger than the cap", + Cap::AttachmentSize => "an attachment larger than the cap", + Cap::Recipients => "more recipients than the cap", + Cap::Attachments => "more attachments than the cap", + Cap::EmbeddedDepth => "messages attached deeper than the cap", + Cap::BTreePages => "a tree with more pages than the cap", + Cap::Subnodes => "an item with more parts than the cap", + Cap::DataBlocks => "data spread over more blocks than its size needs", + Cap::NamedProperties => "a named property's name, or all of them, longer than the cap", + Cap::ListingSize => "a folder list or a list of items larger than the cap", + Cap::EmbeddedMessages => "more attached messages under one item than the cap", + Cap::TotalRead => "more read from the file than its size allows for", + }; + f.write_str(what) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Io(kind) => write!(f, "the file could not be read ({kind})"), + Error::NotPst => f.write_str("not a .pst file"), + Error::UnsupportedVersion(36) => f.write_str("an offline folders file (.ost) with 4 KB pages, not supported (version 36)"), + Error::UnsupportedVersion(37) => f.write_str("a file protected by Windows Information Protection, not supported (version 37)"), + Error::UnsupportedVersion(v) => write!(f, "a .pst file of a version not supported ({v})"), + Error::UnsupportedEncryption(m) => write!(f, "the file's data is encoded in a way not supported ({m:#04x})"), + Error::BadChecksum(what) => write!(f, "damaged file: {what} doesn't match its checksum"), + Error::Corrupt(what) => write!(f, "damaged file: {what}"), + Error::OutOfBounds => f.write_str("damaged file: a part points past its end"), + Error::Cycle(what) => write!(f, "damaged file: {what} leads back to itself"), + Error::NotFound(what) => write!(f, "damaged file: {what} is missing"), + Error::Cap(cap) => write!(f, "left out: {cap}"), + Error::PasswordRequired => f.write_str("the file has a password"), + Error::WrongPassword => f.write_str("the password is not the file's"), + Error::Unsupported(what) => write!(f, "not supported: {what}"), + } + } +} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(e: std::io::Error) -> Self { + match e.kind() { + std::io::ErrorKind::UnexpectedEof => Error::OutOfBounds, + kind => Error::Io(kind), + } + } +} + +pub type Result = std::result::Result; + +/// A part of the file's own structure that could not be read: a page of the node tree (the +/// folders and items under it can't be found) or of the block tree (the data of the nodes that +/// use its blocks can't be found). The page's own count of what it held can't be trusted, so +/// what was lost with it is not known; each folder or item that needed it is named in the +/// `left_out` of the walk, list or message that met it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Damage { + /// Where the page is in the file. + pub offset: u64, + /// "a page of the node tree" or "a page of the block tree". + pub what: &'static str, + pub reason: Error, + /// 1; or, for the last entry once [`MAX_DAMAGE`] pages have been named, how many pages + /// from it on could not be read (at least: past 65,536 they are no longer counted). + pub pages: u64, +} + +/// Damaged pages named one by one; past them they are counted in one entry. +pub const MAX_DAMAGE: usize = 1000; + +/// The caps, checked before anything is read or allocated. [`Limits::default`] is the import's +/// (the file at most 100 GB, 2,000,000 items, 10,000 folders nested at most 64 deep, 256 MB an +/// item, 150 MB an attachment, 1,000 recipients and 1,000 attachments an item, attached +/// messages at most 8 deep, 10,000 of them under one item; a folder walk or a listing at most +/// 1 GB; at most 64 times the file's size plus 256 MB read in all); lower ones serve tests. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Limits { + pub max_file_size: u64, + /// Items listed across the file's folders ([`crate::Pst::items`]). + pub max_items: usize, + pub max_folders: usize, + pub max_folder_depth: usize, + /// One node's data (a message's properties, a table), and all of an item's property values + /// read at once. + pub max_item_bytes: u64, + pub max_attachment_bytes: u64, + pub max_recipients: usize, + pub max_attachments: usize, + pub max_embedded_depth: usize, + /// Pages a walk of the whole node tree may visit. + pub max_btree_pages: usize, + /// Entries in one node's subnode tree. + pub max_subnodes: usize, + /// A property value larger than this is not read with the rest ([`crate::PropValue::Large`]) + /// and is read on its own ([`crate::Pst::read_property`]). + pub max_inline_value: u64, + /// All that one [`crate::Pst::walk_folders`] or one listing of a folder's items may read + /// and keep (tables and values together). + pub max_listing_bytes: u64, + /// A value larger than this in a folder walk or an item list is [`crate::PropValue::Large`]. + pub max_summary_value: u64, + /// Attached messages read under one item, at every depth together. + pub max_embedded_messages: usize, + /// All that may be read from the file while it is open: this many times its size, plus + /// [`Limits::read_floor`], so that the work stays in proportion to the file. It is + /// counted for each [`crate::Pst`] from its opening: an import that reads a file more than + /// once (again after a cancel, say) opens it again. + pub read_multiple: u64, + pub read_floor: u64, +} + +impl Default for Limits { + fn default() -> Self { + Limits { + max_file_size: 100 * 1024 * 1024 * 1024, + max_items: 2_000_000, + max_folders: 10_000, + max_folder_depth: 64, + max_item_bytes: 256 * 1024 * 1024, + max_attachment_bytes: 150 * 1024 * 1024, + max_recipients: 1_000, + max_attachments: 1_000, + max_embedded_depth: 8, + max_btree_pages: 4_000_000, + max_subnodes: 200_000, + max_inline_value: 4 * 1024 * 1024, + max_listing_bytes: 1024 * 1024 * 1024, + max_summary_value: 64 * 1024, + max_embedded_messages: 10_000, + read_multiple: 64, + read_floor: 256 * 1024 * 1024, + } + } +} diff --git a/crates/pst/src/header.rs b/crates/pst/src/header.rs new file mode 100644 index 000000000..866decd20 --- /dev/null +++ b/crates/pst/src/header.rs @@ -0,0 +1,111 @@ +//! The HEADER at offset 0 ([MS-PST] 2.2.2.6) and the ROOT in it (2.2.2.5): which format, how +//! data is encoded, and where the two B-trees start. Both CRCs are checked. + +use crate::bytes::{u8_at, u16_at, u32_at, u64_at}; +use crate::crc::crc32; +use crate::crypt::Encryption; +use crate::error::{Error, Result}; + +/// ANSI (Outlook 97–2002: 32-bit block ids and offsets) or Unicode (64-bit). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Format { + Ansi, + Unicode, +} + +impl Format { + /// Bytes in a block id or a file offset. + pub(crate) fn id_size(self) -> usize { + match self { + Format::Ansi => 4, + Format::Unicode => 8, + } + } + + pub(crate) fn read_id(self, b: &[u8], at: usize) -> Result { + match self { + Format::Ansi => u32_at(b, at).map(u64::from), + Format::Unicode => u64_at(b, at), + } + } + + /// A block reference (`BREF`): id, then offset. + pub(crate) fn read_bref(self, b: &[u8], at: usize) -> Result { + let bid = self.read_id(b, at)?; + let ib = self.read_id(b, at.checked_add(self.id_size()).ok_or(Error::OutOfBounds)?)?; + Ok(Bref { bid, ib }) + } + + /// Bytes of a page trailer, a block trailer (the same sizes). + pub(crate) fn trailer_size(self) -> usize { + match self { + Format::Ansi => 12, + Format::Unicode => 16, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Bref { + pub bid: u64, + pub ib: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Header { + pub format: Format, + pub version: u16, + pub encryption: Encryption, + pub file_eof: u64, + pub nbt: Bref, + pub bbt: Bref, +} + +pub(crate) const HEADER_READ: usize = 564; + +impl Header { + pub(crate) fn parse(b: &[u8]) -> Result
{ + if b.get(0..4) != Some(b"!BDN") || b.get(8..10) != Some(b"SM") { + return Err(Error::NotPst); + } + let version = u16_at(b, 10)?; + let format = match version { + 14 | 15 => Format::Ansi, + // 36 is an offline folders file with 4 KB pages, 37 one protected by Windows + // Information Protection: neither is read. + 23..=35 => Format::Unicode, + v => return Err(Error::UnsupportedVersion(v)), + }; + // dwCRCPartial: the 471 bytes from wMagicClient. + let partial = u32_at(b, 4)?; + if crc32(b.get(8..479).ok_or(Error::NotPst)?) != partial { + return Err(Error::BadChecksum("the file's header")); + } + // ROOT (at 180, or 164 in ANSI): dwReserved, ibFileEof, ibAMapLast, cbAMapFree, + // cbPMapFree, BREFNBT, BREFBBT; then the sentinel and bCryptMethod. + let (eof_at, nbt_at, bbt_at, sentinel, crypt) = match format { + Format::Unicode => (184, 216, 232, 512, 513), + Format::Ansi => (168, 184, 192, 460, 461), + }; + if format == Format::Unicode { + // dwCRCFull: the 516 bytes from wMagicClient to bidNextB. + let full = u32_at(b, 524)?; + if crc32(b.get(8..524).ok_or(Error::NotPst)?) != full { + return Err(Error::BadChecksum("the file's header")); + } + } + if u8_at(b, sentinel)? != 0x80 { + return Err(Error::Corrupt("the header's sentinel")); + } + let encryption = match u8_at(b, crypt)? { + 0 => Encryption::None, + 1 => Encryption::Permute, + 2 => Encryption::Cyclic, + other => return Err(Error::UnsupportedEncryption(other)), + }; + let file_eof = format.read_id(b, eof_at)?; + let nbt = format.read_bref(b, nbt_at)?; + let bbt = format.read_bref(b, bbt_at)?; + Ok(Header { format, version, encryption, file_eof, nbt, bbt }) + } +} diff --git a/crates/pst/src/heap.rs b/crates/pst/src/heap.rs new file mode 100644 index 000000000..c8b031200 --- /dev/null +++ b/crates/pst/src/heap.rs @@ -0,0 +1,83 @@ +//! The heap-on-node ([MS-PST] 2.3.1): a node's data blocks, each with its page map, and the +//! items allocated in them, found by their HID. + +use crate::bytes::{slice, u8_at, u16_at, u32_at}; +use crate::error::{Error, Result}; + +const HN_SIGNATURE: u8 = 0xEC; +pub(crate) const CLIENT_TC: u8 = 0x7C; +pub(crate) const CLIENT_BTH: u8 = 0xB5; +pub(crate) const CLIENT_PC: u8 = 0xBC; + +pub(crate) struct Heap { + blocks: Vec>, + pub client: u8, + pub user_root: u32, +} + +/// Whether an HNID names a heap item (else it names a subnode). +pub(crate) fn is_hid(hnid: u32) -> bool { + hnid & 0x1F == 0 +} + +impl Heap { + pub(crate) fn new(blocks: Vec>) -> Result { + let first = blocks.first().ok_or(Error::Corrupt("an empty heap"))?; + if u8_at(first, 2)? != HN_SIGNATURE { + return Err(Error::Corrupt("a heap's signature")); + } + let client = u8_at(first, 3)?; + let user_root = u32_at(first, 4)?; + if blocks.len() > 1 << 16 { + return Err(Error::Corrupt("a heap of too many blocks")); + } + // Every page map's offsets run forward and end before the map, so that no two items + // share a byte and the items of a heap together are no larger than it. + for block in &blocks { + let map = usize::from(u16_at(block, 0)?); + let count = usize::from(u16_at(block, map)?); + let mut last = 0usize; + for i in 0..=count { + let off = i.checked_mul(2).and_then(|o| o.checked_add(map)).and_then(|o| o.checked_add(4)).ok_or(Error::OutOfBounds)?; + let at = usize::from(u16_at(block, off)?); + if at < last || at > map { + return Err(Error::Corrupt("a heap's page map")); + } + last = at; + } + } + Ok(Heap { blocks, client, user_root }) + } + + /// The bytes of all its blocks. + pub(crate) fn size(&self) -> usize { + self.blocks.iter().map(Vec::len).sum() + } + + /// The item `hid`; an HID of zero is an empty item. + pub(crate) fn item(&self, hid: u32) -> Result<&[u8]> { + if hid == 0 { + return Ok(&[]); + } + if !is_hid(hid) { + return Err(Error::Corrupt("a heap id of the wrong type")); + } + let index = usize::from(((hid >> 5) & 0x7FF) as u16); + let block = self.blocks.get((hid >> 16) as usize).ok_or(Error::Corrupt("a heap id past the heap"))?; + let map = usize::from(u16_at(block, 0)?); + let count = usize::from(u16_at(block, map)?); + if index == 0 || index > count { + return Err(Error::Corrupt("a heap id past the page map")); + } + let at = |i: usize| -> Result { + let off = i.checked_mul(2).and_then(|o| o.checked_add(map)).and_then(|o| o.checked_add(4)).ok_or(Error::OutOfBounds)?; + Ok(usize::from(u16_at(block, off)?)) + }; + let start = at(index.saturating_sub(1))?; + let end = at(index)?; + if start > end || end > map { + return Err(Error::Corrupt("a heap item outside its block")); + } + slice(block, start, end.saturating_sub(start)) + } +} diff --git a/crates/pst/src/lib.rs b/crates/pst/src/lib.rs new file mode 100644 index 000000000..b532fbeaa --- /dev/null +++ b/crates/pst/src/lib.rs @@ -0,0 +1,69 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! A reader of Outlook personal folders files (`.pst`), written from Microsoft's published +//! specification, [MS-PST], and nothing else. +//! +//! [`Pst::open`] reads over any `Read + Seek` and never holds the file in memory: a page or a +//! block is read when it is needed. ANSI and Unicode files are read, with data blocks encoded +//! either way the format allows (or not at all). A file with a password is read only once +//! [`Pst::unlock`] has been given it. +//! +//! The layers are the specification's: the node database (`ndb`: B-trees of nodes and blocks, +//! data trees, subnode trees), lists, tables and properties (`ltp`: the heap on a node, its +//! B-tree, property and table contexts), and the messaging layer ([`Pst`]: the store, folders, +//! items, recipients, attachments, attached messages, named properties). +//! +//! A .pst file is hostile input. Nothing here panics on any bytes: every offset and length is +//! checked against the file, every CRC, signature and block id against what it guards, trees +//! are bounded in depth and read each part once, and the caps of [`Limits`] are checked before +//! anything is read or allocated. Past a cap, a part is left out and named ([`Error::Cap`]). +#![cfg_attr(not(test), deny(clippy::indexing_slicing, clippy::arithmetic_side_effects))] + +mod bth; +mod bytes; +mod crc; +mod crypt; +mod error; +mod header; +mod heap; +mod list_stream; +mod ltp; +mod named; +mod ndb; +mod prop; +mod pst; +pub mod recur; +mod source; +/// The test writer: synthetic .pst files from invented folders and messages, for this crate's +/// tests and, through the `test-writer` feature (a dev-dependency only, never in a build that +/// ships), for the tests of the import that reads them. +#[cfg(any(test, feature = "test-writer"))] +#[doc(hidden)] +#[path = "tests/writer.rs"] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing, + clippy::arithmetic_side_effects, + clippy::new_without_default, + clippy::too_many_arguments, + dead_code +)] +pub mod test_writer; +#[cfg(test)] +mod tests; + +pub use crc::crc32; +pub use crypt::Encryption; +pub use error::{Cap, Damage, Error, Limits, MAX_DAMAGE, Result}; +pub use header::Format; +pub use list_stream::{ListMember, ListStream, MAX_LIST_ENTRY, list_stream}; +pub use named::{NamedMap, NamedProperty, PS_MAPI, PS_PUBLIC_STRINGS, PropName}; +pub use prop::{PropValue, Properties, Property, types}; +pub use pst::{ + Attachment, AttachmentContent, AttachmentReader, AttachmentRef, Folder, FolderEntry, FolderKind, FolderTree, ItemList, ItemSummary, LeftOut, + LeftOutPart, Message, MessageRef, Nid, NodeInfo, Part, Pst, Recipient, RecipientKind, Store, node_types, +}; +pub use recur::{Recur, RecurException, appointment_recurrence, time_zone_key}; diff --git a/crates/pst/src/list_stream.rs b/crates/pst/src/list_stream.rs new file mode 100644 index 000000000..f24d67d64 --- /dev/null +++ b/crates/pst/src/list_stream.rs @@ -0,0 +1,106 @@ +//! The stream a personal distribution list keeps its members in once they pass about 15,000 +//! bytes (PidLidDistributionListStream, [MS-OXOCNTC] 2.2.2.2.4): a header, then one +//! DistListMemberInfo (2.2.2.2.4.1) for each member, each an entry id and a one-off entry id, +//! then two terminators. +//! +//! The bytes are hostile: the count of members is checked against the bytes there are, every +//! length against what is left, nothing is allocated by what the stream says, and at most +//! `max_members` members are kept. + +use crate::bytes::{slice, u16_at, u32_at}; +use crate::error::{Error, Result}; + +/// The header: StreamVersion, Reserved, BuildVersion, DistListStreamFlags, CountOfEntries, +/// TotalEntryIDSize, TotalOneOffSize, TotalExtraSize. +const HEADER: usize = 28; +/// The least a member takes: its three sizes. +const LEAST_MEMBER: usize = 12; +/// Longest an entry id of a member may be (one-off and wrapped entry ids are small). +pub const MAX_LIST_ENTRY: usize = 16 * 1024; + +/// One member of the stream: its entry id (a one-off entry id or a WrappedEntryId) and its +/// one-off entry id (empty when it has none). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListMember<'a> { + pub entry_id: &'a [u8], + pub one_off: &'a [u8], +} + +/// The members a stream holds, as far as they could be read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListStream<'a> { + /// The members read, at most the cap given. + pub members: Vec>, + /// Members read past the cap, counted and not kept. + pub more: usize, + /// Members the stream says it has and that couldn't be read: it ends, or a length runs past + /// it, before them. + pub damaged: usize, + /// Whether the totals of the header disagree with the members read, or the members carry + /// extra information, which they are not to. + pub totals_differ: bool, +} + +const SHORT: Error = Error::Corrupt("a distribution list's stream is shorter than its header"); +const VERSION: Error = Error::Corrupt("a distribution list's stream of a version this reader doesn't know"); +const COUNT: Error = Error::Corrupt("a distribution list's stream counts more members than it has room for"); + +/// The members of a distribution list's stream. Fails when the header can't be believed (too +/// short, another version, more members than the bytes could hold); a member that runs past the +/// end stops the reading there, and the members before it are kept. +pub fn list_stream(b: &[u8], max_members: usize) -> Result> { + if b.len() < HEADER { + return Err(SHORT); + } + if u16_at(b, 0)? != 1 { + return Err(VERSION); + } + let count = usize::try_from(u32_at(b, 12)?).map_err(|_| COUNT)?; + let total_ids = u32_at(b, 16)?; + let total_one_offs = u32_at(b, 20)?; + let total_extra = u32_at(b, 24)?; + let room = b.len().saturating_sub(HEADER) / LEAST_MEMBER; + if count > room { + return Err(COUNT); + } + let mut out = ListStream { members: Vec::new(), more: 0, damaged: 0, totals_differ: false }; + let (mut ids, mut one_offs, mut extra) = (0u64, 0u64, 0u64); + let mut at = HEADER; + for read in 0..count { + let Some((member, extra_len, next)) = member(b, at) else { + out.damaged = count.saturating_sub(read); + break; + }; + at = next; + ids = ids.saturating_add(member.entry_id.len() as u64); + one_offs = one_offs.saturating_add(member.one_off.len() as u64); + extra = extra.saturating_add(extra_len as u64); + if out.members.len() < max_members { + out.members.push(member); + } else { + out.more = out.more.saturating_add(1); + } + } + out.totals_differ = out.damaged == 0 && (ids != u64::from(total_ids) || one_offs != u64::from(total_one_offs) || extra != 0 || total_extra != 0); + Ok(out) +} + +/// The member at `at`, how long its extra information is, and where the next begins, when it +/// fits in `b`. ExtraMemberInfoSize is to be 0; what it names is stepped over when it fits. +fn member(b: &[u8], at: usize) -> Option<(ListMember<'_>, usize, usize)> { + let (entry_id, at) = sized(b, at)?; + let (one_off, at) = sized(b, at)?; + let (extra, at) = sized(b, at)?; + Some((ListMember { entry_id, one_off }, extra.len(), at)) +} + +/// The bytes a 4-byte size at `at` names, at most [`MAX_LIST_ENTRY`], and where they end. +fn sized(b: &[u8], at: usize) -> Option<(&[u8], usize)> { + let len = usize::try_from(u32_at(b, at).ok()?).ok()?; + if len > MAX_LIST_ENTRY { + return None; + } + let start = at.checked_add(4)?; + let bytes = slice(b, start, len).ok()?; + Some((bytes, start.checked_add(len)?)) +} diff --git a/crates/pst/src/ltp.rs b/crates/pst/src/ltp.rs new file mode 100644 index 000000000..8ff1e8e4a --- /dev/null +++ b/crates/pst/src/ltp.rs @@ -0,0 +1,323 @@ +//! Lists, tables and properties ([MS-PST] 2.3): a node's heap with its subnodes, the property +//! context (2.3.3) and the table context (2.3.4), whose row matrix may be spread over blocks and +//! is read a block at a time. + +use std::io::{Read, Seek}; + +use crate::bth; +use crate::bytes::{slice, u8_at, u16_at, u32_at}; +use crate::error::{Cap, Error, Result}; +use crate::header::Format; +use crate::heap::{CLIENT_PC, CLIENT_TC, Heap, is_hid}; +use crate::ndb::{Ndb, Subnodes, is_internal}; +use crate::prop::{self, PropValue, Properties, Property, fixed_size}; + +/// A node's data as a heap, and its subnodes (where values too large for the heap are kept). +pub(crate) struct NodeData { + pub heap: Heap, + pub subnodes: Subnodes, +} + +/// How much more one call may read and keep: every node it loads, every table block and +/// every value, charged as read (so a value named twice costs twice). Past it, `cap`. +pub(crate) struct Budget { + pub left: u64, + /// A value larger than this is not read but left as [`PropValue::Large`]. + pub inline: u64, + pub cap: Cap, +} + +impl Budget { + pub(crate) fn new(left: u64, inline: u64, cap: Cap) -> Budget { + Budget { left, inline, cap } + } + + pub(crate) fn take(&mut self, n: u64) -> Result<()> { + self.left = self.left.checked_sub(n).ok_or(Error::Cap(self.cap))?; + Ok(()) + } + + /// Charges what is kept of a value or a structure: refused past what is left. + pub(crate) fn keep(&mut self, n: usize) -> Result<()> { + self.take(n as u64) + } + + /// Charges what is kept of a note of something left out: never refused (each is one per + /// part already charged, so they are bounded), but what is left goes down. + pub(crate) fn note(&mut self, n: usize) { + self.left = self.left.saturating_sub(n as u64); + } +} + +impl NodeData { + /// A node's heap and subnodes, charged to `budget`. + pub(crate) fn load(ndb: &mut Ndb, data: u64, sub: u64, budget: &mut Budget) -> Result { + let max = ndb.limits.max_item_bytes.min(budget.left); + let pieces = ndb.read_pieces(data, max, budget.cap)?; + budget.take(pieces.iter().map(|p| p.len() as u64).sum())?; + let heap = Heap::new(pieces)?; + let subnodes = ndb.subnodes(sub)?; + Ok(NodeData { heap, subnodes }) + } + + /// The size of the value an HNID names, without reading it. + pub(crate) fn hnid_len(&self, ndb: &mut Ndb, hnid: u32) -> Result { + if is_hid(hnid) { + return Ok(self.heap.item(hnid)?.len() as u64); + } + let entry = self.subnodes.get(&hnid).ok_or(Error::NotFound("a property's subnode"))?; + ndb.data_size(entry.data) + } + + /// The subnode an HNID names, when it names one. + pub(crate) fn subnode(&self, hnid: u32) -> Option { + if is_hid(hnid) { None } else { self.subnodes.get(&hnid).copied() } + } + + /// The value an HNID names, refused past `max` bytes. + pub(crate) fn hnid_bytes(&self, ndb: &mut Ndb, hnid: u32, max: u64, cap: Cap) -> Result> { + if is_hid(hnid) { + let item = self.heap.item(hnid)?; + if item.len() as u64 > max { + return Err(Error::Cap(cap)); + } + return Ok(item.to_vec()); + } + let entry = self.subnodes.get(&hnid).ok_or(Error::NotFound("a property's subnode"))?; + ndb.read_data(entry.data, max, cap) + } + + /// A value stored apart: read when small enough, else [`PropValue::Large`]. + fn apart(&self, ndb: &mut Ndb, ty: u16, hnid: u32, budget: &mut Budget) -> Result { + let inline = budget.inline; + let size = self.hnid_len(ndb, hnid)?; + if size > inline { + return Ok(PropValue::Large { size }); + } + budget.take(size)?; + let b = self.hnid_bytes(ndb, hnid, inline, budget.cap)?; + // What making the value holds must fit in what is left, before it is made. + if prop::building_cost(ty, &b) as u64 > budget.left { + return Err(Error::Cap(budget.cap)); + } + prop::stored(ty, &b) + } +} + +/// A value that failed to read becomes [`PropValue::Damaged`] with its reason, so that one +/// damaged property doesn't cost the rest; a cap, or the reader under the file failing, is +/// passed on. +fn salvage(ty: u16, r: Result) -> Result { + match r { + Ok(v) => Ok(v), + Err(e @ (Error::Cap(_) | Error::Io(_))) => Err(e), + Err(reason) => Ok(PropValue::Damaged { ty, reason }), + } +} + +/// A property context's raw records: id, type and the 4-byte value or HNID. +pub(crate) fn pc_records(node: &NodeData) -> Result> { + if node.heap.client != CLIENT_PC { + return Err(Error::Corrupt("a property context's signature")); + } + // Property ids are 16 bits: more records than that are repeats. + let tree = bth::read(&node.heap, node.heap.user_root, 1 << 16, Error::Corrupt("a property context with more records than ids"))?; + if tree.key_size != 2 || tree.data_size != 6 { + return Err(Error::Corrupt("a property context's record sizes")); + } + tree.records.iter().map(|r| Ok((u16_at(r, 0)?, u16_at(r, 2)?, u32_at(r, 4)?))).collect() +} + +/// One property context value from its record. +pub(crate) fn pc_value(ndb: &mut Ndb, node: &NodeData, ty: u16, raw: u32, budget: &mut Budget) -> Result { + let r = match fixed_size(ty) { + Some(n) if n <= 4 => prop::fixed(ty, &raw.to_le_bytes()), + Some(_) => node.heap.item(raw).and_then(|b| prop::fixed(ty, b)), + None if is_known_variable(ty) => node.apart(ndb, ty, raw, budget), + None => Ok(PropValue::Unknown { ty, data: raw.to_le_bytes().to_vec() }), + }; + salvage(ty, r) +} + +fn is_known_variable(ty: u16) -> bool { + use crate::prop::types::{BINARY, MULTIPLE, OBJECT, RESTRICTION, RULE_ACTION, SERVER_ID, STRING, STRING8}; + ty & MULTIPLE != 0 || matches!(ty, STRING | STRING8 | BINARY | OBJECT | SERVER_ID | RESTRICTION | RULE_ACTION) +} + +/// All of a property context's properties; those in `defer` that are stored apart are left as +/// [`PropValue::Large`] (an attachment's data, read on its own). +pub(crate) fn pc_properties(ndb: &mut Ndb, node: &NodeData, budget: &mut Budget, defer: &[u16]) -> Result { + let mut out = Vec::new(); + for (id, ty, raw) in pc_records(node)? { + let value = if defer.contains(&id) && fixed_size(ty).is_none() && is_known_variable(ty) && ty != crate::prop::types::OBJECT { + salvage(ty, node.hnid_len(ndb, raw).map(|size| PropValue::Large { size }))? + } else { + pc_value(ndb, node, ty, raw, budget)? + }; + let p = Property { id, ty, value }; + budget.keep(p.kept() as usize)?; + out.push(p); + } + out.sort_by_key(|p| p.id); + Ok(Properties(out)) +} + +#[derive(Debug, Clone, Copy)] +struct Column { + id: u16, + ty: u16, + offset: usize, + size: usize, + bit: usize, +} + +enum Matrix { + None, + Heap(Vec), + Blocks { ids: Vec, per_block: usize, cached: Option<(usize, Vec)> }, +} + +/// A table context: its columns, its rows (`(row id, row index)` from the row index), and the +/// row matrix, read a block at a time. +pub(crate) struct Table { + node: NodeData, + cols: Vec, + row_size: usize, + ceb_start: usize, + pub rows: Vec<(u32, u32)>, + matrix: Matrix, +} + +impl Table { + /// A table of at most `max_rows` rows; more are refused (`cap`) as the row index is read. + pub(crate) fn open(ndb: &mut Ndb, node: NodeData, max_rows: usize, cap: Cap) -> Result { + let format = ndb.header.format; + if node.heap.client != CLIENT_TC { + return Err(Error::Corrupt("a table context's signature")); + } + let info = node.heap.item(node.heap.user_root)?; + if u8_at(info, 0)? != CLIENT_TC { + return Err(Error::Corrupt("a table context's header")); + } + let ncols = usize::from(u8_at(info, 1)?); + let rgib = [u16_at(info, 2)?, u16_at(info, 4)?, u16_at(info, 6)?, u16_at(info, 8)?].map(usize::from); + let [b4, b2, b1, bm] = rgib; + if !(b4 <= b2 && b2 <= b1 && b1 <= bm) || bm < 4 { + return Err(Error::Corrupt("a table context's row layout")); + } + let row_index_hid = u32_at(info, 10)?; + let rows_hnid = u32_at(info, 14)?; + let mut cols = Vec::with_capacity(ncols); + for i in 0..ncols { + let c = slice(info, i.saturating_mul(8).saturating_add(22), 8)?; + let tag = u32_at(c, 0)?; + let col = Column { + id: (tag >> 16) as u16, + ty: (tag & 0xFFFF) as u16, + offset: usize::from(u16_at(c, 4)?), + size: usize::from(u8_at(c, 6)?), + bit: usize::from(u8_at(c, 7)?), + }; + if col.offset.saturating_add(col.size) > b1 { + return Err(Error::Corrupt("a table column outside its row")); + } + cols.push(col); + } + let trailer = format.trailer_size(); + let block_capacity = 8192usize.saturating_sub(trailer); + if bm > block_capacity { + return Err(Error::Corrupt("a table row larger than a block")); + } + let tree = bth::read(&node.heap, row_index_hid, max_rows, Error::Cap(cap))?; + let index_size = match format { + Format::Unicode => 4, + Format::Ansi => 2, + }; + if tree.key_size != 4 || tree.data_size != index_size { + return Err(Error::Corrupt("a table's row index")); + } + let mut rows = Vec::with_capacity(tree.records.len()); + for r in &tree.records { + let index = match format { + Format::Unicode => u32_at(r, 4)?, + Format::Ansi => u32::from(u16_at(r, 4)?), + }; + rows.push((u32_at(r, 0)?, index)); + } + let matrix = if rows_hnid == 0 { + Matrix::None + } else if is_hid(rows_hnid) { + Matrix::Heap(node.heap.item(rows_hnid)?.to_vec()) + } else { + let entry = node.subnodes.get(&rows_hnid).ok_or(Error::NotFound("a table's rows"))?; + let max = ndb.limits.max_item_bytes; + let (ids, _) = ndb.data_blocks(entry.data, max, Cap::ItemSize)?; + Matrix::Blocks { ids, per_block: block_capacity.checked_div(bm).unwrap_or(1), cached: None } + }; + if matches!(matrix, Matrix::None) && !rows.is_empty() { + return Err(Error::NotFound("a table's rows")); + } + Ok(Table { node, cols, row_size: bm, ceb_start: b1, rows, matrix }) + } + + /// The bytes of the row at `index` in the row matrix. + fn row_bytes(&mut self, ndb: &mut Ndb, index: u32, budget: &mut Budget) -> Result> { + let index = index as usize; + let size = self.row_size; + match &mut self.matrix { + Matrix::None => Err(Error::NotFound("a table's rows")), + Matrix::Heap(b) => Ok(slice(b, index.checked_mul(size).ok_or(Error::OutOfBounds)?, size)?.to_vec()), + Matrix::Blocks { ids, per_block, cached } => { + let per = (*per_block).max(1); + let block = index.checked_div(per).unwrap_or(0); + let within = index.checked_rem(per).unwrap_or(0); + if cached.as_ref().is_none_or(|(i, _)| *i != block) { + let bid = *ids.get(block).ok_or(Error::Corrupt("a table row past its blocks"))?; + if is_internal(bid) { + return Err(Error::Corrupt("a table row in an internal block")); + } + let b = ndb.block(bid)?; + budget.take(b.len() as u64)?; + *cached = Some((block, b)); + } + let data = cached.as_ref().map(|(_, d)| d.as_slice()).unwrap_or(&[]); + Ok(slice(data, within.saturating_mul(size), size)?.to_vec()) + } + } + } + + /// The row with row id `id` at `index`: its cells, those that exist. Values larger than the + /// inline cap come back as [`PropValue::Large`]. + pub(crate) fn row(&mut self, ndb: &mut Ndb, id: u32, index: u32, budget: &mut Budget) -> Result { + let row = self.row_bytes(ndb, index, budget)?; + if u32_at(&row, 0)? != id { + return Err(Error::Corrupt("a table row under another row's id")); + } + let mut out = Vec::new(); + for col in self.cols.clone() { + let byte = row.get(self.ceb_start.saturating_add(col.bit / 8)).copied().unwrap_or(0); + if byte & (0x80u8 >> (col.bit % 8)) == 0 { + continue; + } + let cell = slice(&row, col.offset, col.size)?; + let value = match fixed_size(col.ty) { + Some(n) if n <= 8 => { + if col.size < n { + PropValue::Unknown { ty: col.ty, data: cell.to_vec() } + } else { + salvage(col.ty, prop::fixed(col.ty, cell))? + } + } + Some(_) => salvage(col.ty, u32_at(cell, 0).and_then(|h| self.node.heap.item(h).and_then(|b| prop::fixed(col.ty, b))))?, + // A cell too short for its reference loses that property alone. + None if is_known_variable(col.ty) => salvage(col.ty, u32_at(cell, 0).and_then(|hnid| self.node.apart(ndb, col.ty, hnid, budget)))?, + None => PropValue::Unknown { ty: col.ty, data: cell.to_vec() }, + }; + let p = Property { id: col.id, ty: col.ty, value }; + budget.keep(p.kept() as usize)?; + out.push(p); + } + out.sort_by_key(|p| p.id); + Ok(Properties(out)) + } +} diff --git a/crates/pst/src/named.rs b/crates/pst/src/named.rs new file mode 100644 index 000000000..92965e5ec --- /dev/null +++ b/crates/pst/src/named.rs @@ -0,0 +1,142 @@ +//! The named property map ([MS-PST] 2.4.7): which name (a property set and a number or a string) +//! each property id from 0x8000 stands for in this file. Read from its entry, GUID and string +//! streams; the hash table is only for writers and is not read. + +use std::collections::BTreeMap; +use std::io::{Read, Seek}; + +use crate::bytes::{array16, slice, u16_at, u32_at, utf16}; +use crate::error::{Cap, Error, Result}; +use crate::ltp::{Budget, NodeData, pc_records}; +use crate::ndb::Ndb; + +/// PS_MAPI ([MS-OXPROPS] 1.3.2), as stored. +pub const PS_MAPI: [u8; 16] = [0x28, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46]; +/// PS_PUBLIC_STRINGS ([MS-OXPROPS] 1.3.2), as stored. +pub const PS_PUBLIC_STRINGS: [u8; 16] = [0x29, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46]; + +/// The ids 0x8000 to 0xFFFF. +const MAX_ENTRIES: usize = 0x8000; +const MAX_STRINGS: u64 = 16 * 1024 * 1024; +/// One name, as stored (UTF-16): longer ones are left out. +pub(crate) const MAX_NAME: usize = 1024; +/// All names together, as stored: each entry may name the same string, so this is counted per +/// entry and not per string. +pub(crate) const MAX_NAMES: usize = 4 * 1024 * 1024; + +/// A named property's name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PropName { + Id(u32), + Name(String), +} + +/// What one property id from 0x8000 stands for. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NamedProperty { + pub id: u16, + /// The property set, as stored (the first three fields little-endian). + pub guid: [u8; 16], + pub name: PropName, +} + +/// The file's named properties, by id. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct NamedMap { + map: BTreeMap, + left_out: Vec<(Option, Error)>, +} + +impl NamedMap { + pub fn get(&self, id: u16) -> Option<&NamedProperty> { + self.map.get(&id) + } + + /// The id that stands for a name in a property set, if the file has one. + pub fn find(&self, guid: &[u8; 16], name: &PropName) -> Option { + self.map.values().find(|p| &p.guid == guid && &p.name == name).map(|p| p.id) + } + + pub fn iter(&self) -> impl Iterator { + self.map.values() + } + + pub fn len(&self) -> usize { + self.map.len() + } + + pub fn is_empty(&self) -> bool { + self.map.is_empty() + } + + /// The ids whose names were left out (damaged, or past a cap), and why; `None` for an + /// entry whose index names no id (0x8000 or more). + pub fn left_out(&self) -> &[(Option, Error)] { + &self.left_out + } +} + +pub(crate) fn read(ndb: &mut Ndb, node: &NodeData) -> Result { + let mut guids = Vec::new(); + let mut entries = Vec::new(); + let mut strings = Vec::new(); + let mut budget = Budget::new(ndb.limits.max_item_bytes, MAX_STRINGS, Cap::NamedProperties); + for (id, ty, raw) in pc_records(node)? { + if ty != crate::prop::types::BINARY { + continue; + } + let max = match id { + 0x0002 => (MAX_ENTRIES as u64).saturating_mul(16), + 0x0003 => (MAX_ENTRIES as u64).saturating_mul(8), + 0x0004 => MAX_STRINGS, + _ => continue, + }; + let b = node.hnid_bytes(ndb, raw, max, Cap::NamedProperties)?; + budget.take(b.len() as u64)?; + match id { + 0x0002 => guids = b, + 0x0003 => entries = b, + _ => strings = b, + } + } + let mut map = BTreeMap::new(); + let mut left_out = Vec::new(); + let mut names = 0usize; + for e in entries.as_chunks::<8>().0 { + let value = u32_at(e, 0)?; + let flags = u16_at(e, 4)?; + let index = u16_at(e, 6)?; + let Some(id) = 0x8000u16.checked_add(index) else { + left_out.push((None, Error::Corrupt("a named property's index"))); + continue; + }; + let is_string = flags & 1 != 0; + let name = || -> Result<(NamedProperty, usize)> { + let guid = match flags >> 1 { + 0 => [0u8; 16], + 1 => PS_MAPI, + 2 => PS_PUBLIC_STRINGS, + n => array16(&guids, usize::from(n.saturating_sub(3)).saturating_mul(16))?, + }; + let name = if is_string { + let at = value as usize; + let len = u32_at(&strings, at)? as usize; + if len > MAX_NAME || names.saturating_add(len) > MAX_NAMES { + return Err(Error::Cap(Cap::NamedProperties)); + } + (PropName::Name(utf16(slice(&strings, at.saturating_add(4), len)?)), len) + } else { + (PropName::Id(value), 0) + }; + Ok((NamedProperty { id, guid, name: name.0 }, name.1)) + }; + match name() { + Ok((p, stored)) => { + names = names.saturating_add(stored); + map.entry(id).or_insert(p); + } + Err(reason) => left_out.push((Some(id), reason)), + } + } + Ok(NamedMap { map, left_out }) +} diff --git a/crates/pst/src/ndb.rs b/crates/pst/src/ndb.rs new file mode 100644 index 000000000..2727e5f5f --- /dev/null +++ b/crates/pst/src/ndb.rs @@ -0,0 +1,546 @@ +//! The node database ([MS-PST] 2.2): the node and block B-trees (pages of 512 bytes), blocks +//! (at most 8 KB, each with its CRC, signature and id checked), data trees (XBLOCK, XXBLOCK) +//! and subnode trees (SLBLOCK, SIBLOCK). +//! +//! A hostile file is kept bounded: a B-tree is at most 9 pages deep and each page must be one +//! level below the page that points to it, so a lookup ends; a walk of the whole node tree +//! visits a page at most once and at most [`Limits::max_btree_pages`] pages; a data tree is at +//! most two levels, its claimed size is checked against the cap before a block is read, and it +//! may not be spread over more blocks than its size needs. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::io::{Read, Seek}; + +use crate::bytes::{slice, u8_at, u16_at, u32_at}; +use crate::crc::crc32; +use crate::crypt::{Encryption, decode}; +use crate::error::{Cap, Damage, Error, Limits, MAX_DAMAGE, Result}; +use crate::header::{Bref, Format, Header}; +use crate::source::Source; + +const PAGE: usize = 512; +const PTYPE_BBT: u8 = 0x80; +const PTYPE_NBT: u8 = 0x81; +/// "Up to 8 intermediate levels" (2.2.2.7.7). +const MAX_LEVEL: u8 = 8; +/// A data tree's average block holds at least this much, or it is refused ([`Cap::DataBlocks`]). +const MIN_AVERAGE_BLOCK: u64 = 64; +/// B-tree pages kept once read (512 bytes each), so that the upper pages, which every lookup +/// passes through, are read from the file once. +const CACHED_PAGES: usize = 256; +/// Damaged pages told apart (so that one met again is not counted again); past them the +/// count of [`Damage::pages`] stops, and is a lower bound. +const MAX_NOTED: usize = 1 << 16; + +/// A leaf of the node B-tree (`NBTENTRY`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct NodeEntry { + pub nid: u32, + pub data: u64, + pub sub: u64, + pub parent: u32, +} + +/// A subnode (`SLENTRY`): its data and its own subnodes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SubEntry { + pub data: u64, + pub sub: u64, +} + +pub(crate) type Subnodes = BTreeMap; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct BlockEntry { + pub ib: u64, + pub cb: u16, +} + +struct Page { + data: Vec, + count: usize, + entry_size: usize, + level: u8, +} + +impl Page { + fn entry(&self, i: usize) -> Result<&[u8]> { + slice(&self.data, i.checked_mul(self.entry_size).ok_or(Error::OutOfBounds)?, self.entry_size) + } +} + +/// The block signature (5.5). +pub(crate) fn signature(ib: u64, bid: u64) -> u16 { + let x = ib ^ bid; + ((x >> 16) as u16) ^ (x as u16) +} + +/// Whether a block id names an internal block (an XBLOCK, XXBLOCK, SLBLOCK or SIBLOCK). +pub(crate) fn is_internal(bid: u64) -> bool { + bid & 2 != 0 +} + +/// Whether an error is damage to the file (noted and skipped) rather than a cap reached or +/// the reader under the file failing (which end what is being done). +fn is_damage(e: &Error) -> bool { + !matches!(e, Error::Cap(_) | Error::Io(_)) +} + +/// The id a block is looked up by: the reserved low bit is ignored (2.2.2.2). +fn key(bid: u64) -> u64 { + bid & !1 +} + +pub(crate) struct Ndb { + src: Source, + pub header: Header, + pub limits: Limits, + /// Pages as read, by offset; checked again each time they are used. + pages: HashMap>, + /// Pages below a tree's root that could not be read, each once. + pub damage: Vec, + noted: HashSet, +} + +impl Ndb { + pub(crate) fn new(src: Source, header: Header, limits: Limits) -> Self { + Ndb { src, header, limits, pages: HashMap::new(), damage: Vec::new(), noted: HashSet::new() } + } + + /// Notes a page below a root that could not be read. A cap or a failure of the reader + /// under the file is not damage to the file, and is not noted. + pub(crate) fn note(&mut self, ib: u64, ptype: u8, e: &Error) { + if !is_damage(e) { + return; + } + if self.damage.len() >= MAX_DAMAGE { + // Past the cap the pages are counted (once each, to the same cap as the walk's). + if self.noted.len() < MAX_NOTED + && self.noted.insert(ib) + && let Some(last) = self.damage.last_mut() + { + last.pages = last.pages.saturating_add(1); + } + return; + } + if !self.noted.insert(ib) { + return; + } + let what = if ptype == PTYPE_NBT { "a page of the node tree" } else { "a page of the block tree" }; + let pages = 1; + self.damage.push(Damage { offset: ib, what, reason: e.clone(), pages }); + if self.damage.len() == MAX_DAMAGE { + // The last entry is the one that counts the rest. + if let Some(last) = self.damage.last_mut() { + last.what = if ptype == PTYPE_NBT { "pages of the node tree, from this one" } else { "pages of the block tree, from this one" }; + } + } + } + + /// The page at `bref`, at the level `expect` when one is expected. + fn page_at(&mut self, bref: Bref, ptype: u8, expect: Option) -> Result { + let page = self.read_page(bref, ptype)?; + if expect.is_some_and(|e| e != page.level) { + return Err(Error::Corrupt("a B-tree page at the wrong level")); + } + Ok(page) + } + + fn page_bytes(&mut self, ib: u64) -> Result> { + if let Some(p) = self.pages.get(&ib) { + return Ok(p.clone()); + } + let data = self.src.read_at(ib, PAGE)?; + // ponytail: emptied when full, not least-recently-used; the root and upper pages come + // back at once. An LRU if lookups in very large files show it matters. + if self.pages.len() >= CACHED_PAGES { + self.pages.clear(); + } + self.pages.insert(ib, data.clone()); + Ok(data) + } + + fn format(&self) -> Format { + self.header.format + } + + fn read_page(&mut self, bref: Bref, ptype: u8) -> Result { + let f = self.format(); + let data = self.page_bytes(bref.ib)?; + let t = PAGE.saturating_sub(f.trailer_size()); + let (crc_at, bid_at) = match f { + Format::Unicode => (t.saturating_add(4), t.saturating_add(8)), + Format::Ansi => (t.saturating_add(8), t.saturating_add(4)), + }; + if u8_at(&data, t)? != ptype || u8_at(&data, t.saturating_add(1))? != ptype { + return Err(Error::Corrupt("a B-tree page of the wrong type")); + } + if crc32(slice(&data, 0, t)?) != u32_at(&data, crc_at)? { + return Err(Error::BadChecksum("a B-tree page")); + } + let bid = f.read_id(&data, bid_at)?; + if key(bid) != key(bref.bid) || u16_at(&data, t.saturating_add(2))? != signature(bref.ib, bid) { + return Err(Error::BadChecksum("a B-tree page's id")); + } + let (area, meta) = match f { + Format::Unicode => (488, 488), + Format::Ansi => (496, 496), + }; + let count = usize::from(u8_at(&data, meta)?); + let entry_size = usize::from(u8_at(&data, meta.saturating_add(2))?); + let level = u8_at(&data, meta.saturating_add(3))?; + // A level the format doesn't have is damage to that page, like a bad count. + if level > MAX_LEVEL { + return Err(Error::Corrupt("a B-tree page's level")); + } + let id = f.id_size(); + let needed = match (level, ptype) { + (1.., _) => id.saturating_mul(3), + (0, PTYPE_NBT) => id.saturating_mul(3).saturating_add(4), + _ => id.saturating_mul(2).saturating_add(4), + }; + if entry_size < needed || count.saturating_mul(entry_size) > area { + return Err(Error::Corrupt("a B-tree page's entries")); + } + Ok(Page { data, count, entry_size, level }) + } + + /// The leaf entry of `target` in the tree at `root`, if there is one. + fn find(&mut self, root: Bref, ptype: u8, target: u64) -> Result>> { + let f = self.format(); + let id = f.id_size(); + let mask = |k: u64| if ptype == PTYPE_BBT { key(k) } else { k }; + let target = mask(target); + let mut bref = root; + let mut expect: Option = None; + loop { + // A page below the root that can't be read loses what is under it, not the file: + // it is noted, and this lookup fails. + let page = match self.page_at(bref, ptype, expect) { + Ok(p) => p, + Err(e) => { + if bref != root { + self.note(bref.ib, ptype, &e); + } + return Err(e); + } + }; + if page.level == 0 { + for i in 0..page.count { + let e = page.entry(i)?; + if mask(f.read_id(e, 0)?) == target { + return Ok(Some(e.to_vec())); + } + } + return Ok(None); + } + let mut best: Option<(u64, Bref)> = None; + for i in 0..page.count { + let e = page.entry(i)?; + let k = mask(f.read_id(e, 0)?); + if k <= target && best.is_none_or(|(b, _)| k >= b) { + best = Some((k, f.read_bref(e, id)?)); + } + } + match best { + Some((_, next)) => bref = next, + None => return Ok(None), + } + expect = page.level.checked_sub(1); + } + } + + fn node_entry(&self, e: &[u8]) -> Result { + let f = self.format(); + let id = f.id_size(); + Ok(NodeEntry { + nid: f.read_id(e, 0)? as u32, + data: f.read_id(e, id)?, + sub: f.read_id(e, id.saturating_mul(2))?, + parent: u32_at(e, id.saturating_mul(3))?, + }) + } + + /// The node `nid` of the node B-tree. + pub(crate) fn node(&mut self, nid: u32) -> Result { + let root = self.header.nbt; + match self.find(root, PTYPE_NBT, u64::from(nid))? { + Some(e) => self.node_entry(&e), + None => Err(Error::NotFound("a node")), + } + } + + pub(crate) fn block_entry(&mut self, bid: u64) -> Result { + let root = self.header.bbt; + let e = self.find(root, PTYPE_BBT, bid)?.ok_or(Error::NotFound("a block"))?; + let f = self.format(); + let bref = f.read_bref(&e, 0)?; + let cb = u16_at(&e, f.id_size().saturating_mul(2))?; + Ok(BlockEntry { ib: bref.ib, cb }) + } + + /// The bytes of block `bid`, decoded when it holds data. + pub(crate) fn block(&mut self, bid: u64) -> Result> { + let f = self.format(); + let entry = self.block_entry(bid)?; + let cb = usize::from(entry.cb); + let ts = f.trailer_size(); + let total = cb.saturating_add(ts).div_ceil(64).saturating_mul(64); + let mut raw = self.src.read_at(entry.ib, total)?; + let t = total.saturating_sub(ts); + let (crc_at, bid_at) = match f { + Format::Unicode => (t.saturating_add(4), t.saturating_add(8)), + Format::Ansi => (t.saturating_add(8), t.saturating_add(4)), + }; + let stored_bid = f.read_id(&raw, bid_at)?; + if usize::from(u16_at(&raw, t)?) != cb || key(stored_bid) != key(bid) { + return Err(Error::BadChecksum("a block's id")); + } + if u16_at(&raw, t.saturating_add(2))? != signature(entry.ib, stored_bid) { + return Err(Error::BadChecksum("a block's signature")); + } + if crc32(slice(&raw, 0, cb)?) != u32_at(&raw, crc_at)? { + return Err(Error::BadChecksum("a block")); + } + raw.truncate(cb); + if !is_internal(bid) { + // The key of cyclic encoding is the block's id without the reserved bit A, which + // a file may set on a reference or on the trailer alone. + decode(self.header.encryption, &mut raw, key(stored_bid)); + } + Ok(raw) + } + + /// The size of the data stream at `bid`, from its top block alone. + pub(crate) fn data_size(&mut self, bid: u64) -> Result { + if bid == 0 { + return Ok(0); + } + if !is_internal(bid) { + return Ok(u64::from(self.block_entry(bid)?.cb)); + } + let top = self.block(bid)?; + if u8_at(&top, 0)? != 1 { + return Err(Error::Corrupt("a data tree's block type")); + } + Ok(u64::from(u32_at(&top, 4)?)) + } + + /// The data blocks of the data tree at `bid`, in order, refused past `max` bytes. + pub(crate) fn data_blocks(&mut self, bid: u64, max: u64, cap: Cap) -> Result<(Vec, u64)> { + if bid == 0 { + return Ok((Vec::new(), 0)); + } + if !is_internal(bid) { + let cb = u64::from(self.block_entry(bid)?.cb); + if cb > max { + return Err(Error::Cap(cap)); + } + return Ok((vec![bid], cb)); + } + let f = self.format(); + let id = f.id_size(); + let top = self.block(bid)?; + if u8_at(&top, 0)? != 1 { + return Err(Error::Corrupt("a data tree's block type")); + } + let level = u8_at(&top, 1)?; + let total = u64::from(u32_at(&top, 4)?); + if total > max { + return Err(Error::Cap(cap)); + } + let allowed = usize::try_from(total / MIN_AVERAGE_BLOCK).unwrap_or(usize::MAX).saturating_add(8); + let ids = |b: &[u8]| -> Result> { + let n = usize::from(u16_at(b, 2)?); + if n > allowed { + return Err(Error::Cap(Cap::DataBlocks)); + } + (0..n).map(|i| f.read_id(b, i.saturating_mul(id).saturating_add(8))).collect() + }; + let mut out = Vec::new(); + // A block named twice in one data tree would be read twice: refused. + let mut seen = HashSet::new(); + let mut once = |bid: u64| if seen.insert(key(bid)) { Ok(()) } else { Err(Error::Cycle("a data tree")) }; + match level { + 1 => { + for child in ids(&top)? { + once(child)?; + if is_internal(child) { + return Err(Error::Corrupt("an XBLOCK pointing at an internal block")); + } + out.push(child); + } + } + 2 => { + for child in ids(&top)? { + once(child)?; + if !is_internal(child) { + return Err(Error::Corrupt("an XXBLOCK pointing at a data block")); + } + let x = self.block(child)?; + if u8_at(&x, 0)? != 1 || u8_at(&x, 1)? != 1 { + return Err(Error::Corrupt("an XXBLOCK pointing at something else than an XBLOCK")); + } + for grandchild in ids(&x)? { + once(grandchild)?; + if is_internal(grandchild) { + return Err(Error::Corrupt("an XBLOCK pointing at an internal block")); + } + if out.len() >= allowed { + return Err(Error::Cap(Cap::DataBlocks)); + } + out.push(grandchild); + } + } + } + _ => return Err(Error::Corrupt("a data tree's level")), + } + Ok((out, total)) + } + + /// The whole data stream at `bid`, one piece per data block, refused past `max` bytes. + pub(crate) fn read_pieces(&mut self, bid: u64, max: u64, cap: Cap) -> Result>> { + let (blocks, total) = self.data_blocks(bid, max, cap)?; + let mut pieces = Vec::with_capacity(blocks.len()); + let mut sum = 0u64; + for b in blocks { + let piece = self.block(b)?; + sum = sum.saturating_add(piece.len() as u64); + if sum > total { + return Err(Error::Corrupt("a data tree longer than it says")); + } + pieces.push(piece); + } + if sum != total { + return Err(Error::Corrupt("a data tree shorter than it says")); + } + Ok(pieces) + } + + /// The whole data stream at `bid` as one buffer, refused past `max` bytes. + pub(crate) fn read_data(&mut self, bid: u64, max: u64, cap: Cap) -> Result> { + Ok(self.read_pieces(bid, max, cap)?.concat()) + } + + /// The subnodes of the subnode tree at `bid`. + pub(crate) fn subnodes(&mut self, bid: u64) -> Result { + let mut out = Subnodes::new(); + if bid == 0 { + return Ok(out); + } + if !is_internal(bid) { + return Err(Error::Corrupt("a subnode tree in a data block")); + } + let top = self.block(bid)?; + if u8_at(&top, 0)? != 2 { + return Err(Error::Corrupt("a subnode tree's block type")); + } + match u8_at(&top, 1)? { + 0 => self.leaf_entries(&top, &mut out)?, + 1 => { + let mut seen = HashSet::new(); + for child in self.intermediate_entries(&top)? { + if !seen.insert(key(child)) { + return Err(Error::Cycle("a subnode tree")); + } + if !is_internal(child) { + return Err(Error::Corrupt("a subnode tree in a data block")); + } + let leaf = self.block(child)?; + if u8_at(&leaf, 0)? != 2 || u8_at(&leaf, 1)? != 0 { + return Err(Error::Corrupt("an SIBLOCK pointing at something else than an SLBLOCK")); + } + self.leaf_entries(&leaf, &mut out)?; + } + } + _ => return Err(Error::Corrupt("a subnode tree's level")), + } + Ok(out) + } + + fn entries_start(&self) -> usize { + match self.format() { + Format::Unicode => 8, + Format::Ansi => 4, + } + } + + fn leaf_entries(&self, b: &[u8], out: &mut Subnodes) -> Result<()> { + let f = self.format(); + let id = f.id_size(); + let size = id.saturating_mul(3); + let n = usize::from(u16_at(b, 2)?); + for i in 0..n { + let at = i.saturating_mul(size).saturating_add(self.entries_start()); + let e = slice(b, at, size)?; + let nid = f.read_id(e, 0)? as u32; + let entry = SubEntry { data: f.read_id(e, id)?, sub: f.read_id(e, id.saturating_mul(2))? }; + if out.len() >= self.limits.max_subnodes { + return Err(Error::Cap(Cap::Subnodes)); + } + out.entry(nid).or_insert(entry); + } + Ok(()) + } + + fn intermediate_entries(&self, b: &[u8]) -> Result> { + let f = self.format(); + let id = f.id_size(); + let size = id.saturating_mul(2); + let n = usize::from(u16_at(b, 2)?); + (0..n).map(|i| f.read_id(b, i.saturating_mul(size).saturating_add(self.entries_start()).saturating_add(id))).collect() + } + + /// Every node of the node B-tree, in key order, until `f` returns false. + /// A page below the root that can't be read, or that is met twice, is noted and skipped + /// with what is under it; the root must be read. + pub(crate) fn for_each_node(&mut self, f: &mut dyn FnMut(NodeEntry) -> bool) -> Result<()> { + let root = self.header.nbt; + let mut visited = HashSet::new(); + let mut stack: Vec<(Bref, Option)> = vec![(root, None)]; + while let Some((bref, expect)) = stack.pop() { + if !visited.insert(bref.ib) { + self.note(bref.ib, PTYPE_NBT, &Error::Cycle("the node tree")); + continue; + } + if visited.len() > self.limits.max_btree_pages { + return Err(Error::Cap(Cap::BTreePages)); + } + let page = match self.page_at(bref, PTYPE_NBT, expect) { + Ok(p) => p, + // Skipped only when it is damage, and named; a cap or the reader under the + // file failing ends the walk. + Err(e) if bref != root && is_damage(&e) => { + self.note(bref.ib, PTYPE_NBT, &e); + continue; + } + Err(e) => return Err(e), + }; + if page.level == 0 { + for i in 0..page.count { + let e = self.node_entry(page.entry(i)?)?; + if !f(e) { + return Ok(()); + } + } + continue; + } + let id = self.format().id_size(); + let below = page.level.checked_sub(1); + // Children pushed last-first so that they come off the stack in key order. + for i in (0..page.count).rev() { + let child = self.format().read_bref(page.entry(i)?, id)?; + stack.push((child, below)); + } + } + Ok(()) + } + + pub(crate) fn encryption(&self) -> Encryption { + self.header.encryption + } + + pub(crate) fn file_size(&self) -> u64 { + self.src.size() + } +} diff --git a/crates/pst/src/prop.rs b/crates/pst/src/prop.rs new file mode 100644 index 000000000..3de6d88ec --- /dev/null +++ b/crates/pst/src/prop.rs @@ -0,0 +1,344 @@ +//! Property values: the types of [MS-OXCDATA] 2.11.1 as a .pst file stores them, single and +//! multi-valued ([MS-PST] 2.3.3.4). + +use crate::bytes::{slice, u16_at, u32_at, u64_at, utf16}; +use crate::error::{Error, Result}; + +/// Property types ([MS-OXCDATA] 2.11.1). +pub mod types { + pub const NULL: u16 = 0x0001; + pub const INT16: u16 = 0x0002; + pub const INT32: u16 = 0x0003; + pub const FLOAT32: u16 = 0x0004; + pub const FLOAT64: u16 = 0x0005; + pub const CURRENCY: u16 = 0x0006; + pub const APP_TIME: u16 = 0x0007; + pub const ERROR: u16 = 0x000A; + pub const BOOLEAN: u16 = 0x000B; + pub const OBJECT: u16 = 0x000D; + pub const INT64: u16 = 0x0014; + pub const STRING8: u16 = 0x001E; + pub const STRING: u16 = 0x001F; + pub const TIME: u16 = 0x0040; + pub const GUID: u16 = 0x0048; + pub const SERVER_ID: u16 = 0x00FB; + pub const RESTRICTION: u16 = 0x00FD; + pub const RULE_ACTION: u16 = 0x00FE; + pub const BINARY: u16 = 0x0102; + /// Set on a type for its multi-valued form. + pub const MULTIPLE: u16 = 0x1000; +} + +use types::{ + APP_TIME, BINARY, BOOLEAN, CURRENCY, ERROR, FLOAT32, FLOAT64, GUID, INT16, INT32, INT64, MULTIPLE, NULL, OBJECT, RESTRICTION, RULE_ACTION, + SERVER_ID, STRING, STRING8, TIME, +}; + +/// A property's value. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum PropValue { + Null, + Int16(i16), + Int32(i32), + Float32(f32), + Float64(f64), + /// Currency, in ten-thousandths. + Currency(i64), + /// An OLE automation date. + AppTime(f64), + Error(u32), + Bool(bool), + Int64(i64), + /// A FILETIME: 100-nanosecond intervals since 1601-01-01 UTC. + Time(u64), + Guid([u8; 16]), + String(String), + /// Text in the item's code page, as stored (ANSI files, and some properties of Unicode ones). + String8(Vec), + Binary(Vec), + /// An object in a subnode (an attached message's data): its subnode id and size. + Object { + nid: u32, + size: u32, + }, + MultiInt16(Vec), + MultiInt32(Vec), + MultiFloat32(Vec), + MultiFloat64(Vec), + MultiCurrency(Vec), + MultiAppTime(Vec), + MultiInt64(Vec), + MultiTime(Vec), + MultiGuid(Vec<[u8; 16]>), + MultiString(Vec), + MultiString8(Vec>), + MultiBinary(Vec>), + /// Larger than [`crate::Limits::max_inline_value`]: not read with the rest; read it with + /// [`crate::Pst::read_property`]. + Large { + size: u64, + }, + /// A type this reader doesn't know, as stored. + Unknown { + ty: u16, + data: Vec, + }, + /// A value that could not be read, and why: kept in its place so that it is accounted for, + /// and named in the `left_out` of whatever holds it. + Damaged { + ty: u16, + reason: crate::Error, + }, +} + +/// One property of an object. +#[derive(Debug, Clone, PartialEq)] +pub struct Property { + pub id: u16, + pub ty: u16, + pub value: PropValue, +} + +impl PropValue { + /// The bytes it holds outside itself (its text, bytes or values). + pub(crate) fn held(&self) -> usize { + use std::mem::size_of_val; + let vecs = |v: &[Vec]| v.iter().map(|b| size_of_val(b).saturating_add(b.capacity())).sum::(); + match self { + PropValue::String(s) => s.capacity(), + PropValue::String8(b) | PropValue::Binary(b) | PropValue::Unknown { data: b, .. } => b.capacity(), + PropValue::MultiInt16(v) => size_of_val(v.as_slice()), + PropValue::MultiInt32(v) => size_of_val(v.as_slice()), + PropValue::MultiFloat32(v) => size_of_val(v.as_slice()), + PropValue::MultiFloat64(v) | PropValue::MultiAppTime(v) => size_of_val(v.as_slice()), + PropValue::MultiCurrency(v) | PropValue::MultiInt64(v) => size_of_val(v.as_slice()), + PropValue::MultiTime(v) => size_of_val(v.as_slice()), + PropValue::MultiGuid(v) => size_of_val(v.as_slice()), + PropValue::MultiString(v) => v.iter().map(|s| size_of_val(s).saturating_add(s.capacity())).sum(), + PropValue::MultiString8(v) | PropValue::MultiBinary(v) => vecs(v), + _ => 0, + } + } +} + +impl Property { + /// What keeping it costs: itself, and what it holds. + pub(crate) fn kept(&self) -> u64 { + std::mem::size_of::().saturating_add(self.value.held()) as u64 + } + + /// The 32-bit property tag (id in the high half, type in the low). + pub fn tag(&self) -> u32 { + (u32::from(self.id) << 16) | u32::from(self.ty) + } +} + +/// An object's properties, in the order of their ids. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Properties(pub Vec); + +impl Properties { + pub fn get(&self, id: u16) -> Option<&PropValue> { + self.0.iter().find(|p| p.id == id).map(|p| &p.value) + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// A text property: Unicode, or 8-bit text read as Latin-1. Latin-1 is right only for + /// part of the code pages an ANSI file may use; the import decodes [`PropValue::String8`] + /// with the item's own code page (`PidTagMessageCodepage`, `PidTagInternetCodepage`) + /// instead of calling this. + pub fn text(&self, id: u16) -> Option { + match self.get(id)? { + PropValue::String(s) => Some(s.clone()), + PropValue::String8(b) => Some(b.iter().map(|&c| char::from(c)).collect()), + _ => None, + } + } + + /// The properties that could not be read ([`PropValue::Damaged`]): id and reason. + pub fn damaged(&self) -> impl Iterator { + self.0.iter().filter_map(|p| match &p.value { + PropValue::Damaged { reason, .. } => Some((p.id, reason)), + _ => None, + }) + } + + pub fn int(&self, id: u16) -> Option { + match self.get(id)? { + PropValue::Int16(v) => Some(i64::from(*v)), + PropValue::Int32(v) => Some(i64::from(*v)), + PropValue::Int64(v) => Some(*v), + PropValue::Error(v) => Some(i64::from(*v)), + _ => None, + } + } + + pub fn bool(&self, id: u16) -> Option { + match self.get(id)? { + PropValue::Bool(v) => Some(*v), + _ => None, + } + } + + pub fn binary(&self, id: u16) -> Option<&[u8]> { + match self.get(id)? { + PropValue::Binary(v) => Some(v), + _ => None, + } + } + + pub fn time(&self, id: u16) -> Option { + match self.get(id)? { + PropValue::Time(v) => Some(*v), + _ => None, + } + } +} + +/// The size of a fixed-size type, `None` for a variable-size one. +pub(crate) fn fixed_size(ty: u16) -> Option { + match ty { + NULL => Some(0), + BOOLEAN => Some(1), + INT16 => Some(2), + INT32 | FLOAT32 | ERROR => Some(4), + FLOAT64 | CURRENCY | APP_TIME | INT64 | TIME => Some(8), + GUID => Some(16), + _ => None, + } +} + +/// A fixed-size value from exactly its bytes (or more: the first are read). +pub(crate) fn fixed(ty: u16, b: &[u8]) -> Result { + Ok(match ty { + NULL => PropValue::Null, + BOOLEAN => PropValue::Bool(b.first().copied().ok_or(Error::Corrupt("a short value"))? != 0), + INT16 => PropValue::Int16(u16_at(b, 0)? as i16), + INT32 => PropValue::Int32(u32_at(b, 0)? as i32), + FLOAT32 => PropValue::Float32(f32::from_bits(u32_at(b, 0)?)), + ERROR => PropValue::Error(u32_at(b, 0)?), + FLOAT64 => PropValue::Float64(f64::from_bits(u64_at(b, 0)?)), + CURRENCY => PropValue::Currency(u64_at(b, 0)? as i64), + APP_TIME => PropValue::AppTime(f64::from_bits(u64_at(b, 0)?)), + INT64 => PropValue::Int64(u64_at(b, 0)? as i64), + TIME => PropValue::Time(u64_at(b, 0)?), + GUID => PropValue::Guid(crate::bytes::array16(b, 0)?), + other => PropValue::Unknown { ty: other, data: b.to_vec() }, + }) +} + +/// What building a value from its `b` bytes holds beyond them, at most: for a list of +/// strings or binaries, each item's own vector and its place while the list is made (an +/// empty item is four bytes in the file and far more in memory). Other values hold no more +/// than their bytes. +pub(crate) fn building_cost(ty: u16, b: &[u8]) -> usize { + let variable = matches!(ty, t if t == STRING | MULTIPLE || t == STRING8 | MULTIPLE || t == BINARY | MULTIPLE); + if !variable { + return 0; + } + let count = u32_at(b, 0).map_or(0, |c| (c as usize).min(b.len() / 4)); + count.saturating_mul(std::mem::size_of::>().saturating_add(std::mem::size_of::<&[u8]>())) +} + +/// A value stored apart (in the heap or a subnode), from all its bytes. +pub(crate) fn stored(ty: u16, b: &[u8]) -> Result { + if ty & MULTIPLE != 0 { + return multi(ty & !MULTIPLE, b); + } + Ok(match ty { + STRING => PropValue::String(utf16(b)), + STRING8 => PropValue::String8(trim_nul(b).to_vec()), + BINARY | SERVER_ID | RESTRICTION | RULE_ACTION => PropValue::Binary(b.to_vec()), + OBJECT => PropValue::Object { nid: u32_at(b, 0)?, size: u32_at(b, 4)? }, + t if fixed_size(t).is_some() => fixed(t, b)?, + other => PropValue::Unknown { ty: other, data: b.to_vec() }, + }) +} + +fn trim_nul(b: &[u8]) -> &[u8] { + let mut end = b.len(); + while end > 0 && b.get(end.saturating_sub(1)) == Some(&0) { + end = end.saturating_sub(1); + } + b.get(..end).unwrap_or(b) +} + +fn multi(base: u16, b: &[u8]) -> Result { + if let Some(size) = fixed_size(base).filter(|&s| s > 1) { + if !b.len().is_multiple_of(size) { + return Err(Error::Corrupt("a multi-valued property's size")); + } + // Each value straight into its own type: no value is held twice on the way. + return Ok(match base { + INT16 => PropValue::MultiInt16(values(b, size, |c| Ok(u16_at(c, 0)? as i16))?), + INT32 => PropValue::MultiInt32(values(b, size, |c| Ok(u32_at(c, 0)? as i32))?), + FLOAT32 => PropValue::MultiFloat32(values(b, size, |c| Ok(f32::from_bits(u32_at(c, 0)?)))?), + FLOAT64 => PropValue::MultiFloat64(values(b, size, |c| Ok(f64::from_bits(u64_at(c, 0)?)))?), + CURRENCY => PropValue::MultiCurrency(values(b, size, |c| Ok(u64_at(c, 0)? as i64))?), + APP_TIME => PropValue::MultiAppTime(values(b, size, |c| Ok(f64::from_bits(u64_at(c, 0)?)))?), + INT64 => PropValue::MultiInt64(values(b, size, |c| Ok(u64_at(c, 0)? as i64))?), + TIME => PropValue::MultiTime(values(b, size, |c| u64_at(c, 0))?), + GUID => PropValue::MultiGuid(values(b, size, |c| crate::bytes::array16(c, 0))?), + _ => PropValue::Unknown { ty: base | MULTIPLE, data: b.to_vec() }, + }); + } + // Only the lists of strings and binaries have their items read; any other is kept as bytes. + if !matches!(base, STRING | STRING8 | BINARY) { + return Ok(PropValue::Unknown { ty: base | MULTIPLE, data: b.to_vec() }); + } + let items = variable_items(b)?; + Ok(match base { + STRING => PropValue::MultiString(items.into_iter().map(utf16).collect()), + STRING8 => PropValue::MultiString8(items.into_iter().map(|i| trim_nul(i).to_vec()).collect()), + _ => PropValue::MultiBinary(items.into_iter().map(<[u8]>::to_vec).collect()), + }) +} + +/// The items of a multi-valued property of a variable-size type (2.3.3.4.2). +/// Each `size` bytes of `b` straight into a value of its own type: none is held twice. +fn values(b: &[u8], size: usize, f: impl Fn(&[u8]) -> Result) -> Result> { + let mut out = Vec::with_capacity(b.len().checked_div(size).unwrap_or(0)); + for c in b.chunks_exact(size) { + out.push(f(c)?); + } + Ok(out) +} + +fn variable_items(b: &[u8]) -> Result> { + if b.is_empty() { + return Ok(Vec::new()); + } + let count = u32_at(b, 0)? as usize; + // Each item has its 4-byte offset, so the count can't pass a quarter of the size. + if count > b.len() / 4 { + return Err(Error::Corrupt("a multi-valued property's count")); + } + let table_end = count.saturating_mul(4).saturating_add(4); + let offset = |i: usize| -> Result { + if i >= count { + return Ok(b.len()); + } + Ok(u32_at(b, i.saturating_mul(4).saturating_add(4))? as usize) + }; + let mut items = Vec::with_capacity(count); + for i in 0..count { + let (start, end) = (offset(i)?, offset(i.saturating_add(1))?); + if start < table_end || end < start || end > b.len() { + return Err(Error::Corrupt("a multi-valued property's offsets")); + } + items.push(slice(b, start, end.saturating_sub(start))?); + } + Ok(items) +} diff --git a/crates/pst/src/pst.rs b/crates/pst/src/pst.rs new file mode 100644 index 000000000..89541786d --- /dev/null +++ b/crates/pst/src/pst.rs @@ -0,0 +1,1069 @@ +//! The messaging layer ([MS-PST] 2.4): the message store, folders, items, recipients, +//! attachments and attached messages, over the node database and its lists, tables and +//! properties. + +use std::collections::{HashMap, HashSet}; +use std::io::{self, Read, Seek}; +use std::mem::size_of; + +use crate::crc::crc32; +use crate::crypt::Encryption; +use crate::error::{Cap, Error, Limits, Result}; +use crate::header::{Format, HEADER_READ, Header}; +use crate::ltp::{Budget, NodeData, Table, pc_properties, pc_records, pc_value}; +use crate::named::{self, NamedMap}; +use crate::ndb::{Ndb, NodeEntry, is_internal}; +use crate::prop::{PropValue, Properties, types}; +use crate::source::Source; + +const NID_MESSAGE_STORE: u32 = 0x21; +const NID_NAME_TO_ID_MAP: u32 = 0x61; +const NID_ROOT_FOLDER: u32 = 0x122; +const NID_ATTACHMENT_TABLE: u32 = 0x671; +const NID_RECIPIENT_TABLE: u32 = 0x692; +/// Rows of a recipient or attachment table read at most: the caps of each are applied to what +/// is read, so a table may be larger than they are, but not without bound. +const MAX_PART_ROWS: usize = 1 << 16; + +/// Node types ([MS-PST] 2.2.2.1), the low five bits of a node id. +pub mod node_types { + pub const NORMAL_FOLDER: u8 = 0x02; + pub const SEARCH_FOLDER: u8 = 0x03; + pub const NORMAL_MESSAGE: u8 = 0x04; + pub const ATTACHMENT: u8 = 0x05; + pub const ASSOC_MESSAGE: u8 = 0x08; + pub const HIERARCHY_TABLE: u8 = 0x0D; + pub const CONTENTS_TABLE: u8 = 0x0E; + pub const ASSOC_CONTENTS_TABLE: u8 = 0x0F; + pub const ATTACHMENT_TABLE: u8 = 0x11; + pub const RECIPIENT_TABLE: u8 = 0x12; +} + +use node_types::{ + ASSOC_CONTENTS_TABLE, ASSOC_MESSAGE, ATTACHMENT_TABLE, CONTENTS_TABLE, HIERARCHY_TABLE, NORMAL_FOLDER, NORMAL_MESSAGE, RECIPIENT_TABLE, + SEARCH_FOLDER, +}; + +/// A node id: unique in the file for a folder or an item, unique within its parent for an +/// attachment or an attached message. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Nid(pub u32); + +impl Nid { + /// The node's type ([`node_types`]). + pub fn kind(self) -> u8 { + (self.0 & 0x1F) as u8 + } + + fn with_kind(self, kind: u8) -> Nid { + Nid((self.0 & !0x1F) | u32::from(kind)) + } +} + +/// The message store's own properties. +#[derive(Debug, Clone, PartialEq)] +pub struct Store { + pub display_name: Option, + /// The file's provider UID (`PidTagRecordKey`), which its entry ids carry. + pub record_key: Option>, + /// The top of the folders the owner sees ("Top of Personal Folders"). + pub ipm_subtree: Option, + pub deleted_items: Option, + /// The root of the search folders. + pub search_root: Option, + pub properties: Properties, +} + +/// What a folder holds, from its container class. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FolderKind { + Mail, + Calendar, + Contacts, + Tasks, + Notes, + Journal, + /// A container class this reader doesn't name. + Other(String), + /// A search folder: its items live in other folders. + Search, +} + +impl FolderKind { + fn of(class: Option<&str>, nid: Nid) -> FolderKind { + if nid.kind() == SEARCH_FOLDER { + return FolderKind::Search; + } + let Some(class) = class else { return FolderKind::Mail }; + let family = |prefix: &str| class == prefix || class.strip_prefix(prefix).is_some_and(|rest| rest.starts_with('.')); + if family("IPF.Note") || family("IPF.Imap") || class == "IPF" { + FolderKind::Mail + } else if family("IPF.Appointment") { + FolderKind::Calendar + } else if family("IPF.Contact") { + FolderKind::Contacts + } else if family("IPF.Task") { + FolderKind::Tasks + } else if family("IPF.StickyNote") { + FolderKind::Notes + } else if family("IPF.Journal") { + FolderKind::Journal + } else { + FolderKind::Other(class.to_string()) + } + } +} + +/// A folder's own properties. +#[derive(Debug, Clone, PartialEq)] +pub struct Folder { + pub id: Nid, + /// None for the root (whose parent is itself). + pub parent: Option, + pub name: String, + /// What the folder says it holds (`PidTagContentCount`); [`Pst::items`] says what it does. + pub content_count: u32, + pub unread_count: u32, + pub has_subfolders: bool, + pub container_class: Option, + pub kind: FolderKind, + pub properties: Properties, +} + +/// A folder in [`FolderTree`]. +#[derive(Debug, Clone, PartialEq)] +pub struct FolderEntry { + pub folder: Folder, + /// 0 for the root. + pub depth: usize, + /// Whether it is the top of the owner's folders or under it; the rest (the root, the search + /// root, views) are the file's own. + pub in_ipm_subtree: bool, +} + +/// What was left out of a walk or a read, and why. +#[derive(Debug, Clone, PartialEq)] +pub struct LeftOut { + pub id: Nid, + pub parent: Option, + /// The one property left out of the folder or item, when that is what was. + pub property: Option, + pub reason: Error, +} + +impl LeftOut { + fn new(id: Nid, parent: Option, reason: Error) -> LeftOut { + LeftOut { id, parent, property: None, reason } + } + + /// One for each damaged property of `properties`. + fn damaged(id: Nid, parent: Option, properties: &Properties) -> impl Iterator + '_ { + properties.damaged().map(move |(p, reason)| LeftOut { id, parent, property: Some(p), reason: reason.clone() }) + } +} + +/// Every folder of the file, parents before children. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct FolderTree { + pub folders: Vec, + /// Folders that could not be read (with their subfolders), or past a cap, and folders' + /// properties that could not be read. A folder whose subfolders were left out for a cap is + /// named with that cap. + pub left_out: Vec, +} + +/// An item as its folder's contents table lists it. +#[derive(Debug, Clone, PartialEq)] +pub struct ItemSummary { + pub id: Nid, + pub class: Option, + /// The table's columns for it (subject, flags, times…); empty when the row could not be + /// read (and then named in [`ItemList::left_out`]). A value larger than + /// [`Limits::max_summary_value`] is [`PropValue::Large`]. + pub properties: Properties, +} + +/// The items of a folder. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ItemList { + pub items: Vec, + /// Items whose row, or a property of it, could not be read (listed all the same); and, + /// once each, the folder itself when items were past [`Limits::max_items`] or past the + /// listing's budget ([`Limits::max_listing_bytes`]). + pub left_out: Vec, + /// How many items were not listed: past [`Limits::max_items`] (counted across the + /// folders listed so far), or past the listing's budget. + pub past_cap: usize, +} + +/// Where a message is: a folder's item, or attached to one (through its attachments). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct MessageRef { + pub node: Nid, + /// For an attached message, pairs of (attachment, object) subnode ids from the item down. + pub path: Vec, +} + +impl MessageRef { + pub fn item(node: Nid) -> MessageRef { + MessageRef { node, path: Vec::new() } + } + + /// How deep it is attached (0 for an item in a folder). + pub fn depth(&self) -> usize { + self.path.len() / 2 + } +} + +/// An attachment of a message. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct AttachmentRef { + pub message: MessageRef, + pub id: Nid, +} + +/// A recipient: its row of the recipient table. +#[derive(Debug, Clone, PartialEq)] +pub struct Recipient { + pub properties: Properties, +} + +/// To, Cc or Bcc (`PidTagRecipientType`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecipientKind { + Originator, + To, + Cc, + Bcc, + Other(i64), +} + +impl Recipient { + pub fn name(&self) -> Option { + self.properties.text(0x3001) + } + + /// The SMTP address when there is one, else the address of its type. + pub fn address(&self) -> Option { + self.properties.text(0x39FE).or_else(|| self.properties.text(0x3003)) + } + + pub fn address_type(&self) -> Option { + self.properties.text(0x3002) + } + + pub fn kind(&self) -> RecipientKind { + match self.properties.int(0x0C15).map(|v| v & 0x0FFF_FFFF) { + Some(0) => RecipientKind::Originator, + Some(1) | None => RecipientKind::To, + Some(2) => RecipientKind::Cc, + Some(3) => RecipientKind::Bcc, + Some(other) => RecipientKind::Other(other), + } + } +} + +/// What an attachment holds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AttachmentContent { + /// Nothing (a link to a file elsewhere, by its path, for one). + None, + /// Bytes, read with [`Pst::attachment_reader`] or [`Pst::attachment_bytes`]. + Data { size: u64 }, + /// An attached message, read with [`Pst::message`]. + Message(MessageRef), + /// An attached message deeper than [`Limits::max_embedded_depth`]: left out. + MessageTooDeep, + /// An object other than a message (an OLE object), read as bytes. + Object { size: u64 }, +} + +/// An attachment's properties. +#[derive(Debug, Clone, PartialEq)] +pub struct Attachment { + pub id: AttachmentRef, + /// `PidTagAttachMethod`: 1 by value, 2–4 by reference, 5 an attached message, 6 OLE. + pub method: Option, + /// The long file name when there is one, else the short. + pub file_name: Option, + pub mime_type: Option, + pub content_id: Option, + pub content: AttachmentContent, + pub properties: Properties, +} + +/// A part of a message that was left out, and why. +#[derive(Debug, Clone, PartialEq)] +pub enum Part { + /// Recipients past [`Limits::max_recipients`]. + Recipients { count: usize }, + /// One damaged recipient row. + Recipient { row: u32 }, + /// The recipient table as a whole. + RecipientTable, + /// Attachments past [`Limits::max_attachments`]. + Attachments { count: usize }, + /// One attachment. + Attachment { id: Nid }, + /// The attachment table as a whole. + AttachmentTable, + /// A property of the message. + Property { id: u16 }, + /// A property of a recipient (by its row). + RecipientProperty { row: u32, id: u16 }, + /// A property of an attachment. + AttachmentProperty { attachment: Nid, id: u16 }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct LeftOutPart { + pub part: Part, + pub reason: Error, +} + +/// A message (or any item) with its recipients and attachments. +#[derive(Debug, Clone, PartialEq)] +pub struct Message { + pub id: MessageRef, + pub class: Option, + pub properties: Properties, + pub recipients: Vec, + pub attachments: Vec, + pub left_out: Vec, +} + +/// A node of the node B-tree, for counting what the file holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NodeInfo { + pub id: Nid, + /// The folder it is in (for a folder or an item), or zero. + pub parent: Nid, +} + +/// A .pst file, read over any `Read + Seek`. +pub struct Pst { + ndb: Ndb, + store: Properties, + password: u32, + unlocked: bool, + named: Option, + listed: HashMap, + listed_total: usize, + /// The item whose attached messages are being read: what is left of its budget, and how + /// many have been read. + tree: ItemTree, +} + +/// What one item and the messages attached to it, together, may still read. +#[derive(Debug, Clone, Copy, Default)] +struct ItemTree { + node: u32, + left: u64, + embedded: usize, +} + +/// `s` in Windows-1252, when every character is in it. +fn windows_1252(s: &str) -> Option> { + // 0x80 to 0x9F; zero where the code page has no character. + const HIGH: [u32; 32] = [ + 0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0x017D, 0, 0, 0x2018, 0x2019, 0x201C, + 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0, 0x017E, 0x0178, + ]; + s.chars() + .map(|c| { + let n = u32::from(c); + if n < 0x80 || (0xA0..=0xFF).contains(&n) { + u8::try_from(n).ok() + } else { + HIGH.iter().position(|&h| h != 0 && h == n).and_then(|i| u8::try_from(i).ok()).map(|i| i | 0x80) + } + }) + .collect() +} + +/// The node id an entry id names (its last four bytes, [MS-PST] 2.4.3.2). +fn entry_id_nid(b: &[u8]) -> Option { + let tail = b.get(20..24)?; + Some(Nid(u32::from_le_bytes([tail.first().copied()?, tail.get(1).copied()?, tail.get(2).copied()?, tail.get(3).copied()?]))) +} + +/// What ends a call rather than leaving one part of it out: the reader under the file failing, +/// the file's read allowance ([`Cap::TotalRead`]) spent, or the call's own budget (`own`: +/// [`Cap::ItemSize`] for a message; a walk names each folder past its budget and a listing +/// counts the items past it). Anything else is damage to that part, or a cap on that part +/// alone (its recipients, attachments, rows, depth, blocks or size), and is named in +/// `left_out` while the rest is read. +fn ends_call(e: &Error, own: Option) -> bool { + match e { + Error::Io(_) | Error::Cap(Cap::TotalRead) => true, + Error::Cap(c) => Some(*c) == own, + _ => false, + } +} + +impl Pst { + /// Opens a file with the import's caps. The header and the message store are read; a file + /// with a password then waits for [`Pst::unlock`]. + pub fn open(reader: R) -> Result { + Self::open_with(reader, Limits::default()) + } + + pub fn open_with(reader: R, limits: Limits) -> Result { + let mut src = Source::new(reader, &limits)?; + let len = HEADER_READ.min(usize::try_from(src.size()).unwrap_or(HEADER_READ)); + let header = Header::parse(&src.read_at(0, len)?)?; + let mut ndb = Ndb::new(src, header, limits); + let entry = ndb.node(NID_MESSAGE_STORE)?; + let mut budget = Budget::new(ndb.limits.max_item_bytes, ndb.limits.max_inline_value, Cap::ItemSize); + let node = NodeData::load(&mut ndb, entry.data, entry.sub, &mut budget)?; + let store = pc_properties(&mut ndb, &node, &mut budget, &[])?; + let password = store.int(0x67FF).map(|v| v as u32).unwrap_or(0); + Ok(Pst { ndb, store, password, unlocked: password == 0, named: None, listed: HashMap::new(), listed_total: 0, tree: ItemTree::default() }) + } + + pub fn format(&self) -> Format { + self.ndb.header.format + } + + pub fn encryption(&self) -> Encryption { + self.ndb.encryption() + } + + /// Whether the file has a password that has not been given yet. + pub fn password_required(&self) -> bool { + !self.unlocked + } + + /// Whether `password` is the file's: its CRC ([MS-PST] 5.3) is what the message store keeps + /// (`PidTagPstPassword`). The specification doesn't say in which encoding the text is + /// hashed, so its Windows-1252 bytes (when it has them), its UTF-8 bytes and its UTF-16LE + /// bytes are all tried. A file without a password takes any. + pub fn check_password(&self, password: &str) -> bool { + if self.password == 0 { + return true; + } + let utf16: Vec = password.encode_utf16().flat_map(u16::to_le_bytes).collect(); + let ansi = windows_1252(password).is_some_and(|b| crc32(&b) == self.password); + ansi || crc32(password.as_bytes()) == self.password || crc32(&utf16) == self.password + } + + /// Gives the file's password; nothing of the file is read until it has been. + pub fn unlock(&mut self, password: &str) -> Result<()> { + if self.check_password(password) { + self.unlocked = true; + Ok(()) + } else { + Err(Error::WrongPassword) + } + } + + fn gate(&self) -> Result<()> { + if self.unlocked { Ok(()) } else { Err(Error::PasswordRequired) } + } + + /// What one item or folder may read. + fn budget(&self) -> Budget { + Budget::new(self.ndb.limits.max_item_bytes, self.ndb.limits.max_inline_value, Cap::ItemSize) + } + + /// What one folder walk or one listing may read, all of it. + fn listing_budget(&self) -> Budget { + let l = &self.ndb.limits; + Budget::new(l.max_listing_bytes, l.max_summary_value.min(l.max_inline_value), Cap::ListingSize) + } + + pub fn store(&self) -> Result { + self.gate()?; + let p = &self.store; + Ok(Store { + display_name: p.text(0x3001), + record_key: p.binary(0x0FF9).map(<[u8]>::to_vec), + ipm_subtree: p.binary(0x35E0).and_then(entry_id_nid), + deleted_items: p.binary(0x35E3).and_then(entry_id_nid), + search_root: p.binary(0x35E7).and_then(entry_id_nid), + properties: p.clone(), + }) + } + + /// The root folder, which holds the top of the owner's folders and the file's own. + pub fn root_folder(&self) -> Nid { + Nid(NID_ROOT_FOLDER) + } + + fn load(&mut self, entry: &NodeEntry, budget: &mut Budget) -> Result { + NodeData::load(&mut self.ndb, entry.data, entry.sub, budget) + } + + /// A folder's properties; a property that could not be read is [`PropValue::Damaged`]. + pub fn folder(&mut self, id: Nid) -> Result { + let mut budget = self.budget(); + self.folder_in(id, &mut budget) + } + + fn folder_in(&mut self, id: Nid, budget: &mut Budget) -> Result { + self.gate()?; + if !matches!(id.kind(), NORMAL_FOLDER | SEARCH_FOLDER) { + return Err(Error::Unsupported("a node that is not a folder")); + } + let entry = self.ndb.node(id.0)?; + let node = self.load(&entry, budget)?; + let properties = pc_properties(&mut self.ndb, &node, budget, &[])?; + let container_class = properties.text(0x3613); + let parent = Some(Nid(entry.parent)).filter(|p| p.0 != 0 && *p != id); + Ok(Folder { + id, + parent, + name: properties.text(0x3001).unwrap_or_default(), + content_count: properties.int(0x3602).map(|v| v as u32).unwrap_or(0), + unread_count: properties.int(0x3603).map(|v| v as u32).unwrap_or(0), + has_subfolders: properties.bool(0x360A).unwrap_or(false), + kind: FolderKind::of(container_class.as_deref(), id), + container_class, + properties, + }) + } + + /// The table at `nid`, or None when the file has no such node (a folder without + /// subfolders, say). Anything that fails after the node is found is an error. + fn table(&mut self, nid: u32, budget: &mut Budget, max_rows: usize, cap: Cap) -> Result> { + let entry = match self.ndb.node(nid) { + Ok(e) => e, + Err(Error::NotFound(_)) => return Ok(None), + Err(e) => return Err(e), + }; + let node = self.load(&entry, budget)?; + Table::open(&mut self.ndb, node, max_rows, cap).map(Some) + } + + /// A folder's subfolders, in the order of its hierarchy table. + pub fn subfolders(&mut self, id: Nid) -> Result> { + let mut budget = self.budget(); + self.subfolders_in(id, &mut budget) + } + + fn subfolders_in(&mut self, id: Nid, budget: &mut Budget) -> Result> { + self.gate()?; + let max = self.ndb.limits.max_folders; + let Some(table) = self.table(id.with_kind(HIERARCHY_TABLE).0, budget, max, Cap::Folders)? else { + return Ok(Vec::new()); + }; + let mut rows = table.rows; + rows.sort_by_key(|&(_, index)| index); + Ok(rows.into_iter().map(|(rowid, _)| Nid(rowid)).collect()) + } + + /// Every folder from the root down, within the caps: at most [`Limits::max_folders`], + /// nested at most [`Limits::max_folder_depth`], each once (a folder met again is a cycle), + /// all of it within [`Limits::max_listing_bytes`] and each value within + /// [`Limits::max_summary_value`] (larger ones are [`PropValue::Large`]; [`Pst::folder`] + /// reads a folder whole). + pub fn walk_folders(&mut self) -> Result { + self.gate()?; + // The store's own properties, read for the one entry id (not a copy of them all). + let ipm = self.store.binary(0x35E0).and_then(entry_id_nid); + let limits = self.ndb.limits.clone(); + let mut budget = self.listing_budget(); + let mut tree = FolderTree::default(); + let mut seen = HashSet::new(); + let mut scheduled = 1usize; + let mut stack = vec![(Nid(NID_ROOT_FOLDER), None::, 0usize, false)]; + while let Some((id, parent, depth, under_ipm)) = stack.pop() { + // What this folder adds to what is kept: its notes of what was left out too. + let noted = tree.left_out.len(); + self.walk_one(&mut tree, &mut stack, &mut seen, &mut scheduled, &mut budget, (id, parent, depth, under_ipm), ipm, &limits)?; + budget.note(tree.left_out.len().saturating_sub(noted).saturating_mul(size_of::())); + } + Ok(tree) + } + + #[allow(clippy::too_many_arguments)] + fn walk_one( + &mut self, + tree: &mut FolderTree, + stack: &mut Vec<(Nid, Option, usize, bool)>, + seen: &mut HashSet, + scheduled: &mut usize, + budget: &mut Budget, + (id, parent, depth, under_ipm): (Nid, Option, usize, bool), + ipm: Option, + limits: &Limits, + ) -> Result<()> { + { + if !seen.insert(id) { + tree.left_out.push(LeftOut::new(id, parent, Error::Cycle("the folder hierarchy"))); + return Ok(()); + } + if depth > limits.max_folder_depth { + tree.left_out.push(LeftOut::new(id, parent, Error::Cap(Cap::FolderDepth))); + return Ok(()); + } + let folder = match self.folder_in(id, budget) { + Ok(f) => f, + Err(reason) if ends_call(&reason, None) => return Err(reason), + Err(reason) => { + tree.left_out.push(LeftOut::new(id, parent, reason)); + return Ok(()); + } + }; + let kept = size_of::() + .saturating_add(folder.name.capacity()) + .saturating_add(folder.container_class.as_ref().map_or(0, String::capacity)); + if let Err(reason) = budget.keep(kept) { + tree.left_out.push(LeftOut::new(id, parent, reason)); + return Ok(()); + } + tree.left_out.extend(LeftOut::damaged(id, parent, &folder.properties)); + let in_ipm = under_ipm || Some(id) == ipm; + let children = if id.kind() == SEARCH_FOLDER { Ok(Vec::new()) } else { self.subfolders_in(id, budget) }; + tree.folders.push(FolderEntry { folder, depth, in_ipm_subtree: in_ipm }); + match children { + Ok(children) => { + let room = limits.max_folders.saturating_sub(*scheduled); + if children.len() > room { + tree.left_out.push(LeftOut::new(id, parent, Error::Cap(Cap::Folders))); + } + let take: Vec = children.into_iter().take(room).collect(); + *scheduled = scheduled.saturating_add(take.len()); + for child in take.into_iter().rev() { + stack.push((child, Some(id), depth.saturating_add(1), in_ipm)); + } + } + Err(reason) if ends_call(&reason, None) => return Err(reason), + Err(reason) => tree.left_out.push(LeftOut::new(id, parent, reason)), + } + } + Ok(()) + } + + /// The items of a folder (its contents table), in the table's order: all of the listing + /// within [`Limits::max_listing_bytes`], each value within [`Limits::max_summary_value`]. + pub fn items(&mut self, folder: Nid) -> Result { + self.list(folder, CONTENTS_TABLE) + } + + /// The folder's associated items (views, rules, forms: the file's own, not the owner's). + pub fn associated_items(&mut self, folder: Nid) -> Result { + self.list(folder, ASSOC_CONTENTS_TABLE) + } + + fn list(&mut self, folder: Nid, kind: u8) -> Result { + self.gate()?; + match folder.kind() { + NORMAL_FOLDER => {} + SEARCH_FOLDER => return Err(Error::Unsupported("a search folder's items")), + _ => return Err(Error::Unsupported("a node that is not a folder")), + } + let nid = folder.with_kind(kind).0; + let mut budget = self.listing_budget(); + let max = self.ndb.limits.max_items; + let Some(mut table) = self.table(nid, &mut budget, max, Cap::Items)? else { + return Ok(ItemList::default()); + }; + let mut rows = std::mem::take(&mut table.rows); + rows.sort_by_key(|&(_, index)| index); + let room = match self.listed.get(&nid) { + Some(&n) => n, + None => self.ndb.limits.max_items.saturating_sub(self.listed_total), + }; + let mut past_cap = rows.len().saturating_sub(room); + let mut left_out = Vec::new(); + if past_cap > 0 { + left_out.push(LeftOut::new(folder, None, Error::Cap(Cap::Items))); + } + rows.truncate(room); + if let std::collections::hash_map::Entry::Vacant(e) = self.listed.entry(nid) { + e.insert(rows.len()); + self.listed_total = self.listed_total.saturating_add(rows.len()); + } + let total = rows.len(); + let mut items = Vec::new(); + for (n, (rowid, index)) in rows.into_iter().enumerate() { + let id = Nid(rowid); + let noted = left_out.len(); + let properties = match table.row(&mut self.ndb, rowid, index, &mut budget) { + Ok(p) => Some(p), + Err(Error::Cap(Cap::ListingSize)) => None, + Err(reason) if ends_call(&reason, None) => return Err(reason), + Err(reason) => { + left_out.push(LeftOut::new(id, Some(folder), reason)); + Some(Properties::default()) + } + }; + let item = properties.map(|properties| { + left_out.extend(LeftOut::damaged(id, Some(folder), &properties)); + ItemSummary { id, class: properties.text(0x001A), properties } + }); + budget.note(left_out.len().saturating_sub(noted).saturating_mul(size_of::())); + let kept = item.as_ref().map(|i| size_of::().saturating_add(i.class.as_ref().map_or(0, String::capacity))); + match (item, kept.map(|k| budget.keep(k))) { + (Some(item), Some(Ok(()))) => items.push(item), + // The budget is spent: the rest are counted, and the folder named once. + _ => { + left_out.push(LeftOut::new(folder, None, Error::Cap(Cap::ListingSize))); + past_cap = past_cap.saturating_add(total.saturating_sub(n)); + break; + } + } + } + Ok(ItemList { items, left_out, past_cap }) + } + + /// The data and subnodes of the object at `path` under the node `node`. + fn locate(&mut self, node: Nid, path: &[Nid]) -> Result<(u64, u64)> { + let chain = self.chain(node, path)?; + chain.last().copied().ok_or(Error::NotFound("a node")) + } + + /// The data and subnodes of each object from `node` down `path`; one met again on the way + /// (an attached message that holds itself) is a cycle. + fn chain(&mut self, node: Nid, path: &[Nid]) -> Result> { + let entry = self.ndb.node(node.0)?; + let mut chain = vec![(entry.data, entry.sub)]; + let mut sub = entry.sub; + for step in path { + let subs = self.ndb.subnodes(sub)?; + let s = subs.get(&step.0).ok_or(Error::NotFound("an attachment"))?; + if chain.contains(&(s.data, s.sub)) { + return Err(Error::Cycle("an attached message")); + } + chain.push((s.data, s.sub)); + sub = s.sub; + } + Ok(chain) + } + + /// A message (or any item) with its recipients and attachments. Property values larger + /// than [`Limits::max_inline_value`] come back as [`PropValue::Large`]; what could not be + /// read is named in `left_out`. + /// + /// An item and the messages attached to it share one budget, for what is read and what + /// is kept: reading the item (a call with an empty path) starts it at + /// [`Limits::max_item_bytes`], and the messages attached to it, read straight after it, + /// take from what is left, as many as [`Limits::max_embedded_messages`]. So the whole of + /// an item's tree is within the cap. Reading another item in between starts the budget + /// for that one: an import reads an item and its attached messages together. + pub fn message(&mut self, id: &MessageRef) -> Result { + self.gate()?; + let limits = self.ndb.limits.clone(); + if id.depth() > limits.max_embedded_depth || !id.path.len().is_multiple_of(2) { + return Err(Error::Cap(Cap::EmbeddedDepth)); + } + if id.path.is_empty() && !matches!(id.node.kind(), NORMAL_MESSAGE | ASSOC_MESSAGE) { + return Err(Error::Unsupported("a node that is not an item")); + } + if id.path.is_empty() || self.tree.node != id.node.0 { + self.tree = ItemTree { node: id.node.0, left: limits.max_item_bytes, embedded: 0 }; + } + if !id.path.is_empty() { + self.tree.embedded = self.tree.embedded.saturating_add(1); + if self.tree.embedded > limits.max_embedded_messages { + return Err(Error::Cap(Cap::EmbeddedMessages)); + } + } + let mut budget = Budget::new(self.tree.left, limits.max_inline_value, Cap::ItemSize); + let read = self.message_in(id, &limits, &mut budget); + // What a failed read took counts too. + self.tree.left = budget.left; + read + } + + fn message_in(&mut self, id: &MessageRef, limits: &Limits, budget: &mut Budget) -> Result { + let chain = self.chain(id.node, &id.path)?; + let (data, sub) = chain.last().copied().ok_or(Error::NotFound("a node"))?; + let node = NodeData::load(&mut self.ndb, data, sub, budget)?; + let properties = pc_properties(&mut self.ndb, &node, budget, &[])?; + let mut left_out: Vec = + properties.damaged().map(|(p, reason)| LeftOutPart { part: Part::Property { id: p }, reason: reason.clone() }).collect(); + + let mut recipients = Vec::new(); + let recipient_table = node + .subnodes + .get(&NID_RECIPIENT_TABLE) + .copied() + .or_else(|| node.subnodes.iter().find(|(nid, _)| Nid(**nid).kind() == RECIPIENT_TABLE).map(|(_, e)| *e)); + if let Some(e) = recipient_table { + match NodeData::load(&mut self.ndb, e.data, e.sub, budget).and_then(|n| Table::open(&mut self.ndb, n, MAX_PART_ROWS, Cap::Recipients)) { + Ok(mut table) => { + let mut rows = table.rows.clone(); + rows.sort_by_key(|&(_, index)| index); + if rows.len() > limits.max_recipients { + left_out.push(LeftOutPart { + part: Part::Recipients { count: rows.len().saturating_sub(limits.max_recipients) }, + reason: Error::Cap(Cap::Recipients), + }); + rows.truncate(limits.max_recipients); + } + for (rowid, index) in rows { + match table.row(&mut self.ndb, rowid, index, budget) { + Ok(properties) => { + budget.keep(size_of::())?; + let row = index; + left_out.extend( + properties + .damaged() + .map(|(p, reason)| LeftOutPart { part: Part::RecipientProperty { row, id: p }, reason: reason.clone() }), + ); + recipients.push(Recipient { properties }); + } + Err(reason) if ends_call(&reason, Some(Cap::ItemSize)) => return Err(reason), + Err(reason) => left_out.push(LeftOutPart { part: Part::Recipient { row: index }, reason }), + } + } + } + Err(reason) if ends_call(&reason, Some(Cap::ItemSize)) => return Err(reason), + Err(reason) => left_out.push(LeftOutPart { part: Part::RecipientTable, reason }), + } + } + + let mut attachments = Vec::new(); + let attachment_table = node + .subnodes + .get(&NID_ATTACHMENT_TABLE) + .copied() + .or_else(|| node.subnodes.iter().find(|(nid, _)| Nid(**nid).kind() == ATTACHMENT_TABLE).map(|(_, e)| *e)); + if let Some(e) = attachment_table { + match NodeData::load(&mut self.ndb, e.data, e.sub, budget).and_then(|n| Table::open(&mut self.ndb, n, MAX_PART_ROWS, Cap::Attachments)) { + Ok(table) => { + let mut rows = table.rows.clone(); + rows.sort_by_key(|&(_, index)| index); + if rows.len() > limits.max_attachments { + left_out.push(LeftOutPart { + part: Part::Attachments { count: rows.len().saturating_sub(limits.max_attachments) }, + reason: Error::Cap(Cap::Attachments), + }); + rows.truncate(limits.max_attachments); + } + // Row ids are unique: the row index's keys strictly increase (bth::read). + for (rowid, _) in rows { + let att = Nid(rowid); + match self.attachment_from(&node, id, &chain, att, budget) { + Ok(a) => { + let strings = + [&a.file_name, &a.mime_type, &a.content_id].iter().map(|s| s.as_ref().map_or(0, String::capacity)).sum::(); + budget.keep(size_of::().saturating_add(strings))?; + left_out.extend(a.properties.damaged().map(|(p, reason)| LeftOutPart { + part: Part::AttachmentProperty { attachment: att, id: p }, + reason: reason.clone(), + })); + attachments.push(a); + } + Err(reason) if ends_call(&reason, Some(Cap::ItemSize)) => return Err(reason), + Err(reason) => left_out.push(LeftOutPart { part: Part::Attachment { id: att }, reason }), + } + } + } + Err(reason) if ends_call(&reason, Some(Cap::ItemSize)) => return Err(reason), + Err(reason) => left_out.push(LeftOutPart { part: Part::AttachmentTable, reason }), + } + } + + budget.note(left_out.len().saturating_mul(size_of::())); + let class = properties.text(0x001A); + budget.keep(size_of::().saturating_add(class.as_ref().map_or(0, String::capacity)))?; + Ok(Message { id: id.clone(), class, properties, recipients, attachments, left_out }) + } + + fn attachment_from(&mut self, message: &NodeData, id: &MessageRef, chain: &[(u64, u64)], att: Nid, budget: &mut Budget) -> Result { + let e = message.subnodes.get(&att.0).copied().ok_or(Error::NotFound("an attachment"))?; + let node = NodeData::load(&mut self.ndb, e.data, e.sub, budget)?; + let properties = pc_properties(&mut self.ndb, &node, budget, &[0x3701])?; + let method = properties.int(0x3705); + let records = pc_records(&node)?; + let data = records.iter().find(|(pid, _, _)| *pid == 0x3701).copied(); + let content = match data { + None => AttachmentContent::None, + Some((_, types::BINARY, raw)) => AttachmentContent::Data { size: node.hnid_len(&mut self.ndb, raw)? }, + Some((_, types::OBJECT, raw)) => { + let object = pc_value(&mut self.ndb, &node, types::OBJECT, raw, budget)?; + let PropValue::Object { nid, size } = object else { + return Err(Error::Corrupt("an attachment's object")); + }; + if method == Some(5) { + // An attached message that is its attachment, or the message it is + // attached to (or one above), would be read for ever. + let again = |o: &crate::ndb::SubEntry| (o.data == e.data && o.sub == e.sub) || chain.contains(&(o.data, o.sub)); + if node.subnodes.get(&nid).is_some_and(again) { + return Err(Error::Cycle("an attached message")); + } + let mut path = id.path.clone(); + path.extend([att, Nid(nid)]); + let child = MessageRef { node: id.node, path }; + if child.depth() > self.ndb.limits.max_embedded_depth { + AttachmentContent::MessageTooDeep + } else { + AttachmentContent::Message(child) + } + } else { + AttachmentContent::Object { size: u64::from(size) } + } + } + Some(_) => AttachmentContent::None, + }; + Ok(Attachment { + id: AttachmentRef { message: id.clone(), id: att }, + method, + file_name: properties.text(0x3707).or_else(|| properties.text(0x3704)), + mime_type: properties.text(0x370E), + content_id: properties.text(0x3712), + content, + properties, + }) + } + + /// An attachment's data, streamed a block at a time; refused past + /// [`Limits::max_attachment_bytes`] before anything is read. + pub fn attachment_reader(&mut self, att: &AttachmentRef) -> Result> { + self.gate()?; + let max = self.ndb.limits.max_attachment_bytes; + let mut path = att.message.path.clone(); + path.push(att.id); + let (data, sub) = self.locate(att.message.node, &path)?; + let mut budget = self.budget(); + let node = NodeData::load(&mut self.ndb, data, sub, &mut budget)?; + let (_, ty, raw) = pc_records(&node)?.into_iter().find(|(id, _, _)| *id == 0x3701).ok_or(Error::NotFound("an attachment's data"))?; + let hnid = match ty { + types::BINARY => raw, + types::OBJECT => { + let b = node.heap.item(raw)?; + crate::bytes::u32_at(b, 0)? + } + _ => return Err(Error::Corrupt("an attachment's data type")), + }; + if let Some(e) = node.subnode(hnid) { + let (blocks, total) = self.ndb.data_blocks(e.data, max, Cap::AttachmentSize)?; + Ok(AttachmentReader { ndb: &mut self.ndb, blocks, next: 0, buf: Vec::new(), pos: 0, total, read: 0 }) + } else { + let buf = node.hnid_bytes(&mut self.ndb, hnid, max, Cap::AttachmentSize)?; + let total = buf.len() as u64; + Ok(AttachmentReader { ndb: &mut self.ndb, blocks: Vec::new(), next: 0, buf, pos: 0, total, read: total }) + } + } + + /// An attachment's data, whole (within [`Limits::max_attachment_bytes`]). + pub fn attachment_bytes(&mut self, att: &AttachmentRef) -> Result> { + let mut r = self.attachment_reader(att)?; + let mut out = Vec::with_capacity(usize::try_from(r.total.min(1 << 20)).unwrap_or(0)); + r.read_to_end(&mut out).map_err(|e| match e.get_ref().and_then(|i| i.downcast_ref::()) { + Some(inner) => inner.clone(), + None => Error::from(e), + })?; + Ok(out) + } + + /// One property of a message, read on its own (for a [`PropValue::Large`]), within `max` + /// bytes and [`Limits::max_item_bytes`]. + pub fn read_property(&mut self, message: &MessageRef, id: u16, max: u64) -> Result> { + self.read_property_at(message.node, &message.path, id, max) + } + + /// One property of an attachment, read on its own. + pub fn read_attachment_property(&mut self, att: &AttachmentRef, id: u16, max: u64) -> Result> { + let mut path = att.message.path.clone(); + path.push(att.id); + self.read_property_at(att.message.node, &path, id, max) + } + + fn read_property_at(&mut self, node: Nid, path: &[Nid], id: u16, max: u64) -> Result> { + self.gate()?; + let (data, sub) = self.locate(node, path)?; + let mut budget = self.budget(); + let n = NodeData::load(&mut self.ndb, data, sub, &mut budget)?; + let Some((_, ty, raw)) = pc_records(&n)?.into_iter().find(|(pid, _, _)| *pid == id) else { + return Ok(None); + }; + let max = max.min(self.ndb.limits.max_item_bytes); + let mut budget = Budget::new(max, max, Cap::ItemSize); + let value = pc_value(&mut self.ndb, &n, ty, raw, &mut budget)?; + // What the value holds beyond the bytes it was made from is within `max` too. + let read = max.saturating_sub(budget.left) as usize; + budget.keep(value.held().saturating_sub(read))?; + Ok(Some(value)) + } + + /// The file's named properties (what the ids from 0x8000 stand for). + pub fn named_properties(&mut self) -> Result<&NamedMap> { + self.gate()?; + if self.named.is_none() { + let map = match self.ndb.node(NID_NAME_TO_ID_MAP) { + Ok(entry) => { + let mut budget = self.budget(); + let node = self.load(&entry, &mut budget)?; + named::read(&mut self.ndb, &node)? + } + Err(Error::NotFound(_)) => NamedMap::default(), + Err(e) => return Err(e), + }; + self.named = Some(map); + } + self.named.as_ref().ok_or(Error::NotFound("the named property map")) + } + + /// Every node of the file's node tree (folders, items and the file's own), in id order, + /// until `f` returns false; for counting what the file holds against what was read. + pub fn for_each_node(&mut self, mut f: impl FnMut(NodeInfo) -> bool) -> Result<()> { + self.gate()?; + self.ndb.for_each_node(&mut |e| f(NodeInfo { id: Nid(e.nid), parent: Nid(e.parent) })) + } + + /// The pages of the file's own trees met so far that could not be read, and why: what was + /// under them is lost, how much is not known. A walk of every node + /// ([`Pst::for_each_node`]) meets every page of the node tree; the block tree's are met + /// as the data that uses them is read. + pub fn damage(&self) -> &[crate::Damage] { + &self.ndb.damage + } + + /// The file's size as the header says it (`ibFileEof`) and as it is. + pub fn sizes(&self) -> (u64, u64) { + (self.ndb.header.file_eof, self.ndb.file_size()) + } +} + +/// An attachment's data, a block at a time ([`Pst::attachment_reader`]). +pub struct AttachmentReader<'a, R> { + ndb: &'a mut Ndb, + blocks: Vec, + next: usize, + buf: Vec, + pos: usize, + total: u64, + read: u64, +} + +impl AttachmentReader<'_, R> { + /// The data's size. + pub fn size(&self) -> u64 { + self.total + } +} + +impl Read for AttachmentReader<'_, R> { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let invalid = |e: Error| io::Error::new(io::ErrorKind::InvalidData, e); + while self.pos >= self.buf.len() { + let Some(&bid) = self.blocks.get(self.next) else { + if self.read != self.total { + return Err(invalid(Error::Corrupt("attachment data shorter than it says"))); + } + return Ok(0); + }; + self.next = self.next.saturating_add(1); + if is_internal(bid) { + return Err(invalid(Error::Corrupt("attachment data in an internal block"))); + } + self.buf = self.ndb.block(bid).map_err(invalid)?; + self.pos = 0; + self.read = self.read.saturating_add(self.buf.len() as u64); + if self.read > self.total { + return Err(invalid(Error::Corrupt("attachment data longer than it says"))); + } + } + let rest = self.buf.get(self.pos..).unwrap_or(&[]); + let n = rest.len().min(out.len()); + if let (Some(dst), Some(src)) = (out.get_mut(..n), rest.get(..n)) { + dst.copy_from_slice(src); + } + self.pos = self.pos.saturating_add(n); + Ok(n) + } +} diff --git a/crates/pst/src/recur.rs b/crates/pst/src/recur.rs new file mode 100644 index 000000000..2c080e57c --- /dev/null +++ b/crates/pst/src/recur.rs @@ -0,0 +1,268 @@ +//! How an appointment repeats, as Outlook keeps it in PidLidAppointmentRecur: the +//! AppointmentRecurrencePattern and RecurrencePattern of [MS-OXOCAL] 2.2.1.44, and the time zone +//! of PidLidAppointmentTimeZoneDefinitionRecur (TZDEFINITION, 2.2.1.41). +//! +//! The bytes are hostile: each count is checked against the bytes there are, nothing is +//! allocated by what the blob says, and a part past its end is an error. The extended +//! exceptions, which only add the wide-character subject and location, may be missing or +//! damaged without the rest being lost. + +use crate::error::{Error, Result}; + +/// Most exceptions kept of a series. +pub const MAX_EXCEPTIONS: usize = 5_000; +/// Longest a time zone's key name may be, in UTF-16 units. +const MAX_ZONE_KEY: usize = 256; + +/// RecurFrequency. +pub const FREQ_DAILY: u16 = 0x200A; +pub const FREQ_WEEKLY: u16 = 0x200B; +pub const FREQ_MONTHLY: u16 = 0x200C; +pub const FREQ_YEARLY: u16 = 0x200D; +/// PatternType. +pub const PATTERN_DAY: u16 = 0; +pub const PATTERN_WEEK: u16 = 1; +pub const PATTERN_MONTH: u16 = 2; +pub const PATTERN_MONTH_NTH: u16 = 3; +pub const PATTERN_MONTH_END: u16 = 4; +/// EndType: on a date, after a number of occurrences. Anything else never ends. +pub const END_DATE: u32 = 0x2021; +pub const END_COUNT: u32 = 0x2022; + +const ARO_SUBJECT: u16 = 0x0001; +const ARO_MEETINGTYPE: u16 = 0x0002; +const ARO_REMINDERDELTA: u16 = 0x0004; +const ARO_REMINDER: u16 = 0x0008; +const ARO_LOCATION: u16 = 0x0010; +const ARO_BUSYSTATUS: u16 = 0x0020; +const ARO_ATTACHMENT: u16 = 0x0040; +const ARO_SUBTYPE: u16 = 0x0080; +const ARO_APPTCOLOR: u16 = 0x0100; + +/// One occurrence of a series that was changed. Times are minutes since 1601 by the clock of +/// the series' zone. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RecurException { + pub start: u32, + pub end: u32, + /// When it would have started. + pub original: u32, + /// In the item's code page. + pub subject_8bit: Option>, + pub location_8bit: Option>, + /// From the extended exceptions. + pub subject: Option, + pub location: Option, + pub reminder_delta: Option, + pub reminder_set: Option, + pub busy: Option, + pub all_day: Option, +} + +/// A repeating appointment. Dates are minutes since 1601, at midnight by the series' clock. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Recur { + pub frequency: u16, + pub pattern: u16, + /// 0 and 1 are the Gregorian calendar. + pub calendar: u16, + /// Minutes for a daily series, weeks for a weekly one, months for the others. + pub period: u32, + /// The days of the week (bit 0 is Sunday), for a weekly or nth-weekday series. + pub weekdays: u32, + /// The day of the month, or which of the weekdays (1 to 4, 5 for the last). + pub day_or_nth: u32, + pub end_type: u32, + pub count: u32, + /// The day weeks start on (0 is Sunday): what an interval of weeks is counted by. + pub first_day_of_week: u32, + /// Days that do not happen as the series says: deleted, and changed ones too. + pub deleted: Vec, + pub modified: Vec, + pub start_date: u32, + pub end_date: u32, + /// Minutes from midnight to the start and to the end of an occurrence. + pub start_offset: u32, + pub end_offset: u32, + pub exceptions: Vec, +} + +struct Cursor<'a> { + b: &'a [u8], + at: usize, +} + +const CUT: Error = Error::Corrupt("recurrence cut short"); + +impl<'a> Cursor<'a> { + fn take(&mut self, n: usize) -> Result<&'a [u8]> { + let end = self.at.checked_add(n).ok_or(CUT)?; + let s = self.b.get(self.at..end).ok_or(CUT)?; + self.at = end; + Ok(s) + } + + fn u16(&mut self) -> Result { + let s = self.take(2)?; + Ok(u16::from_le_bytes(<[u8; 2]>::try_from(s).map_err(|_| CUT)?)) + } + + fn u32(&mut self) -> Result { + let s = self.take(4)?; + Ok(u32::from_le_bytes(<[u8; 4]>::try_from(s).map_err(|_| CUT)?)) + } + + fn left(&self) -> usize { + self.b.len().saturating_sub(self.at) + } + + /// A list of `u32`s after its count, the count checked against the bytes there are. + fn dates(&mut self) -> Result> { + let n = usize::try_from(self.u32()?).map_err(|_| CUT)?; + if n > self.left() / 4 { + return Err(CUT); + } + (0..n).map(|_| self.u32()).collect() + } + + /// A block a size is written before, skipped. + fn skip_block(&mut self) -> Result<()> { + let n = usize::try_from(self.u32()?).map_err(|_| CUT)?; + self.take(n).map(|_| ()) + } + + fn wide(&mut self) -> Result { + let n = usize::from(self.u16()?); + let s = self.take(n.checked_mul(2).ok_or(CUT)?)?; + let units = s.as_chunks::<2>().0.iter().map(|c| u16::from_le_bytes(*c)); + Ok(char::decode_utf16(units).map(|c| c.unwrap_or('\u{FFFD}')).collect()) + } + + /// An exception's 8-bit text: its length plus one, its length, then the bytes. + fn text_8bit(&mut self) -> Result> { + let _with_end = self.u16()?; + let n = usize::from(self.u16()?); + Ok(self.take(n)?.to_vec()) + } +} + +fn exception(c: &mut Cursor) -> Result<(RecurException, u16)> { + let mut e = RecurException { start: c.u32()?, end: c.u32()?, original: c.u32()?, ..Default::default() }; + let flags = c.u16()?; + if flags & ARO_SUBJECT != 0 { + e.subject_8bit = Some(c.text_8bit()?); + } + if flags & ARO_MEETINGTYPE != 0 { + c.u32()?; + } + if flags & ARO_REMINDERDELTA != 0 { + e.reminder_delta = Some(c.u32()?); + } + if flags & ARO_REMINDER != 0 { + e.reminder_set = Some(c.u32()? != 0); + } + if flags & ARO_LOCATION != 0 { + e.location_8bit = Some(c.text_8bit()?); + } + if flags & ARO_BUSYSTATUS != 0 { + e.busy = Some(c.u32()?); + } + if flags & ARO_ATTACHMENT != 0 { + c.u32()?; + } + if flags & ARO_SUBTYPE != 0 { + e.all_day = Some(c.u32()? != 0); + } + if flags & ARO_APPTCOLOR != 0 { + c.u32()?; + } + Ok((e, flags)) +} + +/// One extended exception: the wide-character subject and location of the exception before it. +fn extended(c: &mut Cursor, writer: u32, e: &mut RecurException, flags: u16) -> Result<()> { + if writer >= 0x3009 { + c.skip_block()?; + } + c.skip_block()?; + if flags & (ARO_SUBJECT | ARO_LOCATION) != 0 { + c.take(12)?; + if flags & ARO_SUBJECT != 0 { + e.subject = Some(c.wide()?); + } + if flags & ARO_LOCATION != 0 { + e.location = Some(c.wide()?); + } + c.skip_block()?; + } + Ok(()) +} + +/// The series in an AppointmentRecurrencePattern. +pub fn appointment_recurrence(blob: &[u8]) -> Result { + let mut c = Cursor { b: blob, at: 0 }; + if c.u16()? != 0x3004 || c.u16()? != 0x3004 { + return Err(Error::Corrupt("recurrence of a version not known")); + } + let mut r = Recur { frequency: c.u16()?, pattern: c.u16()?, calendar: c.u16()?, ..Default::default() }; + let _first_date_time = c.u32()?; + r.period = c.u32()?; + let _sliding = c.u32()?; + match r.pattern { + PATTERN_DAY => {} + PATTERN_WEEK => r.weekdays = c.u32()?, + // (The Hijri and Hebrew patterns have the shape of these.) + PATTERN_MONTH | PATTERN_MONTH_END | 0x0A | 0x0C => r.day_or_nth = c.u32()?, + PATTERN_MONTH_NTH | 0x0B => { + r.weekdays = c.u32()?; + r.day_or_nth = c.u32()?; + } + _ => return Err(Error::Corrupt("recurrence of a pattern not known")), + } + r.end_type = c.u32()?; + r.count = c.u32()?; + r.first_day_of_week = c.u32()?; + r.deleted = c.dates()?; + r.modified = c.dates()?; + r.start_date = c.u32()?; + r.end_date = c.u32()?; + let _reader = c.u32()?; + let writer = c.u32()?; + r.start_offset = c.u32()?; + r.end_offset = c.u32()?; + let count = usize::from(c.u16()?); + if count > MAX_EXCEPTIONS { + return Err(Error::Corrupt("recurrence with too many exceptions")); + } + let mut flags = Vec::new(); + for _ in 0..count { + let (e, f) = exception(&mut c)?; + r.exceptions.push(e); + flags.push(f); + } + // The wide-character texts add to what is read: without them the 8-bit ones stand. + if c.skip_block().is_ok() { + for (e, f) in r.exceptions.iter_mut().zip(flags) { + if extended(&mut c, writer, e, f).is_err() { + break; + } + } + } + Ok(r) +} + +/// The Windows name of the time zone in a TZDEFINITION (`W. Europe Standard Time`). +pub fn time_zone_key(blob: &[u8]) -> Result { + let mut c = Cursor { b: blob, at: 0 }; + let head = c.take(2)?; + if head.first() != Some(&2) { + return Err(Error::Corrupt("time zone of a version not known")); + } + let _header_size = c.u16()?; + let _reserved = c.u16()?; + let key = c.wide()?; + if key.is_empty() || key.chars().count() > MAX_ZONE_KEY { + return Err(Error::Corrupt("time zone without a usable name")); + } + Ok(key) +} diff --git a/crates/pst/src/source.rs b/crates/pst/src/source.rs new file mode 100644 index 000000000..2233212d3 --- /dev/null +++ b/crates/pst/src/source.rs @@ -0,0 +1,55 @@ +//! The file under the reader: any `Read + Seek`, read a page or a block at a time and never +//! whole. Every read is checked against the file's size before anything is allocated for it. + +use std::io::{Read, Seek, SeekFrom}; + +use crate::error::{Cap, Error, Result}; + +/// The largest single read: a block ([MS-PST] 2.2.2.8: at most 8 KB). +pub(crate) const MAX_READ: usize = 8192; + +pub(crate) struct Source { + inner: R, + size: u64, + /// Bytes read so far, and how many may be ([`crate::Limits::read_multiple`]). + read: u64, + allowed: u64, +} + +impl Source { + pub(crate) fn new(mut inner: R, limits: &crate::Limits) -> Result { + let size = inner.seek(SeekFrom::End(0))?; + if size > limits.max_file_size { + return Err(Error::Cap(Cap::FileSize)); + } + let allowed = size.saturating_mul(limits.read_multiple).saturating_add(limits.read_floor); + Ok(Source { inner, size, read: 0, allowed }) + } + + pub(crate) fn size(&self) -> u64 { + self.size + } + + /// `len` bytes at `offset`, all of them inside the file. + pub(crate) fn read_at(&mut self, offset: u64, len: usize) -> Result> { + if len > MAX_READ { + return Err(Error::Corrupt("a read larger than a block")); + } + // Counted before it is checked, so that reads past the end aren't free either. + self.read = self.read.saturating_add(len as u64); + if self.read > self.allowed { + return Err(Error::Cap(Cap::TotalRead)); + } + let end = offset.checked_add(len as u64).ok_or(Error::OutOfBounds)?; + if end > self.size { + return Err(Error::OutOfBounds); + } + // Within the file's size, so a short read is the reader failing (the file cut short or + // changed under the import), not damage to the file. + let io = |e: std::io::Error| Error::Io(e.kind()); + self.inner.seek(SeekFrom::Start(offset)).map_err(io)?; + let mut buf = vec![0u8; len]; + self.inner.read_exact(&mut buf).map_err(io)?; + Ok(buf) + } +} diff --git a/crates/pst/src/tests/accounting.rs b/crates/pst/src/tests/accounting.rs new file mode 100644 index 000000000..f2a233dec --- /dev/null +++ b/crates/pst/src/tests/accounting.rs @@ -0,0 +1,177 @@ +//! Nothing that is left out vanishes: a damaged property, row or name is named with its +//! reason. And the file variants the reader is tolerant of, or refuses in words: the reserved +//! bit A of block ids, Windows-1252 passwords, versions 36 and 37. + +use std::io::Cursor; + +use super::writer::{self, Data, Layout, binary, build, hnid_ref, int32, nodes, raw, tc}; +use super::{first_item, note, pattern, sample}; +use crate::crc::crc32; +use crate::prop::types::BINARY; +use crate::{Encryption, Error, Format, MessageRef, Nid, Part, PropValue, Pst}; + +fn open(bytes: &[u8]) -> Pst> { + Pst::open(Cursor::new(bytes)).unwrap() +} + +/// A subnode id that names nothing. +const NOWHERE: u32 = (0x7777 << 5) | 0x1F; + +fn inbox(pst: &mut Pst>) -> Nid { + pst.walk_folders().unwrap().folders.iter().find(|f| f.folder.name == "Inbox").unwrap().folder.id +} + +#[test] +fn a_damaged_property_of_a_message_is_named_in_left_out() { + let mut spec = sample(Layout::new(Format::Unicode, Encryption::Permute), None); + let mut m = note("A property that points nowhere"); + m.extra = vec![(0x6001, hnid_ref(BINARY, NOWHERE)), (0x6002, binary(b"fine"))]; + spec.folders[0].messages = vec![m]; + let bytes = build(&spec); + let mut pst = open(&bytes); + let item = first_item(&mut pst); + let m = pst.message(&MessageRef::item(item)).unwrap(); + assert!(matches!(m.properties.get(0x6001), Some(PropValue::Damaged { ty: BINARY, reason: Error::NotFound(_) }))); + assert_eq!(m.properties.get(0x6002), Some(&PropValue::Binary(b"fine".to_vec()))); + assert_eq!(m.left_out.len(), 1); + assert_eq!(m.left_out[0].part, Part::Property { id: 0x6001 }); +} + +#[test] +fn a_variable_cell_too_short_for_its_reference_loses_that_property_only() { + let spec = sample(Layout::new(Format::Unicode, Encryption::None), None); + let mut all = nodes(&spec); + let (tag, number) = (0x6100_0102u32, 0x6101_0003u32); + let rows = vec![(((0x20000) << 5) | 4, vec![(tag, binary(b"ok")), (number, int32(7))])]; + let (mut blocks, subs) = tc(&spec.layout, &[tag, number], &rows); + // The binary column's description (tag, offset, width, bit): its width cut to 2 bytes. + let heap = &mut blocks[0]; + let at = heap.windows(4).position(|w| w == tag.to_le_bytes()).unwrap(); + assert_eq!(heap[at + 6], 4); + heap[at + 6] = 2; + let table = all.iter_mut().find(|n| n.nid == (0x403 << 5) | 0x0E).unwrap(); + table.data = Data::Blocks(blocks); + table.subnodes = subs; + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes); + let folder = inbox(&mut pst); + let list = pst.items(folder).unwrap(); + assert_eq!(list.items.len(), 1); + let props = &list.items[0].properties; + assert_eq!(props.get(0x6101), Some(&PropValue::Int32(7)), "the rest of the row is read"); + assert!(matches!(props.get(0x6100), Some(PropValue::Damaged { .. }))); + assert_eq!(list.left_out.len(), 1); + assert_eq!((list.left_out[0].property, &list.left_out[0].reason), (Some(0x6100), &Error::Corrupt("a structure runs past its end"))); +} + +#[test] +fn a_damaged_cell_of_an_item_list_is_named_in_left_out() { + let spec = sample(Layout::new(Format::Unicode, Encryption::None), None); + let mut all = nodes(&spec); + let tag = 0x6100_0102u32; + let rows = vec![(((0x20000) << 5) | 4, vec![(tag, hnid_ref(BINARY, NOWHERE))]), (((0x20001) << 5) | 4, vec![(tag, binary(b"ok"))])]; + let (blocks, subs) = tc(&spec.layout, &[tag], &rows); + let table = all.iter_mut().find(|n| n.nid == (0x403 << 5) | 0x0E).unwrap(); + table.data = Data::Blocks(blocks); + table.subnodes = subs; + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes); + let folder = inbox(&mut pst); + let list = pst.items(folder).unwrap(); + assert_eq!(list.items.len(), 2); + assert_eq!(list.left_out.len(), 1); + let l = &list.left_out[0]; + assert_eq!((l.id, l.parent, l.property), (Nid(((0x20000) << 5) | 4), Some(folder), Some(0x6100))); + assert!(matches!(l.reason, Error::NotFound(_))); +} + +#[test] +fn a_row_that_cannot_be_read_is_listed_and_named() { + // The row index names a row id that its row in the matrix doesn't have. + let spec = sample(Layout::new(Format::Unicode, Encryption::None), None); + let mut all = nodes(&spec); + let rows = vec![(((0x20000) << 5) | 4, vec![(0x0E07_0003, int32(1))])]; + let (mut blocks, subs) = tc(&spec.layout, &[0x0E07_0003], &rows); + // The matrix is the heap item holding the row id: change the id in it, not in the index. + let id = (((0x20000u32) << 5) | 4).to_le_bytes(); + let block = &mut blocks[0]; + let at = block.windows(4).rposition(|w| w == id).unwrap(); + block[at] ^= 0x80; + let table = all.iter_mut().find(|n| n.nid == (0x403 << 5) | 0x0E).unwrap(); + table.data = Data::Blocks(blocks); + table.subnodes = subs; + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes); + let folder = inbox(&mut pst); + let list = pst.items(folder).unwrap(); + assert_eq!(list.items.len(), 1); + assert!(list.items[0].properties.is_empty()); + assert_eq!(list.left_out.len(), 1); + assert_eq!(list.left_out[0].property, None); + assert!(matches!(list.left_out[0].reason, Error::Corrupt(_) | Error::NotFound(_)), "{:?}", list.left_out[0].reason); +} + +#[test] +fn passwords_are_also_tried_in_windows_1252() { + for (password, ansi) in [("Müller", true), ("Müller", false), ("plain", true)] { + let mut spec = sample(Layout::new(Format::Ansi, Encryption::Permute), Some(password)); + spec.password_ansi = ansi; + let bytes = build(&spec); + let mut pst = open(&bytes); + assert!(pst.password_required()); + assert_eq!(pst.unlock("Muller"), Err(Error::WrongPassword)); + pst.unlock(password).unwrap(); + } + // A character of the 0x80 to 0x9F range of the code page (the euro sign, 0x80). + let mut spec = sample(Layout::new(Format::Unicode, Encryption::None), Some("x")); + let mut all = nodes(&spec); + spec.password = None; + let mut spill = writer::Spill::new(); + let store = writer::pc(&spec.layout, &[(0x3001, writer::unicode("Personal Folders")), (0x67FF, int32(crc32(&[0x80, b'5']) as i32))], &mut spill); + let n = all.iter_mut().find(|n| n.nid == 0x21).unwrap(); + n.data = Data::Blocks(store); + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes); + assert_eq!(pst.unlock("e5"), Err(Error::WrongPassword)); + pst.unlock("€5").unwrap(); +} + +fn with_version(version: u16) -> Vec { + let mut b = build(&sample(Layout::new(Format::Unicode, Encryption::None), None)); + b[10..12].copy_from_slice(&version.to_le_bytes()); + let full = crc32(&b[8..524]); + b[524..528].copy_from_slice(&full.to_le_bytes()); + let partial = crc32(&b[8..479]); + b[4..8].copy_from_slice(&partial.to_le_bytes()); + b +} + +#[test] +fn offline_files_with_4k_pages_and_protected_files_are_refused_in_words() { + for (version, words) in [(36u16, "4 KB pages"), (37, "Windows Information Protection")] { + let b = with_version(version); + let e = Pst::open(Cursor::new(&b[..])).err().unwrap(); + assert_eq!(e, Error::UnsupportedVersion(version)); + assert!(e.to_string().contains(words), "{e}"); + } + assert!(Pst::open(Cursor::new(&with_version(24)[..])).is_ok(), "other Unicode versions are read"); +} + +#[test] +fn the_reserved_bit_a_of_block_ids_is_ignored() { + for (refs, trailers) in [(true, false), (false, true), (true, true)] { + for format in [Format::Unicode, Format::Ansi] { + let layout = Layout { bit_a_in_references: refs, bit_a_in_trailers: trailers, ..Layout::tiny(format, Encryption::Cyclic) }; + let bytes = build(&sample(layout, None)); + let mut pst = open(&bytes); + let what = format!("{format:?} refs {refs} trailers {trailers}"); + let tree = pst.walk_folders().unwrap(); + assert!(tree.left_out.is_empty(), "{what}: {:?}", tree.left_out); + assert_eq!(tree.folders.len(), 7, "{what}"); + let item = pst.items(tree.folders[2].folder.id).unwrap().items[0].id; + let m = pst.message(&MessageRef::item(item)).unwrap(); + assert!(m.left_out.is_empty(), "{what}: {:?}", m.left_out); + assert_eq!(pst.attachment_bytes(&m.attachments[0].id).unwrap(), pattern(20_000, 7), "{what}"); + } + } +} diff --git a/crates/pst/src/tests/caps.rs b/crates/pst/src/tests/caps.rs new file mode 100644 index 000000000..37c6ad32b --- /dev/null +++ b/crates/pst/src/tests/caps.rs @@ -0,0 +1,195 @@ +//! Each cap of [`Limits`]: past it, the part is left out and named, and nothing panics. + +use std::io::{self, Cursor, Read, Seek, SeekFrom}; + +use super::writer::{AttachSpec, Data, FolderSpec, Layout, Node, build, raw}; +use super::{note, person, sample}; +use crate::{AttachmentContent, Cap, Encryption, Error, Format, Limits, MessageRef, Nid, Part, Pst}; + +fn open(bytes: &[u8], limits: Limits) -> Pst> { + Pst::open_with(Cursor::new(bytes), limits).unwrap() +} + +fn unicode() -> Layout { + Layout::new(Format::Unicode, Encryption::Permute) +} + +/// A reader of a file that says it is 200 GB and holds nothing but zeros. +struct Huge(u64); + +impl Read for Huge { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + buf.fill(0); + Ok(buf.len()) + } +} + +impl Seek for Huge { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + self.0 = match pos { + SeekFrom::End(_) => 200 * 1024 * 1024 * 1024, + SeekFrom::Start(n) => n, + SeekFrom::Current(n) => self.0.saturating_add_signed(n), + }; + Ok(self.0) + } +} + +#[test] +fn a_file_over_100_gb_is_refused_before_it_is_read() { + assert_eq!(Pst::open(Huge(0)).err(), Some(Error::Cap(Cap::FileSize))); + let bytes = build(&sample(unicode(), None)); + let small = Limits { max_file_size: 1000, ..Default::default() }; + assert_eq!(Pst::open_with(Cursor::new(&bytes[..]), small).err(), Some(Error::Cap(Cap::FileSize))); +} + +#[test] +fn items_past_the_cap_are_counted_and_left_out() { + let bytes = build(&sample(unicode(), None)); + let mut pst = open(&bytes, Limits { max_items: 3, ..Default::default() }); + let tree = pst.walk_folders().unwrap(); + let id = |name: &str| tree.folders.iter().find(|f| f.folder.name == name).unwrap().folder.id; + let inbox = pst.items(id("Inbox")).unwrap(); + assert_eq!((inbox.items.len(), inbox.left_out.len()), (2, 0)); + let projects = pst.items(id("Projects")).unwrap(); + assert_eq!((projects.items.len(), projects.left_out.len()), (1, 0)); + let calendar = pst.items(id("Calendar")).unwrap(); + assert_eq!((calendar.items.len(), calendar.past_cap), (0, 1)); + assert_eq!(calendar.left_out, vec![crate::LeftOut { id: id("Calendar"), parent: None, property: None, reason: Error::Cap(Cap::Items) }]); + // A table of more rows than the cap is refused as it is read. + let mut small = open(&bytes, Limits { max_items: 1, ..Default::default() }); + assert_eq!(small.items(id("Inbox")).err(), Some(Error::Cap(Cap::Items))); + // Listing a folder again doesn't count it twice. + let again = pst.items(id("Inbox")).unwrap(); + assert_eq!((again.items.len(), again.left_out.len()), (2, 0)); +} + +#[test] +fn folders_past_the_cap_are_left_out() { + let mut spec = sample(unicode(), None); + spec.folders[2].children = (0..20).map(|i| FolderSpec { name: format!("Old {i}"), ..Default::default() }).collect(); + let bytes = build(&spec); + let mut pst = open(&bytes, Limits { max_folders: 6, ..Default::default() }); + let tree = pst.walk_folders().unwrap(); + assert!(tree.folders.len() <= 6, "{}", tree.folders.len()); + assert!(tree.left_out.iter().any(|l| l.reason == Error::Cap(Cap::Folders))); + // A hierarchy table with more rows than the cap is refused as a whole. + assert_eq!(pst.subfolders(tree.folders.iter().find(|f| f.folder.name == "Contacts").unwrap().folder.id).err(), Some(Error::Cap(Cap::Folders))); +} + +#[test] +fn folders_nested_past_the_cap_are_left_out() { + let mut spec = sample(unicode(), None); + let mut deep = FolderSpec { name: "Level 6".into(), ..Default::default() }; + for level in (2..6).rev() { + deep = FolderSpec { name: format!("Level {level}"), children: vec![deep], ..Default::default() }; + } + spec.folders[2].children = vec![deep]; + let bytes = build(&spec); + let mut pst = open(&bytes, Limits { max_folder_depth: 4, ..Default::default() }); + let tree = pst.walk_folders().unwrap(); + assert!(tree.folders.iter().all(|f| f.depth <= 4)); + let left: Vec<&Error> = tree.left_out.iter().map(|l| &l.reason).collect(); + assert_eq!(left, vec![&Error::Cap(Cap::FolderDepth)]); +} + +#[test] +fn recipients_and_attachments_past_their_caps_are_left_out() { + let bytes = build(&sample(unicode(), None)); + let mut pst = open(&bytes, Limits { max_recipients: 2, max_attachments: 1, ..Default::default() }); + let inbox = pst.walk_folders().unwrap().folders[2].folder.id; + let item = pst.items(inbox).unwrap().items[0].id; + let m = pst.message(&MessageRef::item(item)).unwrap(); + assert_eq!(m.recipients.len(), 2); + assert_eq!(m.attachments.len(), 1); + let parts: Vec<&Part> = m.left_out.iter().map(|p| &p.part).collect(); + assert_eq!(parts, vec![&Part::Recipients { count: 1 }, &Part::Attachments { count: 1 }]); + assert_eq!(m.left_out[0].reason, Error::Cap(Cap::Recipients)); + assert_eq!(m.left_out[1].reason, Error::Cap(Cap::Attachments)); +} + +#[test] +fn an_attachment_past_its_cap_is_refused_before_it_is_read() { + let bytes = build(&sample(unicode(), None)); + let mut pst = open(&bytes, Limits { max_attachment_bytes: 10_000, ..Default::default() }); + let inbox = pst.walk_folders().unwrap().folders[2].folder.id; + let item = pst.items(inbox).unwrap().items[0].id; + let m = pst.message(&MessageRef::item(item)).unwrap(); + assert_eq!(m.attachments[0].content, AttachmentContent::Data { size: 20_000 }); + assert_eq!(pst.attachment_reader(&m.attachments[0].id).err(), Some(Error::Cap(Cap::AttachmentSize))); + assert_eq!(pst.attachment_bytes(&m.attachments[0].id).err(), Some(Error::Cap(Cap::AttachmentSize))); +} + +#[test] +fn an_item_past_its_cap_is_left_out() { + let mut spec = sample(unicode(), None); + spec.folders[0].messages[1].body = "Words and more words. ".repeat(500); + let bytes = build(&spec); + let mut pst = open(&bytes, Limits { max_item_bytes: 4096, ..Default::default() }); + let inbox = pst.walk_folders().unwrap().folders[2].folder.id; + let items = pst.items(inbox).unwrap(); + assert_eq!(pst.message(&MessageRef::item(items.items[1].id)).err(), Some(Error::Cap(Cap::ItemSize))); +} + +#[test] +fn messages_attached_past_the_depth_cap_are_left_out() { + let mut spec = sample(unicode(), None); + let mut nested = note("Innermost"); + for level in 0..4 { + let mut outer = note(&format!("Wrapper {level}")); + outer.attachments = vec![AttachSpec::Message(Box::new(nested))]; + nested = outer; + } + spec.folders[0].messages = vec![nested]; + let bytes = build(&spec); + let mut pst = open(&bytes, Limits { max_embedded_depth: 2, ..Default::default() }); + let inbox = pst.walk_folders().unwrap().folders[2].folder.id; + let first = pst.items(inbox).unwrap().items[0].id; + let mut m = pst.message(&MessageRef::item(first)).unwrap(); + let mut depth = 0; + loop { + match m.attachments[0].content.clone() { + AttachmentContent::Message(inner) => { + depth += 1; + m = pst.message(&inner).unwrap(); + } + AttachmentContent::MessageTooDeep => break, + other => panic!("{other:?}"), + } + } + assert_eq!(depth, 2); + let too_deep = MessageRef { node: m.id.node, path: vec![Nid(1); 6] }; + assert_eq!(pst.message(&too_deep).err(), Some(Error::Cap(Cap::EmbeddedDepth))); +} + +#[test] +fn an_item_with_too_many_parts_is_left_out() { + let mut spec = sample(unicode(), None); + let mut m = note("Many files"); + m.recipients = vec![person("Ada Fernsby", "ada", 1)]; + m.attachments = (0..40).map(|i| AttachSpec::File { name: format!("f{i}.txt"), mime: "text/plain".into(), data: vec![b'x'; 10] }).collect(); + spec.folders[0].messages = vec![m]; + let bytes = build(&spec); + let mut pst = open(&bytes, Limits { max_subnodes: 20, ..Default::default() }); + let inbox = pst.walk_folders().unwrap().folders[2].folder.id; + let item = pst.items(inbox).unwrap().items[0].id; + assert_eq!(pst.message(&MessageRef::item(item)).err(), Some(Error::Cap(Cap::Subnodes))); +} + +#[test] +fn data_spread_over_too_many_blocks_is_refused() { + // The message store's data in 600 blocks of one byte each. + let layout = Layout { block_data: 1, xblock_ids: 1000, ..Layout::new(Format::Unicode, Encryption::None) }; + let nodes = vec![Node { nid: 0x21, parent: 0, data: Data::Stream(vec![0xEC; 600]), subnodes: vec![] }]; + let bytes = raw(&layout, &nodes); + assert_eq!(Pst::open(Cursor::new(&bytes[..])).err(), Some(Error::Cap(Cap::DataBlocks))); +} + +#[test] +fn a_store_past_the_item_cap_is_refused_before_it_is_read() { + let layout = Layout { block_data: 8000, ..Layout::new(Format::Unicode, Encryption::None) }; + let nodes = vec![Node { nid: 0x21, parent: 0, data: Data::Stream(vec![0; 40_000]), subnodes: vec![] }]; + let bytes = raw(&layout, &nodes); + let limits = Limits { max_item_bytes: 30_000, ..Default::default() }; + assert_eq!(Pst::open_with(Cursor::new(&bytes[..]), limits).err(), Some(Error::Cap(Cap::ItemSize))); +} diff --git a/crates/pst/src/tests/cost.rs b/crates/pst/src/tests/cost.rs new file mode 100644 index 000000000..9e5873ac1 --- /dev/null +++ b/crates/pst/src/tests/cost.rs @@ -0,0 +1,583 @@ +//! Files made to cost far more to read than their size: one value or block named many times, +//! loads that were not counted, attached messages that hold themselves or multiply. Each is +//! held to a budget for the whole call, or refused. + +use std::io::Cursor; + +use super::writer::{self, AttachSpec, Data, FolderSpec, Layout, Node, Spill, SubNode, binary, hnid_ref, int32, nodes, pc, raw, tc, unicode}; +use super::{first_item, note, pattern, sample}; +use crate::header::{HEADER_READ, Header}; +use crate::ndb::Ndb; +use crate::prop::types::BINARY; +use crate::source::Source; +use crate::{AttachmentContent, Cap, Encryption, Error, Format, Limits, MessageRef, Nid, Part, PropName, PropValue, Pst}; + +fn unicode_layout() -> Layout { + Layout::new(Format::Unicode, Encryption::Permute) +} + +fn open(bytes: &[u8], limits: Limits) -> Pst> { + Pst::open_with(Cursor::new(bytes), limits).unwrap() +} + +/// The first spilled value of a fresh [`Spill`] (and of a table's): its subnode id. +const FIRST_SPILL: u32 = (0x101 << 5) | 0x1F; + +/// Bytes kept in binary values, to measure what a walk or a listing holds. +fn binary_bytes<'a>(props: impl Iterator) -> usize { + props.map(|p| if let PropValue::Binary(b) = &p.value { b.len() } else { 0 }).sum() +} + +fn inbox(pst: &mut Pst>) -> Nid { + pst.walk_folders().unwrap().folders.iter().find(|f| f.folder.name == "Inbox").unwrap().folder.id +} + +#[test] +fn named_property_names_are_capped_one_by_one_and_in_all() { + // One 2,048-byte name, then 5,000 entries that all name the same 1,000-byte string: a + // stream of 3 KB that would come out as 5 MB of names. + let mut strings = Vec::new(); + for len in [2048usize, 1000] { + strings.extend_from_slice(&(len as u32).to_le_bytes()); + strings.extend(std::iter::repeat_n(b'a', len).enumerate().map(|(i, c)| if i % 2 == 0 { c } else { 0 })); + } + let mut entries = Vec::new(); + for i in 0..5001u32 { + let at: u32 = if i == 0 { 0 } else { 2052 }; + entries.extend_from_slice(&at.to_le_bytes()); + entries.extend_from_slice(&((2u16 << 1) | 1).to_le_bytes()); + entries.extend_from_slice(&(i as u16).to_le_bytes()); + } + let spec = sample(unicode_layout(), None); + let mut all = nodes(&spec); + let mut spill = Spill::new(); + let blocks = pc(&spec.layout, &[(0x0001, int32(251)), (0x0002, binary(&[])), (0x0003, binary(&entries)), (0x0004, binary(&strings))], &mut spill); + let map = all.iter_mut().find(|n| n.nid == 0x61).unwrap(); + map.data = Data::Blocks(blocks); + map.subnodes = spill.subnodes; + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes, Limits::default()); + let named = pst.named_properties().unwrap(); + assert_eq!(named.left_out()[0], (Some(0x8000), Error::Cap(Cap::NamedProperties)), "a name over 1 KB is left out"); + assert!(named.left_out().iter().all(|(_, e)| *e == Error::Cap(Cap::NamedProperties))); + assert_eq!(named.len() + named.left_out().len(), 5001, "every entry is read or named as left out"); + let kept: usize = named.iter().map(|p| if let PropName::Name(n) = &p.name { n.len() * 2 } else { 0 }).sum(); + assert!(kept <= crate::named::MAX_NAMES, "{kept}"); + assert!(named.len() >= 4000, "names within the total are kept: {}", named.len()); +} + +/// The file's folders under Contacts, each with 64 binary properties that all name one value of +/// `size` bytes. +fn aliased_folders(size: usize, count: usize) -> (Layout, Vec) { + let mut spec = sample(unicode_layout(), None); + spec.folders[2].children = (0..count).map(|i| FolderSpec { name: format!("Box {i}"), ..Default::default() }).collect(); + let mut all = nodes(&spec); + for n in all.iter_mut().filter(|n| n.nid & 0x1F == 2 && n.nid != 0x122 && n.nid != writer::IPM_SUBTREE) { + let mut spill = Spill::new(); + let blob = spill.put(0x1F, pattern(size, 3)); + assert_eq!(blob, FIRST_SPILL); + let mut props = vec![(0x3001, unicode("Aliased"))]; + props.extend((0..64u16).map(|i| (0x6000 + i, hnid_ref(BINARY, blob)))); + n.data = Data::Blocks(pc(&spec.layout, &props, &mut spill)); + n.subnodes = spill.subnodes; + } + let bytes = raw(&spec.layout, &all); + (spec.layout, bytes) +} + +#[test] +fn a_folder_walk_reads_within_one_budget() { + // 25 folders × 64 names of one 60 KB value: 96 MB from a file of under 2 MB. + let (_, bytes) = aliased_folders(60_000, 20); + assert!(bytes.len() < 2_000_000); + let limit = 4 * 1024 * 1024; + let mut pst = open(&bytes, Limits { max_listing_bytes: limit, ..Default::default() }); + let tree = pst.walk_folders().unwrap(); + let kept: usize = tree.folders.iter().map(|f| binary_bytes(f.folder.properties.iter())).sum(); + assert!(kept as u64 <= limit, "{kept}"); + assert!(tree.left_out.iter().any(|l| l.reason == Error::Cap(Cap::ListingSize)), "{:?}", tree.left_out); +} + +#[test] +fn a_folder_walk_leaves_large_values_for_later() { + let (_, bytes) = aliased_folders(100_000, 1); + let mut pst = open(&bytes, Limits::default()); + let tree = pst.walk_folders().unwrap(); + let boxed = tree.folders.iter().find(|f| f.folder.name == "Aliased").unwrap().folder.clone(); + assert_eq!(boxed.properties.get(0x6000), Some(&PropValue::Large { size: 100_000 }), "past the summary cap"); + let whole = pst.folder(boxed.id).unwrap(); + assert_eq!(whole.properties.get(0x6000), Some(&PropValue::Binary(pattern(100_000, 3))), "read whole on its own"); +} + +#[test] +fn a_listing_of_items_reads_within_one_budget() { + // 200 rows whose binary cell names the first row's 50 KB value: 10 MB from 60 KB. + let spec = sample(unicode_layout(), None); + let mut all = nodes(&spec); + let tag = 0x6100_0102u32; + let rows: Vec<(u32, Vec<(u32, writer::Value)>)> = (0..200u32) + .map(|i| { + let v = if i == 0 { binary(&pattern(50_000, 9)) } else { hnid_ref(BINARY, FIRST_SPILL) }; + (((0x20000 + i) << 5) | 4, vec![(tag, v)]) + }) + .collect(); + let (blocks, subs) = tc(&spec.layout, &[tag], &rows); + let inbox_contents = (0x403 << 5) | 0x0E; + let table = all.iter_mut().find(|n| n.nid == inbox_contents).unwrap(); + table.data = Data::Blocks(blocks); + table.subnodes = subs; + let bytes = raw(&spec.layout, &all); + assert!(bytes.len() < 200_000, "{}", bytes.len()); + let limit = 1024 * 1024; + let mut pst = open(&bytes, Limits { max_listing_bytes: limit, ..Default::default() }); + let folder = inbox(&mut pst); + let list = pst.items(folder).unwrap(); + // Listed until the budget is spent; the rest are counted, and the folder named once. + assert!(!list.items.is_empty() && list.items.len() < 200, "{}", list.items.len()); + assert_eq!(list.past_cap, 200 - list.items.len()); + let kept: usize = list.items.iter().map(|i| binary_bytes(i.properties.iter())).sum(); + assert!(kept as u64 <= limit, "{kept}"); + let short: Vec<&crate::LeftOut> = list.left_out.iter().filter(|l| l.reason == Error::Cap(Cap::ListingSize)).collect(); + assert_eq!(short.len(), 1); + assert_eq!((short[0].id, short[0].parent), (folder, None)); +} + +#[test] +fn a_data_tree_naming_one_block_twice_is_refused() { + for levels in [1u8, 2] { + let layout = Layout::new(Format::Unicode, Encryption::None); + let store = Node { nid: 0x21, parent: 0, data: Data::Repeated { piece: vec![0xEC; 8000], times: 32, levels }, subnodes: vec![] }; + let bytes = raw(&layout, &[store]); + assert!(bytes.len() < 30_000); + assert_eq!(Pst::open(Cursor::new(&bytes[..])).err(), Some(Error::Cycle("a data tree")), "levels {levels}"); + } +} + +#[test] +fn every_load_of_a_message_is_charged_to_its_budget() { + // 30 attachments whose data (3,000 bytes each) is kept in their own heaps: each heap is + // loaded, and counts, though the data itself is left for later. + let mut spec = sample(unicode_layout(), None); + let mut m = note("Many small files"); + m.attachments = + (0..30).map(|i| AttachSpec::File { name: format!("f{i}.bin"), mime: "application/octet-stream".into(), data: pattern(3000, i) }).collect(); + spec.folders[0].messages = vec![m]; + let bytes = build_spec(&spec); + let mut pst = open(&bytes, Limits::default()); + let item = first_item(&mut pst); + assert_eq!(pst.message(&MessageRef::item(item)).unwrap().attachments.len(), 30); + let mut pst = open(&bytes, Limits { max_item_bytes: 60_000, ..Default::default() }); + let item = first_item(&mut pst); + assert_eq!(pst.message(&MessageRef::item(item)).err(), Some(Error::Cap(Cap::ItemSize))); +} + +fn build_spec(spec: &writer::FileSpec) -> Vec { + writer::build(spec) +} + +#[test] +fn all_that_is_read_is_held_to_the_file_size() { + let bytes = build_spec(&sample(unicode_layout(), None)); + let mut pst = open(&bytes, Limits { read_multiple: 3, read_floor: 0, ..Default::default() }); + let mut stopped = None; + for _ in 0..100 { + // A walk names what it could not read in its left_out, or fails as a whole. + let reason = match pst.walk_folders() { + Err(e) => Some(e), + Ok(tree) => tree.left_out.first().map(|l| l.reason.clone()), + }; + if reason.is_some() { + stopped = reason; + break; + } + } + assert_eq!(stopped, Some(Error::Cap(Cap::TotalRead))); + let mut pst = open(&bytes, Limits::default()); + for _ in 0..20 { + pst.walk_folders().unwrap(); + } +} + +/// A file whose first Inbox item has one attachment: a message whose object is `object`. +fn with_object(object: SubNode) -> Vec { + let mut spec = sample(unicode_layout(), None); + let mut m = note("Holds itself"); + m.attachments = vec![AttachSpec::Object(object)]; + spec.folders[0].messages = vec![m]; + build_spec(&spec) +} + +const OBJECT: u32 = (0x300 << 5) | 0x04; +const ATTACHMENT: u32 = (0x200 << 5) | 0x05; + +/// The data and subnode tree of a node, read from the file's node tree. +fn node_bids(bytes: &[u8], nid: u32) -> (u64, u64) { + let limits = Limits::default(); + let mut src = Source::new(Cursor::new(bytes), &limits).unwrap(); + let header = Header::parse(&src.read_at(0, HEADER_READ).unwrap()).unwrap(); + let e = Ndb::new(src, header, limits).node(nid).unwrap(); + (e.data, e.sub) +} + +#[test] +fn an_attached_message_that_is_the_message_it_is_attached_to_is_a_cycle() { + // First with a stand-in for the object, to learn where the message's own data and subnode + // tree are; then with the object naming them. Nothing else moves between the two. + let first = with_object(SubNode::with_sub_bid(OBJECT, 4, 6)); + let mut pst = open(&first, Limits::default()); + let item = first_item(&mut pst); + let (data, sub) = node_bids(&first, item.0); + let bytes = with_object(SubNode::with_sub_bid(OBJECT, data, sub)); + assert_eq!(node_bids(&bytes, item.0), (data, sub)); + let mut pst = open(&bytes, Limits::default()); + let m = pst.message(&MessageRef::item(item)).unwrap(); + assert!(m.attachments.is_empty()); + assert_eq!(m.left_out[0].part, Part::Attachment { id: Nid(ATTACHMENT) }); + assert_eq!(m.left_out[0].reason, Error::Cycle("an attached message")); + let inner = MessageRef { node: item, path: vec![Nid(ATTACHMENT), Nid(OBJECT)] }; + assert_eq!(pst.message(&inner).err(), Some(Error::Cycle("an attached message"))); +} + +#[test] +fn an_attachment_table_naming_a_row_twice_is_refused() { + let mut spec = sample(unicode_layout(), None); + let mut m = note("Twice"); + m.attachments = vec![ + AttachSpec::File { name: "a.txt".into(), mime: "text/plain".into(), data: b"one".to_vec() }, + AttachSpec::File { name: "b.txt".into(), mime: "text/plain".into(), data: b"two".to_vec() }, + ]; + spec.folders[0].messages = vec![m]; + let mut all = nodes(&spec); + let message = all.iter_mut().find(|n| n.nid & 0x1F == 4).unwrap(); + let rows = vec![(ATTACHMENT, vec![(0x3705_0003, int32(1))]), (ATTACHMENT, vec![(0x3705_0003, int32(1))])]; + let (blocks, subs) = tc(&spec.layout, &[0x3705_0003], &rows); + let table = message.subnodes.iter_mut().find(|s| s.nid == 0x671).unwrap(); + table.data = Data::Blocks(blocks); + table.subnodes = subs; + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes, Limits::default()); + let item = first_item(&mut pst); + // A table's row ids are the keys of its row index, which must strictly increase. + let m = pst.message(&MessageRef::item(item)).unwrap(); + assert!(m.attachments.is_empty()); + assert_eq!(m.left_out[0].part, Part::AttachmentTable); + assert_eq!(m.left_out[0].reason, Error::Corrupt("a B-tree-on-heap's keys out of order")); +} + +#[test] +fn attached_messages_are_counted_per_item() { + // Three attached messages, each with three of its own: twelve under the item. + let mut spec = sample(unicode_layout(), None); + let leaf = note("Leaf"); + let mut middle = note("Middle"); + middle.attachments = vec![AttachSpec::Message(Box::new(leaf)); 3]; + let mut top = note("Top"); + top.attachments = vec![AttachSpec::Message(Box::new(middle)); 3]; + spec.folders[0].messages = vec![top.clone(), top]; + let bytes = build_spec(&spec); + let mut pst = open(&bytes, Limits { max_embedded_messages: 5, ..Default::default() }); + let folder = inbox(&mut pst); + let items = pst.items(folder).unwrap().items; + // The first item twice: reading it again starts its count again. + for item in [&items[0], &items[0], &items[1]] { + let (mut read, mut refused) = (0, 0); + let mut queue = vec![MessageRef::item(item.id)]; + while let Some(m) = queue.pop() { + match pst.message(&m) { + Ok(msg) => { + read += usize::from(!m.path.is_empty()); + for a in msg.attachments { + if let AttachmentContent::Message(inner) = a.content { + queue.push(inner); + } + } + } + Err(e) => { + assert_eq!(e, Error::Cap(Cap::EmbeddedMessages)); + refused += 1; + } + } + } + assert_eq!((read, refused), (5, 4), "counted again for each item"); + } +} + +/// A heap block (not the first) whose page map holds `k` items that all span the same leaf of +/// `l` property records: the odd items run from its start to its end, the even ones back. +fn aliasing_block(k: usize, l: usize) -> Vec { + let mut b = vec![0u8; 2]; + let start = b.len() as u16; + for i in 0..l { + b.extend_from_slice(&((0x6000 + (i % 0x1000)) as u16).to_le_bytes()); + b.extend_from_slice(&0x0003u16.to_le_bytes()); + b.extend_from_slice(&(i as u32).to_le_bytes()); + } + let end = b.len() as u16; + b[0..2].copy_from_slice(&end.to_le_bytes()); + b.extend_from_slice(&((2 * k - 1) as u16).to_le_bytes()); + b.extend_from_slice(&0u16.to_le_bytes()); + for i in 0..2 * k { + b.extend_from_slice(&(if i % 2 == 0 { start } else { end }).to_le_bytes()); + } + b +} + +/// A message store whose property context's index names every aliasing item of `blocks` +/// aliasing blocks: `blocks × k × l` records from a few kilobytes. +fn aliased_store(blocks: usize, k: usize, l: usize) -> Vec { + let mut index = Vec::new(); + for j in 1..=blocks { + for i in 0..k { + index.extend_from_slice(&0x6000u16.to_le_bytes()); + index.extend_from_slice(&(((j as u32) << 16) | (((2 * i + 1) as u32) << 5)).to_le_bytes()); + } + } + let mut b0 = vec![0u8; 12]; + b0[2] = 0xEC; + b0[3] = 0xBC; + b0[4..8].copy_from_slice(&(1u32 << 5).to_le_bytes()); + let s1 = b0.len() as u16; + b0.extend_from_slice(&[0xB5, 2, 6, 1]); + b0.extend_from_slice(&(2u32 << 5).to_le_bytes()); + let s2 = b0.len() as u16; + b0.extend_from_slice(&index); + let s3 = b0.len() as u16; + b0[0..2].copy_from_slice(&s3.to_le_bytes()); + b0.extend_from_slice(&2u16.to_le_bytes()); + b0.extend_from_slice(&0u16.to_le_bytes()); + for o in [s1, s2, s3] { + b0.extend_from_slice(&o.to_le_bytes()); + } + let mut heap = vec![b0]; + heap.extend((0..blocks).map(|_| aliasing_block(k, l))); + let layout = Layout::new(Format::Unicode, Encryption::None); + let mut all = nodes(&sample(layout.clone(), None)); + let store = all.iter_mut().find(|n| n.nid == 0x21).unwrap(); + store.data = Data::Blocks(heap); + store.subnodes = vec![]; + raw(&layout, &all) +} + +#[test] +fn heap_items_that_share_bytes_are_refused() { + // 2 blocks × 500 items × 450 records: 450,000 properties from a file of about 70 KB. + let bytes = aliased_store(2, 500, 450); + assert!(bytes.len() < 100_000, "{}", bytes.len()); + assert_eq!(Pst::open(Cursor::new(&bytes[..])).err(), Some(Error::Corrupt("a heap's page map"))); + // A page map whose offsets run back, alone. + let mut block = vec![12u8, 0, 0xEC, 0xBC, 0, 0, 0, 0, 0, 0, 0, 0]; + block.extend_from_slice(&[2, 0, 0, 0, 12, 0, 8, 0, 12, 0]); + assert!(crate::heap::Heap::new(vec![block]).is_err()); +} + +#[test] +fn a_property_context_naming_an_id_twice_is_refused() { + // 65,537 records: one id is named twice, whatever the order. + let spec = sample(unicode_layout(), None); + let mut all = nodes(&spec); + let props: Vec<(u16, writer::Value)> = (0..=65_536u32).map(|i| (i as u16, int32(1))).collect(); + let mut spill = Spill::new(); + let store = all.iter_mut().find(|n| n.nid == 0x21).unwrap(); + store.data = Data::Blocks(pc(&spec.layout, &props, &mut spill)); + let bytes = raw(&spec.layout, &all); + assert_eq!(Pst::open(Cursor::new(&bytes[..])).err(), Some(Error::Corrupt("a B-tree-on-heap's keys out of order"))); +} + +#[test] +fn a_table_that_cannot_be_read_is_named_not_taken_for_none() { + // The Inbox's hierarchy and contents tables name a block that isn't in the file. + let spec = sample(unicode_layout(), None); + let mut all = nodes(&spec); + for kind in [0x0D, 0x0E] { + let table = all.iter_mut().find(|n| n.nid == (0x403 << 5) | kind).unwrap(); + table.data = Data::Bid(0x9_9990); + table.subnodes = vec![]; + } + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes, Limits::default()); + let tree = pst.walk_folders().unwrap(); + let inbox = Nid((0x403 << 5) | 2); + assert!(tree.folders.iter().any(|f| f.folder.id == inbox)); + assert!(!tree.folders.iter().any(|f| f.folder.name == "Projects")); + assert_eq!(tree.left_out.len(), 1); + assert_eq!((tree.left_out[0].id, &tree.left_out[0].reason), (inbox, &Error::NotFound("a block"))); + assert_eq!(pst.items(inbox).err(), Some(Error::NotFound("a block"))); + // A folder with no tables at all has no subfolders and no items. + let mut all = nodes(&spec); + // The folder made last (Projects), without its tables. + let bare = Nid(all.iter().filter(|n| n.nid & 0x1F == 2).map(|n| n.nid).max().unwrap()); + all.retain(|n| n.nid & !0x1F != bare.0 & !0x1F || n.nid == bare.0); + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes, Limits::default()); + assert_eq!(pst.subfolders(bare), Ok(vec![])); + assert!(pst.items(bare).unwrap().items.is_empty()); +} + +#[test] +fn an_item_and_its_attached_messages_share_one_budget() { + let body = "Words, and more words. ".repeat(1500); + let mut inner = note("Inside"); + inner.body = body.clone(); + let mut top = note("Outside"); + top.body = body; + top.attachments = vec![AttachSpec::Message(Box::new(inner))]; + let mut spec = sample(unicode_layout(), None); + spec.folders[0].messages = vec![top]; + let bytes = build_spec(&spec); + // Each message costs about 105 KB, read and kept: one fits in 150 KB, the two together don't. + let mut pst = open(&bytes, Limits { max_item_bytes: 150_000, ..Default::default() }); + let item = first_item(&mut pst); + let m = pst.message(&MessageRef::item(item)).unwrap(); + let AttachmentContent::Message(inner) = m.attachments[0].content.clone() else { panic!() }; + assert_eq!(pst.message(&inner).err(), Some(Error::Cap(Cap::ItemSize))); + // Reading the item again starts its budget again. + pst.message(&MessageRef::item(item)).unwrap(); + let mut pst = open(&bytes, Limits::default()); + pst.message(&MessageRef::item(item)).unwrap(); + pst.message(&inner).unwrap(); +} + +/// What a call's answer keeps, as the reader charges it: each structure, its strings, and +/// each property with what it holds. (A vector's spare room, at most as much again, is not.) +fn kept_props(p: &crate::Properties) -> usize { + p.iter().map(|p| p.kept() as usize).sum() +} + +fn kept_text(s: &Option) -> usize { + s.as_ref().map_or(0, String::capacity) +} + +#[test] +fn what_a_call_keeps_is_within_its_budget() { + use std::mem::size_of; + // One byte on file becomes a whole property kept: 250 one-byte columns in 2,000 rows, + // 8,000 one-byte properties in a message, 300 folders. + let mut spec = sample(unicode_layout(), None); + spec.folders[0].children = (0..300).map(|i| FolderSpec { name: format!("Folder {i}"), ..Default::default() }).collect(); + let mut all = nodes(&spec); + let columns: Vec = (0..250u32).map(|i| ((0x6800 + i) << 16) | 0x000B).collect(); + let rows: Vec<(u32, Vec<(u32, writer::Value)>)> = + (0..2000u32).map(|i| (((0x20000 + i) << 5) | 4, columns.iter().map(|&c| (c, writer::boolean(true))).collect())).collect(); + let (blocks, subs) = tc(&spec.layout, &columns, &rows); + let table = all.iter_mut().find(|n| n.nid == (0x403 << 5) | 0x0E).unwrap(); + table.data = Data::Blocks(blocks); + table.subnodes = subs; + let props: Vec<(u16, writer::Value)> = (0..8000u16).map(|i| (0x8000 + i, writer::boolean(true))).collect(); + let mut spill = Spill::new(); + let message = all.iter_mut().find(|n| n.nid & 0x1F == 4).unwrap(); + let item = Nid(message.nid); + message.data = Data::Blocks(pc(&spec.layout, &props, &mut spill)); + message.subnodes = spill.subnodes; + let bytes = raw(&spec.layout, &all); + assert!(bytes.len() < 1_500_000, "{}", bytes.len()); + + // A walk of the folders. + let budget = 64 * 1024; + let mut pst = open(&bytes, Limits { max_listing_bytes: budget, ..Default::default() }); + let tree = pst.walk_folders().unwrap(); + let kept: usize = tree + .folders + .iter() + .map(|f| size_of::() + f.folder.name.capacity() + kept_text(&f.folder.container_class) + kept_props(&f.folder.properties)) + .sum::() + + tree.left_out.len() * size_of::(); + assert!(kept <= budget as usize, "{kept}"); + assert!(tree.left_out.iter().any(|l| l.reason == Error::Cap(Cap::ListingSize))); + + // A listing of items. + let inbox = Nid((0x403 << 5) | 2); + let budget = 256 * 1024; + let mut pst = open(&bytes, Limits { max_listing_bytes: budget, ..Default::default() }); + let list = pst.items(inbox).unwrap(); + let kept: usize = list.items.iter().map(|i| size_of::() + kept_text(&i.class) + kept_props(&i.properties)).sum::() + + list.left_out.len() * size_of::(); + assert!(kept <= budget as usize, "{kept}"); + assert!(list.past_cap > 0 && list.items.len() + list.past_cap == 2000); + + // A message: what it keeps is far more than what it reads, and is refused past the cap. + let mut pst = open(&bytes, Limits::default()); + let m = pst.message(&MessageRef::item(item)).unwrap(); + let kept = size_of::() + kept_text(&m.class) + kept_props(&m.properties); + let budget = 128 * 1024; + assert!(kept > budget as usize, "{kept}"); + let mut pst = open(&bytes, Limits { max_item_bytes: budget, ..Default::default() }); + assert_eq!(pst.message(&MessageRef::item(item)).err(), Some(Error::Cap(Cap::ItemSize))); +} + +#[test] +fn a_list_value_holds_no_more_than_its_cap() { + // 100,000 empty binaries: 400 KB in the file, each a vector of its own in memory. And + // 100,000 two-byte numbers, which hold just their bytes. + let count = 100_000u32; + let mut empties = count.to_le_bytes().to_vec(); + for _ in 0..count { + empties.extend_from_slice(&(4 + 4 * count).to_le_bytes()); + } + let numbers: Vec = (0..count).flat_map(|i| (i as u16).to_le_bytes()).collect(); + let (binaries, int16s) = (crate::prop::types::BINARY | 0x1000, crate::prop::types::INT16 | 0x1000); + // A list of booleans (whose items aren't read), with the same table, and with a count + // that no table could hold. + let booleans = crate::prop::types::BOOLEAN | 0x1000; + let props = vec![ + (0x6900u16, writer::Value { ty: binaries, bytes: empties.clone() }), + (0x6901, writer::Value { ty: int16s, bytes: numbers.clone() }), + (0x6902, writer::Value { ty: booleans, bytes: empties.clone() }), + (0x6903, writer::Value { ty: booleans, bytes: vec![0xFF; 8] }), + ]; + let spec = sample(unicode_layout(), None); + let mut all = nodes(&spec); + let mut spill = Spill::new(); + let message = all.iter_mut().find(|n| n.nid & 0x1F == 4).unwrap(); + let item = MessageRef::item(Nid(message.nid)); + message.data = Data::Blocks(pc(&spec.layout, &props, &mut spill)); + message.subnodes = spill.subnodes; + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes, Limits::default()); + // Within a cap of twice its bytes: refused before it is made. + let max = 2 * empties.len() as u64; + assert_eq!(pst.read_property(&item, 0x6900, max).err(), Some(Error::Cap(Cap::ItemSize))); + let whole = pst.read_property(&item, 0x6900, 64 << 20).unwrap().unwrap(); + let PropValue::MultiBinary(v) = &whole else { panic!() }; + assert_eq!(v.len(), count as usize); + assert!(whole.held() as u64 <= 64 << 20); + // What is checked before it is made covers what making it holds. + assert!(crate::prop::building_cost(binaries, &empties) >= whole.held(), "{}", whole.held()); + assert_eq!(crate::prop::building_cost(int16s, &numbers), 0); + // The numbers fit a cap of exactly their bytes. + let Some(PropValue::MultiInt16(n)) = pst.read_property(&item, 0x6901, numbers.len() as u64).unwrap() else { panic!() }; + assert_eq!((n.len(), n.capacity(), n[7]), (count as usize, count as usize, 7), "no spare room past what is charged"); + // Lists of other types are kept as their bytes, within a cap of exactly those, and + // whatever their bytes say. + let max = empties.len() as u64; + assert_eq!(pst.read_property(&item, 0x6902, max).unwrap(), Some(PropValue::Unknown { ty: booleans, data: empties.clone() })); + assert_eq!(pst.read_property(&item, 0x6903, max).unwrap(), Some(PropValue::Unknown { ty: booleans, data: vec![0xFF; 8] })); + assert_eq!(crate::prop::building_cost(booleans, &empties), 0); + // And in a message the list is refused within the item's cap, not made. + let mut pst = open(&bytes, Limits { max_item_bytes: 1 << 20, max_inline_value: 1 << 20, ..Default::default() }); + let m = pst.message(&item); + assert_eq!(m.err(), Some(Error::Cap(Cap::ItemSize))); +} + +#[test] +fn a_named_entry_that_names_no_id_is_left_out_alone() { + // Three numeric names; the middle one's index (0x8000) names no id. + let mut entries = Vec::new(); + for (value, index) in [(0x10u32, 0u16), (0x11, 0x8000), (0x12, 1)] { + entries.extend_from_slice(&value.to_le_bytes()); + entries.extend_from_slice(&(1u16 << 1).to_le_bytes()); + entries.extend_from_slice(&index.to_le_bytes()); + } + let spec = sample(unicode_layout(), None); + let mut all = nodes(&spec); + let mut spill = Spill::new(); + let blocks = pc(&spec.layout, &[(0x0001, int32(251)), (0x0002, binary(&[])), (0x0003, binary(&entries)), (0x0004, binary(&[]))], &mut spill); + let map = all.iter_mut().find(|n| n.nid == 0x61).unwrap(); + map.data = Data::Blocks(blocks); + map.subnodes = spill.subnodes; + let bytes = raw(&spec.layout, &all); + let mut pst = open(&bytes, Limits::default()); + let named = pst.named_properties().unwrap(); + assert_eq!(named.len(), 2); + assert_eq!(named.get(0x8001).unwrap().name, PropName::Id(0x12)); + assert_eq!(named.left_out(), &[(None, Error::Corrupt("a named property's index"))]); +} diff --git a/crates/pst/src/tests/hostile.rs b/crates/pst/src/tests/hostile.rs new file mode 100644 index 000000000..1cfee0632 --- /dev/null +++ b/crates/pst/src/tests/hostile.rs @@ -0,0 +1,197 @@ +//! Damaged files: checksums, offsets, trees that lead back to themselves. Each ends in an +//! error value, never a panic. + +use std::io::Cursor; + +use super::writer::{self, HeapBuilder, Layout, build, nodes, raw}; +use super::{exercise, sample}; +use crate::crc::crc32; +use crate::header::{HEADER_READ, Header}; +use crate::heap::Heap; +use crate::ndb::{Ndb, signature}; +use crate::source::Source; +use crate::{Encryption, Error, Format, Limits, Pst}; + +fn base(format: Format) -> Vec { + build(&sample(Layout::tiny(format, Encryption::Cyclic), None)) +} + +fn open(bytes: &[u8]) -> crate::Result>> { + Pst::open(Cursor::new(bytes)) +} + +/// Puts a page's CRC right after it has been changed. +fn fix_page(bytes: &mut [u8], format: Format, ib: usize) { + let t = 512 - format.trailer_size(); + let crc = crc32(&bytes[ib..ib + t]); + let at = ib + t + if format == Format::Unicode { 4 } else { 8 }; + bytes[at..at + 4].copy_from_slice(&crc.to_le_bytes()); +} + +#[test] +fn not_a_pst_file() { + assert_eq!(open(b"").err(), Some(Error::NotPst)); + assert_eq!(open(b"!BDN").err(), Some(Error::NotPst)); + assert_eq!(open(&[0u8; 600]).err(), Some(Error::NotPst)); + let mut b = base(Format::Unicode); + b[10] = 20; // wVer + assert_eq!(open(&b).err(), Some(Error::UnsupportedVersion(20))); +} + +#[test] +fn a_damaged_header_is_refused() { + for format in [Format::Unicode, Format::Ansi] { + let mut b = base(format); + b[40] ^= 1; + assert_eq!(open(&b).err(), Some(Error::BadChecksum("the file's header"))); + } +} + +#[test] +fn data_encrypted_another_way_is_refused() { + let mut b = base(Format::Unicode); + b[513] = 0x10; + let full = crc32(&b[8..524]); + b[524..528].copy_from_slice(&full.to_le_bytes()); + let partial = crc32(&b[8..479]); + b[4..8].copy_from_slice(&partial.to_le_bytes()); + assert_eq!(open(&b).err(), Some(Error::UnsupportedEncryption(0x10))); +} + +#[test] +fn a_damaged_block_or_page_is_found_by_its_checksum() { + for format in [Format::Unicode, Format::Ansi] { + let clean = base(format); + let h = Header::parse(&clean).unwrap(); + // A byte of the first data block (the store's first heap block, written first). + let mut b = clean.clone(); + b[0x4400 + 3] ^= 0xFF; + assert!(matches!(open(&b).err(), Some(Error::BadChecksum(_))), "{format:?}"); + // A byte of the node tree's root page. + let mut b = clean.clone(); + b[h.nbt.ib as usize + 5] ^= 0xFF; + assert_eq!(open(&b).err(), Some(Error::BadChecksum("a B-tree page"))); + // The page's id. + let mut b = clean.clone(); + let t = h.nbt.ib as usize + 512 - format.trailer_size(); + b[t + 2] ^= 1; // its signature + assert_eq!(open(&b).err(), Some(Error::BadChecksum("a B-tree page's id"))); + } +} + +#[test] +fn offsets_past_the_end_are_refused() { + let clean = base(Format::Unicode); + let h = Header::parse(&clean).unwrap(); + // Cut the file before its node tree. + let cut = &clean[..h.nbt.ib as usize + 100]; + assert_eq!(open(cut).err(), Some(Error::OutOfBounds)); + // The header pointing far past the end. + let mut b = clean.clone(); + b[224..232].copy_from_slice(&(1u64 << 40).to_le_bytes()); + let full = crc32(&b[8..524]); + b[524..528].copy_from_slice(&full.to_le_bytes()); + let partial = crc32(&b[8..479]); + b[4..8].copy_from_slice(&partial.to_le_bytes()); + assert_eq!(open(&b).err(), Some(Error::OutOfBounds)); + // Every cut of the file ends in an error or a partial read, never a panic. + for len in (0..clean.len()).step_by(997) { + exercise(&clean[..len], Limits::default()); + } +} + +#[test] +fn a_node_tree_that_points_at_itself_is_refused() { + for format in [Format::Unicode, Format::Ansi] { + let mut b = base(format); + let h = Header::parse(&b).unwrap(); + let root = h.nbt.ib as usize; + let meta = if format == Format::Unicode { 488 } else { 496 }; + assert!(b[root + meta + 3] >= 1, "the tiny layout gives the node tree several levels"); + // Its first entry's child is the root page itself. + let n = format.id_size(); + let me: Vec = match format { + Format::Unicode => [h.nbt.bid.to_le_bytes(), h.nbt.ib.to_le_bytes()].concat(), + Format::Ansi => [(h.nbt.bid as u32).to_le_bytes(), (h.nbt.ib as u32).to_le_bytes()].concat(), + }; + b[root + n..root + 3 * n].copy_from_slice(&me); + fix_page(&mut b, format, root); + assert!(matches!(open(&b).err(), Some(Error::Corrupt(_) | Error::NotFound(_))), "{format:?}"); + exercise(&b, Limits::default()); + } +} + +#[test] +fn a_page_deeper_than_the_format_allows_is_refused() { + let mut b = base(Format::Unicode); + let h = Header::parse(&b).unwrap(); + let root = h.nbt.ib as usize; + b[root + 488 + 3] = 9; + fix_page(&mut b, Format::Unicode, root); + assert_eq!(open(&b).err(), Some(Error::Corrupt("a B-tree page's level"))); +} + +#[test] +fn a_folder_that_contains_its_parent_is_a_cycle() { + let spec = sample(Layout::new(Format::Unicode, Encryption::None), None); + let mut all = nodes(&spec); + // Inbox's hierarchy table lists the top of the folders, its own parent, as a subfolder. + let inbox = all.iter().find(|n| n.parent == writer::IPM_SUBTREE && n.nid & 0x1F == 2).unwrap().nid; + let l = &spec.layout; + let (blocks, subs) = writer::tc(l, &[0x3001_001F], &[(writer::IPM_SUBTREE, vec![(0x3001_001F, writer::unicode("Again"))])]); + let table = all.iter_mut().find(|n| n.nid == (inbox & !0x1F) | 0x0D).unwrap(); + table.data = writer::Data::Blocks(blocks); + table.subnodes = subs; + let bytes = raw(l, &all); + let mut pst = open(&bytes).unwrap(); + let tree = pst.walk_folders().unwrap(); + assert!(tree.left_out.iter().any(|x| x.id.0 == writer::IPM_SUBTREE && matches!(x.reason, Error::Cycle(_)))); + assert_eq!(tree.folders.iter().filter(|f| f.folder.id.0 == writer::IPM_SUBTREE).count(), 1); +} + +#[test] +fn a_heap_tree_that_points_at_itself_is_refused() { + let mut heap = HeapBuilder::new(0xBC, 3000); + // The first item (HID 0x20) is an index record pointing at itself. + let mut item = 0u16.to_le_bytes().to_vec(); + item.extend_from_slice(&0x20u32.to_le_bytes()); + assert_eq!(heap.alloc(item), 0x20); + let mut header = vec![0xB5, 2, 6, 3]; + header.extend_from_slice(&0x20u32.to_le_bytes()); + heap.user_root = heap.alloc(header); + let heap = Heap::new(heap.finish()).unwrap(); + assert!(matches!(crate::bth::read(&heap, heap.user_root, 1000, Error::Corrupt("")).err(), Some(Error::Cycle(_)))); +} + +#[test] +fn heap_ids_past_the_heap_are_refused() { + let mut heap = HeapBuilder::new(0xBC, 3000); + heap.user_root = heap.alloc(vec![1, 2, 3]); + let heap = Heap::new(heap.finish()).unwrap(); + assert_eq!(heap.item(0x20).unwrap(), &[1, 2, 3]); + assert!(heap.item(0x40).is_err()); + assert!(heap.item(0x1_0020).is_err()); + assert!(heap.item(0x21).is_err()); + assert!(Heap::new(vec![]).is_err()); + assert!(Heap::new(vec![vec![0, 0, 0xEB, 0xBC]]).is_err()); +} + +#[test] +fn block_signatures_are_checked() { + let clean = base(Format::Ansi); + // The message store's block, found through the file's own trees. + let limits = Limits::default(); + let mut src = Source::new(Cursor::new(&clean[..]), &limits).unwrap(); + let header = Header::parse(&src.read_at(0, HEADER_READ).unwrap()).unwrap(); + let mut ndb = Ndb::new(src, header, limits); + let store = ndb.node(0x21).unwrap(); + let entry = ndb.block_entry(store.data).unwrap(); + // Its trailer (ANSI: cb, wSig, bid, CRC): only the signature changed. The CRC covers the + // data alone, so it still holds. + let ts = Format::Ansi.trailer_size(); + let trailer = entry.ib as usize + (usize::from(entry.cb) + ts).div_ceil(64) * 64 - ts; + let mut b = clean.clone(); + b[trailer + 2] ^= 0xFF; + assert_eq!(open(&b).err(), Some(Error::BadChecksum("a block's signature"))); + assert_eq!(signature(0x4400, 4), 0x4404); +} diff --git a/crates/pst/src/tests/list_stream.rs b/crates/pst/src/tests/list_stream.rs new file mode 100644 index 000000000..38df4ce78 --- /dev/null +++ b/crates/pst/src/tests/list_stream.rs @@ -0,0 +1,163 @@ +//! A distribution list's stream ([MS-OXOCNTC] 2.2.2.2.4), as the writer makes it, and hostile. + +use proptest::prelude::*; + +use super::writer::{list_stream_bytes, one_off, wrapped_entry_id}; +use crate::{Error, ListMember, MAX_LIST_ENTRY, list_stream}; + +fn members(n: usize) -> Vec<(Vec, Vec)> { + (0..n) + .map(|i| { + let o = one_off(&format!("Member {i}"), "SMTP", &format!("m{i}@example.org"), true); + if i % 2 == 0 { (o, Vec::new()) } else { (wrapped_entry_id(0xC3, &[i as u8; 46]), o) } + }) + .collect() +} + +#[test] +fn a_stream_reads_back_member_by_member() { + let m = members(40); + let b = list_stream_bytes(&m); + let s = list_stream(&b, 1000).unwrap(); + assert_eq!((s.more, s.damaged, s.totals_differ), (0, 0, false)); + let want: Vec = m.iter().map(|(e, o)| ListMember { entry_id: e, one_off: o }).collect(); + assert_eq!(s.members, want); +} + +#[test] +fn an_empty_list_has_no_members() { + let b = list_stream_bytes(&[]); + let s = list_stream(&b, 1000).unwrap(); + assert!(s.members.is_empty()); + // Without its terminators too. + assert!(list_stream(&b[..28], 1000).unwrap().members.is_empty()); +} + +#[test] +fn past_the_cap_members_are_counted_not_kept() { + let b = list_stream_bytes(&members(25)); + let s = list_stream(&b, 10).unwrap(); + assert_eq!((s.members.len(), s.more, s.damaged), (10, 15, 0)); +} + +#[test] +fn a_short_header_or_another_version_is_refused() { + let b = list_stream_bytes(&members(2)); + assert!(matches!(list_stream(&b[..27], 10), Err(Error::Corrupt(_)))); + assert!(matches!(list_stream(&[], 10), Err(Error::Corrupt(_)))); + let mut v2 = b.clone(); + v2[0] = 2; + assert!(matches!(list_stream(&v2, 10), Err(Error::Corrupt(_)))); +} + +#[test] +fn a_count_the_bytes_cannot_hold_is_refused() { + let mut b = list_stream_bytes(&members(3)); + b[12..16].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(list_stream(&b, 10), Err(Error::Corrupt("a distribution list's stream counts more members than it has room for"))); +} + +#[test] +fn a_stream_cut_short_keeps_the_members_before_the_cut() { + let m = members(6); + let b = list_stream_bytes(&m); + // Cut inside the fifth member. + let fifth = 28 + m[..4].iter().map(|(e, o)| 12 + e.len() + o.len()).sum::(); + let s = list_stream(&b[..fifth + 20], 10).unwrap(); + assert_eq!((s.members.len(), s.damaged), (4, 2)); + assert!(!s.totals_differ); +} + +#[test] +fn a_length_that_runs_past_the_end_stops_the_reading_there() { + let m = members(3); + let mut b = list_stream_bytes(&m); + // The second member's entry id says it is longer than all there is. + let second = 28 + 12 + m[0].0.len() + m[0].1.len(); + b[second..second + 4].copy_from_slice(&0x7FFF_FFFFu32.to_le_bytes()); + let s = list_stream(&b, 10).unwrap(); + assert_eq!((s.members.len(), s.damaged), (1, 2)); + // And one longer than an entry may be, though it would fit. + let big = vec![(vec![0u8; MAX_LIST_ENTRY + 1], Vec::new()), m[0].clone()]; + let b = list_stream_bytes(&big); + let s = list_stream(&b, 10).unwrap(); + assert_eq!((s.members.len(), s.damaged), (0, 2)); +} + +#[test] +fn totals_that_lie_are_noticed_and_read_past() { + let mut b = list_stream_bytes(&members(4)); + b[16..20].copy_from_slice(&7u32.to_le_bytes()); + let s = list_stream(&b, 10).unwrap(); + assert_eq!((s.members.len(), s.totals_differ), (4, true)); +} + +#[test] +fn extra_member_info_is_stepped_over() { + let m = members(2); + let mut b = list_stream_bytes(&m[..1]); + b.truncate(b.len() - 12); + b.extend(5u32.to_le_bytes()); + b.extend([9; 5]); + let rest = list_stream_bytes(&m[1..]); + b.extend_from_slice(&rest[28..]); + b[12..16].copy_from_slice(&2u32.to_le_bytes()); + let s = list_stream(&b, 10).unwrap(); + assert_eq!(s.members.len(), 2); + assert_eq!(s.members[1].one_off, &m[1].1[..]); + assert!(s.totals_differ); +} + +proptest! { + #[test] + fn any_bytes_never_panic(b in proptest::collection::vec(any::(), 0..600), cap in 0usize..20) { + if let Ok(s) = list_stream(&b, cap) { + prop_assert!(s.members.len() <= cap); + } + } + + #[test] + fn a_header_then_any_bytes_never_panic(count in 0u32..64, tail in proptest::collection::vec(any::(), 0..800)) { + let mut b = list_stream_bytes(&[]); + b.truncate(28); + b[12..16].copy_from_slice(&count.to_le_bytes()); + b.extend(tail); + if let Ok(s) = list_stream(&b, 8) { + prop_assert!(s.members.len() + s.more + s.damaged <= count as usize); + } + } + + #[test] + fn what_is_written_reads_back(n in 0usize..30) { + let m = members(n); + let b = list_stream_bytes(&m); + let s = list_stream(&b, 1000).unwrap(); + prop_assert_eq!(s.members.len(), n); + prop_assert!(!s.totals_differ); + } +} + +/// The layout of [MS-OXOCNTC] 2.2.2.2.4 written out by hand, apart from the writer: one member +/// whose entry id is 3 bytes and whose one-off is 2. +#[test] +fn the_header_is_read_where_the_specification_puts_it() { + #[rustfmt::skip] + let b: &[u8] = &[ + 0x01, 0x00, 0x00, 0x00, // StreamVersion, Reserved + 0x00, 0x71, 0x02, 0x00, // BuildVersion + 0x00, 0x00, 0x00, 0x00, // DistListStreamFlags + 0x01, 0x00, 0x00, 0x00, // CountOfEntries + 0x03, 0x00, 0x00, 0x00, // TotalEntryIDSize + 0x02, 0x00, 0x00, 0x00, // TotalOneOffSize + 0x00, 0x00, 0x00, 0x00, // TotalExtraSize + 0x03, 0x00, 0x00, 0x00, 0xAA, 0xBB, 0xCC, // EntryIdSize, EntryIdData + 0x02, 0x00, 0x00, 0x00, 0xDD, 0xEE, // one-offEntryIdSize, one-offEntryIdData + 0x00, 0x00, 0x00, 0x00, // ExtraMemberInfoSize + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Terminator1, Terminator2 + ]; + let s = list_stream(b, 10).unwrap(); + assert_eq!(s.members, [ListMember { entry_id: &[0xAA, 0xBB, 0xCC], one_off: &[0xDD, 0xEE] }]); + assert!(!s.totals_differ); + // And the writer writes the same. + assert_eq!(list_stream_bytes(&[(vec![0xAA, 0xBB, 0xCC], vec![0xDD, 0xEE])]), b); +} diff --git a/crates/pst/src/tests/mod.rs b/crates/pst/src/tests/mod.rs new file mode 100644 index 000000000..f27b2c68a --- /dev/null +++ b/crates/pst/src/tests/mod.rs @@ -0,0 +1,144 @@ +//! Tests of the .pst reader. Every file here is made by the test writer (`writer.rs`) from +//! invented people at `example.org`, or is a sample printed in [MS-PST]; none was made by +//! Outlook, and none is a real archive. + +mod accounting; +mod caps; +mod cost; +mod hostile; +mod list_stream; +mod properties; +mod recur; +mod roundtrip; +mod salvage; +mod spec_samples; +pub use crate::test_writer as writer; + +use std::io::{Cursor, Read}; + +use writer::{AttachSpec, FileSpec, FolderSpec, Layout, MessageSpec, int32, unicode}; + +use crate::{AttachmentContent, Encryption, Format, Limits, MessageRef, Pst}; + +/// 2026-11-04 09:30 UTC as a FILETIME. +pub const DELIVERED: u64 = 134_067_498_000_000_000; + +pub fn person(name: &str, user: &str, kind: i32) -> (String, String, i32) { + (name.to_string(), format!("{user}@example.org"), kind) +} + +/// Bytes that are not all alike, so that a misplaced block shows. +pub fn pattern(len: usize, seed: u8) -> Vec { + (0..len).map(|i| (i as u8).wrapping_mul(31).wrapping_add(seed) ^ (i >> 8) as u8).collect() +} + +pub fn note(subject: &str) -> MessageSpec { + MessageSpec { + class: "IPM.Note".into(), + subject: subject.into(), + body: format!("{subject}\r\n\r\nWritten for the tests."), + read: false, + delivered: DELIVERED, + ..Default::default() + } +} + +/// A file with a few folders, messages with recipients and attachments, and attached messages. +pub fn sample(layout: Layout, password: Option<&str>) -> FileSpec { + let mut report = note("Quarterly figures"); + report.recipients = vec![person("Ada Fernsby", "ada", 1), person("Bram Okonjo", "bram", 2), person("Cleo Varga", "cleo", 3)]; + report.read = true; + let mut forwarded = note("Minutes of the garden committee"); + forwarded.recipients = vec![person("Dee Halloran", "dee", 1)]; + forwarded.attachments = vec![AttachSpec::File { name: "minutes.txt".into(), mime: "text/plain".into(), data: b"Present: everyone.".to_vec() }]; + report.attachments = vec![ + AttachSpec::File { name: "figures.csv".into(), mime: "text/csv".into(), data: pattern(20_000, 7) }, + AttachSpec::Message(Box::new(forwarded)), + ]; + report.extra = vec![(0x8000, int32(42)), (0x0E1D, unicode("Quarterly figures"))]; + let plain = note("Lunch on Friday?"); + let mut appointment = note("Dentist"); + appointment.class = "IPM.Appointment".into(); + FileSpec { + layout, + password: password.map(str::to_string), + password_ansi: false, + name: "Personal Folders".into(), + folders: vec![ + FolderSpec { + name: "Inbox".into(), + class: Some("IPF.Note".into()), + messages: vec![report, plain], + children: vec![FolderSpec { + name: "Projects".into(), + class: Some("IPF.Note".into()), + messages: vec![note("Shed roof")], + children: vec![], + }], + }, + FolderSpec { name: "Calendar".into(), class: Some("IPF.Appointment".into()), messages: vec![appointment], children: vec![] }, + FolderSpec { name: "Contacts".into(), class: Some("IPF.Contact".into()), messages: vec![], children: vec![] }, + ], + named: vec![([0x08, 0x20, 0x06, 0, 0, 0, 0, 0, 0xC0, 0, 0, 0, 0, 0, 0, 0x46], 0x8514)], + named_strings: vec![(crate::PS_PUBLIC_STRINGS, "Keywords".into())], + } +} + +pub fn layouts() -> Vec { + let mut out = Vec::new(); + for format in [Format::Unicode, Format::Ansi] { + for enc in [Encryption::None, Encryption::Permute, Encryption::Cyclic] { + out.push(Layout::new(format, enc)); + out.push(Layout::tiny(format, enc)); + } + } + out +} + +/// Reads everything a file has, through every call of the API, and ignores what fails: the +/// hostile tests only ask that it ends without a panic. +pub fn exercise(bytes: &[u8], limits: Limits) -> usize { + let Ok(mut pst) = Pst::open_with(Cursor::new(bytes), limits) else { return 0 }; + let mut seen = 0usize; + let _ = pst.unlock(""); + let _ = pst.store(); + let _ = pst.named_properties().map(|m| seen += m.len()); + let _ = pst.for_each_node(|_| { + seen += 1; + seen < 100_000 + }); + let Ok(tree) = pst.walk_folders() else { return seen }; + for entry in &tree.folders { + let Ok(items) = pst.items(entry.folder.id) else { continue }; + let _ = pst.associated_items(entry.folder.id); + for item in items.items { + let mut queue = vec![MessageRef::item(item.id)]; + while let Some(m) = queue.pop() { + let Ok(message) = pst.message(&m) else { continue }; + seen += 1 + message.recipients.len(); + let _ = pst.read_property(&m, 0x1000, 1 << 20); + for a in message.attachments { + match &a.content { + AttachmentContent::Message(inner) if queue.len() < 64 => queue.push(inner.clone()), + AttachmentContent::Data { .. } | AttachmentContent::Object { .. } => { + if let Ok(mut r) = pst.attachment_reader(&a.id) { + let mut sink = Vec::new(); + let _ = r.read_to_end(&mut sink); + seen += sink.len().min(1); + } + } + _ => {} + } + } + } + } + } + seen +} + +/// The first item of the folder named Inbox. +pub fn first_item(pst: &mut Pst>) -> crate::Nid { + let walk = pst.walk_folders().unwrap(); + let inbox = walk.folders.iter().find(|f| f.folder.name == "Inbox").unwrap().folder.id; + pst.items(inbox).unwrap().items[0].id +} diff --git a/crates/pst/src/tests/properties.rs b/crates/pst/src/tests/properties.rs new file mode 100644 index 000000000..96c67f187 --- /dev/null +++ b/crates/pst/src/tests/properties.rs @@ -0,0 +1,104 @@ +//! Property tests: arbitrary bytes, and valid files with bytes changed, cut or inserted, are +//! read through every call of the API without a panic and in bounded time. + +use std::sync::OnceLock; + +use proptest::prelude::*; + +use super::writer::{Layout, build}; +use super::{exercise, sample}; +use crate::{Encryption, Format, Limits}; + +fn bases() -> &'static [Vec] { + static BASES: OnceLock>> = OnceLock::new(); + BASES.get_or_init(|| { + vec![ + build(&sample(Layout::tiny(Format::Unicode, Encryption::Permute), None)), + build(&sample(Layout::tiny(Format::Ansi, Encryption::Cyclic), None)), + build(&sample(Layout::new(Format::Unicode, Encryption::None), None)), + ] + }) +} + +/// Small enough that a hostile file can't make a case slow. +fn limits() -> Limits { + Limits { max_item_bytes: 1 << 20, max_attachment_bytes: 1 << 20, max_inline_value: 1 << 16, ..Default::default() } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 200, ..ProptestConfig::default() })] + + #[test] + fn arbitrary_bytes_never_panic(bytes in proptest::collection::vec(any::(), 0..4096)) { + exercise(&bytes, limits()); + } + + #[test] + fn a_header_with_anything_after_it_never_panics(tail in proptest::collection::vec(any::(), 0..20_000), which in 0usize..3) { + let base = &bases()[which]; + let mut bytes = base[..0x4400.min(base.len())].to_vec(); + bytes.extend_from_slice(&tail); + exercise(&bytes, limits()); + } + + #[test] + fn changed_bytes_never_panic(which in 0usize..3, changes in proptest::collection::vec((any::(), any::()), 1..24)) { + let mut bytes = bases()[which].clone(); + let len = bytes.len(); + for (at, v) in changes { + bytes[at % len] = v; + } + exercise(&bytes, limits()); + } + + #[test] + fn changed_bytes_past_the_checksums_never_panic(which in 0usize..3, changes in proptest::collection::vec((any::(), any::()), 1..8)) { + // The changes go to blocks and pages with their CRCs put right, so that what is behind + // the checksums (the trees, heaps and tables) meets damaged structures. + let mut bytes = bases()[which].clone(); + let format = if which == 1 { Format::Ansi } else { Format::Unicode }; + for (at, v) in changes { + let at = 0x4400 + at % (bytes.len() - 0x4400); + bytes[at] = v; + super::properties::fix_around(&mut bytes, format, at); + } + exercise(&bytes, limits()); + } + + #[test] + fn cut_files_never_panic(which in 0usize..3, cut in any::()) { + let base = &bases()[which]; + exercise(&base[..cut % base.len()], limits()); + } +} + +/// Puts right the CRC of the page or block that `at` falls in, when it can tell which. +pub fn fix_around(bytes: &mut [u8], format: Format, at: usize) { + let ts = format.trailer_size(); + // A page: 512-aligned, whose trailer names a B-tree page type twice. + let page = at - at % 512; + if page + 512 <= bytes.len() { + let t = page + 512 - ts; + if matches!(bytes[t], 0x80 | 0x81) && bytes[t] == bytes[t + 1] { + let crc = crc32(&bytes[page..t]); + let c = t + if format == Format::Unicode { 4 } else { 8 }; + bytes[c..c + 4].copy_from_slice(&crc.to_le_bytes()); + return; + } + } + // A block: scan forward to the first 64-aligned end whose trailer's cb fits. + let start = at - at % 64; + for end in (start + 64..=bytes.len().min(start + 8192)).step_by(64) { + let t = end - ts; + let cb = usize::from(u16::from_le_bytes([bytes[t], bytes[t + 1]])); + let Some(block_start) = end.checked_sub((cb + ts).div_ceil(64) * 64) else { continue }; + if block_start <= at && block_start + cb <= t { + let crc = crc32(&bytes[block_start..block_start + cb]); + let c = t + if format == Format::Unicode { 4 } else { 8 }; + bytes[c..c + 4].copy_from_slice(&crc.to_le_bytes()); + return; + } + } +} + +use crate::crc::crc32; diff --git a/crates/pst/src/tests/recur.rs b/crates/pst/src/tests/recur.rs new file mode 100644 index 000000000..1c1e0f5e7 --- /dev/null +++ b/crates/pst/src/tests/recur.rs @@ -0,0 +1,66 @@ +//! How an appointment repeats ([MS-OXOCAL] 2.2.1.44), as the writer makes it, and hostile. + +use proptest::prelude::*; + +use super::writer::{recurrence_bytes, time_zone_bytes}; +use crate::recur::{END_DATE, FREQ_WEEKLY, PATTERN_WEEK}; +use crate::{appointment_recurrence, time_zone_key}; + +fn weekly() -> Vec { + let changed = vec![(2_000_600, 2_000_660, 2_000_540, Some("Moved caf\u{e9}".to_string()))]; + recurrence_bytes(FREQ_WEEKLY, PATTERN_WEEK, 1, &[0b10], (END_DATE, 4, 2_030_000), &[2_000_000, 2_010_000], 1_990_000, (540, 600), &changed) +} + +#[test] +fn a_series_reads_back() { + let r = appointment_recurrence(&weekly()).unwrap(); + assert_eq!((r.frequency, r.pattern, r.period, r.weekdays), (FREQ_WEEKLY, PATTERN_WEEK, 1, 0b10)); + assert_eq!((r.end_type, r.count, r.start_date, r.end_date), (END_DATE, 4, 1_990_000, 2_030_000)); + assert_eq!(r.deleted, vec![2_000_000, 2_010_000]); + assert_eq!((r.start_offset, r.end_offset), (540, 600)); + let x = &r.exceptions[0]; + assert_eq!((x.start, x.end, x.original), (2_000_600, 2_000_660, 2_000_540)); + assert_eq!(x.subject.as_deref(), Some("Moved caf\u{e9}")); +} + +#[test] +fn a_count_past_the_bytes_is_an_error_and_allocates_nothing() { + let mut b = weekly(); + // DeletedInstanceCount, after 22 bytes of header, the one specific word and three more. + b[38..42].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!(appointment_recurrence(&b).is_err()); +} + +#[test] +fn a_cut_extended_part_keeps_the_rest() { + let mut b = weekly(); + b.truncate(b.len() - 10); + let r = appointment_recurrence(&b).unwrap(); + assert_eq!(r.exceptions.len(), 1); + assert_eq!(r.exceptions[0].subject_8bit.as_deref(), Some("Moved caf\u{e9}".as_bytes())); +} + +#[test] +fn a_time_zone_names_its_key() { + assert_eq!(time_zone_key(&time_zone_bytes("W. Europe Standard Time")).unwrap(), "W. Europe Standard Time"); + assert!(time_zone_key(&[1, 1, 0, 0]).is_err()); + assert!(time_zone_key(&time_zone_bytes("")).is_err()); +} + +proptest! { + #[test] + fn any_bytes_never_panic(b in prop::collection::vec(any::(), 0..600)) { + let _ = appointment_recurrence(&b); + let _ = time_zone_key(&b); + } + + #[test] + fn a_damaged_series_never_panics(flips in prop::collection::vec((any::(), any::()), 1..12)) { + let mut b = weekly(); + for (at, v) in flips { + let i = at.index(b.len()); + b[i] = v; + } + let _ = appointment_recurrence(&b); + } +} diff --git a/crates/pst/src/tests/roundtrip.rs b/crates/pst/src/tests/roundtrip.rs new file mode 100644 index 000000000..49d26b12a --- /dev/null +++ b/crates/pst/src/tests/roundtrip.rs @@ -0,0 +1,198 @@ +//! Files from the test writer read back whole, in every format, encoding and layout. + +use std::io::{Cursor, Read}; + +use super::writer::{FolderSpec, Layout, build}; +use super::{DELIVERED, layouts, pattern, sample}; +use crate::{AttachmentContent, Encryption, Error, FolderKind, Format, MessageRef, Nid, PropName, PropValue, Pst, RecipientKind}; + +fn open(bytes: &[u8]) -> Pst> { + Pst::open(Cursor::new(bytes)).unwrap() +} + +#[test] +#[allow(clippy::cognitive_complexity, reason = "one walk over every layout, with its checks in line")] +fn every_layout_reads_back() { + for layout in layouts() { + let what = format!("{:?} {:?} block {}", layout.format, layout.encryption, layout.block_data); + let bytes = build(&sample(layout.clone(), None)); + let mut pst = open(&bytes); + assert_eq!(pst.format(), layout.format, "{what}"); + assert_eq!(pst.encryption(), layout.encryption, "{what}"); + assert!(!pst.password_required()); + + let store = pst.store().unwrap(); + assert_eq!(store.display_name.as_deref(), Some("Personal Folders"), "{what}"); + let ipm = store.ipm_subtree.unwrap(); + + let tree = pst.walk_folders().unwrap(); + assert!(tree.left_out.is_empty(), "{what}: {:?}", tree.left_out); + let names: Vec<(&str, usize, bool)> = tree.folders.iter().map(|f| (f.folder.name.as_str(), f.depth, f.in_ipm_subtree)).collect(); + assert_eq!( + names, + vec![ + ("", 0, false), + ("Top of Personal Folders", 1, true), + ("Inbox", 2, true), + ("Projects", 3, true), + ("Calendar", 2, true), + ("Contacts", 2, true), + ("Search Root", 1, false), + ], + "{what}" + ); + assert_eq!(tree.folders[1].folder.id, ipm); + let kinds: Vec = tree.folders.iter().map(|f| f.folder.kind.clone()).collect(); + assert_eq!(kinds[2], FolderKind::Mail); + assert_eq!(kinds[4], FolderKind::Calendar); + assert_eq!(kinds[5], FolderKind::Contacts); + let inbox = tree.folders[2].folder.clone(); + assert_eq!((inbox.content_count, inbox.unread_count, inbox.has_subfolders), (2, 1, true), "{what}"); + assert_eq!(inbox.parent, Some(ipm)); + + let items = pst.items(inbox.id).unwrap(); + assert!(items.left_out.is_empty(), "{what}: {:?}", items.left_out); + let classes: Vec> = items.items.iter().map(|i| i.class.as_deref()).collect(); + assert_eq!(classes, vec![Some("IPM.Note"), Some("IPM.Note")], "{what}"); + assert_eq!(items.items[0].properties.text(0x0037).as_deref(), Some("Quarterly figures")); + assert!(pst.associated_items(inbox.id).unwrap().items.is_empty()); + + let m = pst.message(&MessageRef::item(items.items[0].id)).unwrap(); + assert!(m.left_out.is_empty(), "{what}: {:?}", m.left_out); + assert_eq!(m.class.as_deref(), Some("IPM.Note")); + assert_eq!(m.properties.text(0x0037).as_deref(), Some("Quarterly figures")); + assert_eq!(m.properties.text(0x1000).as_deref(), Some("Quarterly figures\r\n\r\nWritten for the tests.")); + assert_eq!(m.properties.time(0x0E06), Some(DELIVERED)); + assert_eq!(m.properties.int(0x8000), Some(42)); + let people: Vec<(Option, Option, RecipientKind)> = m.recipients.iter().map(|r| (r.name(), r.address(), r.kind())).collect(); + assert_eq!( + people, + vec![ + (Some("Ada Fernsby".into()), Some("ada@example.org".into()), RecipientKind::To), + (Some("Bram Okonjo".into()), Some("bram@example.org".into()), RecipientKind::Cc), + (Some("Cleo Varga".into()), Some("cleo@example.org".into()), RecipientKind::Bcc), + ], + "{what}" + ); + + assert_eq!(m.attachments.len(), 2, "{what}"); + let csv = &m.attachments[0]; + assert_eq!(csv.file_name.as_deref(), Some("figures.csv")); + assert_eq!(csv.mime_type.as_deref(), Some("text/csv")); + assert_eq!(csv.method, Some(1)); + assert_eq!(csv.content, AttachmentContent::Data { size: 20_000 }); + assert!(matches!(csv.properties.get(0x3701), Some(PropValue::Large { size: 20_000 }))); + assert_eq!(pst.attachment_bytes(&csv.id).unwrap(), pattern(20_000, 7), "{what}"); + let mut r = pst.attachment_reader(&csv.id).unwrap(); + assert_eq!(r.size(), 20_000); + let mut first = [0u8; 10]; + r.read_exact(&mut first).unwrap(); + assert_eq!(first.to_vec(), pattern(10, 7)); + + let AttachmentContent::Message(inner) = &m.attachments[1].content else { panic!("{what}: {:?}", m.attachments[1].content) }; + assert_eq!(inner.depth(), 1); + let fwd = pst.message(inner).unwrap(); + assert_eq!(fwd.properties.text(0x0037).as_deref(), Some("Minutes of the garden committee"), "{what}"); + assert_eq!(fwd.recipients.len(), 1); + assert_eq!(fwd.recipients[0].address().as_deref(), Some("dee@example.org")); + assert_eq!(fwd.attachments.len(), 1); + assert_eq!(pst.attachment_bytes(&fwd.attachments[0].id).unwrap(), b"Present: everyone."); + + let named = pst.named_properties().unwrap(); + let first = named.get(0x8000).unwrap(); + assert_eq!(first.name, PropName::Id(0x8514)); + assert_eq!(named.get(0x8001).unwrap().name, PropName::Name("Keywords".into())); + assert_eq!(named.find(&crate::PS_PUBLIC_STRINGS, &PropName::Name("Keywords".into())), Some(0x8001)); + + let mut messages = 0; + pst.for_each_node(|n| { + messages += usize::from(n.id.kind() == crate::node_types::NORMAL_MESSAGE); + true + }) + .unwrap(); + assert_eq!(messages, 4, "{what}"); + + let calendar = tree.folders[4].folder.id; + let cal = pst.items(calendar).unwrap(); + assert_eq!(cal.items[0].class.as_deref(), Some("IPM.Appointment")); + assert!(pst.items(tree.folders[5].folder.id).unwrap().items.is_empty()); + } +} + +#[test] +fn a_large_body_is_left_for_later_and_read_on_its_own() { + let mut spec = sample(Layout::new(Format::Unicode, Encryption::Cyclic), None); + let body: String = "All work and no play. ".repeat(30_000); + spec.folders[0].messages[1].body = body.clone(); + let bytes = build(&spec); + let mut pst = Pst::open_with(Cursor::new(&bytes[..]), crate::Limits { max_inline_value: 64 * 1024, ..Default::default() }).unwrap(); + let inbox = pst.walk_folders().unwrap().folders[2].folder.id; + let item = pst.items(inbox).unwrap().items[1].id; + let m = pst.message(&MessageRef::item(item)).unwrap(); + let size = (body.len() * 2) as u64; + assert_eq!(m.properties.get(0x1000), Some(&PropValue::Large { size })); + assert_eq!(pst.read_property(&m.id, 0x1000, 4 << 20).unwrap(), Some(PropValue::String(body))); + assert_eq!(pst.read_property(&m.id, 0x1000, 1000).unwrap(), Some(PropValue::Large { size })); +} + +#[test] +fn many_folders_and_items_spread_over_pages_and_blocks() { + for format in [Format::Unicode, Format::Ansi] { + let mut spec = sample(Layout::tiny(format, Encryption::Permute), None); + spec.folders[2].children = (0..30).map(|i| FolderSpec { name: format!("Box {i}"), ..Default::default() }).collect(); + spec.folders[1].messages = (0..120).map(|i| super::note(&format!("Visit {i}"))).collect(); + let bytes = build(&spec); + let mut pst = open(&bytes); + let tree = pst.walk_folders().unwrap(); + assert!(tree.left_out.is_empty()); + let boxes: Vec<&str> = + tree.folders.iter().filter(|f| f.depth == 3 && f.folder.name.starts_with("Box")).map(|f| f.folder.name.as_str()).collect(); + assert_eq!(boxes.len(), 30); + assert_eq!(boxes[29], "Box 29"); + let calendar = tree.folders.iter().find(|f| f.folder.name == "Calendar").unwrap().folder.id; + let items = pst.items(calendar).unwrap(); + assert_eq!(items.items.len(), 120); + assert_eq!(items.items[119].properties.text(0x0037).as_deref(), Some("Visit 119")); + let last = pst.message(&MessageRef::item(items.items[119].id)).unwrap(); + assert_eq!(last.properties.text(0x0037).as_deref(), Some("Visit 119")); + } +} + +#[test] +fn a_password_is_asked_for_before_anything_is_read() { + for format in [Format::Unicode, Format::Ansi] { + let bytes = build(&sample(Layout::new(format, Encryption::Permute), Some("tomato soup"))); + let mut pst = open(&bytes); + assert!(pst.password_required()); + assert_eq!(pst.store().err(), Some(Error::PasswordRequired)); + assert_eq!(pst.walk_folders().err(), Some(Error::PasswordRequired)); + assert_eq!(pst.folder(Nid(0x122)).err(), Some(Error::PasswordRequired)); + assert_eq!(pst.items(Nid(0x8022)).err(), Some(Error::PasswordRequired)); + assert_eq!(pst.message(&MessageRef::item(Nid(0x200004))).err(), Some(Error::PasswordRequired)); + assert_eq!(pst.named_properties().err(), Some(Error::PasswordRequired)); + assert_eq!(pst.for_each_node(|_| true).err(), Some(Error::PasswordRequired)); + assert!(!pst.check_password("")); + assert!(!pst.check_password("tomato")); + assert_eq!(pst.unlock("Tomato soup"), Err(Error::WrongPassword)); + assert!(pst.password_required()); + assert!(pst.check_password("tomato soup")); + pst.unlock("tomato soup").unwrap(); + assert_eq!(pst.walk_folders().unwrap().folders.len(), 7); + } +} + +#[test] +fn a_file_without_a_password_takes_any() { + let bytes = build(&sample(Layout::new(Format::Unicode, Encryption::None), None)); + let mut pst = open(&bytes); + assert!(pst.check_password("anything")); + pst.unlock("anything").unwrap(); +} + +#[test] +fn a_search_folder_lists_no_items() { + let bytes = build(&sample(Layout::new(Format::Unicode, Encryption::None), None)); + let mut pst = open(&bytes); + assert_eq!(pst.items(Nid((0x500 << 5) | 0x03)).err(), Some(Error::Unsupported("a search folder's items"))); + assert_eq!(pst.items(Nid(0x21)).err(), Some(Error::Unsupported("a node that is not a folder"))); +} diff --git a/crates/pst/src/tests/salvage.rs b/crates/pst/src/tests/salvage.rs new file mode 100644 index 000000000..7d5d391ea --- /dev/null +++ b/crates/pst/src/tests/salvage.rs @@ -0,0 +1,284 @@ +//! A damaged page or block loses what it holds and no more: the rest of the file reads, and +//! each loss is named (in a walk's, list's or message's `left_out`, and a page of the file's +//! own trees in [`Pst::damage`]). + +use std::io::Cursor; + +use super::sample; +use super::writer::{Layout, build}; +use crate::header::{HEADER_READ, Header}; +use crate::ndb::Ndb; +use crate::source::Source; +use crate::{Encryption, Error, Format, Limits, MessageRef, Nid, Pst}; + +fn base() -> Vec { + build(&sample(Layout::tiny(Format::Unicode, Encryption::Permute), None)) +} + +/// The leaf pages of one tree (0x81 the node tree, 0x80 the block tree): offset and first key. +fn leaves(bytes: &[u8], ptype: u8) -> Vec<(usize, u64)> { + (0x4400..bytes.len().saturating_sub(511)) + .step_by(512) + .filter(|&ib| bytes[ib + 496] == ptype && bytes[ib + 497] == ptype && bytes[ib + 491] == 0) + .map(|ib| (ib, u64::from_le_bytes(bytes[ib..ib + 8].try_into().unwrap()))) + .collect() +} + +fn ndb(bytes: &[u8]) -> Ndb> { + let limits = Limits::default(); + let mut src = Source::new(Cursor::new(bytes), &limits).unwrap(); + let header = Header::parse(&src.read_at(0, HEADER_READ).unwrap()).unwrap(); + Ndb::new(src, header, limits) +} + +/// Every item of every folder: read, or the reason it wasn't. +fn read_all(pst: &mut Pst>) -> (usize, Vec<(Nid, Error)>) { + let tree = pst.walk_folders().unwrap(); + let (mut read, mut lost) = (0, Vec::new()); + for f in &tree.folders { + let Ok(list) = pst.items(f.folder.id) else { continue }; + for item in list.items { + match pst.message(&MessageRef::item(item.id)) { + Ok(_) => read += 1, + Err(e) => lost.push((item.id, e)), + } + } + } + (read, lost) +} + +#[test] +fn a_damaged_leaf_page_of_the_node_tree_loses_its_nodes_only() { + let mut bytes = base(); + // The leaf with the highest ids holds messages alone. + let &(ib, first) = leaves(&bytes, 0x81).iter().max_by_key(|l| l.1).unwrap(); + assert_eq!(first & 0x1F, 4, "a leaf of items"); + bytes[ib + 20] ^= 0xFF; + let mut pst = Pst::open(Cursor::new(&bytes[..])).unwrap(); + let tree = pst.walk_folders().unwrap(); + assert_eq!(tree.folders.len(), 7, "every folder still read"); + let (read, lost) = read_all(&mut pst); + assert!(read >= 1 && !lost.is_empty(), "{read} read, {lost:?}"); + assert!(lost.iter().all(|(_, e)| *e == Error::BadChecksum("a B-tree page"))); + let mut nodes = 0; + pst.for_each_node(|_| { + nodes += 1; + true + }) + .unwrap(); + assert!(nodes > 0); + assert_eq!(pst.damage().len(), 1); + assert_eq!((pst.damage()[0].offset, pst.damage()[0].what), (ib as u64, "a page of the node tree")); +} + +#[test] +fn a_damaged_leaf_page_of_the_block_tree_loses_the_data_it_finds_only() { + let mut bytes = base(); + let all = leaves(&bytes, 0x80); + // The last leaf: blocks written last, of the search root's own tables. + let &(ib, _) = all.iter().max_by_key(|l| l.1).unwrap(); + bytes[ib + 20] ^= 0xFF; + let mut pst = Pst::open(Cursor::new(&bytes[..])).unwrap(); + let tree = pst.walk_folders().unwrap(); + assert!(tree.folders.len() >= 5, "{}", tree.folders.len()); + let (read, _) = read_all(&mut pst); + assert!(read >= 3, "{read}"); + let lost_folders = tree.left_out.iter().filter(|l| l.reason == Error::BadChecksum("a B-tree page")).count(); + assert!(lost_folders >= 1 || pst.damage().iter().any(|d| d.offset == ib as u64), "{:?}", tree.left_out); + assert!(pst.damage().iter().all(|d| d.what == "a page of the block tree" && d.offset == ib as u64), "{:?}", pst.damage()); + assert_eq!(pst.damage().len(), 1); +} + +#[test] +fn a_damaged_block_of_one_message_loses_that_message_only() { + let clean = base(); + let mut pst = Pst::open(Cursor::new(&clean[..])).unwrap(); + let item = super::first_item(&mut pst); + let mut n = ndb(&clean); + let entry = n.node(item.0).unwrap(); + // Its top block: its one data block, or the block of its data tree. + let ib = n.block_entry(entry.data).unwrap().ib as usize; + let mut bytes = clean.clone(); + bytes[ib + 1] ^= 0xFF; + let mut pst = Pst::open(Cursor::new(&bytes[..])).unwrap(); + let tree = pst.walk_folders().unwrap(); + assert!(tree.left_out.is_empty()); + let (read, lost) = read_all(&mut pst); + assert_eq!(lost, vec![(item, Error::BadChecksum("a block"))]); + assert_eq!(read, 3, "the file's other items"); + assert!(pst.damage().is_empty(), "a block of a message is the message's loss, not the file's"); + // Its other parts are still there to be read. + let list = pst.items(tree.folders[2].folder.id).unwrap(); + assert_eq!(list.items[0].properties.text(0x0037).as_deref(), Some("Quarterly figures")); +} + +#[test] +fn a_damaged_folder_is_left_out_and_the_rest_read() { + let clean = base(); + let mut pst = Pst::open(Cursor::new(&clean[..])).unwrap(); + let calendar = pst.walk_folders().unwrap().folders.iter().find(|f| f.folder.name == "Calendar").unwrap().folder.id; + let mut n = ndb(&clean); + let entry = n.node(calendar.0).unwrap(); + let mut bid = entry.data; + if crate::ndb::is_internal(bid) { + // The first data block under the data tree's top block. + let top = n.block(bid).unwrap(); + bid = u64::from_le_bytes(top[8..16].try_into().unwrap()); + } + let ib = n.block_entry(bid).unwrap().ib as usize; + let mut bytes = clean.clone(); + bytes[ib + 1] ^= 0xFF; + let mut pst = Pst::open(Cursor::new(&bytes[..])).unwrap(); + let tree = pst.walk_folders().unwrap(); + let names: Vec<&str> = tree.folders.iter().map(|f| f.folder.name.as_str()).collect(); + assert_eq!(names, vec!["", "Top of Personal Folders", "Inbox", "Projects", "Contacts", "Search Root"]); + assert_eq!(tree.left_out.len(), 1); + assert_eq!((tree.left_out[0].id, &tree.left_out[0].reason), (calendar, &Error::BadChecksum("a block"))); + let (read, lost) = read_all(&mut pst); + assert_eq!((read, lost.len()), (3, 0), "the items of the other folders"); +} + +#[test] +fn damaged_pages_are_named_to_a_cap_and_counted_past_it() { + // Each named once, and quickly (a set, not a scan): 200,000 notes, 50,000 pages. + let bytes = base(); + let mut ndb = ndb(&bytes); + let e = Error::BadChecksum("a B-tree page"); + for i in 0..200_000u64 { + ndb.note((i % 50_000) * 512, 0x81, &e); + } + // A cap reached, or the reader failing, is not damage. + ndb.note(1 << 40, 0x81, &Error::Cap(crate::Cap::TotalRead)); + assert_eq!(ndb.damage.len(), crate::MAX_DAMAGE); + let last = ndb.damage.last().unwrap(); + assert_eq!(last.pages, 50_000 - crate::MAX_DAMAGE as u64 + 1); + assert_eq!(last.what, "pages of the node tree, from this one"); + assert!(ndb.damage[..crate::MAX_DAMAGE - 1].iter().all(|d| d.pages == 1)); +} + +#[test] +fn a_walk_of_the_nodes_that_reaches_a_cap_says_so() { + // At every allowance a walk either meets every node or ends with the cap: never fewer + // nodes and Ok. + let bytes = base(); + let count = |pst: &mut Pst>| { + let mut nodes = 0usize; + pst.for_each_node(|_| { + nodes += 1; + true + }) + .map(|()| nodes) + }; + let all = count(&mut Pst::open(Cursor::new(&bytes[..])).unwrap()).unwrap(); + let mut capped = 0; + for floor in (0..bytes.len() as u64).step_by(512) { + let limits = Limits { read_multiple: 0, read_floor: floor, ..Default::default() }; + let Ok(mut pst) = Pst::open_with(Cursor::new(&bytes[..]), limits) else { continue }; + match count(&mut pst) { + Ok(n) => assert_eq!(n, all, "at {floor}"), + Err(e) => { + assert_eq!(e, Error::Cap(crate::Cap::TotalRead), "at {floor}"); + capped += 1; + } + } + assert!(pst.damage().is_empty(), "a cap is not damage, at {floor}"); + } + assert!(capped > 0); +} + +/// A file whose reader fails once, on its `fail`th read: with an error, or (`short`) by +/// giving nothing, as a file cut short under the import does. +struct Flaky<'a> { + inner: Cursor<&'a [u8]>, + reads: usize, + fail: usize, + short: bool, +} + +impl std::io::Read for Flaky<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.reads += 1; + if self.reads == self.fail { + return if self.short { Ok(0) } else { Err(std::io::Error::other("the disk")) }; + } + self.inner.read(buf) + } +} + +impl std::io::Seek for Flaky<'_> { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self.inner.seek(pos) + } +} + +/// Each answer of a walk, every listing and every message, as text, and how many reads it took. +fn answers(pst: &mut Pst, folders: &[Nid], items: &[Nid]) -> Vec> { + let mut out = vec![pst.walk_folders().map(|t| format!("{t:?}"))]; + out.extend(folders.iter().map(|&f| pst.items(f).map(|l| format!("{l:?}")))); + out.extend(items.iter().map(|&i| pst.message(&MessageRef::item(i)).map(|m| format!("{m:?}")))); + out +} + +/// Whatever goes wrong with the reader under the file, or however little it may read, each +/// call is either the whole answer or that error: never an answer with parts missing. +#[test] +fn a_reader_that_fails_or_runs_out_ends_the_call() { + let bytes = base(); + let mut pst = Pst::open(Cursor::new(&bytes[..])).unwrap(); + let tree = pst.walk_folders().unwrap(); + let folders: Vec = tree.folders.iter().map(|f| f.folder.id).collect(); + let mut items = Vec::new(); + for &f in &folders { + items.extend(pst.items(f).unwrap().items.iter().map(|i| i.id)); + } + assert!(items.len() >= 3); + let whole = answers(&mut pst, &folders, &items); + assert!(whole.iter().all(Result::is_ok)); + let mut flaky = Flaky { inner: Cursor::new(&bytes[..]), reads: 0, fail: 0, short: false }; + let mut fresh = Pst::open(&mut flaky).unwrap(); + answers(&mut fresh, &folders, &items); + drop(fresh); + let reads = flaky.reads; + let check = |got: Vec>, expected: fn(&Error) -> bool, at: &str| { + for (n, (got, whole)) in got.iter().zip(&whole).enumerate() { + match got { + Ok(s) => assert_eq!(Some(s), whole.as_ref().ok(), "call {n} {at}"), + Err(e) => assert!(expected(e), "call {n} {at}: {e:?}"), + } + } + }; + let mut failed = 0; + for (fail, short) in (1..=reads).flat_map(|n| [(n, false), (n, true)]) { + let Ok(mut pst) = Pst::open(Flaky { inner: Cursor::new(&bytes[..]), reads: 0, fail, short }) else { continue }; + let got = answers(&mut pst, &folders, &items); + failed += got.iter().filter(|r| r.is_err()).count(); + check(got, |e| matches!(e, Error::Io(_)), &format!("failing read {fail}, short {short}")); + assert!(pst.damage().is_empty(), "failing read {fail}, short {short}"); + } + assert!(failed > 0); + let mut capped = 0; + for floor in (0..bytes.len() as u64 * 4).step_by(256) { + let limits = Limits { read_multiple: 0, read_floor: floor, ..Default::default() }; + let Ok(mut pst) = Pst::open_with(Cursor::new(&bytes[..]), limits) else { continue }; + let got = answers(&mut pst, &folders, &items); + capped += got.iter().filter(|r| r.is_err()).count(); + check(got, |e| *e == Error::Cap(crate::Cap::TotalRead), &format!("allowed {floor}")); + } + assert!(capped > 0); +} + +#[test] +fn a_page_below_the_root_with_a_level_the_format_lacks_is_damage() { + let mut bytes = base(); + let &(ib, first) = leaves(&bytes, 0x81).iter().max_by_key(|l| l.1).unwrap(); + assert_eq!(first & 0x1F, 4, "a leaf of items"); + bytes[ib + 488 + 3] = 9; + let t = 512 - Format::Unicode.trailer_size(); + let crc = crate::crc32(&bytes[ib..ib + t]); + bytes[ib + t + 4..ib + t + 8].copy_from_slice(&crc.to_le_bytes()); + let mut pst = Pst::open(Cursor::new(&bytes[..])).unwrap(); + pst.for_each_node(|_| true).unwrap(); + assert_eq!(pst.damage().len(), 1); + assert_eq!((pst.damage()[0].offset, &pst.damage()[0].reason), (ib as u64, &Error::Corrupt("a B-tree page's level"))); + assert_eq!(pst.walk_folders().unwrap().folders.len(), 7, "every folder still read"); +} diff --git a/crates/pst/src/tests/spec_samples.rs b/crates/pst/src/tests/spec_samples.rs new file mode 100644 index 000000000..0d88f51b9 --- /dev/null +++ b/crates/pst/src/tests/spec_samples.rs @@ -0,0 +1,160 @@ +//! The reader against the sample structures printed in [MS-PST] section 3 (a header, the heap +//! of a message store's property context, the heap of a hierarchy table), and the algorithms +//! of section 5 against the values the specification gives. + +use crate::crc::{crc32, table_head}; +use crate::crypt::{Encryption, decode, encode, tables}; +use crate::error::Error; +use crate::header::{Format, Header}; +use crate::heap::Heap; +use crate::ndb::signature; +use crate::prop::PropValue; + +#[test] +fn crc_table_is_the_specifications() { + assert_eq!(table_head(), [0x0000_0000, 0x7707_3096, 0xEE0E_612C, 0x2D02_EF8D]); +} + +#[test] +fn sample_header_checks_out() { + // 3.2: dwCRCPartial 0x379aa90e, dwCRCFull 0x1fd283d6. + assert_eq!(crc32(&HEADER[8..479]), 0x379A_A90E); + assert_eq!(crc32(&HEADER[8..524]), 0x1FD2_83D6); + let h = Header::parse(HEADER).unwrap(); + assert_eq!(h.format, Format::Unicode); + assert_eq!(h.version, 23); + assert_eq!(h.encryption, Encryption::Permute); + assert_eq!(h.file_eof, 0x9F_2400); + assert_eq!((h.nbt.bid, h.nbt.ib), (0x24B, 0x90_5200)); + assert_eq!((h.bbt.bid, h.bbt.ib), (0x253, 0x90_0A00)); +} + +#[test] +fn sample_page_signatures() { + // 3.3 and 3.5: the intermediate page at 0x8200 (bid 0x206) and the BBT leaf at 0x900200 + // (bid 0x246) carry 0x8006 and 0x00D6. + assert_eq!(signature(0x8200, 0x206), 0x8006); + assert_eq!(signature(0x90_0200, 0x246), 0x00D6); +} + +#[test] +fn permutation_tables_undo_each_other() { + let (r, s, i) = tables(); + for b in 0..=255u8 { + assert_eq!(i[usize::from(r[usize::from(b)])], b); + assert_eq!(s[usize::from(s[usize::from(b)])], b); + } + for method in [Encryption::Permute, Encryption::Cyclic] { + let original: Vec = (0..=255u8).cycle().take(1000).collect(); + let mut data = original.clone(); + encode(method, &mut data, 0x1234_5678); + assert_ne!(data, original); + decode(method, &mut data, 0x1234_5678); + assert_eq!(data, original); + } +} + +#[test] +fn sample_message_store_heap() { + // 3.8–3.10: the message store's PC of a Unicode file named "UNICODE1". + let heap = Heap::new(vec![STORE_HEAP.to_vec()]).unwrap(); + assert_eq!(heap.client, 0xBC); + assert_eq!(heap.user_root, 0x20); + let node = crate::ltp::NodeData { heap, subnodes: Default::default() }; + let records = crate::ltp::pc_records(&node).unwrap(); + assert_eq!(records.len(), 11); + let ids: Vec = records.iter().map(|r| r.0).collect(); + assert!(ids.contains(&0x3001) && ids.contains(&0x67FF) && ids.contains(&0x35E0)); + let (_, ty, raw) = records.iter().find(|r| r.0 == 0x3001).copied().unwrap(); + assert_eq!(ty, 0x001F); + let name = crate::prop::stored(ty, node.heap.item(raw).unwrap()).unwrap(); + assert_eq!(name, PropValue::String("UNICODE1".into())); + let (_, _, raw) = records.iter().find(|r| r.0 == 0x0FF9).copied().unwrap(); + assert_eq!(node.heap.item(raw).unwrap(), &[0x22, 0x9D, 0xB5, 0x0A, 0xDC, 0xD9, 0x94, 0x43, 0x85, 0xDE, 0x90, 0xAE, 0xB0, 0x7D, 0x12, 0x70]); + let (_, _, raw) = records.iter().find(|r| r.0 == 0x35DF).copied().unwrap(); + assert_eq!(raw, 0x89); + let (_, _, raw) = records.iter().find(|r| r.0 == 0x67FF).copied().unwrap(); + assert_eq!(raw, 0); +} + +#[test] +fn sample_hierarchy_table_heap() { + // 3.11–3.12: the root folder's hierarchy table, three rows in the heap. + let heap = Heap::new(vec![HIERARCHY_HEAP.to_vec()]).unwrap(); + assert_eq!(heap.client, 0x7C); + let info = heap.item(heap.user_root).unwrap(); + assert_eq!(info[1], 0x0D); + let index = crate::bth::read(&heap, 0x20, 1000, Error::Corrupt("")).unwrap(); + let rows: Vec<(u32, u32)> = + index.records.iter().map(|r| (u32::from_le_bytes(r[0..4].try_into().unwrap()), u32::from_le_bytes(r[4..8].try_into().unwrap()))).collect(); + assert_eq!(rows, vec![(0x2223, 2), (0x8022, 0), (0x8042, 1)]); + let matrix = heap.item(0x80).unwrap(); + assert_eq!(matrix.len(), 3 * 55); + // Row 0's display name (column 0x3001 at offset 8) is "Top of Personal Folders". + let hid = u32::from_le_bytes(matrix[8..12].try_into().unwrap()); + assert_eq!(crate::bytes::utf16(heap.item(hid).unwrap()), "Top of Personal Folders"); +} + +pub const HEADER: &[u8] = &[ + 0x21, 0x42, 0x44, 0x4E, 0x0E, 0xA9, 0x9A, 0x37, 0x53, 0x4D, 0x17, 0x00, 0x13, 0x00, 0x01, 0x01, 0x5C, 0x07, 0x00, 0x00, 0xD0, 0x7B, 0x99, 0x0B, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x04, 0x04, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x0F, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x9F, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x44, 0x9B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0xF2, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x4B, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x0A, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x80, 0x01, 0x00, 0x00, 0x34, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0x83, 0xD2, 0x1F, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +pub const STORE_HEAP: &[u8] = &[ + 0xEC, 0x00, 0xEC, 0xBC, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB5, 0x02, 0x06, 0x00, 0x40, 0x00, 0x00, 0x00, 0x34, 0x0E, 0x02, 0x01, + 0xA0, 0x00, 0x00, 0x00, 0x38, 0x0E, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0x0F, 0x02, 0x01, 0x60, 0x00, 0x00, 0x00, 0x01, 0x30, 0x1F, 0x00, + 0x80, 0x00, 0x00, 0x00, 0xDF, 0x35, 0x03, 0x00, 0x89, 0x00, 0x00, 0x00, 0xE0, 0x35, 0x02, 0x01, 0xC0, 0x00, 0x00, 0x00, 0xE3, 0x35, 0x02, 0x01, + 0x00, 0x01, 0x00, 0x00, 0xE7, 0x35, 0x02, 0x01, 0xE0, 0x00, 0x00, 0x00, 0x33, 0x66, 0x0B, 0x00, 0x01, 0x00, 0x00, 0x00, 0xFA, 0x66, 0x03, 0x00, + 0x0D, 0x00, 0x0E, 0x00, 0xFF, 0x67, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x9D, 0xB5, 0x0A, 0xDC, 0xD9, 0x94, 0x43, 0x85, 0xDE, 0x90, 0xAE, + 0xB0, 0x7D, 0x12, 0x70, 0x55, 0x00, 0x4E, 0x00, 0x49, 0x00, 0x43, 0x00, 0x4F, 0x00, 0x44, 0x00, 0x45, 0x00, 0x31, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xF5, 0x5E, 0xF6, 0x66, 0x95, 0x69, 0xCC, 0x4C, 0x83, 0xD1, 0xD8, 0x73, 0x98, 0x99, 0x02, 0x85, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x22, 0x9D, 0xB5, 0x0A, 0xDC, 0xD9, 0x94, 0x43, 0x85, 0xDE, 0x90, 0xAE, 0xB0, 0x7D, 0x12, 0x70, 0x22, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x22, 0x9D, 0xB5, 0x0A, 0xDC, 0xD9, 0x94, 0x43, 0x85, 0xDE, 0x90, 0xAE, 0xB0, 0x7D, 0x12, 0x70, 0x42, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x22, 0x9D, 0xB5, 0x0A, 0xDC, 0xD9, 0x94, 0x43, 0x85, 0xDE, 0x90, 0xAE, 0xB0, 0x7D, 0x12, 0x70, 0x62, 0x80, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x0C, 0x00, 0x14, 0x00, 0x6C, 0x00, 0x7C, 0x00, 0x8C, 0x00, 0xA4, 0x00, 0xBC, 0x00, 0xD4, 0x00, 0xEC, 0x00, +]; + +pub const HIERARCHY_HEAP: &[u8] = &[ + 0xBC, 0x01, 0xEC, 0x7C, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB5, 0x04, 0x04, 0x00, 0x60, 0x00, 0x00, 0x00, 0x7C, 0x0D, 0x34, 0x00, + 0x34, 0x00, 0x35, 0x00, 0x37, 0x00, 0x20, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x01, 0x30, 0x0E, 0x14, 0x00, + 0x04, 0x06, 0x14, 0x00, 0x33, 0x0E, 0x18, 0x00, 0x08, 0x07, 0x02, 0x01, 0x34, 0x0E, 0x20, 0x00, 0x04, 0x08, 0x03, 0x00, 0x38, 0x0E, 0x24, 0x00, + 0x04, 0x09, 0x1F, 0x00, 0x01, 0x30, 0x08, 0x00, 0x04, 0x02, 0x03, 0x00, 0x02, 0x36, 0x0C, 0x00, 0x04, 0x03, 0x03, 0x00, 0x03, 0x36, 0x10, 0x00, + 0x04, 0x04, 0x0B, 0x00, 0x0A, 0x36, 0x34, 0x00, 0x01, 0x05, 0x1F, 0x00, 0x13, 0x36, 0x28, 0x00, 0x04, 0x0A, 0x03, 0x00, 0x35, 0x66, 0x2C, 0x00, + 0x04, 0x0B, 0x03, 0x00, 0x36, 0x66, 0x30, 0x00, 0x04, 0x0C, 0x03, 0x00, 0xF2, 0x67, 0x00, 0x00, 0x04, 0x00, 0x03, 0x00, 0xF3, 0x67, 0x04, 0x00, + 0x04, 0x01, 0x23, 0x22, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x22, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x80, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x22, 0x80, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0xA0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xFC, 0x00, 0x42, 0x80, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFC, 0x00, 0x23, 0x22, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, + 0xE0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFC, 0x00, 0x54, + 0x00, 0x6F, 0x00, 0x70, 0x00, 0x20, 0x00, 0x6F, 0x00, 0x66, 0x00, 0x20, 0x00, 0x50, 0x00, 0x65, 0x00, 0x72, 0x00, 0x73, 0x00, 0x6F, 0x00, 0x6E, + 0x00, 0x61, 0x00, 0x6C, 0x00, 0x20, 0x00, 0x46, 0x00, 0x6F, 0x00, 0x6C, 0x00, 0x64, 0x00, 0x65, 0x00, 0x72, 0x00, 0x73, 0x00, 0x53, 0x00, 0x65, + 0x00, 0x61, 0x00, 0x72, 0x00, 0x63, 0x00, 0x68, 0x00, 0x20, 0x00, 0x52, 0x00, 0x6F, 0x00, 0x6F, 0x00, 0x74, 0x00, 0x53, 0x00, 0x50, 0x00, 0x41, + 0x00, 0x4D, 0x00, 0x20, 0x00, 0x53, 0x00, 0x65, 0x00, 0x61, 0x00, 0x72, 0x00, 0x63, 0x00, 0x68, 0x00, 0x20, 0x00, 0x46, 0x00, 0x6F, 0x00, 0x6C, + 0x00, 0x64, 0x00, 0x65, 0x00, 0x72, 0x00, 0x20, 0x00, 0x32, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x14, 0x00, 0x92, 0x00, 0xAA, 0x00, + 0x4F, 0x01, 0x7D, 0x01, 0x93, 0x01, 0xBB, 0x01, +]; diff --git a/crates/pst/src/tests/writer.rs b/crates/pst/src/tests/writer.rs new file mode 100644 index 000000000..0df91cdde --- /dev/null +++ b/crates/pst/src/tests/writer.rs @@ -0,0 +1,1312 @@ +//! A small .pst writer, for tests only: it builds synthetic files from [MS-PST] (ANSI or +//! Unicode, each encoding, with or without a password) holding invented folders, messages, +//! recipients, attachments and attached messages, so that the reader has something to read. +//! It writes no allocation maps (the reader doesn't use them) and can be told to split data +//! into small blocks and pages, so that data trees, subnode trees, heaps over several blocks, +//! row matrices over several blocks and B-trees of several levels come out of small files. + +use std::collections::BTreeMap; + +use crate::crc::crc32; +use crate::crypt::{Encryption, encode}; +use crate::header::Format; +use crate::ndb::signature; +use crate::prop::types::{BINARY, BOOLEAN, INT32, OBJECT, STRING, STRING8, TIME}; + +// --------------------------------------------------------------------------------------------- +// The node database. + +/// A node's data: given as the blocks it is to be stored in (a heap), or as one stream that +/// the writer splits. +#[derive(Clone, Debug)] +pub enum Data { + Blocks(Vec>), + Stream(Vec), + /// A block id given as it is, for nothing written (to name another node's data). + Bid(u64), + /// One data block named `times` times by an XBLOCK, and (with `levels` 2) that XBLOCK + /// named `times` times by an XXBLOCK: a data tree that reads one block over and over. + Repeated { + piece: Vec, + times: usize, + levels: u8, + }, +} + +#[derive(Clone, Debug)] +pub struct SubNode { + pub nid: u32, + pub data: Data, + pub subnodes: Vec, +} + +/// Marks a subnode's own subnode tree as a block id given as it is ([`SubNode::with_sub_bid`]). +const SUB_BID: u32 = u32::MAX; + +impl SubNode { + /// A subnode whose data and subnode tree are the block ids given, for nothing written. + pub fn with_sub_bid(nid: u32, data: u64, sub: u64) -> SubNode { + SubNode { nid, data: Data::Bid(data), subnodes: vec![SubNode { nid: SUB_BID, data: Data::Bid(sub), subnodes: Vec::new() }] } + } +} + +#[derive(Clone, Debug)] +pub struct Node { + pub nid: u32, + pub parent: u32, + pub data: Data, + pub subnodes: Vec, +} + +/// How the writer lays things out; the small sizes make deep structures from little data. +#[derive(Clone, Debug)] +pub struct Layout { + pub format: Format, + pub encryption: Encryption, + /// Data bytes per data block of a stream (at most a block's capacity). + pub block_data: usize, + /// Ids per XBLOCK / XXBLOCK. + pub xblock_ids: usize, + /// Entries per SLBLOCK / SIBLOCK. + pub sl_entries: usize, + /// Entries per B-tree page. + pub page_entries: usize, + /// Bytes of heap items per heap block. + pub heap_block: usize, + /// Records per BTH leaf item (an index level above when there are more). + pub bth_leaf: usize, + /// Store a table's rows in a subnode even when they would fit the heap. + pub rows_in_subnode: bool, + /// Set the reserved bit A (bit 0) of every block id where a block is referred to (the + /// B-trees, data and subnode trees), or in each block's own trailer. + pub bit_a_in_references: bool, + pub bit_a_in_trailers: bool, +} + +impl Layout { + pub fn new(format: Format, encryption: Encryption) -> Layout { + let cap = 8192 - format.trailer_size(); + Layout { + format, + encryption, + block_data: cap, + xblock_ids: (cap - 8) / format.id_size(), + sl_entries: 300, + page_entries: 15, + heap_block: 3800, + bth_leaf: 400, + rows_in_subnode: false, + bit_a_in_references: false, + bit_a_in_trailers: false, + } + } + + /// Everything as small as it goes, to exercise every level of every tree. + pub fn tiny(format: Format, encryption: Encryption) -> Layout { + Layout { + block_data: 100, + xblock_ids: 3, + sl_entries: 2, + page_entries: 3, + heap_block: 160, + bth_leaf: 3, + rows_in_subnode: true, + ..Layout::new(format, encryption) + } + } +} + +struct Ndb { + layout: Layout, + out: Vec, + next_block: u64, + next_page: u64, + blocks: Vec<(u64, u64, u16)>, +} + +fn put_id(out: &mut Vec, f: Format, v: u64) { + match f { + Format::Unicode => out.extend_from_slice(&v.to_le_bytes()), + Format::Ansi => out.extend_from_slice(&(v as u32).to_le_bytes()), + } +} + +impl Ndb { + fn new(layout: Layout) -> Ndb { + Ndb { layout, out: vec![0; 0x4400], next_block: 1, next_page: 1, blocks: Vec::new() } + } + + /// A block id as it is written where the block is referred to. + fn r(&self, bid: u64) -> u64 { + if self.layout.bit_a_in_references && bid != 0 { bid | 1 } else { bid } + } + + fn align(&mut self, to: usize) { + while !self.out.len().is_multiple_of(to) { + self.out.push(0); + } + } + + fn block(&mut self, data: &[u8], internal: bool) -> u64 { + let f = self.layout.format; + let bid = (self.next_block << 2) | if internal { 2 } else { 0 }; + self.next_block += 1; + let mut stored = data.to_vec(); + if !internal { + encode(self.layout.encryption, &mut stored, bid); + } + self.align(64); + let ib = self.out.len() as u64; + let ts = f.trailer_size(); + let total = (stored.len() + ts).div_ceil(64) * 64; + assert!(total <= 8192, "block too large"); + let own = if self.layout.bit_a_in_trailers { bid | 1 } else { bid }; + let mut b = stored.clone(); + b.resize(total - ts, 0xAB); + b.extend_from_slice(&(stored.len() as u16).to_le_bytes()); + b.extend_from_slice(&signature(ib, own).to_le_bytes()); + let crc = crc32(&stored); + match f { + Format::Unicode => { + b.extend_from_slice(&crc.to_le_bytes()); + b.extend_from_slice(&own.to_le_bytes()); + } + Format::Ansi => { + b.extend_from_slice(&(own as u32).to_le_bytes()); + b.extend_from_slice(&crc.to_le_bytes()); + } + } + self.out.extend_from_slice(&b); + self.blocks.push((bid, ib, stored.len() as u16)); + bid + } + + fn stream(&mut self, data: &Data) -> u64 { + let pieces: Vec> = match data { + Data::Bid(bid) => return *bid, + Data::Repeated { piece, times, levels } => { + let f = self.layout.format; + let one = self.block(piece, false); + let mut x = vec![1u8, 1]; + x.extend_from_slice(&(*times as u16).to_le_bytes()); + x.extend_from_slice(&((piece.len() * times) as u32).to_le_bytes()); + for _ in 0..*times { + put_id(&mut x, f, self.r(one)); + } + let xb = self.block(&x, true); + if *levels < 2 { + return xb; + } + let mut xx = vec![1u8, 2]; + xx.extend_from_slice(&(*times as u16).to_le_bytes()); + xx.extend_from_slice(&((piece.len() * times * times) as u32).to_le_bytes()); + for _ in 0..*times { + put_id(&mut xx, f, self.r(xb)); + } + return self.block(&xx, true); + } + Data::Blocks(b) => b.clone(), + Data::Stream(s) if s.is_empty() => vec![Vec::new()], + Data::Stream(s) => s.chunks(self.layout.block_data).map(<[u8]>::to_vec).collect(), + }; + if pieces.len() == 1 { + return self.block(&pieces[0], false); + } + let f = self.layout.format; + let per = self.layout.xblock_ids; + let mut xblocks = Vec::new(); + for group in pieces.chunks(per) { + let total: usize = group.iter().map(Vec::len).sum(); + let ids: Vec = group.iter().map(|p| self.block(p, false)).collect(); + let mut x = vec![1u8, 1]; + x.extend_from_slice(&(ids.len() as u16).to_le_bytes()); + x.extend_from_slice(&(total as u32).to_le_bytes()); + for id in ids { + put_id(&mut x, f, self.r(id)); + } + xblocks.push((self.block(&x, true), total)); + } + if xblocks.len() == 1 { + return xblocks[0].0; + } + assert!(xblocks.len() <= (8192 - 8 - f.trailer_size()) / f.id_size(), "data too large for an XXBLOCK"); + let total: usize = xblocks.iter().map(|x| x.1).sum(); + let mut xx = vec![1u8, 2]; + xx.extend_from_slice(&(xblocks.len() as u16).to_le_bytes()); + xx.extend_from_slice(&(total as u32).to_le_bytes()); + for (id, _) in xblocks { + put_id(&mut xx, f, self.r(id)); + } + self.block(&xx, true) + } + + fn subnodes(&mut self, subs: &[SubNode]) -> u64 { + if subs.is_empty() { + return 0; + } + let f = self.layout.format; + let mut entries: Vec<(u32, u64, u64)> = subs + .iter() + .map(|s| { + let sub = match s.subnodes.as_slice() { + [SubNode { nid: SUB_BID, data: Data::Bid(bid), .. }] => *bid, + other => self.subnodes(other), + }; + (s.nid, self.stream(&s.data), sub) + }) + .collect(); + entries.sort_by_key(|e| e.0); + let head = |level: u8, n: usize| -> Vec { + let mut b = vec![2u8, level]; + b.extend_from_slice(&(n as u16).to_le_bytes()); + if f == Format::Unicode { + b.extend_from_slice(&[0; 4]); + } + b + }; + let mut leaves = Vec::new(); + for group in entries.chunks(self.layout.sl_entries) { + let mut b = head(0, group.len()); + for &(nid, data, sub) in group { + put_id(&mut b, f, u64::from(nid)); + put_id(&mut b, f, self.r(data)); + put_id(&mut b, f, self.r(sub)); + } + leaves.push((group[0].0, self.block(&b, true))); + } + if leaves.len() == 1 { + return leaves[0].1; + } + let mut b = head(1, leaves.len()); + for (nid, bid) in leaves { + put_id(&mut b, f, u64::from(nid)); + put_id(&mut b, f, self.r(bid)); + } + self.block(&b, true) + } + + /// A B-tree over `leaves` (key, entry bytes), returning the root's (bid, ib). + fn btree(&mut self, ptype: u8, leaves: Vec<(u64, Vec)>, leaf_size: usize) -> (u64, u64) { + let f = self.layout.format; + let mut level_entries: Vec<(u64, Vec)> = leaves; + let mut level = 0u8; + let mut entry_size = leaf_size; + loop { + let mut next = Vec::new(); + let chunks: Vec)>> = + if level_entries.is_empty() { vec![Vec::new()] } else { level_entries.chunks(self.layout.page_entries).map(<[_]>::to_vec).collect() }; + let single = chunks.len() == 1; + for chunk in chunks { + let (bid, ib) = self.page(ptype, level, entry_size, &chunk); + let key = chunk.first().map_or(0, |e| e.0); + if single { + return (bid, ib); + } + let mut e = Vec::new(); + put_id(&mut e, f, key); + put_id(&mut e, f, bid); + put_id(&mut e, f, ib); + next.push((key, e)); + } + level_entries = next; + level += 1; + entry_size = 3 * f.id_size(); + } + } + + fn page(&mut self, ptype: u8, level: u8, entry_size: usize, entries: &[(u64, Vec)]) -> (u64, u64) { + let f = self.layout.format; + self.align(512); + let ib = self.out.len() as u64; + let bid = self.next_page; + self.next_page += 1; + let area = if f == Format::Unicode { 488 } else { 496 }; + let mut p = Vec::new(); + for (_, e) in entries { + let mut e = e.clone(); + e.resize(entry_size, 0); + p.extend_from_slice(&e); + } + assert!(p.len() <= area); + p.resize(area, 0); + p.push(entries.len() as u8); + p.push((area / entry_size) as u8); + p.push(entry_size as u8); + p.push(level); + if f == Format::Unicode { + p.extend_from_slice(&[0; 4]); + } + let crc = crc32(&p); + p.push(ptype); + p.push(ptype); + p.extend_from_slice(&signature(ib, bid).to_le_bytes()); + match f { + Format::Unicode => { + p.extend_from_slice(&crc.to_le_bytes()); + p.extend_from_slice(&bid.to_le_bytes()); + } + Format::Ansi => { + p.extend_from_slice(&(bid as u32).to_le_bytes()); + p.extend_from_slice(&crc.to_le_bytes()); + } + } + assert_eq!(p.len(), 512); + self.out.extend_from_slice(&p); + (bid, ib) + } + + fn finish(mut self, nodes: &[Node]) -> Vec { + let f = self.layout.format; + let mut nbt = Vec::new(); + for n in nodes { + let data = self.stream(&n.data); + let sub = self.subnodes(&n.subnodes); + let mut e = Vec::new(); + put_id(&mut e, f, u64::from(n.nid)); + put_id(&mut e, f, self.r(data)); + put_id(&mut e, f, self.r(sub)); + e.extend_from_slice(&n.parent.to_le_bytes()); + nbt.push((u64::from(n.nid), e)); + } + nbt.sort_by_key(|e| e.0); + let mut bbt: Vec<(u64, Vec)> = self + .blocks + .iter() + .map(|&(bid, ib, cb)| { + let mut e = Vec::new(); + put_id(&mut e, f, self.r(bid)); + put_id(&mut e, f, ib); + e.extend_from_slice(&cb.to_le_bytes()); + e.extend_from_slice(&2u16.to_le_bytes()); + (bid, e) + }) + .collect(); + bbt.sort_by_key(|e| e.0); + let (nbt_size, bbt_size) = match f { + Format::Unicode => (32, 24), + Format::Ansi => (16, 12), + }; + let nbt_root = self.btree(0x81, nbt, nbt_size); + let bbt_root = self.btree(0x80, bbt, bbt_size); + let eof = self.out.len() as u64; + let header = header(f, self.layout.encryption, eof, nbt_root, bbt_root, self.next_block << 2, self.next_page); + self.out[..header.len()].copy_from_slice(&header); + self.out + } +} + +/// The HEADER, with both CRCs. +pub fn header(f: Format, enc: Encryption, eof: u64, nbt: (u64, u64), bbt: (u64, u64), next_b: u64, next_p: u64) -> Vec { + let crypt = match enc { + Encryption::None => 0u8, + Encryption::Permute => 1, + Encryption::Cyclic => 2, + }; + let mut h = Vec::new(); + h.extend_from_slice(b"!BDN"); + h.extend_from_slice(&[0; 4]); // dwCRCPartial + h.extend_from_slice(b"SM"); + h.extend_from_slice(&(if f == Format::Unicode { 23u16 } else { 14 }).to_le_bytes()); + h.extend_from_slice(&19u16.to_le_bytes()); + h.extend_from_slice(&[1, 1]); + h.extend_from_slice(&[0; 8]); // dwReserved1, dwReserved2 + let mut root = Vec::new(); + root.extend_from_slice(&[0; 4]); + put_id(&mut root, f, eof); + put_id(&mut root, f, 0); // ibAMapLast + put_id(&mut root, f, 0); // cbAMapFree + put_id(&mut root, f, 0); // cbPMapFree + put_id(&mut root, f, nbt.0); + put_id(&mut root, f, nbt.1); + put_id(&mut root, f, bbt.0); + put_id(&mut root, f, bbt.1); + root.extend_from_slice(&[2, 0, 0, 0]); + let mut rgnid = Vec::new(); + for _ in 0..32 { + rgnid.extend_from_slice(&0x400u32.to_le_bytes()); + } + match f { + Format::Unicode => { + h.extend_from_slice(&[0; 8]); // bidUnused + h.extend_from_slice(&next_p.to_le_bytes()); + h.extend_from_slice(&1u32.to_le_bytes()); // dwUnique + h.extend_from_slice(&rgnid); + h.extend_from_slice(&[0; 8]); // qwUnused + h.extend_from_slice(&root); + h.extend_from_slice(&[0; 4]); // dwAlign + h.extend_from_slice(&[0xFF; 256]); + h.extend_from_slice(&[0x80, crypt, 0, 0]); + h.extend_from_slice(&next_b.to_le_bytes()); + h.extend_from_slice(&[0; 4]); // dwCRCFull + h.extend_from_slice(&[0; 36]); + assert_eq!(h.len(), 564); + let full = crc32(&h[8..524]); + h[524..528].copy_from_slice(&full.to_le_bytes()); + } + Format::Ansi => { + h.extend_from_slice(&(next_b as u32).to_le_bytes()); + h.extend_from_slice(&(next_p as u32).to_le_bytes()); + h.extend_from_slice(&1u32.to_le_bytes()); + h.extend_from_slice(&rgnid); + h.extend_from_slice(&root); + h.extend_from_slice(&[0xFF; 256]); + h.extend_from_slice(&[0x80, crypt, 0, 0]); + h.extend_from_slice(&[0; 12]); + h.extend_from_slice(&[0; 36]); + assert_eq!(h.len(), 512); + } + } + let partial = crc32(&h[8..479]); + h[4..8].copy_from_slice(&partial.to_le_bytes()); + h +} + +// --------------------------------------------------------------------------------------------- +// Lists, tables and properties. + +/// A heap being filled: items go into blocks of at most `capacity` bytes. +pub struct HeapBuilder { + client: u8, + capacity: usize, + blocks: Vec>>, + pub user_root: u32, +} + +impl HeapBuilder { + pub fn new(client: u8, capacity: usize) -> HeapBuilder { + HeapBuilder { client, capacity, blocks: vec![Vec::new()], user_root: 0 } + } + + pub fn alloc(&mut self, item: Vec) -> u32 { + let used: usize = self.blocks.last().unwrap().iter().map(Vec::len).sum(); + let count = self.blocks.last().unwrap().len(); + if count > 0 && (used + item.len() > self.capacity || count >= 2000) { + self.blocks.push(Vec::new()); + } + let block = self.blocks.len() - 1; + self.blocks[block].push(item); + let index = self.blocks[block].len(); + ((block as u32) << 16) | ((index as u32) << 5) + } + + pub fn finish(self) -> Vec> { + let mut out = Vec::new(); + for (i, items) in self.blocks.iter().enumerate() { + let header_len = if i == 0 { + 12 + } else if i == 8 || (i > 8 && (i - 8) % 128 == 0) { + 66 + } else { + 2 + }; + let mut b = vec![0u8; header_len]; + let mut offsets = Vec::new(); + for item in items { + offsets.push(b.len() as u16); + b.extend_from_slice(item); + } + offsets.push(b.len() as u16); + if b.len() % 2 == 1 { + b.push(0); + } + let map = b.len() as u16; + b[0..2].copy_from_slice(&map.to_le_bytes()); + if i == 0 { + b[2] = 0xEC; + b[3] = self.client; + b[4..8].copy_from_slice(&self.user_root.to_le_bytes()); + } + b.extend_from_slice(&((offsets.len() - 1) as u16).to_le_bytes()); + b.extend_from_slice(&0u16.to_le_bytes()); + for o in offsets { + b.extend_from_slice(&o.to_le_bytes()); + } + out.push(b); + } + out + } +} + +/// A BTH in `heap` over `records` (sorted by key), returning its header's HID. +pub fn bth(heap: &mut HeapBuilder, key: usize, ent: usize, records: &[Vec], leaf: usize) -> u32 { + if records.is_empty() { + return heap.alloc(vec![0xB5, key as u8, ent as u8, 0, 0, 0, 0, 0]); + } + let mut level: Vec<(Vec, u32)> = records.chunks(leaf).map(|c| (c[0][..key].to_vec(), heap.alloc(c.concat()))).collect(); + let mut levels = 0u8; + while level.len() > 1 { + levels += 1; + level = level + .chunks(leaf) + .map(|c| { + let mut item = Vec::new(); + for (k, hid) in c { + item.extend_from_slice(k); + item.extend_from_slice(&hid.to_le_bytes()); + } + (c[0].0.clone(), heap.alloc(item)) + }) + .collect(); + } + let mut h = vec![0xB5, key as u8, ent as u8, levels]; + h.extend_from_slice(&level[0].1.to_le_bytes()); + heap.alloc(h) +} + +/// A property value as the writer is given it. +#[derive(Clone, Debug)] +pub struct Value { + pub ty: u16, + pub bytes: Vec, +} + +/// What [`hnid_ref`] puts before the HNID. +const HNID_REF: &[u8; 8] = b"\xFFHNIDREF"; + +/// A variable-size value whose record names `hnid` as it is (an HNID another record names +/// too, or one that names nothing). +pub fn hnid_ref(ty: u16, hnid: u32) -> Value { + let mut bytes = HNID_REF.to_vec(); + bytes.extend_from_slice(&hnid.to_le_bytes()); + Value { ty, bytes } +} + +fn as_ref(v: &Value) -> Option { + let rest = v.bytes.strip_prefix(HNID_REF)?; + Some(u32::from_le_bytes(rest.try_into().ok()?)) +} + +pub fn int32(v: i32) -> Value { + Value { ty: INT32, bytes: v.to_le_bytes().to_vec() } +} +pub fn boolean(v: bool) -> Value { + Value { ty: BOOLEAN, bytes: vec![u8::from(v)] } +} +pub fn time(v: u64) -> Value { + Value { ty: TIME, bytes: v.to_le_bytes().to_vec() } +} +pub fn binary(v: &[u8]) -> Value { + Value { ty: BINARY, bytes: v.to_vec() } +} +pub fn unicode(s: &str) -> Value { + Value { ty: STRING, bytes: s.encode_utf16().flat_map(u16::to_le_bytes).collect() } +} +pub fn string8(s: &str) -> Value { + Value { ty: STRING8, bytes: s.chars().map(|c| c as u8).collect() } +} + +/// A one-off entry id ([MS-OXCDATA] 2.2.5.1): a display name, an address type and an address, +/// in UTF-16 or in single bytes. +pub fn one_off(name: &str, kind: &str, address: &str, unicode: bool) -> Vec { + let mut e = vec![0u8; 4]; + e.extend([0x81, 0x2B, 0x1F, 0xA4, 0xBE, 0xA3, 0x10, 0x19, 0x9D, 0x6E, 0x00, 0xDD, 0x01, 0x0F, 0x54, 0x02]); + e.extend([0, 0]); + e.extend(if unicode { 0x8000u16 } else { 0 }.to_le_bytes()); + for t in [name, kind, address] { + if unicode { + e.extend(t.encode_utf16().flat_map(u16::to_le_bytes)); + e.extend([0, 0]); + } else { + e.extend(t.bytes()); + e.push(0); + } + } + e +} + +/// A WrappedEntryId ([MS-OXOCNTC] 2.2.2.2.4.1.1): `kind` (0x00 a one-off, 0xC3 a contact's +/// first email address, 0xB4 a list…) and the entry id it wraps. +pub fn wrapped_entry_id(kind: u8, embedded: &[u8]) -> Vec { + let mut e = vec![0u8; 4]; + e.extend([0xC0, 0x91, 0xAD, 0xD3, 0x51, 0x9D, 0xCF, 0x11, 0xA4, 0xA9, 0x00, 0xAA, 0x00, 0x47, 0xFA, 0xA4]); + e.push(kind); + e.extend_from_slice(embedded); + e +} + +/// One changed occurrence of a series: start, end, original start, new subject. +pub type ExceptionSpec = (u32, u32, u32, Option); + +/// An AppointmentRecurrencePattern ([MS-OXOCAL] 2.2.1.44.5). `specific` is the PatternTypeSpecific +/// words; dates are minutes since 1601 by the series' clock. +#[allow(clippy::too_many_arguments)] +pub fn recurrence_bytes( + frequency: u16, + pattern: u16, + period: u32, + specific: &[u32], + end: (u32, u32, u32), + deleted: &[u32], + start_date: u32, + offsets: (u32, u32), + exceptions: &[ExceptionSpec], +) -> Vec { + let mut b = Vec::new(); + let w16 = |b: &mut Vec, v: u16| b.extend_from_slice(&v.to_le_bytes()); + let w32 = |b: &mut Vec, v: u32| b.extend_from_slice(&v.to_le_bytes()); + w16(&mut b, 0x3004); + w16(&mut b, 0x3004); + w16(&mut b, frequency); + w16(&mut b, pattern); + w16(&mut b, 0); + w32(&mut b, 0); + w32(&mut b, period); + w32(&mut b, 0); + specific.iter().for_each(|v| w32(&mut b, *v)); + w32(&mut b, end.0); + w32(&mut b, end.1); + w32(&mut b, 1); + w32(&mut b, deleted.len() as u32); + deleted.iter().for_each(|v| w32(&mut b, *v)); + w32(&mut b, exceptions.len() as u32); + exceptions.iter().for_each(|x| w32(&mut b, x.2)); + w32(&mut b, start_date); + w32(&mut b, end.2); + w32(&mut b, 0x3006); + w32(&mut b, 0x3008); + w32(&mut b, offsets.0); + w32(&mut b, offsets.1); + w16(&mut b, exceptions.len() as u16); + for (s, e, o, subject) in exceptions { + w32(&mut b, *s); + w32(&mut b, *e); + w32(&mut b, *o); + w16(&mut b, u16::from(subject.is_some())); + if let Some(t) = subject { + w16(&mut b, t.len() as u16 + 1); + w16(&mut b, t.len() as u16); + b.extend_from_slice(t.as_bytes()); + } + } + w32(&mut b, 0); + for (s, e, o, subject) in exceptions { + w32(&mut b, 0); + if let Some(t) = subject { + w32(&mut b, *s); + w32(&mut b, *e); + w32(&mut b, *o); + let wide: Vec = t.encode_utf16().collect(); + w16(&mut b, wide.len() as u16); + wide.iter().for_each(|u| w16(&mut b, *u)); + w32(&mut b, 0); + } + } + w32(&mut b, 0); + b +} + +/// A TZDEFINITION with a key name and no rules. +pub fn time_zone_bytes(key: &str) -> Vec { + let mut b = vec![2, 1]; + let wide: Vec = key.encode_utf16().collect(); + b.extend_from_slice(&((6 + wide.len() * 2) as u16).to_le_bytes()); + b.extend_from_slice(&2u16.to_le_bytes()); + b.extend_from_slice(&(wide.len() as u16).to_le_bytes()); + wide.iter().for_each(|u| b.extend_from_slice(&u.to_le_bytes())); + b.extend_from_slice(&0u16.to_le_bytes()); + b +} + +/// The bytes of a distribution list's stream ([MS-OXOCNTC] 2.2.2.2.4): the header, each member's +/// entry id and one-off entry id (empty for none), and the two terminators. +pub fn list_stream_bytes(members: &[(Vec, Vec)]) -> Vec { + let mut b = Vec::new(); + b.extend(1u16.to_le_bytes()); + b.extend([0, 0]); + b.extend(160_000u32.to_le_bytes()); + b.extend(0u32.to_le_bytes()); + b.extend((members.len() as u32).to_le_bytes()); + b.extend((members.iter().map(|m| m.0.len()).sum::() as u32).to_le_bytes()); + b.extend((members.iter().map(|m| m.1.len()).sum::() as u32).to_le_bytes()); + b.extend(0u32.to_le_bytes()); + for (id, one_off) in members { + b.extend((id.len() as u32).to_le_bytes()); + b.extend_from_slice(id); + b.extend((one_off.len() as u32).to_le_bytes()); + b.extend_from_slice(one_off); + b.extend(0u32.to_le_bytes()); + } + b.extend([0; 8]); + b +} + +/// A distribution list's stream as a binary property value. +pub fn list_stream(members: &[(Vec, Vec)]) -> Value { + binary(&list_stream_bytes(members)) +} + +/// Where a value too large for the heap goes: a subnode, with a fresh local id. +pub struct Spill { + pub subnodes: Vec, + next: u32, +} + +impl Spill { + pub fn new() -> Spill { + Spill { subnodes: Vec::new(), next: 0x100 } + } + + pub fn put(&mut self, kind: u8, data: Vec) -> u32 { + self.next += 1; + let nid = (self.next << 5) | u32::from(kind); + self.subnodes.push(SubNode { nid, data: Data::Stream(data), subnodes: Vec::new() }); + nid + } +} + +const HEAP_VALUE_MAX: usize = 3580; + +fn hnid(heap: &mut HeapBuilder, spill: &mut Spill, bytes: &[u8]) -> u32 { + if bytes.len() <= HEAP_VALUE_MAX.min(heap.capacity) { heap.alloc(bytes.to_vec()) } else { spill.put(0x1F, bytes.to_vec()) } +} + +fn fixed_len(ty: u16) -> Option { + crate::prop::fixed_size(ty) +} + +/// A property context: its heap blocks; values too large go to `spill`. +pub fn pc(layout: &Layout, props: &[(u16, Value)], spill: &mut Spill) -> Vec> { + let mut heap = HeapBuilder::new(0xBC, layout.heap_block); + let mut sorted = props.to_vec(); + sorted.sort_by_key(|p| p.0); + let mut records = Vec::new(); + for (id, v) in &sorted { + let raw: u32 = match fixed_len(v.ty) { + Some(n) if n <= 4 => { + let mut b = v.bytes.clone(); + b.resize(4, 0); + u32::from_le_bytes(b.try_into().unwrap()) + } + Some(_) => heap.alloc(v.bytes.clone()), + None if v.ty == OBJECT => heap.alloc(v.bytes.clone()), + None => as_ref(v).unwrap_or_else(|| hnid(&mut heap, spill, &v.bytes)), + }; + let mut r = id.to_le_bytes().to_vec(); + r.extend_from_slice(&v.ty.to_le_bytes()); + r.extend_from_slice(&raw.to_le_bytes()); + records.push(r); + } + heap.user_root = bth(&mut heap, 2, 6, &records, layout.bth_leaf); + heap.finish() +} + +/// A table context's column: its tag and width in the row. +fn width(ty: u16) -> usize { + match fixed_len(ty) { + Some(n) if n <= 8 => n.max(1), + _ => 4, + } +} + +/// A table context over `columns` (tags) and `rows` (row id, values by tag); its heap blocks +/// and its subnodes (the row matrix, when it goes there, and large values). +pub fn tc(layout: &Layout, columns: &[u32], rows: &[(u32, Vec<(u32, Value)>)]) -> (Vec>, Vec) { + let f = layout.format; + let mut cols: Vec = vec![0x67F2_0003, 0x67F3_0003]; + for c in columns { + if !cols.contains(c) { + cols.push(*c); + } + } + // Offsets: row id, row version, then 8- and 4-byte, 2-byte and 1-byte columns. + let mut place: BTreeMap = BTreeMap::new(); + place.insert(0x67F2_0003, (0, 4, 0)); + place.insert(0x67F3_0003, (4, 4, 1)); + let mut at = 8; + let mut ends = [0usize; 3]; + for (g, widths) in [(0, 4usize..=8), (1, 2..=2), (2, 1..=1)] { + for (i, &c) in cols.iter().enumerate().skip(2) { + let w = width((c & 0xFFFF) as u16); + if widths.contains(&w) { + place.insert(c, (at, w, i)); + at += w; + } + } + ends[g] = at; + } + let [end4, end2, end1] = ends; + let row_size = end1 + cols.len().div_ceil(8); + + let mut heap = HeapBuilder::new(0x7C, layout.heap_block); + let mut spill = Spill::new(); + let mut matrix = Vec::new(); + for (rowid, values) in rows { + let mut row = vec![0u8; row_size]; + row[0..4].copy_from_slice(&rowid.to_le_bytes()); + let mut set = vec![0usize, 1]; + for (tag, v) in values { + let Some(&(off, w, bit)) = place.get(tag) else { panic!("no column {tag:#x}") }; + let ty = (tag & 0xFFFF) as u16; + match fixed_len(ty) { + Some(n) if n <= 8 => { + let mut b = v.bytes.clone(); + b.resize(w, 0); + row[off..off + w].copy_from_slice(&b); + } + _ => { + let h = as_ref(v).unwrap_or_else(|| hnid(&mut heap, &mut spill, &v.bytes)); + row[off..off + 4].copy_from_slice(&h.to_le_bytes()); + } + } + set.push(bit); + } + for bit in set { + row[end1 + bit / 8] |= 0x80 >> (bit % 8); + } + matrix.push(row); + } + // The row index. + let mut index: Vec> = rows + .iter() + .enumerate() + .map(|(i, (rowid, _))| { + let mut r = rowid.to_le_bytes().to_vec(); + match f { + Format::Unicode => r.extend_from_slice(&(i as u32).to_le_bytes()), + Format::Ansi => r.extend_from_slice(&(i as u16).to_le_bytes()), + } + r + }) + .collect(); + index.sort_by_key(|r| u32::from_le_bytes([r[0], r[1], r[2], r[3]])); + let ent = if f == Format::Unicode { 4 } else { 2 }; + let row_index = bth(&mut heap, 4, ent, &index, layout.bth_leaf); + let matrix_bytes: usize = matrix.iter().map(Vec::len).sum(); + let rows_hnid = if matrix.is_empty() { + 0 + } else if !layout.rows_in_subnode && matrix_bytes <= HEAP_VALUE_MAX.min(layout.heap_block) { + heap.alloc(matrix.concat()) + } else { + let per_block = ((8192 - f.trailer_size()) / row_size).max(1); + let pieces: Vec> = matrix.chunks(per_block).map(|c| c.concat()).collect(); + spill.next += 1; + let nid = (spill.next << 5) | 0x1F; + spill.subnodes.push(SubNode { nid, data: Data::Blocks(pieces), subnodes: Vec::new() }); + nid + }; + let mut info = vec![0x7C, cols.len() as u8]; + for v in [end4, end2, end1, row_size] { + info.extend_from_slice(&(v as u16).to_le_bytes()); + } + info.extend_from_slice(&row_index.to_le_bytes()); + info.extend_from_slice(&rows_hnid.to_le_bytes()); + info.extend_from_slice(&0u32.to_le_bytes()); + let mut sorted = cols.clone(); + sorted.sort(); + for c in sorted { + let (off, w, bit) = place[&c]; + info.extend_from_slice(&c.to_le_bytes()); + info.extend_from_slice(&(off as u16).to_le_bytes()); + info.push(w as u8); + info.push(bit as u8); + } + heap.user_root = heap.alloc(info); + (heap.finish(), spill.subnodes) +} + +// --------------------------------------------------------------------------------------------- +// The messaging layer. + +#[derive(Clone, Debug, Default)] +pub struct MessageSpec { + pub class: String, + pub subject: String, + pub body: String, + pub read: bool, + pub delivered: u64, + pub extra: Vec<(u16, Value)>, + pub recipients: Vec<(String, String, i32)>, + /// PidTagRecipientFlags and PidTagRecipientTrackStatus of each recipient, by position. + pub recipient_states: Vec<(i32, i32)>, + pub attachments: Vec, +} + +#[derive(Clone, Debug)] +pub enum AttachSpec { + File { + name: String, + mime: String, + data: Vec, + }, + Message(Box), + /// An attached message (method 5) whose object is this subnode, as it is. + Object(SubNode), + /// An attachment with exactly these properties (a linked file, an inline picture with its + /// content id, a hidden one). + Props(Vec<(u16, Value)>), + /// An OLE object (method 6): these bytes in a subnode of its own. + Ole(Vec), +} + +#[derive(Clone, Debug, Default)] +pub struct FolderSpec { + pub name: String, + pub class: Option, + pub messages: Vec, + pub children: Vec, +} + +#[derive(Clone, Debug)] +pub struct FileSpec { + pub layout: Layout, + pub password: Option, + /// The password's CRC taken of its Windows-1252 bytes (its characters as bytes) rather + /// than its UTF-8. + pub password_ansi: bool, + pub name: String, + pub folders: Vec, + /// Named properties: (property set, numeric name). + pub named: Vec<([u8; 16], u32)>, + pub named_strings: Vec<([u8; 16], String)>, +} + +pub const RECORD_KEY: [u8; 16] = [0x5A, 0x11, 0x0E, 0x93, 0x42, 0x7C, 0x4B, 0x1D, 0x8E, 0x22, 0x61, 0x0C, 0x3F, 0x77, 0x90, 0x05]; + +pub fn entry_id(nid: u32) -> Vec { + let mut e = vec![0u8; 4]; + e.extend_from_slice(&RECORD_KEY); + e.extend_from_slice(&nid.to_le_bytes()); + e +} + +/// A container class written `hidden:IPF.Contact` is a hidden folder (PidTagAttributeHidden) of +/// that class. +fn hidden_class(class: &str) -> (&str, bool) { + match class.strip_prefix("hidden:") { + Some(rest) => (rest, true), + None => (class, false), + } +} + +pub struct Ids { + folder: u32, + message: u32, +} + +impl Ids { + fn folder(&mut self) -> u32 { + self.folder += 1; + (self.folder << 5) | 0x02 + } + fn message(&mut self) -> u32 { + self.message += 1; + (self.message << 5) | 0x04 + } +} + +pub const IPM_SUBTREE: u32 = (0x401 << 5) | 0x02; +pub const SEARCH_ROOT: u32 = (0x402 << 5) | 0x02; + +fn text(layout: &Layout, s: &str) -> Value { + if layout.format == Format::Ansi { string8(s) } else { unicode(s) } +} + +fn text_tag(layout: &Layout, id: u16) -> u32 { + (u32::from(id) << 16) | u32::from(if layout.format == Format::Ansi { STRING8 } else { STRING }) +} + +/// The columns this writer puts in a hierarchy table. +fn hierarchy_columns(l: &Layout) -> Vec { + vec![text_tag(l, 0x3001), 0x3602_0003, 0x3603_0003, 0x360A_000B, text_tag(l, 0x3613)] +} + +fn contents_columns(l: &Layout) -> Vec { + vec![0x0017_0003, text_tag(l, 0x001A), text_tag(l, 0x0037), 0x0E06_0040, 0x0E07_0003] +} + +/// Builds the message `m`: its PC blocks and its subnodes. +fn message(l: &Layout, m: &MessageSpec) -> (Vec>, Vec) { + let flags = i32::from(m.read) | if m.attachments.is_empty() { 0 } else { 0x10 }; + let mut props = vec![ + (0x001A, text(l, &m.class)), + (0x0037, text(l, &m.subject)), + (0x1000, text(l, &m.body)), + (0x0E07, int32(flags)), + (0x0E06, time(m.delivered)), + (0x0017, int32(1)), + ]; + // A property in `extra` takes the place of the one given above. + props.retain(|(id, _)| !m.extra.iter().any(|(e, _)| e == id)); + props.extend(m.extra.iter().cloned()); + let mut spill = Spill::new(); + let pc_blocks = pc(l, &props, &mut spill); + let mut subs = spill.subnodes; + + // Recipients. + let rcols = vec![0x0C15_0003, text_tag(l, 0x3001), text_tag(l, 0x3003), text_tag(l, 0x3002), text_tag(l, 0x39FE), 0x5FFD_0003, 0x5FFF_0003]; + let rrows: Vec<(u32, Vec<(u32, Value)>)> = m + .recipients + .iter() + .enumerate() + .map(|(i, (name, email, kind))| { + let mut row = vec![ + (0x0C15_0003, int32(*kind)), + (text_tag(l, 0x3001), text(l, name)), + (text_tag(l, 0x3003), text(l, email)), + (text_tag(l, 0x3002), text(l, "SMTP")), + ]; + if let Some((flags, track)) = m.recipient_states.get(i) { + row.push((0x5FFD_0003, int32(*flags))); + row.push((0x5FFF_0003, int32(*track))); + } + (i as u32 + 1, row) + }) + .collect(); + let (rblocks, rsubs) = tc(l, &rcols, &rrows); + subs.push(SubNode { nid: 0x692, data: Data::Blocks(rblocks), subnodes: rsubs }); + + // Attachments. + if !m.attachments.is_empty() { + let acols = vec![0x0E20_0003, text_tag(l, 0x3704), 0x3705_0003, 0x370B_0003]; + let mut arows = Vec::new(); + for (i, a) in m.attachments.iter().enumerate() { + let nid = ((0x200 + i as u32) << 5) | 0x05; + let mut aspill = Spill::new(); + let (aprops, size, method) = match a { + AttachSpec::File { name, mime, data } => { + let props = vec![ + (0x3705, int32(1)), + (0x3707, text(l, name)), + (0x3704, text(l, name)), + (0x370E, text(l, mime)), + (0x0E20, int32(data.len() as i32)), + (0x370B, int32(-1)), + (0x3701, binary(data)), + ]; + (props, data.len(), 1) + } + AttachSpec::Message(inner) => { + let (mblocks, msubs) = message(l, inner); + let size: usize = mblocks.iter().map(Vec::len).sum(); + aspill.next += 1; + let onid = (aspill.next << 5) | 0x04; + aspill.subnodes.push(SubNode { nid: onid, data: Data::Blocks(mblocks), subnodes: msubs }); + let mut obj = onid.to_le_bytes().to_vec(); + obj.extend_from_slice(&(size as u32).to_le_bytes()); + let props = vec![ + (0x3705, int32(5)), + (0x3704, text(l, &inner.subject)), + (0x0E20, int32(size as i32)), + (0x370B, int32(-1)), + (0x3701, Value { ty: OBJECT, bytes: obj }), + ]; + (props, size, 5) + } + AttachSpec::Object(object) => { + aspill.subnodes.push(object.clone()); + let mut obj = object.nid.to_le_bytes().to_vec(); + obj.extend_from_slice(&0u32.to_le_bytes()); + let props = vec![(0x3705, int32(5)), (0x3704, text(l, "object")), (0x3701, Value { ty: OBJECT, bytes: obj })]; + (props, 0, 5) + } + AttachSpec::Props(props) => { + let method = props.iter().find(|(id, _)| *id == 0x3705).map_or(1, |(_, v)| i32::from_le_bytes(v.bytes[..4].try_into().unwrap())); + (props.clone(), 0, method) + } + AttachSpec::Ole(data) => { + let onid = aspill.put(0x01, data.clone()); + let mut obj = onid.to_le_bytes().to_vec(); + obj.extend_from_slice(&(data.len() as u32).to_le_bytes()); + let props = vec![(0x3705, int32(6)), (0x3704, text(l, "Picture (Metafile)")), (0x3701, Value { ty: OBJECT, bytes: obj })]; + (props, data.len(), 6) + } + }; + let ablocks = pc(l, &aprops, &mut aspill); + subs.push(SubNode { nid, data: Data::Blocks(ablocks), subnodes: aspill.subnodes }); + let name = match a { + AttachSpec::File { name, .. } => name.clone(), + AttachSpec::Message(inner) => inner.subject.clone(), + AttachSpec::Object(_) => "object".into(), + AttachSpec::Props(_) => "attachment".into(), + AttachSpec::Ole(_) => "Picture (Metafile)".into(), + }; + arows.push(( + nid, + vec![ + (0x0E20_0003, int32(size as i32)), + (text_tag(l, 0x3704), text(l, &name)), + (0x3705_0003, int32(method)), + (0x370B_0003, int32(-1)), + ], + )); + } + let (ablocks, asubs) = tc(l, &acols, &arows); + subs.push(SubNode { nid: 0x671, data: Data::Blocks(ablocks), subnodes: asubs }); + } + (pc_blocks, subs) +} + +/// The four nodes of a folder (and those of its messages and subfolders, recursively). +fn folder(l: &Layout, ids: &mut Ids, nid: u32, parent: u32, spec: &FolderSpec, nodes: &mut Vec) { + let mut child_rows = Vec::new(); + let mut child_ids = Vec::new(); + for c in &spec.children { + let cid = ids.folder(); + child_ids.push(cid); + let mut vals = vec![ + (text_tag(l, 0x3001), text(l, &c.name)), + (0x3602_0003, int32(c.messages.len() as i32)), + (0x3603_0003, int32(c.messages.iter().filter(|m| !m.read).count() as i32)), + (0x360A_000B, boolean(!c.children.is_empty())), + ]; + if let Some(class) = &c.class { + vals.push((text_tag(l, 0x3613), text(l, hidden_class(class).0))); + } + child_rows.push((cid, vals)); + } + let mut props = vec![ + (0x3001, text(l, &spec.name)), + (0x3602, int32(spec.messages.len() as i32)), + (0x3603, int32(spec.messages.iter().filter(|m| !m.read).count() as i32)), + (0x360A, boolean(!spec.children.is_empty())), + ]; + if let Some(class) = &spec.class { + let (class, hidden) = hidden_class(class); + props.push((0x3613, text(l, class))); + if hidden { + props.push((0x10F4, boolean(true))); + } + } + let mut spill = Spill::new(); + let blocks = pc(l, &props, &mut spill); + nodes.push(Node { nid, parent, data: Data::Blocks(blocks), subnodes: spill.subnodes }); + let (h, hs) = tc(l, &hierarchy_columns(l), &child_rows); + nodes.push(Node { nid: (nid & !0x1F) | 0x0D, parent: 0, data: Data::Blocks(h), subnodes: hs }); + let mut rows = Vec::new(); + for m in &spec.messages { + let mid = ids.message(); + let (blocks, subs) = message(l, m); + nodes.push(Node { nid: mid, parent: nid, data: Data::Blocks(blocks), subnodes: subs }); + rows.push(( + mid, + vec![ + (0x0017_0003, int32(1)), + (text_tag(l, 0x001A), text(l, &m.class)), + (text_tag(l, 0x0037), text(l, &m.subject)), + (0x0E06_0040, time(m.delivered)), + (0x0E07_0003, int32(i32::from(m.read))), + ], + )); + } + let (c, cs) = tc(l, &contents_columns(l), &rows); + nodes.push(Node { nid: (nid & !0x1F) | 0x0E, parent: 0, data: Data::Blocks(c), subnodes: cs }); + let (a, asubs) = tc(l, &[text_tag(l, 0x001A)], &[]); + nodes.push(Node { nid: (nid & !0x1F) | 0x0F, parent: 0, data: Data::Blocks(a), subnodes: asubs }); + for (c, cid) in spec.children.iter().zip(child_ids) { + folder(l, ids, cid, nid, c, nodes); + } +} + +/// The name-to-id map's PC. +fn named_map(spec: &FileSpec) -> (Vec>, Vec) { + let l = &spec.layout; + let mut guids: Vec<[u8; 16]> = Vec::new(); + let mut guid_index = |g: &[u8; 16]| -> u16 { + if *g == crate::named::PS_MAPI { + return 1; + } + if *g == crate::named::PS_PUBLIC_STRINGS { + return 2; + } + if let Some(i) = guids.iter().position(|x| x == g) { + return 3 + i as u16; + } + guids.push(*g); + 2 + guids.len() as u16 + }; + let mut entries = Vec::new(); + let mut strings = Vec::new(); + let mut index = 0u16; + for (g, id) in &spec.named { + let gi = guid_index(g); + entries.extend_from_slice(&id.to_le_bytes()); + entries.extend_from_slice(&(gi << 1).to_le_bytes()); + entries.extend_from_slice(&index.to_le_bytes()); + index += 1; + } + for (g, name) in &spec.named_strings { + let gi = guid_index(g); + let at = strings.len() as u32; + let units: Vec = name.encode_utf16().flat_map(u16::to_le_bytes).collect(); + strings.extend_from_slice(&(units.len() as u32).to_le_bytes()); + strings.extend_from_slice(&units); + while strings.len() % 4 != 0 { + strings.push(0); + } + entries.extend_from_slice(&at.to_le_bytes()); + entries.extend_from_slice(&((gi << 1) | 1).to_le_bytes()); + entries.extend_from_slice(&index.to_le_bytes()); + index += 1; + } + let props = vec![(0x0001, int32(251)), (0x0002, binary(&guids.concat())), (0x0003, binary(&entries)), (0x0004, binary(&strings))]; + let mut spill = Spill::new(); + let blocks = pc(l, &props, &mut spill); + (blocks, spill.subnodes) +} + +/// The whole file. +pub fn build(spec: &FileSpec) -> Vec { + raw(&spec.layout, &nodes(spec)) +} + +/// The file's nodes, for tests that change some before writing them. +pub fn nodes(spec: &FileSpec) -> Vec { + let l = &spec.layout; + let mut nodes = Vec::new(); + let mut store = vec![ + (0x0FF9, binary(&RECORD_KEY)), + (0x3001, text(l, &spec.name)), + (0x35E0, binary(&entry_id(IPM_SUBTREE))), + (0x35E7, binary(&entry_id(SEARCH_ROOT))), + ]; + // A top-level folder named Deleted Items is the store's (its ids run from 0x403). + if let Some(i) = spec.folders.iter().position(|f| f.name == "Deleted Items") { + store.push((0x35E3, binary(&entry_id(((0x403 + i as u32) << 5) | 0x02)))); + } + if let Some(p) = &spec.password { + let bytes: Vec = if spec.password_ansi { p.chars().map(|c| c as u8).collect() } else { p.as_bytes().to_vec() }; + store.push((0x67FF, int32(crc32(&bytes) as i32))); + } + let mut spill = Spill::new(); + let blocks = pc(l, &store, &mut spill); + nodes.push(Node { nid: 0x21, parent: 0, data: Data::Blocks(blocks), subnodes: spill.subnodes }); + let (nb, ns) = named_map(spec); + nodes.push(Node { nid: 0x61, parent: 0, data: Data::Blocks(nb), subnodes: ns }); + + let mut ids = Ids { folder: 0x402, message: 0x10000 }; + let root = FolderSpec { + name: String::new(), + class: None, + messages: Vec::new(), + children: vec![ + FolderSpec { name: "Top of Personal Folders".into(), class: None, messages: Vec::new(), children: spec.folders.clone() }, + FolderSpec { name: "Search Root".into(), class: None, messages: Vec::new(), children: Vec::new() }, + ], + }; + folder_fixed(l, &mut ids, &root, &mut nodes); + nodes +} + +/// The root (0x122, its own parent), the IPM subtree and the search root with fixed ids. +fn folder_fixed(l: &Layout, ids: &mut Ids, root: &FolderSpec, nodes: &mut Vec) { + let rows = vec![ + (IPM_SUBTREE, vec![(text_tag(l, 0x3001), text(l, &root.children[0].name)), (0x360A_000B, boolean(!root.children[0].children.is_empty()))]), + (SEARCH_ROOT, vec![(text_tag(l, 0x3001), text(l, &root.children[1].name)), (0x360A_000B, boolean(false))]), + ]; + let mut spill = Spill::new(); + let blocks = pc(l, &[(0x3001, text(l, "")), (0x360A, boolean(true)), (0x3602, int32(0)), (0x3603, int32(0))], &mut spill); + nodes.push(Node { nid: 0x122, parent: 0x122, data: Data::Blocks(blocks), subnodes: spill.subnodes }); + let (h, hs) = tc(l, &hierarchy_columns(l), &rows); + nodes.push(Node { nid: 0x12D, parent: 0, data: Data::Blocks(h), subnodes: hs }); + let (c, cs) = tc(l, &contents_columns(l), &[]); + nodes.push(Node { nid: 0x12E, parent: 0, data: Data::Blocks(c), subnodes: cs }); + let (a, asubs) = tc(l, &[text_tag(l, 0x001A)], &[]); + nodes.push(Node { nid: 0x12F, parent: 0, data: Data::Blocks(a), subnodes: asubs }); + folder(l, ids, IPM_SUBTREE, 0x122, &root.children[0], nodes); + folder(l, ids, SEARCH_ROOT, 0x122, &root.children[1], nodes); +} + +/// Writes raw nodes as a file (for tests that build broken structures themselves). +pub fn raw(layout: &Layout, nodes: &[Node]) -> Vec { + Ndb::new(layout.clone()).finish(nodes) +}