diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 3d51a65a8..36893f43c 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -108,9 +108,24 @@ implicit `@`, the spaces and line breaks typed in the formula, numbers as Excel `FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the file keeps the tokens once. A formula whose decoding needs something the reader does not read keeps -only its saved result, with a warning saying why: a defined name left out, table references, other -workbooks, Python code stored outside the cell, data tables. The XLSB function table -(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. +only its saved result, with a warning saying why: a defined name left out, a table whose part can't +be read, other workbooks, Python code stored outside the cell, data tables. + +A binary workbook's tables (`tableN.bin`, listed by the sheet's `BrtListPart`s) read through +`Package::tables` into the same `Table`, as Excel writes the table in `.xlsx`: the calculated and +totals formulas decoded as cell formulas are, an XML map's column properties, and the AutoFilter and +sort state built as the `autoFilter` and `sortState` elements, with numbers in a filter spelled as +Excel writes them there (17 significant digits; Excel writes a few values shorter, such as `0.1`, +which read back the same). A filter column the reader can't spell is left out with a warning (Excel +keeps date-group filters in an extension the reader skips; a custom filter on a Boolean value has no +`.xlsx` spelling Excel has shown), as is a sort by stroke or PinYin order. The revision ids Excel +writes (`xr:uid`) are in alternate-content blocks, which are skipped. A structured reference in a +formula (`PtgList`) is spelled as `.xlsx` has it (`Sales[Val]`, `Sales[[#Headers],[Key]]`, +`Sales[[#This Row],[A]:[B]]`, with Excel's spacing flags), from the table's id, which is +workbook-wide: the workbook's table parts are indexed when its workbook part is read. A reference to +a table that can't be read, to a deleted table or into another workbook keeps the cell's saved +result. The XLSB function table (`xlsb/ftab.rs`) and the expression printer are this crate's own, +apart from `craft-xls`'s. ## Hostile input diff --git a/crates/xlsx/src/package.rs b/crates/xlsx/src/package.rs index 62076b3fc..a75874e71 100644 --- a/crates/xlsx/src/package.rs +++ b/crates/xlsx/src/package.rs @@ -12,7 +12,7 @@ use crate::pivot::{CacheRecord, PivotCacheDefinition, PivotTableDefinition, read use crate::sheet::{Row, RowReader, SheetPart}; use crate::strings::StringItem; use crate::styles::Styles; -use crate::workbook::{SheetEntry, Workbook}; +use crate::workbook::{SheetEntry, SheetKind, Workbook}; use crate::{Error, Limits, Warnings, xlsb}; /// Most bytes of `[Content_Types].xml` read from a binary workbook. @@ -149,7 +149,12 @@ impl<'a> Package<'a> { if self.binary.is_some() { let bytes = self.required(&self.workbook_part.clone())?; let rels = self.workbook_rels()?; - let (wb, links) = xlsb::workbook(&bytes, &rels, &self.limits, &self.budget, &mut self.warnings)?; + let (mut wb, mut links, names) = xlsb::workbook(&bytes, &rels, &self.limits, &self.budget, &mut self.warnings)?; + if let Some(links) = links.as_mut() { + self.index_tables(&wb, links)?; + } + // Names after tables: a name's formula may use a structured reference. + xlsb::read_names(&mut wb, names, links.as_mut(), &self.limits, &self.budget, &mut self.warnings); self.binary_links = Some(links); return Ok(wb); } @@ -314,10 +319,31 @@ impl<'a> Package<'a> { Ok(Some((part, Drawing::read(&root), rels))) } - /// The tables of a worksheet read by [`Package::sheet`], in the order of its `tableParts`, - /// each with its part name. A table part that does not read is skipped with a warning. + /// The tables of a worksheet read by [`Package::sheet`], in the order of its `tableParts` + /// (its `BrtListPart`s in a binary workbook), each with its part name. A table part that does + /// not read is skipped with a warning. pub fn tables(&mut self, sheet: &SheetPart) -> Result, Error> { let mut out = Vec::new(); + if self.binary.is_some() { + let rels = self.opc.rels(&sheet.part)?; + for id in &sheet.features.table_parts { + let Some(rel) = rels.by_id(id).filter(|r| !r.external && r.kind() == "table") else { + self.warn(format!("XLSB: {}: a table part relationship is missing", sheet.part)); + continue; + }; + let part = rel.target.clone(); + let read = self.binary_part(&part, xlsb::TABLE).and_then(|bytes| { + let links = self.binary_links.as_ref().and_then(Option::as_ref); + xlsb::table(&bytes, links, &self.limits, &self.budget, &mut self.warnings) + }); + match read { + Ok(t) => out.push((part, t)), + Err(Error::TooLarge(m)) => return Err(Error::TooLarge(m)), + Err(e) => self.warn(format!("skipped {part}: {e}")), + } + } + return Ok(out); + } for id in &sheet.features.table_parts { let Some(rel) = sheet.rels.by_id(id).filter(|r| !r.external) else { continue }; let part = rel.target.clone(); @@ -410,6 +436,33 @@ impl<'a> Package<'a> { Ok(root.map(|r| craft_drawingml::theme::read_theme(&mut craft_drawingml::part::NoSource, &r))) } + /// A part of a binary workbook, checked against its content type. + fn binary_part(&mut self, part: &str, content_type: &str) -> Result, Error> { + if let Some(types) = &self.binary { + xlsb::check_type(types, part, content_type)?; + } + self.required(part) + } + + /// Indexes the table parts of a binary workbook's worksheets, for the structured references + /// in its formulas. A table part that does not read is left out with a warning: formulas + /// using it keep their saved results. + fn index_tables(&mut self, wb: &Workbook, links: &mut xlsb::Links) -> Result<(), Error> { + for entry in wb.sheets.iter().filter(|e| e.kind == SheetKind::Worksheet) { + let Some(part) = &entry.part else { continue }; + let rels = self.opc.rels(part)?; + for rel in rels.list.iter().filter(|r| r.kind() == "table" && !r.external) { + let read = self.binary_part(&rel.target, xlsb::TABLE).and_then(|bytes| xlsb::index_table(links, &bytes, &self.limits, &self.budget)); + match read { + Ok(()) => {} + Err(Error::TooLarge(m)) => return Err(Error::TooLarge(m)), + Err(e) => self.warn(format!("XLSB: {} could not be read ({e}); formulas using its table keep their saved results", rel.target)), + } + } + } + Ok(()) + } + fn required(&mut self, part: &str) -> Result, Error> { self.opc.read(part)?.ok_or_else(|| Error::Invalid(format!("missing part {part}"))) } diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index 02087ed4a..b45b48808 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -1,14 +1,15 @@ //! Binary workbooks ([MS-XLSB]): the workbook part with its defined names, shared strings and //! worksheets (rows, cells, columns, default sizes, merged cells), from their records. Cell //! formulas are decoded from their tokens ([`formula`]) into the text an `.xlsx` file holds; -//! one that can't be keeps only its saved result, with a warning saying why. The styles part -//! is read into the same [`crate::styles::Styles`] ([`styles`]). A record that does not parse -//! fails the read. +//! one that can't be keeps only its saved result, with a warning saying why. Table parts are +//! read in `table`; the styles part is read into the same [`crate::styles::Styles`] +//! ([`styles`]). A record that does not parse fails the read. mod expr; mod formula; mod ftab; mod styles; +mod table; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -24,6 +25,7 @@ use crate::{Error, Limits, Warnings}; pub(crate) use formula::Links; use formula::{Book, Mode, Name, Xti}; pub(crate) use styles::styles; +pub(crate) use table::{TABLE, index_table, table}; pub(crate) const MAIN: &str = "application/vnd.ms-excel.main"; const BINARY_MAIN: &str = "application/vnd.ms-excel.sheet.binary.macroEnabled.main"; @@ -37,6 +39,8 @@ const MAX_SHEETS: usize = 1024; const MAX_BOOKS: usize = 65_536; /// Most merged ranges in one worksheet. const MAX_MERGES: usize = 100_000; +/// Table parts one worksheet lists at most. +const MAX_TABLE_PARTS: usize = 65_536; /// Most column records in one worksheet: one per column. const MAX_COLUMNS: usize = MAX_COLS as usize; /// The tallest row, in twips (409.6 points). @@ -189,11 +193,13 @@ struct Records<'a, 'l> { /// Read the alternate content meant for Excel 2007 (version 12), as an `.xlsx` reader sees /// the main attribute where Excel keeps a newer spelling in an extension. excel_2007: bool, + /// Future-record and alternate-content blocks skipped so far. + skipped: usize, } impl<'a, 'l> Records<'a, 'l> { fn new(bytes: &'a [u8], limits: &'l Limits, budget: &'l Budget) -> Self { - Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![], excel_2007: false } + Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![], excel_2007: false, skipped: 0 } } fn next(&mut self, warnings: &mut Warnings) -> Result, Error> { @@ -246,6 +252,7 @@ impl<'a, 'l> Records<'a, 'l> { let read = read && self.blocks.iter().all(|(_, r)| *r); self.blocks.push((id + 1, read)); if !read { + self.skipped += 1; warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped"); } } @@ -282,14 +289,16 @@ impl<'a, 'l> Records<'a, 'l> { /// The workbook part: BrtBeginBook, BrtWbProp, the BrtBundleSh list. /// The workbook part, and the tables its formulas refer to ([`Links`]): `None` when those can't -/// be read, so that the workbook still opens, with every formula kept as its saved result. +/// be read, so that the workbook still opens, with every formula kept as its saved result. Its +/// defined names come back undecoded ([`Names`]): [`read_names`] decodes them once the workbook's +/// tables are indexed, so a name's formula can use a structured reference. pub(crate) fn workbook( bytes: &[u8], rels: &Rels, limits: &Limits, budget: &Budget, warnings: &mut Warnings, -) -> Result<(Workbook, Option), Error> { +) -> Result<(Workbook, Option, Names), Error> { let mut records = Records::new(bytes, limits, budget); records.first(131, warnings)?; let mut wb = Workbook::default(); @@ -313,11 +322,7 @@ pub(crate) fn workbook( if wb.sheets.is_empty() { return Err(invalid("no supported worksheets")); } - match links.as_mut() { - Some(l) => wb.defined_names = defined_names(pending, &bundle_sheets, l, limits, budget, warnings), - None => warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"), - } - return Ok((wb, links)); + return Ok((wb, links, Names { pending, sheets: bundle_sheets })); } 39 | 355 | 357 | 358 | 362 | 667 => { if let Some(l) = links.as_mut() @@ -416,6 +421,22 @@ struct PendingName { help: Option, } +/// The workbook part's defined names, not yet decoded, and each BrtBundleSh's place in +/// [`Workbook::sheets`] (`None` for a skipped one). +pub(crate) struct Names { + pending: Vec, + sheets: Vec>, +} + +/// Decodes the workbook part's defined names `names` into `wb`, after the tables in `links` are +/// indexed. Without `links` none can be read, and formulas keep their saved results. +pub(crate) fn read_names(wb: &mut Workbook, names: Names, links: Option<&mut Links>, limits: &Limits, budget: &Budget, warnings: &mut Warnings) { + match links { + Some(l) => wb.defined_names = defined_names(names.pending, &names.sheets, l, limits, budget, warnings), + None => warnings.warn("XLSB: the workbook's sheet links or names could not be read; formulas keep their saved results"), + } +} + /// The workbook's defined names, as `.xlsx` has them in `definedNames`: every `BrtName` but the /// hidden ones for newer functions and LET parameters, which Excel doesn't write there. `sheets` /// maps a name's `itab` (its sheet's place in `BrtBundleSh` order) to [`Workbook::sheets`]. @@ -1216,6 +1237,18 @@ pub(crate) fn worksheet( }); col = Some(c); } + // BrtListPart: one of the sheet's tables, by relationship id. + 661 => { + if data { + return Err(invalid("table part inside worksheet data")); + } + let rel = b.text(255)?; + b.finish()?; + if sheet.features.table_parts.len() >= MAX_TABLE_PARTS { + return Err(Error::TooLarge(format!("XLSB exceeds {MAX_TABLE_PARTS} tables on a sheet"))); + } + sheet.features.table_parts.push(rel); + } 485 => { if data_seen { return Err(invalid("sheet format after worksheet data")); @@ -1264,3 +1297,25 @@ pub(crate) fn worksheet( } Err(invalid("missing worksheet end record")) } + +#[cfg(test)] +mod records_merge_tests { + use super::*; + + #[test] + fn table_skip_count_preserves_excel_2007_style_blocks() { + // BrtACBegin for Excel 2007, an empty record, BrtACEnd. + let bytes = [37, 6, 1, 0, 12, 0, 0, 0, 1, 0, 38, 0]; + let limits = Limits::default(); + let budget = Budget::default(); + let mut warnings = Warnings::default(); + let mut styles = Records::new(&bytes, &limits, &budget); + styles.excel_2007 = true; + assert_eq!(styles.next(&mut warnings).unwrap(), Some((1, &[][..]))); + assert_eq!(styles.skipped, 0, "supported style content was read, not skipped"); + assert!(styles.next(&mut warnings).unwrap().is_none()); + let mut tables = Records::new(&bytes, &limits, &budget); + assert!(tables.next(&mut warnings).unwrap().is_none()); + assert_eq!(tables.skipped, 1, "table filters still detect skipped alternate content"); + } +} diff --git a/crates/xlsx/src/xlsb/expr.rs b/crates/xlsx/src/xlsb/expr.rs index 4c5efe4f7..7d51551d9 100644 --- a/crates/xlsx/src/xlsb/expr.rs +++ b/crates/xlsx/src/xlsb/expr.rs @@ -125,6 +125,8 @@ pub(crate) enum Expr { /// Spaces and line breaks typed before and after an expression (`PtgAttrSpace`), which /// `.xlsx` formula text keeps. Spaced(String, Box, String), + /// A structured reference to a table, as written (`Sales[[#Headers],[Key]]`). + Structured(String), Unary(UnOp, Box), Binary(BinOp, Box, Box), Call(String, Vec), @@ -327,6 +329,7 @@ fn write_expr(s: &mut String, e: &Expr) { } None => s.push_str(n), }, + Expr::Structured(t) => s.push_str(t), Expr::Unary(UnOp::Neg, x) => { s.push('-'); write_expr(s, x); diff --git a/crates/xlsx/src/xlsb/formula.rs b/crates/xlsx/src/xlsb/formula.rs index 7e628b197..40d461db1 100644 --- a/crates/xlsx/src/xlsb/formula.rs +++ b/crates/xlsx/src/xlsb/formula.rs @@ -5,11 +5,13 @@ //! ([`super::ftab`]) and its own copies of craft-xls's depth guards and limits. Anything outside the //! token set it knows is reported as unsupported, and the caller keeps the cell's cached value. //! So is a formula that needs something the reader doesn't read: a defined name whose own -//! formula could not be read, or Python code stored outside the cell. Defined names' formulas -//! are decoded here too ([`Mode::Name`]). +//! formula could not be read, a table whose part could not be read, or Python code stored +//! outside the cell. Defined names' formulas are decoded here too ([`Mode::Name`]), and +//! structured references (`PtgList`) are spelled from the tables in [`Links`]. //! //! Names keep their file spelling (`_xlfn.CONCAT`, `_xlpm.x`), as `.xlsx` formula text has them. +use std::collections::HashMap; use std::sync::Arc; use craft_sheet::{CellError, CellRef, MAX_COLS, MAX_ROWS}; @@ -169,6 +171,15 @@ pub(crate) struct Links { pub(super) names: Vec, pub(super) books: Vec, pub(super) xti: Vec, + /// Every table by its `idList`, which `PtgList` names: `None` for an id two tables share. + pub(super) tables: HashMap>, +} + +/// A table as structured references need it: its display name and its columns' names. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct ListInfo { + pub name: String, + pub columns: Vec, } /// Whether a name stands for a LET parameter (`_xlpm.x`), which only the formula itself defines. @@ -587,7 +598,13 @@ pub(super) fn decode_using(rgce: &[u8], rgcb: &[u8], links: &Links, mode: Mode, } 0x01 => return Err(Unsupported("the shared or array formula form")), 0x02 => return Err(Unsupported("what-if data tables")), - 0x18 => return Err(Unsupported("structured table references")), + 0x18 => { + if t.u8()? != 0x19 { + return Err(Unsupported("unknown formula token")); + } + let text = list(&mut t, links)?; + push(&mut stack, kids, Expr::Structured(text))?; + } 0x2C => { if mode == Mode::Cell { return Err(Unsupported("relative references in a cell formula")); @@ -631,6 +648,87 @@ fn bare_function(name: String) -> String { } } +/// `PtgList` ([MS-XLSB] 2.5.98.52), after its two token bytes: a structured reference, spelled +/// as `.xlsx` has it (`Sales[Val]`, `Sales[[#Headers],[Key]]`, `Sales[[#This Row],[A]:[B]]`). +/// The two spacing flags put a space inside the outer brackets and after each comma, as Excel +/// shows the reference. +fn list(t: &mut Bytes<'_>, links: &Links) -> Result { + let ixti = t.u16()?; + let bits = t.u16()?; + let index = t.u32()?; + let (first, last) = (usize::from(t.u16()?), usize::from(t.u16()?)); + if bits & 0x3000 != 0 { + return Err(Unsupported("structured references to deleted tables or to tables in other workbooks")); + } + // The table's sheet link: a table is always in this workbook, on one of its sheets. + links.sheet_sel(ixti)?; + let table = links.tables.get(&index).and_then(Option::as_ref).ok_or(Unsupported("structured references to tables the reader could not read"))?; + let rows: &[&str] = match (bits >> 2) & 0x1F { + 0x00 => &[], + 0x01 => &["#All"], + 0x02 => &["#Headers"], + 0x04 => &["#Data"], + 0x06 => &["#Headers", "#Data"], + 0x08 => &["#Totals"], + 0x0C => &["#Data", "#Totals"], + 0x10 => &["#This Row"], + _ => return Err(Unsupported("unknown structured reference rows")), + }; + let column = |i: usize| table.columns.get(i).map(|c| escape_column(c)).ok_or(Unsupported("structured reference to a missing table column")); + let columns = match bits & 3 { + 0 => None, + 1 => Some((column(first)?, None)), + 2 if first <= last => Some((column(first)?, Some(column(last)?))), + _ => return Err(Unsupported("invalid structured reference columns")), + }; + let pad = if bits & 0x80 != 0 { " " } else { "" }; + let mut s = table.name.clone(); + match (rows, &columns) { + ([], None) => s.push_str("[]"), + ([], Some((c, None))) => { + s.push('['); + s.push_str(pad); + s.push_str(c); + s.push_str(pad); + s.push(']'); + } + ([row], None) => { + s.push('['); + s.push_str(pad); + s.push_str(row); + s.push_str(pad); + s.push(']'); + } + _ => { + let mut items: Vec = rows.iter().map(|r| format!("[{r}]")).collect(); + match &columns { + Some((a, Some(b))) => items.push(format!("[{a}]:[{b}]")), + Some((a, None)) => items.push(format!("[{a}]")), + None => {} + } + let comma = if bits & 0x100 != 0 { ", " } else { "," }; + s.push('['); + s.push_str(pad); + s.push_str(&items.join(comma)); + s.push_str(pad); + s.push(']'); + } + } + Ok(s) +} + +/// A column name in a structured reference: `'` escapes `[`, `]`, `#` and `'` (`It''s '['#1']`). +fn escape_column(name: &str) -> String { + let mut s = String::with_capacity(name.len()); + for c in name.chars() { + if matches!(c, '[' | ']' | '#' | '\'') { + s.push('\''); + } + s.push(c); + } + s +} + /// Text written out in the formula: a string, or strings joined with `&` (how Python code /// longer than a string constant is written). fn literal_text(e: &Expr) -> bool { diff --git a/crates/xlsx/src/xlsb/table.rs b/crates/xlsx/src/xlsb/table.rs new file mode 100644 index 000000000..71e8f3d92 --- /dev/null +++ b/crates/xlsx/src/xlsb/table.rs @@ -0,0 +1,770 @@ +//! Table parts ([MS-XLSB] 2.1.7.51, `tableN.bin`) into the [`Table`] a `.xlsx` table part reads +//! as. The table's AutoFilter and sort state are held as `.xlsx` elements, so they are built here +//! as Excel writes them. A filter column the reader can't spell is left out with a warning, as +//! is a sort state; the table keeps the rest. + +use craft_sheet::{CellRef, MAX_COLS, MAX_ROWS, Range}; +use craft_xml::El; + +use super::formula::{self, ListInfo, Mode}; +use super::{Budget, Bytes, Links, Records, expr, invalid}; +use crate::features::{Table, TableColumn, TableStyleInfo}; +use crate::{Error, Limits, Warnings}; + +/// The content type of a table part. +pub(crate) const TABLE: &str = "application/vnd.ms-excel.table"; + +/// `RfX` / `UncheckedRfX` ([MS-XLSB] 2.5.118, 2.5.154): first and last row, first and last column. +fn rfx(b: &mut Bytes<'_>) -> Result { + let (r0, r1, c0, c1) = (b.u32()?, b.u32()?, b.u32()?, b.u32()?); + if r0 > r1 || c0 > c1 || r1 >= MAX_ROWS || c1 >= MAX_COLS { + return Err(invalid("invalid table range")); + } + Ok(Range::new(CellRef::new(r0, c0), CellRef::new(r1, c1))) +} + +/// A `DXFId`, `None` for 0xFFFFFFFF (no differential format). +fn dxf(id: u32) -> Option { + (id != u32::MAX).then_some(id) +} + +/// An `XLNullableWideString` of at most `max` characters, with the empty string as none: Excel +/// writes an empty comment where `.xlsx` has no attribute. +fn text(b: &mut Bytes<'_>, max: u32) -> Result, Error> { + Ok(b.string(max, true)?.filter(|s| !s.is_empty())) +} + +/// What a table part holds of formulas, for the reader's index of tables: its id, display name +/// and column names. +fn list_info(bytes: &[u8], limits: &Limits, budget: &Budget) -> Result<(u32, ListInfo), Error> { + let t = table(bytes, None, limits, budget, &mut Warnings::default())?; + let id = t.id.ok_or_else(|| invalid("table without an id"))?; + let name = t.display_name.or(t.name).ok_or_else(|| invalid("table without a name"))?; + // A column without a name can't be spelled in a structured reference. + let columns = t + .columns + .into_iter() + .map(|c| c.name.filter(|n| !n.is_empty())) + .collect::>() + .ok_or_else(|| invalid("table column without a name"))?; + Ok((id, ListInfo { name, columns })) +} + +/// Adds a table part to `links`, so formulas can name it. Two tables with one id name neither. +pub(crate) fn index_table(links: &mut Links, bytes: &[u8], limits: &Limits, budget: &Budget) -> Result<(), Error> { + let (id, info) = list_info(bytes, limits, budget)?; + links.tables.entry(id).and_modify(|t| *t = None).or_insert(Some(info)); + Ok(()) +} + +/// A table part. `links` decodes its calculated-column and totals-row formulas; without them +/// (or when one can't be decoded) a formula is left out, with a warning. +pub(crate) fn table(bytes: &[u8], links: Option<&Links>, limits: &Limits, budget: &Budget, warnings: &mut Warnings) -> Result { + let mut records = Records::new(bytes, limits, budget); + let mut table = match records.next(warnings)? { + Some((343, payload)) => begin_list(payload)?, + _ => return Err(invalid("missing table begin record")), + }; + let mut cols_open = false; + let mut cols_seen = false; + let mut column: Option = None; + while let Some((id, payload)) = records.next(warnings)? { + let mut b = Bytes::new(payload); + match id { + 161 if !cols_seen && table.auto_filter.is_none() => { + let range = rfx(&mut b)?; + b.finish()?; + table.auto_filter = Some(auto_filter(range, &mut records, warnings)?); + } + 530 if !cols_seen && table.sort_state.is_none() => table.sort_state = sort_state(payload, &mut records, warnings)?, + 345 if !cols_seen => { + b.u32()?; + b.finish()?; + cols_open = true; + cols_seen = true; + } + 346 if cols_open && column.is_none() => { + b.finish()?; + cols_open = false; + } + 347 if cols_open && column.is_none() => { + if table.columns.len() >= MAX_COLS as usize { + return Err(invalid("too many table columns")); + } + column = Some(begin_list_col(payload)?); + } + 351 | 352 if column.is_some() => { + let f = list_formula(payload, links, limits, budget, warnings)?; + if let Some(c) = column.as_mut() { + *(if id == 351 { &mut c.calculated_formula } else { &mut c.totals_row_formula }) = f; + } + } + 348 => { + b.finish()?; + table.columns.push(column.take().ok_or_else(|| invalid("table column end without a column"))?); + } + 349 => { + let c = column.as_mut().ok_or_else(|| invalid("XML column properties outside a table column"))?; + c.other.push(xml_column(payload)?); + } + 513 if !cols_open && cols_seen => { + let flags = b.u16()?; + let name = b.string(255, true)?; + b.finish()?; + let bit = |n: u16| Some(flags & (1 << n) != 0); + table.style = Some(TableStyleInfo { + name, + show_first_column: bit(0), + show_last_column: bit(1), + show_row_stripes: bit(2), + show_column_stripes: bit(3), + ..TableStyleInfo::default() + }); + } + 344 => { + if cols_open || !cols_seen || column.is_some() { + return Err(invalid("incomplete table")); + } + records.end(payload, warnings)?; + return Ok(table); + } + // The end of an XML map's column; anything else out of place is damage. + 350 => b.finish()?, + _ => return Err(invalid(format!("unexpected record {id} in a table"))), + } + } + Err(invalid("missing table end record")) +} + +/// `BrtBeginList` ([MS-XLSB] 2.4.100). `.xlsx` leaves out what is the default: one header row, +/// no totals row, a totals row that has been shown, a worksheet table. +fn begin_list(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let range = rfx(&mut b)?; + let table_type = match b.u32()? { + 0 => None, + 2 => Some("xml".to_string()), + 3 => Some("queryTable".to_string()), + _ => return Err(invalid("invalid table type")), + }; + let id = b.u32()?; + if id == 0 || id == u32::MAX { + return Err(invalid("invalid table id")); + } + let header = b.u32()?; + let totals = b.u32()?; + let flags = b.u32()?; + // fSingleCell: a single-cell table belongs in its own part type, not here. + if flags & 2 != 0 { + return Err(invalid("single-cell table in a table part")); + } + let dxfs: Vec> = (0..6).map(|_| b.u32().map(dxf)).collect::>()?; + let connection = b.u32()?; + let name = b.string(255, true)?; + let display_name = b.string(255, true)?; + let comment = text(&mut b, 255)?; + let header_row_cell_style = b.string(255, true)?; + let data_cell_style = b.string(255, true)?; + let totals_row_cell_style = b.string(255, true)?; + b.finish()?; + let set = |bit: u32| (flags & bit != 0).then_some(true); + Ok(Table { + id: Some(id), + name, + display_name, + comment, + table_type, + header_row_count: (header != 1).then_some(header), + insert_row: set(4), + insert_row_shift: set(8), + totals_row_count: (totals != 0).then_some(totals), + totals_row_shown: (flags & 1 == 0).then_some(false), + published: set(16), + header_row_dxf_id: dxfs[0], + data_dxf_id: dxfs[1], + totals_row_dxf_id: dxfs[2], + table_border_dxf_id: dxfs[3], + header_row_border_dxf_id: dxfs[4], + totals_row_border_dxf_id: dxfs[5], + header_row_cell_style, + data_cell_style, + totals_row_cell_style, + connection_id: (connection != 0).then_some(connection), + range, + ..Table::default() + }) +} + +/// `BrtBeginListCol` ([MS-XLSB] 2.4.101): `stName` is `.xlsx`'s `uniqueName`, `stCaption` its +/// `name`, and the insert row's format and style are the data's. +fn begin_list_col(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let id = b.u32()?; + let function = match b.u32()? { + 0 => None, + 1 => Some("average"), + 2 => Some("count"), + 3 => Some("countNums"), + 4 => Some("max"), + 5 => Some("min"), + 6 => Some("sum"), + 7 => Some("stdDev"), + 8 => Some("var"), + 9 => Some("custom"), + _ => return Err(invalid("invalid totals row function")), + }; + let header_row_dxf_id = dxf(b.u32()?); + let data_dxf_id = dxf(b.u32()?); + let totals_row_dxf_id = dxf(b.u32()?); + let query = b.u32()?; + let unique_name = b.string(255, true)?; + let name = b.string(255, true)?; + let totals_row_label = b.string(8189, true)?; + let header_row_cell_style = b.string(255, true)?; + let data_cell_style = b.string(255, true)?; + let totals_row_cell_style = b.string(255, true)?; + b.finish()?; + Ok(TableColumn { + id: Some(id), + unique_name, + name, + totals_row_function: function.map(str::to_string), + totals_row_label, + query_table_field_id: (query != 0).then_some(query), + header_row_dxf_id, + data_dxf_id, + totals_row_dxf_id, + header_row_cell_style, + data_cell_style, + totals_row_cell_style, + ..TableColumn::default() + }) +} + +/// `XmlDataType` ([MS-XLSB] 2.5.170) from 1, as `.xlsx` names the XML Schema types. +const XML_TYPES: [&str; 45] = [ + "string", + "normalizedString", + "token", + "byte", + "unsignedByte", + "base64Binary", + "hexBinary", + "integer", + "positiveInteger", + "negativeInteger", + "nonPositiveInteger", + "nonNegativeInteger", + "int", + "unsignedInt", + "long", + "unsignedLong", + "short", + "unsignedShort", + "decimal", + "float", + "double", + "boolean", + "time", + "dateTime", + "duration", + "date", + "gMonth", + "gYear", + "gYearMonth", + "gDay", + "gMonthDay", + "Name", + "QName", + "NCName", + "anyURI", + "language", + "ID", + "IDREF", + "IDREFS", + "ENTITY", + "ENTITIES", + "NOTATION", + "NMTOKEN", + "NMTOKENS", + "anyType", +]; + +/// `BrtBeginListXmlCPr` ([MS-XLSB] 2.4.104): a column's place in an XML map, as `xmlColumnPr`. +/// `fCanBeSingle` (one node for every row) is `.xlsx`'s `denormalized`. +fn xml_column(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let map = b.u32()?; + let flags = b.u32()?; + let kind = b.u32()?; + let xpath = b.text(31_999)?; + b.finish()?; + let kind = kind.checked_sub(1).and_then(|k| XML_TYPES.get(k as usize)).ok_or_else(|| invalid("invalid XML data type"))?; + let mut el = El::new("xmlColumnPr").with_attr("mapId", map).with_attr("xpath", xpath); + if flags & 2 != 0 { + el.set_attr("denormalized", 1); + } + el.set_attr("xmlDataType", kind); + Ok(el) +} + +/// `BrtListCCFmla` / `BrtListTrFmla` ([MS-XLSB] 2.4.706, 2.4.708): a flag byte (`fArray`, bit 1) +/// and a `ListParsedFormula`. Its references hold coordinates, as a cell formula's do. +fn list_formula( + payload: &[u8], + links: Option<&Links>, + limits: &Limits, + budget: &Budget, + warnings: &mut Warnings, +) -> Result, Error> { + let mut b = Bytes::new(payload); + let array = b.u8()? & 2 != 0; + let cce = b.u32()?; + if !(1..=16384).contains(&cce) { + return Err(invalid("invalid formula token length")); + } + let rgce = b.take(cce as usize)?; + let cb = b.u32()?; + let rgcb = b.take(cb as usize)?; + b.finish()?; + let Some(links) = links else { return Ok(None) }; + let text = match formula::decode(rgce, rgcb, links, Mode::Cell) { + Ok(e) => expr::print(&e), + Err(formula::Unsupported(why)) => { + warnings.warn(format!("XLSB: some table column formulas were left out ({why})")); + return Ok(None); + } + }; + if !budget.try_formula_text(text.len(), limits) { + warnings.warn("XLSB: some table column formulas were left out (more formula text than the limits allow)"); + return Ok(None); + } + Ok(Some((text, array))) +} + +/// A number in a filter, as Excel writes it in `.xlsx`, which differs from a formula's: 17 +/// significant digits (`1234.5678901234501`, `1.0000000000000001E-5`), written out from 0.01 up to +/// 10^16 unless that takes more than 17 decimals, and otherwise with an exponent of no fixed +/// width (`1E-4`, `1E+20`). Excel writes a few values with fewer digits (`0.1`, not +/// `0.10000000000000001`); both read back as the same number. +fn filter_number(n: f64) -> String { + if n == 0.0 || !n.is_finite() { + return "0".into(); + } + let sci = format!("{:.16e}", n.abs()); + let Some((mantissa, exp)) = sci.split_once('e') else { return sci }; + let e: i32 = exp.parse().unwrap_or(0); + let digits = mantissa.replace('.', ""); + let digits = digits.trim_end_matches('0'); + let digits = if digits.is_empty() { "0" } else { digits }; + let decimals = i32::try_from(digits.len()).unwrap_or(i32::MAX).saturating_sub(1).saturating_sub(e); + let mut s = String::from(if n < 0.0 { "-" } else { "" }); + if !(-2..=15).contains(&e) || decimals > 17 { + s.push_str(&digits[..1]); + if digits.len() > 1 { + s.push('.'); + s.push_str(&digits[1..]); + } + s.push_str(&format!("E{}{}", if e < 0 { '-' } else { '+' }, e.unsigned_abs())); + } else if e >= 0 { + let whole = e as usize + 1; + if digits.len() <= whole { + s.push_str(digits); + s.push_str(&"0".repeat(whole - digits.len())); + } else { + s.push_str(&digits[..whole]); + s.push('.'); + s.push_str(&digits[whole..]); + } + } else { + s.push_str("0."); + s.push_str(&"0".repeat((-e - 1) as usize)); + s.push_str(digits); + } + s +} + +/// Warns that a table's filter column was left out, and why. +fn left_out(warnings: &mut Warnings, why: &str) { + warnings.warn(format!("XLSB: some table filters were left out ({why})")); +} + +/// The records after `BrtBeginAFilter` ([MS-XLSB] 2.4.8) up to `BrtEndAFilter`, as `autoFilter`. +fn auto_filter(range: Range, records: &mut Records<'_, '_>, warnings: &mut Warnings) -> Result { + let mut el = El::new("autoFilter").with_attr("ref", range.a1()); + while let Some((id, payload)) = records.next(warnings)? { + match id { + 163 => { + if let Some(c) = filter_column(payload, records, warnings)? { + el.push(c); + } + } + 162 => { + Bytes::new(payload).finish()?; + return Ok(el); + } + _ => return Err(invalid(format!("unexpected record {id} in a table filter"))), + } + } + Err(invalid("missing table filter end record")) +} + +/// A `BrtBeginFilterColumn` ([MS-XLSB] 2.4.84) and its criteria, up to `BrtEndFilterColumn`, as +/// `filterColumn`. `None` when its criteria can't be spelled as `.xlsx` has them, or are kept +/// in an extension the reader skips (Excel's date-group filters). +fn filter_column(payload: &[u8], records: &mut Records<'_, '_>, warnings: &mut Warnings) -> Result, Error> { + let mut b = Bytes::new(payload); + let col = b.u32()?; + let flags = b.u16()?; + b.finish()?; + let mut el = El::new("filterColumn").with_attr("colId", col); + if flags & 1 != 0 { + el.set_attr("hiddenButton", 1); + } + if flags & 2 != 0 { + el.set_attr("showButton", 0); + } + let skipped = records.skipped; + let mut keep = true; + while let Some((id, payload)) = records.next(warnings)? { + let mut b = Bytes::new(payload); + match id { + 165 => { + let blank = b.u32()?; + b.u32()?; + b.finish()?; + let mut filters = El::new("filters"); + if blank == 1 { + filters.set_attr("blank", 1); + } + loop { + let (id, payload) = records.next(warnings)?.ok_or_else(|| invalid("missing filter list end record"))?; + let mut b = Bytes::new(payload); + match id { + 167 => { + let val = b.text(255)?; + b.finish()?; + filters.push(El::new("filter").with_attr("val", val)); + } + 175 => filters.push(date_group_item(payload)?), + 166 => { + b.finish()?; + break; + } + _ => return Err(invalid(format!("unexpected record {id} in a filter list"))), + } + } + el.push(filters); + } + 172 => { + let and = b.u32()?; + b.finish()?; + // Excel writes 1 where `.xlsx` has `and="1"` ([MS-XLSB]'s table has them the other way round). + let mut custom = El::new("customFilters"); + if and == 1 { + custom.set_attr("and", 1); + } + loop { + let (id, payload) = records.next(warnings)?.ok_or_else(|| invalid("missing custom filter end record"))?; + match id { + 174 => match custom_filter(payload)? { + Some(f) => custom.push(f), + None => { + left_out(warnings, "custom filters on a Boolean value"); + keep = false; + } + }, + 173 => { + Bytes::new(payload).finish()?; + break; + } + _ => return Err(invalid(format!("unexpected record {id} in custom filters"))), + } + } + el.push(custom); + } + 170 => { + let flags = b.u8()?; + let val = b.f64()?; + let filter_val = b.f64()?; + b.finish()?; + let mut top = El::new("top10"); + if flags & 1 == 0 { + top.set_attr("top", 0); + } + if flags & 2 != 0 { + top.set_attr("percent", 1); + } + top.set_attr("val", filter_number(val)); + if flags & 4 != 0 { + top.set_attr("filterVal", filter_number(filter_val)); + } + el.push(top); + } + 171 => el.push(dynamic_filter(payload)?), + 168 => { + let dxf_id = b.u32()?; + let cell = b.u32()?; + b.finish()?; + let mut color = El::new("colorFilter").with_attr("dxfId", dxf_id); + if cell == 0 { + color.set_attr("cellColor", 0); + } + el.push(color); + } + 169 => { + let set = b.u32()?; + let icon = b.u32()?; + b.finish()?; + // KPINIL: no filter. + if set != u32::MAX { + let name = ICON_SETS.get(set as usize).ok_or_else(|| invalid("invalid filter icon set"))?; + el.push(El::new("iconFilter").with_attr("iconSet", name).with_attr("iconId", icon)); + } + } + 164 => { + b.finish()?; + if records.skipped != skipped { + left_out(warnings, "filters kept in extensions the reader skips, such as date groups"); + return Ok(None); + } + return Ok(keep.then_some(el)); + } + _ => return Err(invalid(format!("unexpected record {id} in a filter column"))), + } + } + Err(invalid("missing filter column end record")) +} + +/// `KPISets` ([MS-XLSB] 2.5.86), as `.xlsx` names them. +const ICON_SETS: [&str; 17] = [ + "3Arrows", + "3ArrowsGray", + "3Flags", + "3TrafficLights1", + "3TrafficLights2", + "3Signs", + "3Symbols", + "3Symbols2", + "4Arrows", + "4ArrowsGray", + "4RedToBlack", + "4Rating", + "4TrafficLights", + "5Arrows", + "5ArrowsGray", + "5Rating", + "5Quarters", +]; + +/// `BrtCustomFilter` ([MS-XLSB] 2.4.348) as `customFilter`; `None` for a Boolean value, whose +/// `.xlsx` spelling no Excel file has shown. A blank or non-blank test is `val=" "`, as Excel +/// writes it. +fn custom_filter(payload: &[u8]) -> Result, Error> { + let mut b = Bytes::new(payload); + let vts = b.u8()?; + let operator = match b.u8()? { + 1 => Some("lessThan"), + 2 => None, + 3 => Some("lessThanOrEqual"), + 4 => Some("greaterThan"), + 5 => Some("notEqual"), + 6 => Some("greaterThanOrEqual"), + _ => return Err(invalid("invalid custom filter operator")), + }; + let union = b.take(8)?; + let val = match vts { + 4 => { + let n = f64::from_le_bytes(union.try_into().map_err(|_| invalid("truncated number"))?); + if !n.is_finite() { + return Err(invalid("nonfinite numeric value")); + } + filter_number(n) + } + 6 => b.text(255)?, + 8 => return Ok(None), + 0x0C | 0x0E => " ".to_string(), + _ => return Err(invalid("invalid custom filter type")), + }; + b.finish()?; + let mut el = El::new("customFilter"); + if let Some(op) = operator { + el.set_attr("operator", op); + } + el.set_attr("val", val); + Ok(Some(el)) +} + +/// `BrtDynamicFilter` ([MS-XLSB] 2.4.362) as `dynamicFilter`: the computed values only once the +/// filter has been applied, and the maximum only for date periods. +fn dynamic_filter(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let cft = b.u32()?; + let applied = b.u8()? & 1 != 0; + let val = b.f64()?; + let max = b.f64()?; + b.finish()?; + const PERIODS: [&str; 16] = [ + "tomorrow", + "today", + "yesterday", + "nextWeek", + "thisWeek", + "lastWeek", + "nextMonth", + "thisMonth", + "lastMonth", + "nextQuarter", + "thisQuarter", + "lastQuarter", + "nextYear", + "thisYear", + "lastYear", + "yearToDate", + ]; + let kind = match cft { + 0 => "null".to_string(), + 1 => "aboveAverage".to_string(), + 2 => "belowAverage".to_string(), + 8..=0x17 => PERIODS[(cft - 8) as usize].to_string(), + 0x18..=0x1B => format!("Q{}", cft - 0x17), + 0x1C..=0x27 => format!("M{}", cft - 0x1B), + _ => return Err(invalid("invalid dynamic filter type")), + }; + let mut el = El::new("dynamicFilter").with_attr("type", kind); + if applied && (1..=0x17).contains(&cft) { + el.set_attr("val", filter_number(val)); + if cft >= 8 { + el.set_attr("maxVal", filter_number(max)); + } + } + Ok(el) +} + +/// `BrtAFilterDateGroupItem` ([MS-XLSB] 2.4.5) as `dateGroupItem`, with the parts of the date +/// its grouping compares. +fn date_group_item(payload: &[u8]) -> Result { + let mut b = Bytes::new(payload); + let year = b.u16()?; + let month = b.u16()?; + let day = b.u32()?; + let (hour, minute, second) = (b.u16()?, b.u16()?, b.u16()?); + b.take(6)?; + let grouping = b.u32()?; + b.finish()?; + const GROUPS: [&str; 6] = ["year", "month", "day", "hour", "minute", "second"]; + let name = GROUPS.get(grouping as usize).ok_or_else(|| invalid("invalid date group"))?; + let mut el = El::new("dateGroupItem").with_attr("year", year); + let parts = [("month", u32::from(month)), ("day", day), ("hour", u32::from(hour)), ("minute", u32::from(minute)), ("second", u32::from(second))]; + for (name, value) in parts.iter().take(grouping as usize) { + el.set_attr(name, value); + } + el.set_attr("dateTimeGrouping", name); + Ok(el) +} + +/// `BrtBeginSortState` ([MS-XLSB] 2.4.227) and its `BrtBeginSortCond`s, up to `BrtEndSortState`, +/// as `sortState`. `None`, with a warning, for a sort by another method than character order, +/// which `.xlsx` names (stroke or PinYin) and the record does not. +fn sort_state(payload: &[u8], records: &mut Records<'_, '_>, warnings: &mut Warnings) -> Result, Error> { + let mut b = Bytes::new(payload); + let flags = b.u16()?; + let range = rfx(&mut b)?; + b.finish()?; + let mut el = El::new("sortState"); + if flags & 1 != 0 { + el.set_attr("columnSort", 1); + } + if flags & 2 != 0 { + el.set_attr("caseSensitive", 1); + } + el.set_attr("ref", range.a1()); + let keep = flags & 4 == 0; + if !keep { + warnings.warn("XLSB: some table sort states were left out (sorts by stroke or PinYin order)"); + } + while let Some((id, payload)) = records.next(warnings)? { + let mut b = Bytes::new(payload); + match id { + 532 => { + let flags = b.u16()?; + let range = rfx(&mut b)?; + let (first, second) = (b.u32()?, b.u32()?); + let list = b.string(32_767, true)?; + b.finish()?; + let mut cond = El::new("sortCondition"); + if flags & 1 != 0 { + cond.set_attr("descending", 1); + } + let on = (flags >> 1) & 0xF; + match on { + 0 => {} + 1 => cond.set_attr("sortBy", "cellColor"), + 2 => cond.set_attr("sortBy", "fontColor"), + 3 => cond.set_attr("sortBy", "icon"), + _ => return Err(invalid("invalid sort condition")), + } + cond.set_attr("ref", range.a1()); + match on { + 0 => { + if let Some(list) = list { + cond.set_attr("customList", list); + } + } + 1 | 2 => cond.set_attr("dxfId", first), + _ => { + let name = ICON_SETS.get(first as usize).ok_or_else(|| invalid("invalid sort icon set"))?; + cond.set_attr("iconSet", name); + cond.set_attr("iconId", second); + } + } + el.push(cond); + } + 533 => b.finish()?, + 531 => { + b.finish()?; + return Ok(keep.then_some(el)); + } + _ => return Err(invalid(format!("unexpected record {id} in a sort state"))), + } + } + Err(invalid("missing sort state end record")) +} + +#[cfg(test)] +mod tests { + use super::filter_number; + + #[test] + fn filter_numbers_are_spelled_as_excel_writes_them() { + // Excel's `.xlsx` text for each value. + let cases = [ + (20.0, "20"), + (23.25, "23.25"), + (46340.0, "46340"), + (-1.5, "-1.5"), + (1e15, "1000000000000000"), + (999_999_999_999_999.0, "999999999999999"), + (1.234_567_890_123_45e17, "1.2345678901234499E+17"), + (1e16, "1E+16"), + (-1e16, "-1E+16"), + (1e20, "1E+20"), + (2.5e300, "2.5000000000000001E+300"), + (1_234.567_890_123_45, "1234.5678901234501"), + (12_345_678_901_234.5, "12345678901234.5"), + (99_999_999_999_999.9, "99999999999999.906"), + (1.1, "1.1000000000000001"), + (0.57, "0.56999999999999995"), + (0.01, "0.01"), + (0.035, "3.5000000000000003E-2"), + (0.001, "1E-3"), + (1e-4, "1E-4"), + (1e-5, "1.0000000000000001E-5"), + (1.5e-10, "1.5E-10"), + (1e-19, "9.9999999999999998E-20"), + ]; + for (n, want) in cases { + assert_eq!(filter_number(n), want, "{n}"); + } + } +} diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 96c7cff5d..a7253df22 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -7,7 +7,7 @@ use common::{read, read_with}; use craft_xlsx::testing::zip; use craft_xlsx::{ CellError, CellRef, Color, ColorSource, Error, Fill, FormulaKind, Limits, Package, Pattern, PatternFill, Protection, Range, Row, SheetKind, - SheetPart, Styles, Value, Visibility, is_xlsb, + SheetPart, Styles, Table, Value, Visibility, is_xlsb, }; const ROOT: &str = r#""#; @@ -590,7 +590,7 @@ fn cell_formulas_are_read_in_their_xlsx_spelling() { assert_eq!(sheet.cell_metadata.iter().collect::>(), [(&CellRef::new(0, 14), &1)]); // Every formula cell keeps its saved result as its value. assert!((1..=15).all(|c| sheet.value(CellRef::new(0, c)) == Value::Number(0.0))); - for why in ["other workbooks", "missing from the file", "structured table references"] { + for why in ["other workbooks", "missing from the file", "tables the reader could not read"] { assert!(warned(&wb, why), "{why}: {:?}", wb.warnings); } } @@ -1616,6 +1616,325 @@ fn references_to_macro_sheets_and_spans_over_chart_sheets_are_named() { assert!(warned(&wb, "references to sheets with no cells"), "{:?}", wb.warnings); } +// Tables ([MS-XLSB] 2.1.7.51): table parts assembled from the specification's records, and the +// `` Excel writes for the same table in `.xlsx`. + +fn rfx(r0: u32, r1: u32, c0: u32, c1: u32) -> Vec { + [r0, r1, c0, c1].iter().flat_map(|n| n.to_le_bytes()).collect() +} +/// `BrtBeginList`: a worksheet table with no formats or cell styles, its name as its display name. +fn begin_list(range: &[u8], id: u32, header: u32, totals: u32, flags: u32, name: &str) -> Vec { + begin_list_of(0, range, id, header, totals, flags, name) +} +/// `BrtBeginList` of a table type (`lt`): 0 a worksheet table, 2 an XML table. +fn begin_list_of(lt: u32, range: &[u8], id: u32, header: u32, totals: u32, flags: u32, name: &str) -> Vec { + let mut data = range.to_vec(); + for n in [lt, id, header, totals, flags] { + data.extend(n.to_le_bytes()); + } + data.extend(u32::MAX.to_le_bytes().repeat(6)); + data.extend(0u32.to_le_bytes()); + data.extend([wide(name), wide(name), wide("")].concat()); + data.extend(u32::MAX.to_le_bytes().repeat(3)); + record(343, &data) +} +/// `BrtBeginListCol` with its totals function (`ilta`), data format and totals label. +fn list_col(id: u32, ilta: u32, data_dxf: u32, name: &str, total: Option<&str>) -> Vec { + let mut data: Vec = [id, ilta, u32::MAX, data_dxf, u32::MAX, 0, u32::MAX].iter().flat_map(|n| n.to_le_bytes()).collect(); + data.extend(wide(name)); + data.extend(total.map_or_else(|| u32::MAX.to_le_bytes().to_vec(), wide)); + data.extend(u32::MAX.to_le_bytes().repeat(3)); + record(347, &data) +} +/// `BrtListCCFmla` (351) or `BrtListTrFmla` (352). +fn list_formula(id: u16, rgce: &[u8]) -> Vec { + record(id, &[&[0u8][..], &(rgce.len() as u32).to_le_bytes(), rgce, &0u32.to_le_bytes()].concat()) +} +/// `PtgList`: the flags hold the columns (bits 0-1), the rows (2-6) and the two spacing bits. +fn ptg_list(ixti: u16, flags: u16, list: u32, first: u16, last: u16) -> Vec { + [&[0x18u8, 0x19][..], &ixti.to_le_bytes(), &flags.to_le_bytes(), &list.to_le_bytes(), &first.to_le_bytes(), &last.to_le_bytes()].concat() +} +fn filter_column(col: u32, flags: u16, criteria: &[Vec]) -> Vec { + [record(163, &[&col.to_le_bytes()[..], &flags.to_le_bytes()].concat()), criteria.concat(), record(164, &[])].concat() +} +fn custom_filter(vts: u8, op: u8, union: [u8; 8], text: Option<&str>) -> Vec { + record(174, &[&[vts, op][..], &union, &text.map(wide).unwrap_or_default()].concat()) +} +fn table_style(flags: u16, name: &str) -> Vec { + record(513, &[&flags.to_le_bytes()[..], &wide(name)].concat()) +} +/// The rows of a `PtgList` (bits 2-6). +const fn rows(t: u16) -> u16 { + t << 2 +} + +/// The filters of Sales: a value list with blanks, two custom criteria (OR), a bottom-percent +/// filter applied with its button hidden, and "today" applied. +fn sales_filters() -> Vec> { + vec![ + filter_column(0, 0, &[record(165, &[1, 0, 0, 0, 0, 0, 0, 0]), record(167, &wide("k3")), record(167, &wide("k4")), record(166, &[])]), + filter_column( + 1, + 0, + &[ + record(172, &0u32.to_le_bytes()), + custom_filter(4, 1, 20f64.to_le_bytes(), None), + custom_filter(6, 2, [0; 8], Some("k*")), + record(173, &[]), + ], + ), + filter_column(2, 1, &[record(170, &[&[0b110u8][..], &10f64.to_le_bytes(), &23.25f64.to_le_bytes()].concat())]), + filter_column(3, 0, &[record(171, &[&9u32.to_le_bytes()[..], &[1], &46340f64.to_le_bytes(), &46341f64.to_le_bytes()].concat())]), + ] +} + +/// Sales (B2:E8, id 1) with a totals row, a calculated column, a column name needing escapes, +/// `filters` and a sort. +fn sales(filters: &[Vec]) -> Vec { + [ + begin_list(&rfx(1, 7, 1, 4), 1, 1, 1, 1, "Sales"), + record(161, &rfx(1, 6, 1, 4)), + filters.concat(), + record(162, &[]), + record(530, &[&0u16.to_le_bytes()[..], &rfx(2, 6, 1, 4)].concat()), + record(532, &[&1u16.to_le_bytes()[..], &rfx(1, 7, 1, 1), &[0; 8], &u32::MAX.to_le_bytes()].concat()), + record(533, &[]), + record(531, &[]), + record(345, &4u32.to_le_bytes()), + list_col(1, 0, u32::MAX, "Key", Some("Total")), + record(348, &[]), + list_col(2, 6, u32::MAX, "Val", None), + record(348, &[]), + list_col(3, 0, 2, "It's [#1]", None), + record(348, &[]), + list_col(4, 9, u32::MAX, "Calc", None), + list_formula(351, &[ptg_list(0, 1 | rows(0x10), 1, 1, 1), vec![0x1E, 2, 0, 0x05]].concat()), + list_formula(352, &[ptg_list(0, 1, 1, 3, 3), vec![0x42, 1, 7, 0]].concat()), + record(348, &[]), + record(346, &[]), + table_style(0b101, "TableStyleMedium9"), + record(344, &[]), + ] + .concat() +} + +/// NoHead (H10:I11, id 2): no header row, a totals row never shown, no stripes. +fn no_head() -> Vec { + [ + begin_list(&rfx(9, 10, 7, 8), 2, 0, 0, 0, "NoHead"), + record(345, &2u32.to_le_bytes()), + list_col(1, 0, u32::MAX, "Column1", None), + record(348, &[]), + list_col(2, 0, u32::MAX, "Column2", None), + record(348, &[]), + record(346, &[]), + table_style(0, "TableStyleMedium2"), + record(344, &[]), + ] + .concat() +} + +/// XmlT (K1:L2, id 3): an XML table, its columns mapped to an XML map's elements. +fn xml_table_part() -> Vec { + let col = |id: u32, name: &str, kind: u32, xpath: &str| { + let mut data: Vec = [id, 0, u32::MAX, u32::MAX, u32::MAX, 0].iter().flat_map(|n| n.to_le_bytes()).collect(); + data.extend([wide(name), wide(name)].concat()); + data.extend(u32::MAX.to_le_bytes().repeat(4)); + let map = [&1u32.to_le_bytes()[..], &(if id == 2 { 2u32 } else { 0 }).to_le_bytes(), &kind.to_le_bytes(), &wide(xpath)].concat(); + [record(347, &data), record(349, &map), record(350, &[]), record(348, &[])].concat() + }; + [ + begin_list_of(2, &rfx(0, 1, 10, 11), 3, 1, 0, 0, "XmlT"), + record(345, &2u32.to_le_bytes()), + col(1, "a", 0x0D, "/rows/r/a"), + col(2, "b", 0x01, "/rows/@b"), + record(346, &[]), + table_style(0b100, "TableStyleMedium2"), + record(344, &[]), + ] + .concat() +} + +const XML_TABLE_XML: &str = r#"
"#; +const SALES_XML: &str = r#"Sales[[#This Row],[Val]]*2MAX(Sales[Calc])
"#; +const NO_HEAD_XML: &str = r#"
"#; + +fn xml_table(xml: &str) -> Table { + let root = craft_xml::Options::canonical(&craft_xlsx::ns::CANONICAL).parse(xml.as_bytes()).unwrap(); + Table::read(&root).unwrap() +} + +/// `book_with_links(&xti())` whose sheet one lists `tables` (`BrtListPart`, relationship ids +/// t1, t2, ...) after `data`. +fn book_with_tables(data: &[u8], tables: &[Vec]) -> Vec { + let mut list = vec![record(660, &(tables.len() as u32).to_le_bytes())]; + let mut rels = String::from(r#""#); + let mut types = TYPES.trim_end_matches("").to_string(); + for i in 1..=tables.len() { + list.push(record(661, &wide(&format!("t{i}")))); + rels.push_str(&format!(r#""#)); + types.push_str(&format!(r#""#)); + } + list.push(record(662, &[])); + rels.push_str(""); + types.push_str(""); + let names: Vec = (1..=tables.len()).map(|i| format!("tables/t{i}.bin")).collect(); + let (book, one, two, strings) = (book_with_links(&xti()), worksheet(data, &list.concat()), worksheet(&[], &[]), sst()); + let mut entries: Vec<(&str, &[u8])> = vec![ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", ROOT.as_bytes()), + ("records/book.bin", &book), + ("records/_rels/book.bin.rels", RELS.as_bytes()), + ("data/one.bin", &one), + ("data/_rels/one.bin.rels", rels.as_bytes()), + ("data/two.bin", &two), + ("text/strings.bin", &strings), + ]; + entries.extend(names.iter().map(String::as_str).zip(tables.iter().map(Vec::as_slice))); + zip(&entries) +} + +/// Sheet one's tables through `Package`, and its warnings. +fn tables_of(bytes: &[u8]) -> (Vec, Vec) { + let mut p = Package::open(bytes, &Limits::default()).unwrap(); + let wb = p.workbook().unwrap(); + let part = p.sheet(&wb.sheets[0], |_| Ok(())).unwrap(); + let tables = p.tables(&part).unwrap().into_iter().map(|(_, t)| t).collect(); + (tables, p.warnings().to_vec()) +} + +#[test] +fn a_name_using_a_structured_reference_is_decoded_after_tables_are_indexed() { + // `Vals` = `Sales[Val]`. Names are decoded after the workbook's tables are indexed, so + // the formula and a cell that uses it are kept, instead of only their saved results. + let name = brt_name(0, 0, WORKBOOK, "Vals", &ptg_list(0, 1, 1, 1, 1), None); + let list = [record(660, &1u32.to_le_bytes()), record(661, &wide("t1")), record(662, &[])]; + let mut data = row(0); + data.extend(formula_cell(1, &[0x23, 5, 0, 0, 0], &[])); // =Vals, after the four names of `book_with_links` + let types = { + let t = TYPES.trim_end_matches(""); + format!(r#"{t}"#) + }; + let sheet_rels = r#""#; + let (one, two, strings, table) = (worksheet(&data, &list.concat()), worksheet(&[], &[]), sst(), sales(&sales_filters())); + let book = { + let plain = book_with_links(&xti()); + let end = record(132, &[]); + [&plain[..plain.len() - end.len()], name.as_slice(), end.as_slice()].concat() + }; + let bytes = zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", ROOT.as_bytes()), + ("records/book.bin", book.as_slice()), + ("records/_rels/book.bin.rels", RELS.as_bytes()), + ("data/one.bin", one.as_slice()), + ("data/_rels/one.bin.rels", sheet_rels.as_bytes()), + ("data/two.bin", two.as_slice()), + ("text/strings.bin", strings.as_slice()), + ("tables/t1.bin", table.as_slice()), + ]); + let names = Package::open(&bytes, &Limits::default()).unwrap().workbook().unwrap().defined_names; + assert_eq!(names.iter().find(|n| n.name == "Vals").map(|n| n.formula.as_str()), Some("Sales[Val]")); + let wb = read(&bytes).unwrap(); + assert_eq!(wb.sheets[0].formula(CellRef::new(0, 1)), Some("Vals"), "{:?}", wb.warnings); +} + +#[test] +fn simple_table_references_preserve_square_bracket_spacing() { + let cases = [ + (1, "Sales[Val]"), + (1 | 0x80, "Sales[ Val ]"), + (rows(1), "Sales[#All]"), + (rows(1) | 0x80, "Sales[ #All ]"), + (1 | 0x100, "Sales[Val]"), + (rows(1) | 0x100, "Sales[#All]"), + ]; + let mut data = row(0); + for (i, (bits, _)) in cases.iter().enumerate() { + data.extend(formula_cell(i as u32 + 1, &ptg_list(0, *bits, 1, 1, 1), &[])); + } + let bytes = book_with_tables(&data, &[sales(&sales_filters())]); + let wb = read(&bytes).unwrap(); + for (i, (_, want)) in cases.iter().enumerate() { + assert_eq!(wb.sheets[0].formula(CellRef::new(0, i as u32 + 1)), Some(*want), "{:?}", wb.warnings); + } +} + +#[test] +fn tables_are_read_as_their_xlsx_spelling() { + // Spacing flags: 0x80 inside the outer brackets, 0x100 after commas. + let cells: Vec> = vec![ + ptg_list(0, 1, 1, 1, 1), + ptg_list(0, 1 | rows(2), 1, 0, 0), + ptg_list(0, 2 | rows(4), 1, 1, 3), + ptg_list(0, rows(1), 1, 0, 0), + ptg_list(0, 0, 1, 0, 0), + ptg_list(0, 2 | rows(0x10), 1, 1, 3), + ptg_list(0, 1, 1, 2, 2), + ptg_list(0, 1 | rows(4) | 0x180, 1, 1, 1), + ptg_list(0, rows(0x10), 2, 0, 0), + ptg_list(0, 1 | rows(0xC) | 0x100, 1, 0, 0), + ptg_list(0, rows(6), 1, 0, 0), + // Kept as their saved results: a table that isn't there, a deleted table, a column past + // the table's last. + ptg_list(0, 0, 9, 0, 0), + ptg_list(0, 0x1000, 1, 0, 0), + ptg_list(0, 1, 1, 4, 4), + ]; + let mut data = row(0); + for (i, rgce) in cells.iter().enumerate() { + data.extend(formula_cell(i as u32 + 1, rgce, &[])); + } + let bytes = book_with_tables(&data, &[sales(&sales_filters()), no_head(), xml_table_part()]); + let (tables, _) = tables_of(&bytes); + assert_eq!(tables, [xml_table(SALES_XML), xml_table(NO_HEAD_XML), xml_table(XML_TABLE_XML)]); + let wb = read(&bytes).unwrap(); + let got: Vec<_> = (1..=14).map(|c| wb.sheets[0].formula(CellRef::new(0, c))).collect(); + let want = [ + Some("Sales[Val]"), + Some("Sales[[#Headers],[Key]]"), + Some("Sales[[#Data],[Val]:[Calc]]"), + Some("Sales[#All]"), + Some("Sales[]"), + Some("Sales[[#This Row],[Val]:[Calc]]"), + Some("Sales[It''s '['#1']]"), + Some("Sales[ [#Data], [Val] ]"), + Some("NoHead[#This Row]"), + Some("Sales[[#Data], [#Totals], [Key]]"), + Some("Sales[[#Headers],[#Data]]"), + None, + None, + None, + ]; + assert_eq!(got, want); + for why in ["tables the reader could not read", "deleted tables", "missing table column"] { + assert!(warned(&wb, why), "{why}: {:?}", wb.warnings); + } +} + +#[test] +fn table_filters_and_parts_the_reader_cannot_spell_are_left_out() { + // A filter column whose criteria are in a future-record block (where Excel keeps date + // groups), and one on a Boolean value: both left out, the table kept. + let frt = [record(35, &[3, 0, 2, 0]), record(5094, &[0; 32]), record(36, &[])].concat(); + let mut filters = sales_filters(); + filters.push(filter_column(4, 0, &[frt])); + filters.push(filter_column(5, 0, &[record(172, &0u32.to_le_bytes()), custom_filter(8, 2, [1, 0, 0, 0, 0, 0, 0, 0], None), record(173, &[])])); + // A truncated table part: skipped, and formulas using it keep their saved results. + let mut broken = no_head(); + broken.truncate(20); + let data = [row(0), formula_cell(1, &ptg_list(0, 0, 2, 0, 0), &[]), formula_cell(2, &ptg_list(0, 1, 1, 1, 1), &[])].concat(); + let bytes = book_with_tables(&data, &[sales(&filters), broken]); + let (tables, warnings) = tables_of(&bytes); + assert_eq!(tables, [xml_table(SALES_XML)]); + for why in ["filters kept in extensions", "custom filters on a Boolean value", "skipped tables/t2.bin"] { + assert!(warnings.iter().any(|w| w.contains(why)), "{why}: {warnings:?}"); + } + let wb = read(&bytes).unwrap(); + assert_eq!([1, 2].map(|c| wb.sheets[0].formula(CellRef::new(0, c))), [None, Some("Sales[Val]")]); + assert!(warned(&wb, "tables the reader could not read"), "{:?}", wb.warnings); +} + #[test] fn vba_function_names_without_macro_strings_are_read() { // Excel writes a VBA function a cell calls as a BrtName with fFunc, fOB and