From d1f10a9612d41e38687da3b8b78c7cb4be3c750c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 22:13:49 -0500 Subject: [PATCH] craft-xlsx: PivotTables (caches, records, tables) read and written (#1074) * craft-xlsx: read parts with canonical prefixes so kept trees can be written back The part layer read every part by local name, so extLst children (x14:, xr:, xm:), mc:AlternateContent and r:id lost their namespaces and could not be written back. Parts are now read with one fixed prefix per namespace (the main namespace unprefixed), mc:Ignorable and mc:Choice/@Requires are rewritten to match, and ns::declare prepares a kept tree for writing. craft-xml: a namespace listed with the empty prefix reads unprefixed. * craft-xlsx: write SpreadsheetML workbooks (Document read and write) Document holds a whole workbook: the workbook part, shared strings, styles, each worksheet's rows and the rest of its part, and every other part kept as bytes. write regenerates the workbook part, worksheets, shared strings and styles from the model, keeps unknown children in place (schema order for regenerated ones), keeps part names and relationship ids, and writes kept parts unchanged. Writers: Styles::to_xml and to_el for each table entry, StringItem::to_el (phonetic runs checked as on reading), SheetPart::to_xml with rows streamed, Workbook::to_el, XlsxKind. Row and DefinedName keep their other attributes; CellStyle reads iLevel. craft-opc: LazyPackage::canonical gives a part's stored name. * craft-xlsx: typed worksheet features and comments, written back SheetPart::features lifts sheet views, print setup, data validation, conditional formats, hyperlinks and the legacy drawing into types that mirror their elements (optional attributes, unnamed ones kept) and writes them in schema order. Comments come from the comments part, note shapes from the VML drawing; a sheet that gains comments without a VML drawing gets one. A hyperlink to a new URL gets its own relationship. craft-sheet: Range derives Default. * craft-xlsx: drawings, charts and the theme on craft-drawingml Drawing reads a sheet's xdr:wsDr into anchors with typed placements and objects kept as written; Anchor gives the object's kind, properties, chart or image relationship and xfrm. Package::drawing, Package::chart (c: and cx: parts through craft-drawingml) and Package::theme. Document worksheets hold their drawing part, regenerated with its relationships kept by id; add_chart and add_picture add new chart parts (chartEx behind mc:AlternateContent) and images with their anchors. More drawing namespaces get prefixes (a16, asvg, aink, am3d, cx1-cx4, dgm). * craft-xlsx: tables (ListObjects) read and written back Table mirrors CT_Table: every attribute, columns with calculated and totals formulas, the style info, autoFilter/sortState/extensions kept as written. Package::tables reads a sheet's table parts; Document worksheets hold them as TableParts, regenerated with their relationships and tableParts entries. Part names a document keeps are reserved before new parts are named, so a new part never takes a name a later sheet keeps. * craft-xlsx: PivotTables (caches, records, tables) read and written Typed pivot cache definitions (source, fields, shared items, grouping), streamed cache records and PivotTable definitions (location, fields and items, axes, page and data fields, style), with every unmodelled child and attribute kept. Workbook::pivot_caches lifts pivotCaches. A Document holds caches and tables and regenerates their parts, relationships and pivotCaches entries; an OLAP cache or a table on one stays a kept part. SharedItems::describe gives Excel's shared-item flags and records_xml writes a records part, after GridCraft's pivot writer. --- crates/xlsx/README.md | 9 + crates/xlsx/src/document.rs | 168 +++++++- crates/xlsx/src/features.rs | 6 +- crates/xlsx/src/lib.rs | 10 +- crates/xlsx/src/package.rs | 59 +++ crates/xlsx/src/pivot.rs | 769 ++++++++++++++++++++++++++++++++++++ crates/xlsx/src/workbook.rs | 30 +- crates/xlsx/tests/pivot.rs | 221 +++++++++++ 8 files changed, 1258 insertions(+), 14 deletions(-) create mode 100644 crates/xlsx/src/pivot.rs create mode 100644 crates/xlsx/tests/pivot.rs diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 3541dcb26..e5fe94683 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -64,6 +64,15 @@ worksheet holds them as `TablePart`s, regenerated on writing with their relation reserved before new parts are named, so a new part never takes a name another sheet's part keeps. +PivotTables: `Workbook::pivot_caches` lists the caches by id; `Package::pivot_cache` reads a cache +definition (`PivotCacheDefinition`: source, fields with shared items and grouping), +`Package::pivot_records` streams its records, and `Package::pivot_tables` reads a sheet's +PivotTables (`PivotTableDefinition`: location, fields and items, axes, page and data fields, +style) with the cache each uses. `SharedItems::describe` gives the flags Excel writes for a field's +values, and `records_xml` writes a records part. A `Document` holds caches as `PivotCachePart`s +and a worksheet's tables as `PivotTablePart`s, regenerated with their relationships and +`pivotCaches` entries; an OLAP cache (and a table on one) stays a kept part. + The parts this crate writes use the transitional namespaces; a strict file's kept parts stay as they were. A binary workbook is not written. diff --git a/crates/xlsx/src/document.rs b/crates/xlsx/src/document.rs index dc75a9728..def3c4b92 100644 --- a/crates/xlsx/src/document.rs +++ b/crates/xlsx/src/document.rs @@ -7,10 +7,11 @@ use craft_drawingml::chart::Chart; use crate::drawing::{Anchor, Drawing, Item, Placement}; use crate::features::{Comments, NoteShape, Table, notes_vml}; +use crate::pivot::{PivotCacheDefinition, PivotTableDefinition}; use crate::sheet::{CellValue, Row, SheetPart}; use crate::strings::StringItem; use crate::styles::Styles; -use crate::workbook::{SheetKind, Workbook}; +use crate::workbook::{PivotCacheEntry, SheetKind, Workbook}; use crate::{Error, Limits, Package}; const CT_WORKSHEET: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"; @@ -22,6 +23,9 @@ const CT_TABLE: &str = "application/vnd.openxmlformats-officedocument.spreadshee const CT_DRAWING: &str = "application/vnd.openxmlformats-officedocument.drawing+xml"; const CT_CHART: &str = "application/vnd.openxmlformats-officedocument.drawingml.chart+xml"; const CT_CHARTEX: &str = "application/vnd.ms-office.chartex+xml"; +const CT_PIVOT_TABLE: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotTable+xml"; +const CT_PIVOT_CACHE: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheDefinition+xml"; +const CT_PIVOT_RECORDS: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheRecords+xml"; const RT_CHARTEX: &str = "http://schemas.microsoft.com/office/2014/relationships/chartEx"; /// The relationship type namespace of strict files: kept relationships are written with the /// transitional one, as the parts this crate writes are. @@ -89,6 +93,33 @@ pub struct Worksheet { pub drawing: Option, /// The table parts, written from here, in the order of the sheet's `tableParts`. pub tables: Vec, + /// The PivotTable parts, written from here. A PivotTable that did not read, or whose cache + /// did not, is a kept part. + pub pivot_tables: Vec, +} + +/// A PivotTable part. It is related to the cache of [`Document::pivot_caches`] its `cache_id` +/// names. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PivotTablePart { + /// The part's name; a new name is chosen on writing when empty. + pub part: String, + pub table: PivotTableDefinition, +} + +/// A pivot cache: its definition part and its records. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PivotCachePart { + /// The definition part's name; a new name is chosen on writing when empty. + pub part: String, + /// The id the workbook and its PivotTables know the cache by. + pub cache_id: u32, + pub definition: PivotCacheDefinition, + /// The records part's name; a new name is chosen on writing when empty. + pub records_part: String, + /// The records part as written ([`crate::records_xml`] makes one); `None` for a cache saved + /// without records (`saveData="0"`). + pub records: Option>, } /// A table part. Its relationships (a query table) are kept parts. @@ -163,6 +194,10 @@ pub struct Document { /// One per [`Workbook::sheets`] entry: the worksheet, or `None` for a sheet kept as its /// parts (a chart sheet, a macro sheet, a worksheet whose part could not be read). pub worksheets: Vec>, + /// The pivot caches, written from here with their entries in [`Workbook::pivot_caches`]. A + /// cache that did not read (OLAP data, relationships other than its records) is a kept part + /// whose entry stays. + pub pivot_caches: Vec, pub parts: Vec, /// What reading skipped or repaired. pub warnings: Vec, @@ -181,6 +216,18 @@ impl Document { let workbook_rels = p.workbook_rels()?; let shared_strings = p.shared_strings()?; let styles = p.styles()?; + let mut pivot_caches = Vec::new(); + for entry in &workbook.pivot_caches { + let Some(rel) = workbook_rels.by_id(&entry.rel_id).filter(|r| !r.external && r.kind() == "pivotCacheDefinition") else { continue }; + let rels = p.opc().rels(&rel.target)?; + if rels.list.iter().any(|r| r.kind() != "pivotCacheRecords" || r.external) { + continue; + } + let Some(definition) = p.pivot_cache(&rel.target)? else { continue }; + let records_part = rels.first("pivotCacheRecords").map(|r| r.target.clone()).unwrap_or_default(); + let records = if records_part.is_empty() { None } else { p.opc().read(&records_part)? }; + pivot_caches.push(PivotCachePart { part: rel.target.clone(), cache_id: entry.cache_id, definition, records_part, records }); + } let mut worksheets = Vec::with_capacity(workbook.sheets.len()); for entry in &workbook.sheets { if entry.kind != SheetKind::Worksheet || entry.part.is_none() { @@ -197,7 +244,16 @@ impl Document { let notes = if comments.is_some() { p.note_shapes(&part)? } else { Vec::new() }; let drawing = p.drawing(&part)?.map(|(name, drawing, rels)| DrawingPart { part: name, drawing, rels: rels.list }); let tables = p.tables(&part)?.into_iter().map(|(part, table)| TablePart { part, table }).collect(); - worksheets.push(Some(Worksheet { rows, part, comments, notes, drawing, tables })); + let mut pivot_tables = Vec::new(); + for (name, table, cache) in p.pivot_tables(&workbook, &part)? { + // Typed only when its cache is, by the same id, and it relates to nothing else. + let typed = pivot_caches.iter().any(|c| Some(&c.part) == cache.as_ref() && Some(c.cache_id) == table.cache_id); + let rels = p.opc().rels(&name)?; + if typed && rels.list.iter().all(|r| r.kind() == "pivotCacheDefinition" && !r.external) { + pivot_tables.push(PivotTablePart { part: name, table }); + } + } + worksheets.push(Some(Worksheet { rows, part, comments, notes, drawing, tables, pivot_tables })); } Err(Error::TooLarge(m)) => return Err(Error::TooLarge(m)), Err(e) => { @@ -219,6 +275,10 @@ impl Document { made.push(rels_path(&d.part)); } made.extend(ws.tables.iter().map(|t| t.part.clone())); + made.extend(ws.pivot_tables.iter().flat_map(|t| [t.part.clone(), rels_path(&t.part)])); + } + for c in &pivot_caches { + made.extend([c.part.clone(), rels_path(&c.part), c.records_part.clone()]); } made.extend(workbook_rels.list.iter().filter(|r| !r.external && regenerated(r)).map(|r| r.target.clone())); let opc = p.opc(); @@ -232,7 +292,18 @@ impl Document { parts.push(KeptPart { content_type: types.of(&name).map(str::to_string), name, bytes }); } let kind = types.of(&workbook_part).and_then(XlsxKind::of_content_type).unwrap_or_default(); - Ok(Document { kind, workbook_part, workbook, workbook_rels, shared_strings, styles, worksheets, parts, warnings: p.warnings().to_vec() }) + Ok(Document { + kind, + workbook_part, + workbook, + workbook_rels, + shared_strings, + styles, + worksheets, + pivot_caches, + parts, + warnings: p.warnings().to_vec(), + }) } /// Adds a chart to worksheet `sheet` (an index into [`Workbook::sheets`]): a new chart part @@ -318,7 +389,14 @@ impl Document { // A worksheet's relationship is kept only for a sheet kept as its parts: the others are // made again, and one whose sheet was removed goes with it. let unread = |r: &Rel| self.workbook.sheets.iter().zip(&self.worksheets).any(|(e, ws)| ws.is_none() && e.rel_id == r.id); - for r in self.workbook_rels.list.iter().filter(|r| !regenerated(r) && (r.kind() != "worksheet" || unread(r))) { + let kept_part = |r: &Rel| self.parts.iter().any(|p| p.name.eq_ignore_ascii_case(&r.target)); + let keep = |r: &&Rel| match r.kind() { + "worksheet" => unread(r), + // A cache this writes is related again; one removed goes with its relationship. + "pivotCacheDefinition" => r.external || kept_part(r), + _ => !regenerated(r), + }; + for r in self.workbook_rels.list.iter().filter(keep) { rels.items.push(kept(r)); } let mut workbook = self.workbook.clone(); @@ -329,7 +407,11 @@ impl Document { for ws in self.worksheets.iter().flatten() { let comments = ws.comments.as_ref().and(ws.part.rels.first("comments")).map(|r| r.target.clone()); let own = [ws.part.part.clone()].into_iter().chain(comments).chain(ws.drawing.iter().map(|d| d.part.clone())); - taken.extend(own.chain(ws.tables.iter().map(|t| t.part.clone())).filter(|n| !n.is_empty()).map(|n| n.to_ascii_lowercase())); + let own = own.chain(ws.tables.iter().map(|t| t.part.clone())).chain(ws.pivot_tables.iter().map(|t| t.part.clone())); + taken.extend(own.filter(|n| !n.is_empty()).map(|n| n.to_ascii_lowercase())); + } + for c in &self.pivot_caches { + taken.extend([&c.part, &c.records_part].into_iter().filter(|n| !n.is_empty()).map(|n| n.to_ascii_lowercase())); } for (i, (entry, ws)) in workbook.sheets.iter_mut().zip(&self.worksheets).enumerate() { let Some(ws) = ws else { @@ -354,6 +436,52 @@ impl Document { }); sheet_parts.push((name, ws)); } + // Pivot caches: each written cache related again (with the id it had), and the entries of + // kept caches whose relationship stays. + let mut caches: Vec<(u32, String)> = Vec::new(); + let mut entries: Vec = Vec::new(); + for c in &self.pivot_caches { + if caches.iter().any(|(id, _)| *id == c.cache_id) { + return Err(Error::Invalid(format!("two pivot caches have the id {}", c.cache_id))); + } + let name = if c.part.is_empty() { fresh(&taken, |n| format!("xl/pivotCache/pivotCacheDefinition{n}.xml")) } else { c.part.clone() }; + taken.push(name.to_ascii_lowercase()); + let old = self + .workbook_rels + .list + .iter() + .find(|r| r.kind() == "pivotCacheDefinition" && r.target.eq_ignore_ascii_case(&name) && !rels.items.iter().any(|o| o.id == r.id)); + let id = old.map_or_else(|| fresh_id(&rels), |r| r.id.clone()); + rels.items.push(RelOut { + id: id.clone(), + ty: format!("{RT_BASE}pivotCacheDefinition"), + target: relative(book_part, &name), + external: false, + }); + entries.push(PivotCacheEntry { cache_id: c.cache_id, rel_id: id }); + let mut crels = RelsOut::default(); + let records_rel = c.records.as_ref().map(|bytes| { + let rec = if c.records_part.is_empty() { + fresh(&taken, |n| format!("xl/pivotCache/pivotCacheRecords{n}.xml")) + } else { + c.records_part.clone() + }; + taken.push(rec.to_ascii_lowercase()); + out.binary(&rec, CT_PIVOT_RECORDS, bytes.clone()); + crels.add("pivotCacheRecords", &relative(&name, &rec), false) + }); + out.xml_part(&name, CT_PIVOT_CACHE, c.definition.to_xml(records_rel.as_deref())); + out.rels(&name, &crels); + caches.push((c.cache_id, name)); + } + for e in &self.workbook.pivot_caches { + let target = rels.items.iter().find(|r| r.id == e.rel_id).and_then(|_| self.workbook_rels.by_id(&e.rel_id)); + if let Some(r) = target.filter(|_| !caches.iter().any(|(id, _)| *id == e.cache_id)) { + caches.push((e.cache_id, r.target.clone())); + entries.push(e.clone()); + } + } + workbook.pivot_caches = entries; let mut add = |kind: &str, default: &str, taken: &mut Vec| { let old = self.workbook_rels.list.iter().find(|r| r.kind() == kind && !r.external).map(|r| r.target.clone()); let name = old.filter(|n| !taken.contains(&n.to_ascii_lowercase())).unwrap_or_else(|| default.to_string()); @@ -379,7 +507,7 @@ impl Document { let mut blocks = vml_blocks(&self.parts); let kept_names: Vec = self.parts.iter().map(|p| p.name.to_ascii_lowercase()).collect(); for (name, ws) in &sheet_parts { - write_sheet(&mut out, name, ws, &kept_names, &mut taken, &mut blocks); + write_sheet(&mut out, name, ws, &kept_names, &mut taken, &mut blocks, &caches)?; } if let Some(name) = &strings { let count: usize = @@ -406,7 +534,15 @@ impl Document { /// A worksheet, its relationships, and its comments with a VML drawing to show them when it has /// none. A hyperlink to a URL its relationship does not lead to gets a relationship of its own. -fn write_sheet(out: &mut PackageOut, name: &str, ws: &Worksheet, kept_names: &[String], taken: &mut Vec, blocks: &mut Vec) { +fn write_sheet( + out: &mut PackageOut, + name: &str, + ws: &Worksheet, + kept_names: &[String], + taken: &mut Vec, + blocks: &mut Vec, + caches: &[(u32, String)], +) -> Result<(), Error> { let mut part = ws.part.clone(); let mut rels = RelsOut::default(); let kept_part = |r: &Rel| kept_names.contains(&r.target.to_ascii_lowercase()); @@ -415,6 +551,7 @@ fn write_sheet(out: &mut PackageOut, name: &str, ws: &Worksheet, kept_names: &[S "comments" => ws.comments.is_some(), "drawing" => ws.drawing.as_ref().map_or(kept_part(r), |d| d.part.eq_ignore_ascii_case(&r.target)), "table" => ws.tables.iter().any(|t| t.part.eq_ignore_ascii_case(&r.target)) || kept_part(r), + "pivotTable" => ws.pivot_tables.iter().any(|t| t.part.eq_ignore_ascii_case(&r.target)) || kept_part(r), _ => true, }; rels.items.extend(ws.part.rels.list.iter().filter(stays).map(kept)); @@ -496,8 +633,25 @@ fn write_sheet(out: &mut PackageOut, name: &str, ws: &Worksheet, kept_names: &[S part.features.table_parts.iter().filter(|id| !table_ids.contains(id) && rels.items.iter().any(|r| r.id == **id && r.ty.ends_with("/table"))); table_ids.extend(kept_tables.cloned().collect::>()); part.features.table_parts = table_ids; + for t in &ws.pivot_tables { + let cache = t.table.cache_id.and_then(|id| caches.iter().find(|(c, _)| *c == id)); + let Some((_, cache)) = cache else { + return Err(Error::Invalid(format!("PivotTable {:?} names no pivot cache of the workbook", t.table.name.as_deref().unwrap_or("")))); + }; + let target = if t.part.is_empty() { fresh(taken, |n| format!("xl/pivotTables/pivotTable{n}.xml")) } else { t.part.clone() }; + taken.push(target.to_ascii_lowercase()); + if !ws.part.rels.list.iter().any(|r| r.kind() == "pivotTable" && r.target.eq_ignore_ascii_case(&target)) { + let id = fresh_id(&rels); + rels.items.push(RelOut { id, ty: format!("{RT_BASE}pivotTable"), target: relative(name, &target), external: false }); + } + out.xml_part(&target, CT_PIVOT_TABLE, t.table.to_xml()); + let mut trels = RelsOut::default(); + trels.add("pivotCacheDefinition", &relative(&target, cache), false); + out.rels(&target, &trels); + } out.xml_part(name, CT_WORKSHEET, part.to_xml(&ws.rows)); out.rels(name, &rels); + Ok(()) } /// The shape-id blocks (`o:idmap`) the kept VML drawings use, so a new drawing takes another. diff --git a/crates/xlsx/src/features.rs b/crates/xlsx/src/features.rs index a23ef6118..17c2c2aab 100644 --- a/crates/xlsx/src/features.rs +++ b/crates/xlsx/src/features.rs @@ -14,7 +14,7 @@ use crate::styles::Color; use crate::write::num; /// An attribute value type. -trait Attr: Sized { +pub(crate) trait Attr: Sized { fn parse(s: &str) -> Option; fn spell(&self) -> String; } @@ -62,7 +62,7 @@ impl Attr for String { /// The attributes of `e` not in `named`. A container's `count` (written from the items) and /// the `ref` and `sqref` each type reads into a field of its own are left out. -fn rest(e: &El, named: &[&str]) -> Vec<(String, String)> { +pub(crate) fn rest(e: &El, named: &[&str]) -> Vec<(String, String)> { e.attrs.iter().filter(|(k, _)| !matches!(k.as_str(), "count" | "ref" | "sqref") && !named.contains(&k.as_str())).cloned().collect() } @@ -120,7 +120,7 @@ fn sqref(ranges: &[Range]) -> String { ranges.iter().map(range_ref).collect::>().join(" ") } -fn kids(e: &El, except: &[&str]) -> Vec { +pub(crate) fn kids(e: &El, except: &[&str]) -> Vec { e.elements().filter(|c| !except.contains(&c.name.as_str())).cloned().collect() } diff --git a/crates/xlsx/src/lib.rs b/crates/xlsx/src/lib.rs index ae94c376b..257cbb9ab 100644 --- a/crates/xlsx/src/lib.rs +++ b/crates/xlsx/src/lib.rs @@ -51,9 +51,11 @@ mod document; mod drawing; +#[macro_use] mod features; pub mod ns; mod package; +mod pivot; mod sheet; mod strings; mod styles; @@ -68,7 +70,7 @@ pub use craft_sheet::{ BorderStyle, CellError, CellRef, DEFAULT_PALETTE, HAlign, MAX_COLS, MAX_ROWS, Pattern, Range, Script, Underline, VAlign, Value, Visibility, builtin_format, builtin_format_id, quote_sheet, sheet_needs_quotes, }; -pub use document::{Document, DrawingPart, KeptPart, TablePart, Worksheet, XlsxKind}; +pub use document::{Document, DrawingPart, KeptPart, PivotCachePart, PivotTablePart, TablePart, Worksheet, XlsxKind}; pub use drawing::{Anchor, Drawing, Item, Marker, ObjectKind, Placement}; pub use features::{ Break, CfRule, Cfvo, ColorScale, Comment, Comments, ConditionalFormatting, DataBar, DataValidation, DataValidations, Features, HeaderFooter, @@ -76,12 +78,16 @@ pub use features::{ notes_vml, parse_sqref, range_ref, }; pub use package::{Package, is_xlsb}; +pub use pivot::{ + CacheField, CacheItem, CacheRecord, CacheSource, CacheValue, DataField, FieldGroup, Location, PageField, PivotCacheDefinition, PivotField, + PivotItem, PivotTableDefinition, PivotTableStyleInfo, RangePr, SharedItems, WorksheetSource, records_xml, +}; pub use sheet::{Cell, CellFormula, CellValue, Column, FormulaKind, Row, SheetPart, iso_date_serial}; pub use strings::{PhoneticProps, PhoneticRun, StringItem, TextRun, decode_xstring, encode_xstring, rich_text, t_el}; pub use styles::{ Alignment, Border, BorderEdge, CellStyle, Color, ColorSource, Dxf, Fill, Font, FontScheme, GradientFill, PatternFill, Protection, Styles, Xf, }; -pub use workbook::{DefinedName, SheetEntry, SheetKind, Workbook}; +pub use workbook::{DefinedName, PivotCacheEntry, SheetEntry, SheetKind, Workbook}; pub use write::num; /// Why a workbook could not be read. diff --git a/crates/xlsx/src/package.rs b/crates/xlsx/src/package.rs index 5d415f813..1c133d0ad 100644 --- a/crates/xlsx/src/package.rs +++ b/crates/xlsx/src/package.rs @@ -8,6 +8,7 @@ use craft_drawingml::theme::Theme; use crate::drawing::Drawing; use crate::features::{Comments, NoteShape, Table}; +use crate::pivot::{CacheRecord, PivotCacheDefinition, PivotTableDefinition, read_record}; use crate::sheet::{Row, RowReader, SheetPart}; use crate::strings::StringItem; use crate::styles::Styles; @@ -300,6 +301,64 @@ impl<'a> Package<'a> { Ok(out) } + /// The pivot cache definition in `part` (a workbook relationship that + /// [`Workbook::pivot_caches`] names). `None`, with a warning, when it is missing or does not + /// read. + pub fn pivot_cache(&mut self, part: &str) -> Result, Error> { + let Some(x) = self.optional_xml(part)? else { return Ok(None) }; + let def = PivotCacheDefinition::read(&x); + if def.is_none() { + self.warn(format!("{part}: a pivot cache this crate does not read (OLAP data, or not a cache definition)")); + } + Ok(def) + } + + /// The records of the pivot cache in `cache_part`, one at a time, so a cache of any size is + /// read without holding it. Nothing when it saved none. + pub fn pivot_records(&mut self, cache_part: &str, mut each: impl FnMut(CacheRecord) -> Result<(), Error>) -> Result<(), Error> { + let rels = self.opc.rels(cache_part)?; + let Some(part) = rels.first("pivotCacheRecords").filter(|r| !r.external).map(|r| r.target.clone()) else { return Ok(()) }; + let Some(bytes) = self.opc.read(&part)? else { + self.warn(format!("missing part {part}")); + return Ok(()); + }; + let (_, err) = self.xml_options().parse_streaming_partial(&bytes, &["r"], |e| each(read_record(&e)).map_err(Stop::Caller)); + match err { + Some(Stop::Caller(e)) => Err(e), + Some(Stop::Xml(e)) if e.is_limit() => Err(e.into()), + Some(Stop::Xml(e)) => { + self.warn(format!("{part} is damaged ({e}); its records were read up to there")); + Ok(()) + } + None => Ok(()), + } + } + + /// The PivotTables of a worksheet read by [`Package::sheet`], each with its part name and + /// the part of its cache: the one the table relates to, else the one `workbook` names by + /// its `cacheId`. A part that does not read is skipped with a warning. + pub fn pivot_tables(&mut self, workbook: &Workbook, sheet: &SheetPart) -> Result)>, Error> { + let mut out = Vec::new(); + for rel in sheet.rels.list.iter().filter(|r| r.kind() == "pivotTable" && !r.external) { + let Some(x) = self.optional_xml(&rel.target)? else { continue }; + let Some(table) = PivotTableDefinition::read(&x) else { + self.warn(format!("skipped {}: not a PivotTable with a location", rel.target)); + continue; + }; + let own = self.opc.rels(&rel.target)?.first("pivotCacheDefinition").filter(|r| !r.external).map(|r| r.target.clone()); + let cache = match own { + Some(c) => Some(c), + None => { + let book = self.workbook_rels()?; + let entry = workbook.pivot_caches.iter().find(|c| Some(c.cache_id) == table.cache_id); + entry.and_then(|e| book.by_id(&e.rel_id)).filter(|r| !r.external).map(|r| r.target.clone()) + } + }; + out.push((rel.target.clone(), table, cache)); + } + Ok(out) + } + /// A chart part (`c:chartSpace`, or `cx:chartSpace` of an Office 2016 chart) through /// `craft-drawingml`. `None` when the part is missing or holds no chart it can read. pub fn chart(&mut self, part: &str) -> Result, Error> { diff --git a/crates/xlsx/src/pivot.rs b/crates/xlsx/src/pivot.rs new file mode 100644 index 000000000..93eda35cb --- /dev/null +++ b/crates/xlsx/src/pivot.rs @@ -0,0 +1,769 @@ +//! PivotTables (ECMA-376 Part 1, 18.10): the pivot cache definition (the source and its fields +//! with their shared items and grouping), the cache records, and the PivotTable definition (its +//! location, fields, axes, values and style). +//! +//! As for the worksheet features, an attribute the file may leave out is an `Option`, the +//! attributes a type does not name are kept in its `attrs`, and the children it does not model +//! (OLAP hierarchies, formats, filters, extensions) are kept as written. A cache whose items +//! carry OLAP member data does not read: it stays a kept part, so nothing is lost. A part is +//! read as one tree, so [`crate::Limits::xml`] bounds every list in it; records are streamed. + +use craft_sheet::Range; +use craft_xml::{El, Node}; + +use crate::features::{Attr, kids, rest}; +use crate::write::num; + +/// One value of a pivot cache: a shared or group item, or a field of a record. +#[derive(Clone, Debug, PartialEq)] +pub enum CacheValue { + /// ``: no value. + Missing, + /// `` + Number(f64), + /// `` + Bool(bool), + /// ``: the error as written (`#N/A`). + Error(String), + /// `` + Text(String), + /// ``: an ISO 8601 date-time as written (`2024-03-05T00:00:00`). + Date(String), + /// ``: an index into the field's shared items (records only). + Index(u32), +} + +impl CacheValue { + /// `None` for an element that is not a value, or one with children (OLAP member tuples). + fn read(e: &El) -> Option { + if e.elements().next().is_some() { + return None; + } + let v = e.attr("v"); + Some(match e.name.as_str() { + "m" => CacheValue::Missing, + "n" => CacheValue::Number(craft_xml::value::to_f64(v?)?), + "b" => CacheValue::Bool(craft_xml::value::to_bool(v?)?), + "e" => CacheValue::Error(v?.to_string()), + "s" => CacheValue::Text(v?.to_string()), + "d" => CacheValue::Date(v?.to_string()), + "x" => CacheValue::Index(craft_xml::value::to_u32(v.unwrap_or("0"))?), + _ => return None, + }) + } + + fn to_el(&self) -> El { + match self { + CacheValue::Missing => El::new("m"), + CacheValue::Number(n) => El::new("n").with_attr("v", num(*n)), + CacheValue::Bool(b) => El::new("b").with_attr("v", u8::from(*b)), + CacheValue::Error(s) => El::new("e").with_attr("v", s), + CacheValue::Text(s) => El::new("s").with_attr("v", s), + CacheValue::Date(s) => El::new("d").with_attr("v", s), + CacheValue::Index(i) => El::new("x").with_attr("v", i), + } + } +} + +/// A shared or group item: its value and its other attributes (`u` unused, `f` formula, `c` +/// calculated, `cp`, `in`, ...). +#[derive(Clone, Debug, PartialEq)] +pub struct CacheItem { + pub value: CacheValue, + pub attrs: Vec<(String, String)>, +} + +impl CacheItem { + pub fn new(value: CacheValue) -> CacheItem { + CacheItem { value, attrs: Vec::new() } + } + + fn read(e: &El) -> Option { + Some(CacheItem { value: CacheValue::read(e)?, attrs: rest(e, &["v"]) }) + } + + fn to_el(&self) -> El { + let mut e = self.value.to_el(); + e.attrs.extend(self.attrs.iter().cloned()); + e + } +} + +fn read_items(e: &El) -> Option> { + e.elements().map(CacheItem::read).collect() +} + +fn items_el(name: &str, items: &[CacheItem]) -> El { + let mut e = El::new(name).with_attr("count", items.len()); + items.iter().for_each(|i| e.push(i.to_el())); + e +} + +element! { + /// ``: what a field's values are, and the items an axis refers to by index. + pub struct SharedItems { + contains_semi_mixed_types: bool = "containsSemiMixedTypes", + contains_non_date: bool = "containsNonDate", + contains_date: bool = "containsDate", + contains_string: bool = "containsString", + contains_blank: bool = "containsBlank", + contains_mixed_types: bool = "containsMixedTypes", + contains_number: bool = "containsNumber", + contains_integer: bool = "containsInteger", + min_value: f64 = "minValue", + max_value: f64 = "maxValue", + min_date: String = "minDate", + max_date: String = "maxDate", + long_text: bool = "longText", + } + + { + items: Vec, + } +} + +impl SharedItems { + /// The flags Excel writes for a field holding `values` (no items: add those that an axis + /// needs). Booleans and errors count as strings, as Excel counts them; a field of dates + /// (and blanks) gets the date flags and its first and last date. + pub fn describe<'a>(values: impl IntoIterator) -> SharedItems { + let (mut blank, mut strings, mut long, mut dates, mut non_int) = (false, false, false, false, false); + let mut range: Option<(f64, f64)> = None; + let mut date_range: Option<(&str, &str)> = None; + for v in values { + match v { + CacheValue::Missing => blank = true, + CacheValue::Number(n) => { + non_int |= n.fract() != 0.0; + range = Some(range.map_or((*n, *n), |(a, b)| (a.min(*n), b.max(*n)))); + } + CacheValue::Text(t) => { + strings = true; + long |= t.chars().count() > 255; + } + CacheValue::Bool(_) | CacheValue::Error(_) => strings = true, + CacheValue::Date(d) => { + dates = true; + let d = d.as_str(); + date_range = Some(date_range.map_or((d, d), |(a, b)| (a.min(d), b.max(d)))); + } + CacheValue::Index(_) => {} + } + } + let number = range.is_some(); + let some = |b: bool| b.then_some(true); + if dates && !strings && !number { + return SharedItems { + contains_semi_mixed_types: Some(false), + contains_non_date: Some(false), + contains_date: Some(true), + contains_string: Some(false), + contains_blank: some(blank), + min_date: date_range.map(|r| r.0.to_string()), + max_date: date_range.map(|r| r.1.to_string()), + ..SharedItems::default() + }; + } + let kinds = usize::from(strings) + usize::from(number) + usize::from(dates); + SharedItems { + contains_semi_mixed_types: (!strings && !blank).then_some(false), + contains_date: some(dates), + contains_string: (!strings).then_some(false), + contains_blank: some(blank), + contains_mixed_types: some(kinds > 1), + contains_number: some(number), + contains_integer: some(number && !non_int), + min_value: range.map(|r| r.0), + max_value: range.map(|r| r.1), + min_date: date_range.map(|r| r.0.to_string()), + max_date: date_range.map(|r| r.1.to_string()), + long_text: some(long), + ..SharedItems::default() + } + } +} + +element! { + /// ``: grouping by numeric or date ranges. + pub struct RangePr { + auto_start: bool = "autoStart", + auto_end: bool = "autoEnd", + /// `range`, `seconds`, `minutes`, `hours`, `days`, `months`, `quarters`, `years`. + group_by: String = "groupBy", + start_num: f64 = "startNum", + end_num: f64 = "endNum", + start_date: String = "startDate", + end_date: String = "endDate", + group_interval: f64 = "groupInterval", + } +} + +element! { + /// ``: a field grouped from another (`base`), into ranges or chosen items. + pub struct FieldGroup { + /// The next coarser grouping field of the same base. + par: u32 = "par", + base: u32 = "base", + } + + { + range: Option, + /// `discretePr`: for each base item, the group item it falls in. + discrete: Option>, + /// `groupItems`: the groups' captions. + items: Option>, + } +} + +element! { + /// ``: one column of the source, or a field grouped or calculated from others. + pub struct CacheField { + name: String = "name", + caption: String = "caption", + num_fmt_id: u32 = "numFmtId", + /// A calculated field's formula, without `=`. + formula: String = "formula", + /// `false` for a grouping or calculated field. + database_field: bool = "databaseField", + sql_type: i32 = "sqlType", + hierarchy: i32 = "hierarchy", + level: u32 = "level", + } + + { + shared_items: Option, + group: Option, + /// `mpMap` and `extLst`, as written. + other: Vec, + } +} + +element! { + /// ``: a range on a sheet (`reference` and `sheet`), or a defined name or + /// table (`name`); `r:id` for a source in another workbook. + pub struct WorksheetSource { + reference: String = "ref", + name: String = "name", + sheet: String = "sheet", + rel_id: String = "r:id", + } +} + +element! { + /// ``. + pub struct CacheSource { + /// `worksheet`, `external`, `consolidation`, `scenario`. + source_type: String = "type", + connection_id: u32 = "connectionId", + } + + { + worksheet: Option, + /// `consolidation` and `extLst`, as written. + other: Vec, + } +} + +element! { + /// A pivot cache definition (``, 18.10.1.67). Its records part is + /// related from it (`r:id`), which [`crate::Document`] writes. + pub struct PivotCacheDefinition { + invalid: bool = "invalid", + /// `false`: no records are saved, and the cache is refreshed before use. + save_data: bool = "saveData", + refresh_on_load: bool = "refreshOnLoad", + optimize_memory: bool = "optimizeMemory", + enable_refresh: bool = "enableRefresh", + refreshed_by: String = "refreshedBy", + refreshed_date: f64 = "refreshedDate", + background_query: bool = "backgroundQuery", + missing_items_limit: u32 = "missingItemsLimit", + created_version: u32 = "createdVersion", + refreshed_version: u32 = "refreshedVersion", + min_refreshable_version: u32 = "minRefreshableVersion", + record_count: u32 = "recordCount", + upgrade_on_refresh: bool = "upgradeOnRefresh", + } + + { + source: CacheSource, + fields: Vec, + /// `cacheHierarchies`, `kpis`, `tupleCache`, `calculatedItems`, `calculatedMembers`, + /// `dimensions`, `measureGroups`, `maps`, `extLst`, as written. + other: Vec, + } +} + +impl PivotCacheDefinition { + /// A part read with [`crate::ns::CANONICAL`]. `None` when it is not a cache definition, or + /// an item holds OLAP member data this model does not keep. + pub fn read(x: &El) -> Option { + if x.name != "pivotCacheDefinition" { + return None; + } + let source = x.child("cacheSource").map_or_else(CacheSource::default, |s| CacheSource { + worksheet: s.child("worksheetSource").map(WorksheetSource::read_attrs), + other: kids(s, &["worksheetSource"]), + ..CacheSource::read_attrs(s) + }); + let mut fields = Vec::new(); + for f in x.child("cacheFields").into_iter().flat_map(|c| c.children("cacheField")) { + let shared_items = match f.child("sharedItems") { + Some(s) => Some(SharedItems { items: read_items(s)?, ..SharedItems::read_attrs(s) }), + None => None, + }; + let group = match f.child("fieldGroup") { + Some(g) => Some(FieldGroup { + range: g.child("rangePr").map(RangePr::read_attrs), + discrete: g.child("discretePr").map(|d| d.children("x").filter_map(|x| x.attr_u32("v")).collect()), + items: match g.child("groupItems") { + Some(i) => Some(read_items(i)?), + None => None, + }, + ..FieldGroup::read_attrs(g) + }), + None => None, + }; + fields.push(CacheField { shared_items, group, other: kids(f, &["sharedItems", "fieldGroup"]), ..CacheField::read_attrs(f) }); + } + let mut def = PivotCacheDefinition { source, fields, other: kids(x, &["cacheSource", "cacheFields"]), ..PivotCacheDefinition::read_attrs(x) }; + // The records relationship is the writer's. + def.attrs.retain(|(k, _)| k != "r:id"); + Some(def) + } + + /// The part, related to its records by `records_rel` when it has them. + pub fn to_xml(&self, records_rel: Option<&str>) -> Vec { + let mut e = El::new("pivotCacheDefinition"); + if let Some(id) = records_rel { + e.set_attr("r:id", id); + } + self.write_attrs(&mut e); + let mut src = El::new("cacheSource"); + self.source.write_attrs(&mut src); + if let Some(w) = &self.source.worksheet { + let mut we = El::new("worksheetSource"); + w.write_attrs(&mut we); + src.push(we); + } + src.children.extend(self.source.other.iter().cloned().map(Node::El)); + e.push(src); + let mut fields = El::new("cacheFields").with_attr("count", self.fields.len()); + for f in &self.fields { + let mut fe = El::new("cacheField"); + f.write_attrs(&mut fe); + if let Some(s) = &f.shared_items { + let mut se = El::new("sharedItems"); + s.write_attrs(&mut se); + if !s.items.is_empty() { + se.set_attr("count", s.items.len()); + s.items.iter().for_each(|i| se.push(i.to_el())); + } + fe.push(se); + } + if let Some(g) = &f.group { + let mut ge = El::new("fieldGroup"); + g.write_attrs(&mut ge); + if let Some(r) = &g.range { + let mut re = El::new("rangePr"); + r.write_attrs(&mut re); + ge.push(re); + } + if let Some(d) = &g.discrete { + let mut de = El::new("discretePr").with_attr("count", d.len()); + d.iter().for_each(|v| de.push(El::new("x").with_attr("v", v))); + ge.push(de); + } + if let Some(items) = &g.items { + ge.push(items_el("groupItems", items)); + } + fe.push(ge); + } + fe.children.extend(f.other.iter().cloned().map(Node::El)); + fields.push(fe); + } + e.push(fields); + e.children.extend(self.other.iter().cloned().map(Node::El)); + document(e) + } +} + +/// One record of a pivot cache: a value per database field, in field order. +pub type CacheRecord = Vec; + +/// A pivot cache records part (``) for `records`. +pub fn records_xml(records: &[CacheRecord]) -> Vec { + let mut w = craft_xml::Writer::document(); + w.tag("pivotCacheRecords").attr("xmlns", crate::ns::MAIN).attr("xmlns:r", crate::ns::R).attr("count", records.len()).open(); + for r in records { + let mut e = El::new("r"); + r.iter().for_each(|v| e.push(v.to_el())); + w.tree(&e); + } + w.into_bytes() +} + +/// One `` of a records part, read with [`crate::ns::CANONICAL`]; values with OLAP data and +/// unknown elements are left out, so a field may be missing. +pub(crate) fn read_record(r: &El) -> CacheRecord { + r.elements().filter_map(CacheValue::read).collect() +} + +element! { + /// ``: where the PivotTable is (its body: report filters sit above it). + pub struct Location { + first_header_row: u32 = "firstHeaderRow", + first_data_row: u32 = "firstDataRow", + first_data_col: u32 = "firstDataCol", + row_page_count: u32 = "rowPageCount", + col_page_count: u32 = "colPageCount", + } + + { + range: Range, + } +} + +element! { + /// `` of a pivot field: a shared item shown (`x`), or a subtotal (`t`). + pub struct PivotItem { + /// The index of the item in the cache field's shared or group items. + x: u32 = "x", + /// `data` (the default), `default`, `sum`, `countA`, `avg`, ..., `grand`, `blank`. + item_type: String = "t", + /// Hidden (filtered out). + h: bool = "h", + /// Details shown (expanded); `false` is collapsed. + sd: bool = "sd", + /// A caption replacing the item's. + n: String = "n", + s: bool = "s", + f: bool = "f", + m: bool = "m", + c: bool = "c", + d: bool = "d", + e: bool = "e", + } +} + +element! { + /// ``: how one cache field is used (by position: field `i` is cache field `i`). + pub struct PivotField { + name: String = "name", + /// `axisRow`, `axisCol`, `axisPage`, `axisValues`; `None` when the field is on no axis. + axis: String = "axis", + data_field: bool = "dataField", + num_fmt_id: u32 = "numFmtId", + compact: bool = "compact", + outline: bool = "outline", + subtotal_top: bool = "subtotalTop", + show_all: bool = "showAll", + /// `manual` (the default), `ascending`, `descending`. + sort_type: String = "sortType", + default_subtotal: bool = "defaultSubtotal", + multiple_item_selection_allowed: bool = "multipleItemSelectionAllowed", + drag_to_row: bool = "dragToRow", + drag_to_col: bool = "dragToCol", + drag_to_page: bool = "dragToPage", + drag_to_data: bool = "dragToData", + insert_blank_row: bool = "insertBlankRow", + insert_page_break: bool = "insertPageBreak", + } + + { + /// `None` when the field has no ``. + items: Option>, + /// `autoSortScope` and `extLst`, as written. + other: Vec, + } +} + +element! { + /// ``: a report filter. + pub struct PageField { + fld: i32 = "fld", + /// The one item shown; `None` shows all (or those not hidden in the field's items). + item: u32 = "item", + hier: i32 = "hier", + name: String = "name", + cap: String = "cap", + } + + { + ext: Option, + } +} + +element! { + /// ``: a value field. + pub struct DataField { + name: String = "name", + fld: u32 = "fld", + /// `sum` (the default), `count`, `average`, `max`, `min`, `product`, `countNums`, + /// `stdDev`, `stdDevp`, `var`, `varp`. + subtotal: String = "subtotal", + /// `normal`, `difference`, `percent`, `percentDiff`, `runTotal`, `percentOfRow`, + /// `percentOfCol`, `percentOfTotal`, `index`. + show_data_as: String = "showDataAs", + base_field: i32 = "baseField", + base_item: u32 = "baseItem", + num_fmt_id: u32 = "numFmtId", + } + + { + /// `extLst`, as written: `x14:dataField` holds the Excel 2010 show-values-as kinds + /// (`pivotShowAs="rankDescending"`). + ext: Option, + } +} + +element! { + /// ``. + pub struct PivotTableStyleInfo { + name: String = "name", + show_row_headers: bool = "showRowHeaders", + show_col_headers: bool = "showColHeaders", + show_row_stripes: bool = "showRowStripes", + show_col_stripes: bool = "showColStripes", + show_last_column: bool = "showLastColumn", + } +} + +/// The children of a PivotTable definition in schema order (`CT_pivotTableDefinition`). +const TABLE: &[&str] = &[ + "location", + "pivotFields", + "rowFields", + "rowItems", + "colFields", + "colItems", + "pageFields", + "dataFields", + "formats", + "conditionalFormats", + "chartFormats", + "pivotHierarchies", + "pivotTableStyleInfo", + "filters", + "rowHierarchiesUsage", + "colHierarchiesUsage", + "extLst", +]; + +element! { + /// A PivotTable (``, 18.10.1.73) on the cache `cache_id` names. + pub struct PivotTableDefinition { + name: String = "name", + cache_id: u32 = "cacheId", + /// Σ Values is on the rows axis. + data_on_rows: bool = "dataOnRows", + data_position: u32 = "dataPosition", + data_caption: String = "dataCaption", + grand_total_caption: String = "grandTotalCaption", + error_caption: String = "errorCaption", + show_error: bool = "showError", + missing_caption: String = "missingCaption", + show_missing: bool = "showMissing", + apply_number_formats: bool = "applyNumberFormats", + apply_border_formats: bool = "applyBorderFormats", + apply_font_formats: bool = "applyFontFormats", + apply_pattern_formats: bool = "applyPatternFormats", + apply_alignment_formats: bool = "applyAlignmentFormats", + apply_width_height_formats: bool = "applyWidthHeightFormats", + updated_version: u32 = "updatedVersion", + min_refreshable_version: u32 = "minRefreshableVersion", + enable_drill: bool = "enableDrill", + use_auto_formatting: bool = "useAutoFormatting", + /// The grand total row below the report. + row_grand_totals: bool = "rowGrandTotals", + /// The grand total column at its right. + col_grand_totals: bool = "colGrandTotals", + item_print_titles: bool = "itemPrintTitles", + created_version: u32 = "createdVersion", + indent: u32 = "indent", + show_headers: bool = "showHeaders", + compact: bool = "compact", + outline: bool = "outline", + outline_data: bool = "outlineData", + compact_data: bool = "compactData", + grid_drop_zones: bool = "gridDropZones", + multiple_field_filters: bool = "multipleFieldFilters", + row_header_caption: String = "rowHeaderCaption", + col_header_caption: String = "colHeaderCaption", + } + + { + location: Location, + fields: Vec, + /// `rowFields`: cache field indices, `-2` for Σ Values. + row_fields: Vec, + col_fields: Vec, + page_fields: Vec, + data_fields: Vec, + style: Option, + /// `rowItems`, `colItems`, `formats`, `conditionalFormats`, `chartFormats`, + /// `pivotHierarchies`, `filters`, the hierarchy usages and `extLst`, as written. + other: Vec, + } +} + +impl PivotTableDefinition { + /// A part read with [`crate::ns::CANONICAL`]. `None` when it is not a PivotTable definition + /// or has no readable location. + pub fn read(x: &El) -> Option { + if x.name != "pivotTableDefinition" { + return None; + } + let loc = x.child("location")?; + let location = Location { range: Range::parse(&loc.attr("ref")?.replace('$', ""))?, ..Location::read_attrs(loc) }; + let list = |name: &str| -> Vec { + x.child(name).map(|e| e.children("field").filter_map(|f| f.attr("x").and_then(craft_xml::value::to_i32)).collect()).unwrap_or_default() + }; + let fields = x + .child("pivotFields") + .map(|p| { + p.children("pivotField") + .map(|f| PivotField { + items: f.child("items").map(|i| i.children("item").map(PivotItem::read_attrs).collect()), + other: kids(f, &["items"]), + ..PivotField::read_attrs(f) + }) + .collect() + }) + .unwrap_or_default(); + Some(PivotTableDefinition { + location, + fields, + row_fields: list("rowFields"), + col_fields: list("colFields"), + page_fields: x + .child("pageFields") + .map(|p| p.children("pageField").map(|f| PageField { ext: f.child("extLst").cloned(), ..PageField::read_attrs(f) }).collect()) + .unwrap_or_default(), + data_fields: x + .child("dataFields") + .map(|p| p.children("dataField").map(|f| DataField { ext: f.child("extLst").cloned(), ..DataField::read_attrs(f) }).collect()) + .unwrap_or_default(), + style: x.child("pivotTableStyleInfo").map(PivotTableStyleInfo::read_attrs), + other: kids(x, &["location", "pivotFields", "rowFields", "colFields", "pageFields", "dataFields", "pivotTableStyleInfo"]), + ..PivotTableDefinition::read_attrs(x) + }) + } + + /// The part. + pub fn to_xml(&self) -> Vec { + let mut e = El::new("pivotTableDefinition"); + self.write_attrs(&mut e); + let mut kids: Vec = Vec::new(); + let mut loc = El::new("location").with_attr("ref", self.location.range.a1()); + self.location.write_attrs(&mut loc); + kids.push(loc); + if !self.fields.is_empty() { + let mut pf = El::new("pivotFields").with_attr("count", self.fields.len()); + for f in &self.fields { + let mut fe = El::new("pivotField"); + f.write_attrs(&mut fe); + if let Some(items) = &f.items { + let mut ie = El::new("items").with_attr("count", items.len()); + for i in items { + let mut it = El::new("item"); + i.write_attrs(&mut it); + ie.push(it); + } + fe.push(ie); + } + fe.children.extend(f.other.iter().cloned().map(Node::El)); + pf.push(fe); + } + kids.push(pf); + } + for (name, list) in [("rowFields", &self.row_fields), ("colFields", &self.col_fields)] { + if !list.is_empty() { + let mut l = El::new(name).with_attr("count", list.len()); + list.iter().for_each(|x| l.push(El::new("field").with_attr("x", x))); + kids.push(l); + } + } + if !self.page_fields.is_empty() { + let mut l = El::new("pageFields").with_attr("count", self.page_fields.len()); + for p in &self.page_fields { + let mut pe = El::new("pageField"); + p.write_attrs(&mut pe); + pe.children.extend(p.ext.iter().cloned().map(Node::El)); + l.push(pe); + } + kids.push(l); + } + if !self.data_fields.is_empty() { + let mut l = El::new("dataFields").with_attr("count", self.data_fields.len()); + for d in &self.data_fields { + let mut de = El::new("dataField"); + d.write_attrs(&mut de); + de.children.extend(d.ext.iter().cloned().map(Node::El)); + l.push(de); + } + kids.push(l); + } + if let Some(s) = &self.style { + let mut se = El::new("pivotTableStyleInfo"); + s.write_attrs(&mut se); + kids.push(se); + } + kids.extend(self.other.iter().cloned()); + let pos = |n: &str| TABLE.iter().position(|o| *o == n).unwrap_or(TABLE.len()); + kids.sort_by_key(|k| pos(&k.name)); + e.children.extend(kids.into_iter().map(Node::El)); + document(e) + } +} + +fn document(mut root: El) -> Vec { + crate::ns::declare(&mut root); + let mut w = craft_xml::Writer::document(); + w.tree(&root); + w.into_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn values(v: &[CacheValue]) -> SharedItems { + SharedItems::describe(v) + } + + #[test] + fn shared_item_flags_follow_excel() { + use CacheValue::*; + let s = values(&[Number(1.0), Number(2.5)]); + assert_eq!( + (s.contains_semi_mixed_types, s.contains_string, s.contains_number, s.contains_integer, s.min_value, s.max_value), + (Some(false), Some(false), Some(true), None, Some(1.0), Some(2.5)) + ); + let s = values(&[Text("a".into()), Number(3.0), Missing]); + assert_eq!( + (s.contains_semi_mixed_types, s.contains_string, s.contains_blank, s.contains_mixed_types, s.contains_integer), + (None, None, Some(true), Some(true), Some(true)) + ); + // Booleans and errors count as strings. + let s = values(&[Bool(true), Error("#N/A".into())]); + assert_eq!((s.contains_string, s.contains_number, s.contains_mixed_types), (None, None, None)); + let s = values(&[Text("x".repeat(256))]); + assert_eq!(s.long_text, Some(true)); + let s = values(&[Date("2024-03-05T00:00:00".into()), Missing, Date("2023-01-01T00:00:00".into())]); + assert_eq!( + (s.contains_semi_mixed_types, s.contains_non_date, s.contains_date, s.contains_string, s.contains_blank), + (Some(false), Some(false), Some(true), Some(false), Some(true)) + ); + assert_eq!((s.min_date.as_deref(), s.max_date.as_deref()), (Some("2023-01-01T00:00:00"), Some("2024-03-05T00:00:00"))); + } + + #[test] + fn olap_items_do_not_read() { + let x = craft_xml::parse( + br#""#, + ) + .unwrap(); + assert!(PivotCacheDefinition::read(&x).is_none()); + let bad = craft_xml::parse(br#""#).unwrap(); + assert!(PivotCacheDefinition::read(&bad).is_none()); + } + + #[test] + fn a_table_without_location_does_not_read() { + let x = craft_xml::parse(br#""#).unwrap(); + assert!(PivotTableDefinition::read(&x).is_none()); + let x = craft_xml::parse(br#""#).unwrap(); + assert!(PivotTableDefinition::read(&x).is_none()); + } +} diff --git a/crates/xlsx/src/workbook.rs b/crates/xlsx/src/workbook.rs index 09745e3ae..f3ef9bafb 100644 --- a/crates/xlsx/src/workbook.rs +++ b/crates/xlsx/src/workbook.rs @@ -88,6 +88,14 @@ pub struct DefinedName { pub attrs: Vec<(String, String)>, } +/// ``: a pivot cache of the workbook, by the id its PivotTables name it with. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PivotCacheEntry { + pub cache_id: u32, + /// The relationship to the cache definition part. + pub rel_id: String, +} + /// The workbook part. #[derive(Clone, Debug, Default, PartialEq)] pub struct Workbook { @@ -97,6 +105,8 @@ pub struct Workbook { pub defined_names: Vec, /// `activeTab` of the first visible workbook view (of the first view when none is visible). pub active_tab: Option, + /// `pivotCaches`, in order. + pub pivot_caches: Vec, /// The whole part, for everything not lifted out above (calculation settings, views, /// protection, pivot caches, external references). Empty for a binary workbook. pub root: El, @@ -136,6 +146,12 @@ impl Workbook { }); } } + if let Some(caches) = root.child("pivotCaches") { + wb.pivot_caches = caches + .children("pivotCache") + .filter_map(|c| Some(PivotCacheEntry { cache_id: c.attr_u32("cacheId")?, rel_id: c.attr_or_plain("r:id")?.to_string() })) + .collect(); + } if let Some(views) = root.child("bookViews") { let view = views .children("workbookView") @@ -149,8 +165,8 @@ impl Workbook { /// The workbook part's root: `sheets` and `definedNames` regenerated from the fields, the /// date system set in `workbookPr` and the active tab in the view it was read from; - /// everything else in [`Workbook::root`] as kept. Each sheet's `r:id` is written as - /// [`SheetEntry::rel_id`] holds it. + /// `pivotCaches` from [`Workbook::pivot_caches`]; everything else in [`Workbook::root`] as + /// kept. Each sheet's `r:id` is written as [`SheetEntry::rel_id`] holds it. pub fn to_el(&self) -> El { let root = if self.root.name.is_empty() { El::new("workbook") } else { self.root.clone() }; let mut pr = root.child("workbookPr").cloned().unwrap_or_else(|| El::new("workbookPr")); @@ -195,6 +211,16 @@ impl Workbook { ("bookViews", views), ("sheets", Some(sheets)), ("definedNames", (!self.defined_names.is_empty()).then_some(names)), + ( + "pivotCaches", + (!self.pivot_caches.is_empty()).then(|| { + let mut e = El::new("pivotCaches"); + self.pivot_caches + .iter() + .for_each(|c| e.push(El::new("pivotCache").with_attr("cacheId", c.cache_id).with_attr("r:id", &c.rel_id))); + e + }), + ), ]; assemble(&root, WORKBOOK, owned) } diff --git a/crates/xlsx/tests/pivot.rs b/crates/xlsx/tests/pivot.rs new file mode 100644 index 000000000..4d2fd00d5 --- /dev/null +++ b/crates/xlsx/tests/pivot.rs @@ -0,0 +1,221 @@ +//! PivotTables: caches, records and tables read, written back, and made new. + +use craft_xlsx::testing::zip; +use craft_xlsx::{ + CacheItem, CacheValue, DataField, Document, Error, Limits, Location, Package, PivotCacheDefinition, PivotCachePart, PivotField, PivotItem, + PivotTableDefinition, PivotTablePart, Range, SharedItems, records_xml, +}; + +const MAIN: &str = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; +const RT: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + +const CACHE: &str = r#""#; +const RECORDS: &str = r#""#; +const TABLE: &str = r#""#; +/// An OLAP cache: its items carry member tuples, which this crate does not model. +const OLAP: &str = r#""#; +const OLAP_TABLE: &str = r#""#; + +fn rels(items: &[(&str, &str, &str)]) -> String { + let r: String = items.iter().map(|(id, ty, t)| format!(r#""#)).collect(); + format!(r#"{r}"#) +} + +/// A workbook with a pivot cache (with records) whose table relates to it, and an OLAP cache +/// whose table is found by its `cacheId` only. +fn with_pivots() -> Vec { + let book = format!( + r#""# + ); + let sheet = format!( + r#"Region"# + ); + let types = r#""#; + zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", rels(&[("rId1", "officeDocument", "xl/workbook.xml")]).as_bytes()), + ("xl/workbook.xml", book.as_bytes()), + ( + "xl/_rels/workbook.xml.rels", + rels(&[ + ("rId1", "worksheet", "worksheets/sheet1.xml"), + ("rId3", "pivotCacheDefinition", "pivotCache/pivotCacheDefinition1.xml"), + ("rId4", "pivotCacheDefinition", "pivotCache/pivotCacheDefinition2.xml"), + ]) + .as_bytes(), + ), + ("xl/worksheets/sheet1.xml", sheet.as_bytes()), + ( + "xl/worksheets/_rels/sheet1.xml.rels", + rels(&[("rId2", "pivotTable", "../pivotTables/pivotTable1.xml"), ("rId3", "pivotTable", "../pivotTables/pivotTable2.xml")]).as_bytes(), + ), + ("xl/pivotTables/pivotTable1.xml", TABLE.as_bytes()), + ( + "xl/pivotTables/_rels/pivotTable1.xml.rels", + rels(&[("rId1", "pivotCacheDefinition", "../pivotCache/pivotCacheDefinition1.xml")]).as_bytes(), + ), + ("xl/pivotTables/pivotTable2.xml", OLAP_TABLE.as_bytes()), + ("xl/pivotCache/pivotCacheDefinition1.xml", CACHE.as_bytes()), + ("xl/pivotCache/_rels/pivotCacheDefinition1.xml.rels", rels(&[("rId1", "pivotCacheRecords", "pivotCacheRecords1.xml")]).as_bytes()), + ("xl/pivotCache/pivotCacheRecords1.xml", RECORDS.as_bytes()), + ("xl/pivotCache/pivotCacheDefinition2.xml", OLAP.as_bytes()), + ]) +} + +#[test] +fn the_package_reads_caches_records_and_tables() { + let bytes = with_pivots(); + let mut p = Package::open(&bytes, &Limits::default()).unwrap(); + let wb = p.workbook().unwrap(); + assert_eq!(wb.pivot_caches.iter().map(|c| (c.cache_id, c.rel_id.as_str())).collect::>(), [(5, "rId3"), (7, "rId4")]); + let def = p.pivot_cache("xl/pivotCache/pivotCacheDefinition1.xml").unwrap().unwrap(); + let ws = def.source.worksheet.as_ref().unwrap(); + assert_eq!((ws.reference.as_deref(), ws.sheet.as_deref()), (Some("A1:B3"), Some("Data"))); + assert_eq!((def.refresh_on_load, def.record_count, def.attrs.clone()), (Some(true), Some(2), vec![("xr:uid".to_string(), "{C}".to_string())])); + let names: Vec<_> = def.fields.iter().map(|f| f.name.as_deref().unwrap_or("")).collect(); + assert_eq!(names, ["Region", "Day", "Ratio"]); + let region = def.fields[0].shared_items.as_ref().unwrap(); + assert_eq!(region.items[1], CacheItem { value: CacheValue::Text("West".into()), attrs: vec![("u".into(), "1".into())] }); + let group = def.fields[1].group.as_ref().unwrap(); + assert_eq!((group.par, group.base, group.range.as_ref().and_then(|r| r.group_by.as_deref())), (Some(2), Some(1), Some("months"))); + assert_eq!(group.items.as_ref().map(|i| i.len()), Some(3)); + assert_eq!((def.fields[2].formula.as_deref(), def.fields[2].database_field), (Some("Day/2"), Some(false))); + assert_eq!(def.other.len(), 1); + + let mut records = Vec::new(); + p.pivot_records("xl/pivotCache/pivotCacheDefinition1.xml", |r| { + records.push(r); + Ok(()) + }) + .unwrap(); + assert_eq!( + records, + [ + vec![CacheValue::Index(0), CacheValue::Date("2024-01-01T00:00:00".into())], + vec![CacheValue::Index(1), CacheValue::Date("2024-02-01T00:00:00".into())] + ] + ); + // The caller can stop the records. + let stopped = p.pivot_records("xl/pivotCache/pivotCacheDefinition1.xml", |_| Err(Error::TooLarge("enough".into()))); + assert_eq!(stopped, Err(Error::TooLarge("enough".into()))); + + assert!(p.pivot_cache("xl/pivotCache/pivotCacheDefinition2.xml").unwrap().is_none()); + assert!(p.warnings().iter().any(|w| w.contains("pivotCacheDefinition2"))); + + let sheet = p.sheet(&wb.sheets[0], |_| Ok(())).unwrap(); + let tables = p.pivot_tables(&wb, &sheet).unwrap(); + assert_eq!(tables.len(), 2); + let (part, t, cache) = &tables[0]; + assert_eq!((part.as_str(), cache.as_deref()), ("xl/pivotTables/pivotTable1.xml", Some("xl/pivotCache/pivotCacheDefinition1.xml"))); + assert_eq!( + (t.name.as_deref(), t.cache_id, t.row_grand_totals, t.location.range.a1()), + (Some("PivotTable1"), Some(5), Some(false), "D1:E4".into()) + ); + assert_eq!(t.row_fields, [0, -2]); + let items = t.fields[0].items.as_ref().unwrap(); + assert_eq!((items[0].h, items[1].sd, items[2].item_type.as_deref()), (Some(true), Some(false), Some("default"))); + assert_eq!((t.fields[0].sort_type.as_deref(), t.fields[0].axis.as_deref()), (Some("descending"), Some("axisRow"))); + assert_eq!((t.page_fields[0].fld, t.page_fields[0].item), (Some(1), Some(0))); + assert!(t.data_fields[0].ext.as_ref().is_some_and(|e| e.find("x14:dataField").is_some())); + assert_eq!(t.style.as_ref().and_then(|s| s.name.as_deref()), Some("PivotStyleLight16")); + assert_eq!(t.other.iter().map(|e| e.name.as_str()).collect::>(), ["rowItems", "formats"]); + // The second table names its cache by id only. + assert_eq!(tables[1].2.as_deref(), Some("xl/pivotCache/pivotCacheDefinition2.xml")); +} + +fn read(bytes: &[u8]) -> Document { + Document::read(bytes, &Limits::default()).unwrap() +} + +#[test] +fn pivots_round_trip() { + let doc = read(&with_pivots()); + // The OLAP cache and its table are kept parts; the others are typed. + assert_eq!(doc.pivot_caches.len(), 1); + let ws = doc.worksheets[0].as_ref().unwrap(); + assert_eq!(ws.pivot_tables.len(), 1); + let mut kept: Vec<&str> = doc.parts.iter().map(|p| p.name.as_str()).filter(|n| n.contains("pivot")).collect(); + kept.sort(); + assert_eq!(kept, ["xl/pivotCache/pivotCacheDefinition2.xml", "xl/pivotTables/pivotTable2.xml"]); + let c = &doc.pivot_caches[0]; + assert_eq!((c.cache_id, c.records_part.as_str(), c.records.as_deref()), (5, "xl/pivotCache/pivotCacheRecords1.xml", Some(RECORDS.as_bytes()))); + + let bytes = doc.write().unwrap(); + let again = read(&bytes); + assert_eq!(again.pivot_caches, doc.pivot_caches); + assert_eq!(again.worksheets[0].as_ref().unwrap().pivot_tables, ws.pivot_tables); + assert_eq!(again.workbook.pivot_caches, doc.workbook.pivot_caches); + assert_eq!(again.parts.iter().filter(|p| p.name.contains("pivot")).count(), 2); + // The kept table still finds the kept cache by its id. + let mut p = Package::open(&bytes, &Limits::default()).unwrap(); + let wb = p.workbook().unwrap(); + let sheet = p.sheet(&wb.sheets[0], |_| Ok(())).unwrap(); + let tables = p.pivot_tables(&wb, &sheet).unwrap(); + assert_eq!( + tables.iter().map(|t| t.2.clone().unwrap_or_default()).collect::>(), + ["xl/pivotCache/pivotCacheDefinition1.xml", "xl/pivotCache/pivotCacheDefinition2.xml"] + ); + assert_eq!(read(&again.write().unwrap()).pivot_caches, doc.pivot_caches); +} + +#[test] +fn a_new_pivot_table() { + let mut doc = read(&craft_xlsx::testing::workbook(&[("Data", r#"1"#), ("Report", "")], &[])); + let values = [CacheValue::Number(1.0), CacheValue::Number(2.0)]; + let definition = PivotCacheDefinition { + refresh_on_load: Some(true), + source: craft_xlsx::CacheSource { + source_type: Some("worksheet".into()), + worksheet: Some(craft_xlsx::WorksheetSource { reference: Some("A1:A3".into()), sheet: Some("Data".into()), ..Default::default() }), + ..Default::default() + }, + fields: vec![craft_xlsx::CacheField { + name: Some("N".into()), + num_fmt_id: Some(0), + shared_items: Some(SharedItems { items: values.iter().cloned().map(CacheItem::new).collect(), ..SharedItems::describe(&values) }), + ..Default::default() + }], + ..Default::default() + }; + let records = records_xml(&[vec![CacheValue::Index(0)], vec![CacheValue::Index(1)]]); + doc.pivot_caches.push(PivotCachePart { cache_id: 1, definition, records: Some(records), ..Default::default() }); + let table = PivotTableDefinition { + name: Some("P".into()), + cache_id: Some(1), + location: Location { range: Range::parse("A3:B6").unwrap(), first_header_row: Some(1), ..Default::default() }, + fields: vec![PivotField { + axis: Some("axisRow".into()), + data_field: Some(true), + items: Some(vec![PivotItem { x: Some(0), ..Default::default() }, PivotItem { x: Some(1), ..Default::default() }]), + ..Default::default() + }], + row_fields: vec![0], + data_fields: vec![DataField { name: Some("Sum of N".into()), fld: Some(0), ..Default::default() }], + ..Default::default() + }; + let ws = doc.worksheets[1].as_mut().unwrap(); + ws.pivot_tables.push(PivotTablePart { table: table.clone(), ..Default::default() }); + let back = read(&doc.write().unwrap()); + let c = &back.pivot_caches[0]; + assert_eq!((c.part.as_str(), c.records_part.as_str()), ("xl/pivotCache/pivotCacheDefinition1.xml", "xl/pivotCache/pivotCacheRecords1.xml")); + assert_eq!(c.definition, doc.pivot_caches[0].definition); + assert_eq!(c.definition.fields[0].shared_items.as_ref().map(|s| (s.contains_integer, s.min_value)), Some((Some(true), Some(1.0)))); + assert_eq!(back.workbook.pivot_caches.len(), 1); + let t = &back.worksheets[1].as_ref().unwrap().pivot_tables[0]; + assert_eq!((t.part.as_str(), &t.table), ("xl/pivotTables/pivotTable1.xml", &table)); + + // Removing the cache removes its relationship and entry; a table left naming it is refused. + let mut gone = back.clone(); + gone.pivot_caches.clear(); + assert!(matches!(gone.write(), Err(Error::Invalid(_)))); + gone.worksheets[1].as_mut().unwrap().pivot_tables.clear(); + let after = read(&gone.write().unwrap()); + assert!(after.workbook.pivot_caches.is_empty() && after.pivot_caches.is_empty()); + assert!(!after.workbook_rels.list.iter().any(|r| r.kind() == "pivotCacheDefinition")); + assert!(!after.worksheets[1].as_ref().unwrap().part.rels.list.iter().any(|r| r.kind() == "pivotTable")); + + // Two caches may not share an id. + let mut twice = back.clone(); + twice.pivot_caches.push(PivotCachePart { part: String::new(), ..back.pivot_caches[0].clone() }); + assert!(matches!(twice.write(), Err(Error::Invalid(_)))); +}