diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index 3541dcb26..e5fe94683 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -64,6 +64,15 @@ worksheet holds them as `TablePart`s, regenerated on writing with their relation reserved before new parts are named, so a new part never takes a name another sheet's part keeps. +PivotTables: `Workbook::pivot_caches` lists the caches by id; `Package::pivot_cache` reads a cache +definition (`PivotCacheDefinition`: source, fields with shared items and grouping), +`Package::pivot_records` streams its records, and `Package::pivot_tables` reads a sheet's +PivotTables (`PivotTableDefinition`: location, fields and items, axes, page and data fields, +style) with the cache each uses. `SharedItems::describe` gives the flags Excel writes for a field's +values, and `records_xml` writes a records part. A `Document` holds caches as `PivotCachePart`s +and a worksheet's tables as `PivotTablePart`s, regenerated with their relationships and +`pivotCaches` entries; an OLAP cache (and a table on one) stays a kept part. + The parts this crate writes use the transitional namespaces; a strict file's kept parts stay as they were. A binary workbook is not written. diff --git a/crates/xlsx/src/document.rs b/crates/xlsx/src/document.rs index dc75a9728..def3c4b92 100644 --- a/crates/xlsx/src/document.rs +++ b/crates/xlsx/src/document.rs @@ -7,10 +7,11 @@ use craft_drawingml::chart::Chart; use crate::drawing::{Anchor, Drawing, Item, Placement}; use crate::features::{Comments, NoteShape, Table, notes_vml}; +use crate::pivot::{PivotCacheDefinition, PivotTableDefinition}; use crate::sheet::{CellValue, Row, SheetPart}; use crate::strings::StringItem; use crate::styles::Styles; -use crate::workbook::{SheetKind, Workbook}; +use crate::workbook::{PivotCacheEntry, SheetKind, Workbook}; use crate::{Error, Limits, Package}; const CT_WORKSHEET: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"; @@ -22,6 +23,9 @@ const CT_TABLE: &str = "application/vnd.openxmlformats-officedocument.spreadshee const CT_DRAWING: &str = "application/vnd.openxmlformats-officedocument.drawing+xml"; const CT_CHART: &str = "application/vnd.openxmlformats-officedocument.drawingml.chart+xml"; const CT_CHARTEX: &str = "application/vnd.ms-office.chartex+xml"; +const CT_PIVOT_TABLE: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotTable+xml"; +const CT_PIVOT_CACHE: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheDefinition+xml"; +const CT_PIVOT_RECORDS: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheRecords+xml"; const RT_CHARTEX: &str = "http://schemas.microsoft.com/office/2014/relationships/chartEx"; /// The relationship type namespace of strict files: kept relationships are written with the /// transitional one, as the parts this crate writes are. @@ -89,6 +93,33 @@ pub struct Worksheet { pub drawing: Option, /// The table parts, written from here, in the order of the sheet's `tableParts`. pub tables: Vec, + /// The PivotTable parts, written from here. A PivotTable that did not read, or whose cache + /// did not, is a kept part. + pub pivot_tables: Vec, +} + +/// A PivotTable part. It is related to the cache of [`Document::pivot_caches`] its `cache_id` +/// names. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PivotTablePart { + /// The part's name; a new name is chosen on writing when empty. + pub part: String, + pub table: PivotTableDefinition, +} + +/// A pivot cache: its definition part and its records. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PivotCachePart { + /// The definition part's name; a new name is chosen on writing when empty. + pub part: String, + /// The id the workbook and its PivotTables know the cache by. + pub cache_id: u32, + pub definition: PivotCacheDefinition, + /// The records part's name; a new name is chosen on writing when empty. + pub records_part: String, + /// The records part as written ([`crate::records_xml`] makes one); `None` for a cache saved + /// without records (`saveData="0"`). + pub records: Option>, } /// A table part. Its relationships (a query table) are kept parts. @@ -163,6 +194,10 @@ pub struct Document { /// One per [`Workbook::sheets`] entry: the worksheet, or `None` for a sheet kept as its /// parts (a chart sheet, a macro sheet, a worksheet whose part could not be read). pub worksheets: Vec>, + /// The pivot caches, written from here with their entries in [`Workbook::pivot_caches`]. A + /// cache that did not read (OLAP data, relationships other than its records) is a kept part + /// whose entry stays. + pub pivot_caches: Vec, pub parts: Vec, /// What reading skipped or repaired. pub warnings: Vec, @@ -181,6 +216,18 @@ impl Document { let workbook_rels = p.workbook_rels()?; let shared_strings = p.shared_strings()?; let styles = p.styles()?; + let mut pivot_caches = Vec::new(); + for entry in &workbook.pivot_caches { + let Some(rel) = workbook_rels.by_id(&entry.rel_id).filter(|r| !r.external && r.kind() == "pivotCacheDefinition") else { continue }; + let rels = p.opc().rels(&rel.target)?; + if rels.list.iter().any(|r| r.kind() != "pivotCacheRecords" || r.external) { + continue; + } + let Some(definition) = p.pivot_cache(&rel.target)? else { continue }; + let records_part = rels.first("pivotCacheRecords").map(|r| r.target.clone()).unwrap_or_default(); + let records = if records_part.is_empty() { None } else { p.opc().read(&records_part)? }; + pivot_caches.push(PivotCachePart { part: rel.target.clone(), cache_id: entry.cache_id, definition, records_part, records }); + } let mut worksheets = Vec::with_capacity(workbook.sheets.len()); for entry in &workbook.sheets { if entry.kind != SheetKind::Worksheet || entry.part.is_none() { @@ -197,7 +244,16 @@ impl Document { let notes = if comments.is_some() { p.note_shapes(&part)? } else { Vec::new() }; let drawing = p.drawing(&part)?.map(|(name, drawing, rels)| DrawingPart { part: name, drawing, rels: rels.list }); let tables = p.tables(&part)?.into_iter().map(|(part, table)| TablePart { part, table }).collect(); - worksheets.push(Some(Worksheet { rows, part, comments, notes, drawing, tables })); + let mut pivot_tables = Vec::new(); + for (name, table, cache) in p.pivot_tables(&workbook, &part)? { + // Typed only when its cache is, by the same id, and it relates to nothing else. + let typed = pivot_caches.iter().any(|c| Some(&c.part) == cache.as_ref() && Some(c.cache_id) == table.cache_id); + let rels = p.opc().rels(&name)?; + if typed && rels.list.iter().all(|r| r.kind() == "pivotCacheDefinition" && !r.external) { + pivot_tables.push(PivotTablePart { part: name, table }); + } + } + worksheets.push(Some(Worksheet { rows, part, comments, notes, drawing, tables, pivot_tables })); } Err(Error::TooLarge(m)) => return Err(Error::TooLarge(m)), Err(e) => { @@ -219,6 +275,10 @@ impl Document { made.push(rels_path(&d.part)); } made.extend(ws.tables.iter().map(|t| t.part.clone())); + made.extend(ws.pivot_tables.iter().flat_map(|t| [t.part.clone(), rels_path(&t.part)])); + } + for c in &pivot_caches { + made.extend([c.part.clone(), rels_path(&c.part), c.records_part.clone()]); } made.extend(workbook_rels.list.iter().filter(|r| !r.external && regenerated(r)).map(|r| r.target.clone())); let opc = p.opc(); @@ -232,7 +292,18 @@ impl Document { parts.push(KeptPart { content_type: types.of(&name).map(str::to_string), name, bytes }); } let kind = types.of(&workbook_part).and_then(XlsxKind::of_content_type).unwrap_or_default(); - Ok(Document { kind, workbook_part, workbook, workbook_rels, shared_strings, styles, worksheets, parts, warnings: p.warnings().to_vec() }) + Ok(Document { + kind, + workbook_part, + workbook, + workbook_rels, + shared_strings, + styles, + worksheets, + pivot_caches, + parts, + warnings: p.warnings().to_vec(), + }) } /// Adds a chart to worksheet `sheet` (an index into [`Workbook::sheets`]): a new chart part @@ -318,7 +389,14 @@ impl Document { // A worksheet's relationship is kept only for a sheet kept as its parts: the others are // made again, and one whose sheet was removed goes with it. let unread = |r: &Rel| self.workbook.sheets.iter().zip(&self.worksheets).any(|(e, ws)| ws.is_none() && e.rel_id == r.id); - for r in self.workbook_rels.list.iter().filter(|r| !regenerated(r) && (r.kind() != "worksheet" || unread(r))) { + let kept_part = |r: &Rel| self.parts.iter().any(|p| p.name.eq_ignore_ascii_case(&r.target)); + let keep = |r: &&Rel| match r.kind() { + "worksheet" => unread(r), + // A cache this writes is related again; one removed goes with its relationship. + "pivotCacheDefinition" => r.external || kept_part(r), + _ => !regenerated(r), + }; + for r in self.workbook_rels.list.iter().filter(keep) { rels.items.push(kept(r)); } let mut workbook = self.workbook.clone(); @@ -329,7 +407,11 @@ impl Document { for ws in self.worksheets.iter().flatten() { let comments = ws.comments.as_ref().and(ws.part.rels.first("comments")).map(|r| r.target.clone()); let own = [ws.part.part.clone()].into_iter().chain(comments).chain(ws.drawing.iter().map(|d| d.part.clone())); - taken.extend(own.chain(ws.tables.iter().map(|t| t.part.clone())).filter(|n| !n.is_empty()).map(|n| n.to_ascii_lowercase())); + let own = own.chain(ws.tables.iter().map(|t| t.part.clone())).chain(ws.pivot_tables.iter().map(|t| t.part.clone())); + taken.extend(own.filter(|n| !n.is_empty()).map(|n| n.to_ascii_lowercase())); + } + for c in &self.pivot_caches { + taken.extend([&c.part, &c.records_part].into_iter().filter(|n| !n.is_empty()).map(|n| n.to_ascii_lowercase())); } for (i, (entry, ws)) in workbook.sheets.iter_mut().zip(&self.worksheets).enumerate() { let Some(ws) = ws else { @@ -354,6 +436,52 @@ impl Document { }); sheet_parts.push((name, ws)); } + // Pivot caches: each written cache related again (with the id it had), and the entries of + // kept caches whose relationship stays. + let mut caches: Vec<(u32, String)> = Vec::new(); + let mut entries: Vec = Vec::new(); + for c in &self.pivot_caches { + if caches.iter().any(|(id, _)| *id == c.cache_id) { + return Err(Error::Invalid(format!("two pivot caches have the id {}", c.cache_id))); + } + let name = if c.part.is_empty() { fresh(&taken, |n| format!("xl/pivotCache/pivotCacheDefinition{n}.xml")) } else { c.part.clone() }; + taken.push(name.to_ascii_lowercase()); + let old = self + .workbook_rels + .list + .iter() + .find(|r| r.kind() == "pivotCacheDefinition" && r.target.eq_ignore_ascii_case(&name) && !rels.items.iter().any(|o| o.id == r.id)); + let id = old.map_or_else(|| fresh_id(&rels), |r| r.id.clone()); + rels.items.push(RelOut { + id: id.clone(), + ty: format!("{RT_BASE}pivotCacheDefinition"), + target: relative(book_part, &name), + external: false, + }); + entries.push(PivotCacheEntry { cache_id: c.cache_id, rel_id: id }); + let mut crels = RelsOut::default(); + let records_rel = c.records.as_ref().map(|bytes| { + let rec = if c.records_part.is_empty() { + fresh(&taken, |n| format!("xl/pivotCache/pivotCacheRecords{n}.xml")) + } else { + c.records_part.clone() + }; + taken.push(rec.to_ascii_lowercase()); + out.binary(&rec, CT_PIVOT_RECORDS, bytes.clone()); + crels.add("pivotCacheRecords", &relative(&name, &rec), false) + }); + out.xml_part(&name, CT_PIVOT_CACHE, c.definition.to_xml(records_rel.as_deref())); + out.rels(&name, &crels); + caches.push((c.cache_id, name)); + } + for e in &self.workbook.pivot_caches { + let target = rels.items.iter().find(|r| r.id == e.rel_id).and_then(|_| self.workbook_rels.by_id(&e.rel_id)); + if let Some(r) = target.filter(|_| !caches.iter().any(|(id, _)| *id == e.cache_id)) { + caches.push((e.cache_id, r.target.clone())); + entries.push(e.clone()); + } + } + workbook.pivot_caches = entries; let mut add = |kind: &str, default: &str, taken: &mut Vec| { let old = self.workbook_rels.list.iter().find(|r| r.kind() == kind && !r.external).map(|r| r.target.clone()); let name = old.filter(|n| !taken.contains(&n.to_ascii_lowercase())).unwrap_or_else(|| default.to_string()); @@ -379,7 +507,7 @@ impl Document { let mut blocks = vml_blocks(&self.parts); let kept_names: Vec = self.parts.iter().map(|p| p.name.to_ascii_lowercase()).collect(); for (name, ws) in &sheet_parts { - write_sheet(&mut out, name, ws, &kept_names, &mut taken, &mut blocks); + write_sheet(&mut out, name, ws, &kept_names, &mut taken, &mut blocks, &caches)?; } if let Some(name) = &strings { let count: usize = @@ -406,7 +534,15 @@ impl Document { /// A worksheet, its relationships, and its comments with a VML drawing to show them when it has /// none. A hyperlink to a URL its relationship does not lead to gets a relationship of its own. -fn write_sheet(out: &mut PackageOut, name: &str, ws: &Worksheet, kept_names: &[String], taken: &mut Vec, blocks: &mut Vec) { +fn write_sheet( + out: &mut PackageOut, + name: &str, + ws: &Worksheet, + kept_names: &[String], + taken: &mut Vec, + blocks: &mut Vec, + caches: &[(u32, String)], +) -> Result<(), Error> { let mut part = ws.part.clone(); let mut rels = RelsOut::default(); let kept_part = |r: &Rel| kept_names.contains(&r.target.to_ascii_lowercase()); @@ -415,6 +551,7 @@ fn write_sheet(out: &mut PackageOut, name: &str, ws: &Worksheet, kept_names: &[S "comments" => ws.comments.is_some(), "drawing" => ws.drawing.as_ref().map_or(kept_part(r), |d| d.part.eq_ignore_ascii_case(&r.target)), "table" => ws.tables.iter().any(|t| t.part.eq_ignore_ascii_case(&r.target)) || kept_part(r), + "pivotTable" => ws.pivot_tables.iter().any(|t| t.part.eq_ignore_ascii_case(&r.target)) || kept_part(r), _ => true, }; rels.items.extend(ws.part.rels.list.iter().filter(stays).map(kept)); @@ -496,8 +633,25 @@ fn write_sheet(out: &mut PackageOut, name: &str, ws: &Worksheet, kept_names: &[S part.features.table_parts.iter().filter(|id| !table_ids.contains(id) && rels.items.iter().any(|r| r.id == **id && r.ty.ends_with("/table"))); table_ids.extend(kept_tables.cloned().collect::>()); part.features.table_parts = table_ids; + for t in &ws.pivot_tables { + let cache = t.table.cache_id.and_then(|id| caches.iter().find(|(c, _)| *c == id)); + let Some((_, cache)) = cache else { + return Err(Error::Invalid(format!("PivotTable {:?} names no pivot cache of the workbook", t.table.name.as_deref().unwrap_or("")))); + }; + let target = if t.part.is_empty() { fresh(taken, |n| format!("xl/pivotTables/pivotTable{n}.xml")) } else { t.part.clone() }; + taken.push(target.to_ascii_lowercase()); + if !ws.part.rels.list.iter().any(|r| r.kind() == "pivotTable" && r.target.eq_ignore_ascii_case(&target)) { + let id = fresh_id(&rels); + rels.items.push(RelOut { id, ty: format!("{RT_BASE}pivotTable"), target: relative(name, &target), external: false }); + } + out.xml_part(&target, CT_PIVOT_TABLE, t.table.to_xml()); + let mut trels = RelsOut::default(); + trels.add("pivotCacheDefinition", &relative(&target, cache), false); + out.rels(&target, &trels); + } out.xml_part(name, CT_WORKSHEET, part.to_xml(&ws.rows)); out.rels(name, &rels); + Ok(()) } /// The shape-id blocks (`o:idmap`) the kept VML drawings use, so a new drawing takes another. diff --git a/crates/xlsx/src/features.rs b/crates/xlsx/src/features.rs index a23ef6118..17c2c2aab 100644 --- a/crates/xlsx/src/features.rs +++ b/crates/xlsx/src/features.rs @@ -14,7 +14,7 @@ use crate::styles::Color; use crate::write::num; /// An attribute value type. -trait Attr: Sized { +pub(crate) trait Attr: Sized { fn parse(s: &str) -> Option; fn spell(&self) -> String; } @@ -62,7 +62,7 @@ impl Attr for String { /// The attributes of `e` not in `named`. A container's `count` (written from the items) and /// the `ref` and `sqref` each type reads into a field of its own are left out. -fn rest(e: &El, named: &[&str]) -> Vec<(String, String)> { +pub(crate) fn rest(e: &El, named: &[&str]) -> Vec<(String, String)> { e.attrs.iter().filter(|(k, _)| !matches!(k.as_str(), "count" | "ref" | "sqref") && !named.contains(&k.as_str())).cloned().collect() } @@ -120,7 +120,7 @@ fn sqref(ranges: &[Range]) -> String { ranges.iter().map(range_ref).collect::>().join(" ") } -fn kids(e: &El, except: &[&str]) -> Vec { +pub(crate) fn kids(e: &El, except: &[&str]) -> Vec { e.elements().filter(|c| !except.contains(&c.name.as_str())).cloned().collect() } diff --git a/crates/xlsx/src/lib.rs b/crates/xlsx/src/lib.rs index ae94c376b..257cbb9ab 100644 --- a/crates/xlsx/src/lib.rs +++ b/crates/xlsx/src/lib.rs @@ -51,9 +51,11 @@ mod document; mod drawing; +#[macro_use] mod features; pub mod ns; mod package; +mod pivot; mod sheet; mod strings; mod styles; @@ -68,7 +70,7 @@ pub use craft_sheet::{ BorderStyle, CellError, CellRef, DEFAULT_PALETTE, HAlign, MAX_COLS, MAX_ROWS, Pattern, Range, Script, Underline, VAlign, Value, Visibility, builtin_format, builtin_format_id, quote_sheet, sheet_needs_quotes, }; -pub use document::{Document, DrawingPart, KeptPart, TablePart, Worksheet, XlsxKind}; +pub use document::{Document, DrawingPart, KeptPart, PivotCachePart, PivotTablePart, TablePart, Worksheet, XlsxKind}; pub use drawing::{Anchor, Drawing, Item, Marker, ObjectKind, Placement}; pub use features::{ Break, CfRule, Cfvo, ColorScale, Comment, Comments, ConditionalFormatting, DataBar, DataValidation, DataValidations, Features, HeaderFooter, @@ -76,12 +78,16 @@ pub use features::{ notes_vml, parse_sqref, range_ref, }; pub use package::{Package, is_xlsb}; +pub use pivot::{ + CacheField, CacheItem, CacheRecord, CacheSource, CacheValue, DataField, FieldGroup, Location, PageField, PivotCacheDefinition, PivotField, + PivotItem, PivotTableDefinition, PivotTableStyleInfo, RangePr, SharedItems, WorksheetSource, records_xml, +}; pub use sheet::{Cell, CellFormula, CellValue, Column, FormulaKind, Row, SheetPart, iso_date_serial}; pub use strings::{PhoneticProps, PhoneticRun, StringItem, TextRun, decode_xstring, encode_xstring, rich_text, t_el}; pub use styles::{ Alignment, Border, BorderEdge, CellStyle, Color, ColorSource, Dxf, Fill, Font, FontScheme, GradientFill, PatternFill, Protection, Styles, Xf, }; -pub use workbook::{DefinedName, SheetEntry, SheetKind, Workbook}; +pub use workbook::{DefinedName, PivotCacheEntry, SheetEntry, SheetKind, Workbook}; pub use write::num; /// Why a workbook could not be read. diff --git a/crates/xlsx/src/package.rs b/crates/xlsx/src/package.rs index 5d415f813..1c133d0ad 100644 --- a/crates/xlsx/src/package.rs +++ b/crates/xlsx/src/package.rs @@ -8,6 +8,7 @@ use craft_drawingml::theme::Theme; use crate::drawing::Drawing; use crate::features::{Comments, NoteShape, Table}; +use crate::pivot::{CacheRecord, PivotCacheDefinition, PivotTableDefinition, read_record}; use crate::sheet::{Row, RowReader, SheetPart}; use crate::strings::StringItem; use crate::styles::Styles; @@ -300,6 +301,64 @@ impl<'a> Package<'a> { Ok(out) } + /// The pivot cache definition in `part` (a workbook relationship that + /// [`Workbook::pivot_caches`] names). `None`, with a warning, when it is missing or does not + /// read. + pub fn pivot_cache(&mut self, part: &str) -> Result, Error> { + let Some(x) = self.optional_xml(part)? else { return Ok(None) }; + let def = PivotCacheDefinition::read(&x); + if def.is_none() { + self.warn(format!("{part}: a pivot cache this crate does not read (OLAP data, or not a cache definition)")); + } + Ok(def) + } + + /// The records of the pivot cache in `cache_part`, one at a time, so a cache of any size is + /// read without holding it. Nothing when it saved none. + pub fn pivot_records(&mut self, cache_part: &str, mut each: impl FnMut(CacheRecord) -> Result<(), Error>) -> Result<(), Error> { + let rels = self.opc.rels(cache_part)?; + let Some(part) = rels.first("pivotCacheRecords").filter(|r| !r.external).map(|r| r.target.clone()) else { return Ok(()) }; + let Some(bytes) = self.opc.read(&part)? else { + self.warn(format!("missing part {part}")); + return Ok(()); + }; + let (_, err) = self.xml_options().parse_streaming_partial(&bytes, &["r"], |e| each(read_record(&e)).map_err(Stop::Caller)); + match err { + Some(Stop::Caller(e)) => Err(e), + Some(Stop::Xml(e)) if e.is_limit() => Err(e.into()), + Some(Stop::Xml(e)) => { + self.warn(format!("{part} is damaged ({e}); its records were read up to there")); + Ok(()) + } + None => Ok(()), + } + } + + /// The PivotTables of a worksheet read by [`Package::sheet`], each with its part name and + /// the part of its cache: the one the table relates to, else the one `workbook` names by + /// its `cacheId`. A part that does not read is skipped with a warning. + pub fn pivot_tables(&mut self, workbook: &Workbook, sheet: &SheetPart) -> Result)>, Error> { + let mut out = Vec::new(); + for rel in sheet.rels.list.iter().filter(|r| r.kind() == "pivotTable" && !r.external) { + let Some(x) = self.optional_xml(&rel.target)? else { continue }; + let Some(table) = PivotTableDefinition::read(&x) else { + self.warn(format!("skipped {}: not a PivotTable with a location", rel.target)); + continue; + }; + let own = self.opc.rels(&rel.target)?.first("pivotCacheDefinition").filter(|r| !r.external).map(|r| r.target.clone()); + let cache = match own { + Some(c) => Some(c), + None => { + let book = self.workbook_rels()?; + let entry = workbook.pivot_caches.iter().find(|c| Some(c.cache_id) == table.cache_id); + entry.and_then(|e| book.by_id(&e.rel_id)).filter(|r| !r.external).map(|r| r.target.clone()) + } + }; + out.push((rel.target.clone(), table, cache)); + } + Ok(out) + } + /// A chart part (`c:chartSpace`, or `cx:chartSpace` of an Office 2016 chart) through /// `craft-drawingml`. `None` when the part is missing or holds no chart it can read. pub fn chart(&mut self, part: &str) -> Result, Error> { diff --git a/crates/xlsx/src/pivot.rs b/crates/xlsx/src/pivot.rs new file mode 100644 index 000000000..93eda35cb --- /dev/null +++ b/crates/xlsx/src/pivot.rs @@ -0,0 +1,769 @@ +//! PivotTables (ECMA-376 Part 1, 18.10): the pivot cache definition (the source and its fields +//! with their shared items and grouping), the cache records, and the PivotTable definition (its +//! location, fields, axes, values and style). +//! +//! As for the worksheet features, an attribute the file may leave out is an `Option`, the +//! attributes a type does not name are kept in its `attrs`, and the children it does not model +//! (OLAP hierarchies, formats, filters, extensions) are kept as written. A cache whose items +//! carry OLAP member data does not read: it stays a kept part, so nothing is lost. A part is +//! read as one tree, so [`crate::Limits::xml`] bounds every list in it; records are streamed. + +use craft_sheet::Range; +use craft_xml::{El, Node}; + +use crate::features::{Attr, kids, rest}; +use crate::write::num; + +/// One value of a pivot cache: a shared or group item, or a field of a record. +#[derive(Clone, Debug, PartialEq)] +pub enum CacheValue { + /// ``: no value. + Missing, + /// `` + Number(f64), + /// `` + Bool(bool), + /// ``: the error as written (`#N/A`). + Error(String), + /// `` + Text(String), + /// ``: an ISO 8601 date-time as written (`2024-03-05T00:00:00`). + Date(String), + /// ``: an index into the field's shared items (records only). + Index(u32), +} + +impl CacheValue { + /// `None` for an element that is not a value, or one with children (OLAP member tuples). + fn read(e: &El) -> Option { + if e.elements().next().is_some() { + return None; + } + let v = e.attr("v"); + Some(match e.name.as_str() { + "m" => CacheValue::Missing, + "n" => CacheValue::Number(craft_xml::value::to_f64(v?)?), + "b" => CacheValue::Bool(craft_xml::value::to_bool(v?)?), + "e" => CacheValue::Error(v?.to_string()), + "s" => CacheValue::Text(v?.to_string()), + "d" => CacheValue::Date(v?.to_string()), + "x" => CacheValue::Index(craft_xml::value::to_u32(v.unwrap_or("0"))?), + _ => return None, + }) + } + + fn to_el(&self) -> El { + match self { + CacheValue::Missing => El::new("m"), + CacheValue::Number(n) => El::new("n").with_attr("v", num(*n)), + CacheValue::Bool(b) => El::new("b").with_attr("v", u8::from(*b)), + CacheValue::Error(s) => El::new("e").with_attr("v", s), + CacheValue::Text(s) => El::new("s").with_attr("v", s), + CacheValue::Date(s) => El::new("d").with_attr("v", s), + CacheValue::Index(i) => El::new("x").with_attr("v", i), + } + } +} + +/// A shared or group item: its value and its other attributes (`u` unused, `f` formula, `c` +/// calculated, `cp`, `in`, ...). +#[derive(Clone, Debug, PartialEq)] +pub struct CacheItem { + pub value: CacheValue, + pub attrs: Vec<(String, String)>, +} + +impl CacheItem { + pub fn new(value: CacheValue) -> CacheItem { + CacheItem { value, attrs: Vec::new() } + } + + fn read(e: &El) -> Option { + Some(CacheItem { value: CacheValue::read(e)?, attrs: rest(e, &["v"]) }) + } + + fn to_el(&self) -> El { + let mut e = self.value.to_el(); + e.attrs.extend(self.attrs.iter().cloned()); + e + } +} + +fn read_items(e: &El) -> Option> { + e.elements().map(CacheItem::read).collect() +} + +fn items_el(name: &str, items: &[CacheItem]) -> El { + let mut e = El::new(name).with_attr("count", items.len()); + items.iter().for_each(|i| e.push(i.to_el())); + e +} + +element! { + /// ``: what a field's values are, and the items an axis refers to by index. + pub struct SharedItems { + contains_semi_mixed_types: bool = "containsSemiMixedTypes", + contains_non_date: bool = "containsNonDate", + contains_date: bool = "containsDate", + contains_string: bool = "containsString", + contains_blank: bool = "containsBlank", + contains_mixed_types: bool = "containsMixedTypes", + contains_number: bool = "containsNumber", + contains_integer: bool = "containsInteger", + min_value: f64 = "minValue", + max_value: f64 = "maxValue", + min_date: String = "minDate", + max_date: String = "maxDate", + long_text: bool = "longText", + } + + { + items: Vec, + } +} + +impl SharedItems { + /// The flags Excel writes for a field holding `values` (no items: add those that an axis + /// needs). Booleans and errors count as strings, as Excel counts them; a field of dates + /// (and blanks) gets the date flags and its first and last date. + pub fn describe<'a>(values: impl IntoIterator) -> SharedItems { + let (mut blank, mut strings, mut long, mut dates, mut non_int) = (false, false, false, false, false); + let mut range: Option<(f64, f64)> = None; + let mut date_range: Option<(&str, &str)> = None; + for v in values { + match v { + CacheValue::Missing => blank = true, + CacheValue::Number(n) => { + non_int |= n.fract() != 0.0; + range = Some(range.map_or((*n, *n), |(a, b)| (a.min(*n), b.max(*n)))); + } + CacheValue::Text(t) => { + strings = true; + long |= t.chars().count() > 255; + } + CacheValue::Bool(_) | CacheValue::Error(_) => strings = true, + CacheValue::Date(d) => { + dates = true; + let d = d.as_str(); + date_range = Some(date_range.map_or((d, d), |(a, b)| (a.min(d), b.max(d)))); + } + CacheValue::Index(_) => {} + } + } + let number = range.is_some(); + let some = |b: bool| b.then_some(true); + if dates && !strings && !number { + return SharedItems { + contains_semi_mixed_types: Some(false), + contains_non_date: Some(false), + contains_date: Some(true), + contains_string: Some(false), + contains_blank: some(blank), + min_date: date_range.map(|r| r.0.to_string()), + max_date: date_range.map(|r| r.1.to_string()), + ..SharedItems::default() + }; + } + let kinds = usize::from(strings) + usize::from(number) + usize::from(dates); + SharedItems { + contains_semi_mixed_types: (!strings && !blank).then_some(false), + contains_date: some(dates), + contains_string: (!strings).then_some(false), + contains_blank: some(blank), + contains_mixed_types: some(kinds > 1), + contains_number: some(number), + contains_integer: some(number && !non_int), + min_value: range.map(|r| r.0), + max_value: range.map(|r| r.1), + min_date: date_range.map(|r| r.0.to_string()), + max_date: date_range.map(|r| r.1.to_string()), + long_text: some(long), + ..SharedItems::default() + } + } +} + +element! { + /// ``: grouping by numeric or date ranges. + pub struct RangePr { + auto_start: bool = "autoStart", + auto_end: bool = "autoEnd", + /// `range`, `seconds`, `minutes`, `hours`, `days`, `months`, `quarters`, `years`. + group_by: String = "groupBy", + start_num: f64 = "startNum", + end_num: f64 = "endNum", + start_date: String = "startDate", + end_date: String = "endDate", + group_interval: f64 = "groupInterval", + } +} + +element! { + /// ``: a field grouped from another (`base`), into ranges or chosen items. + pub struct FieldGroup { + /// The next coarser grouping field of the same base. + par: u32 = "par", + base: u32 = "base", + } + + { + range: Option, + /// `discretePr`: for each base item, the group item it falls in. + discrete: Option>, + /// `groupItems`: the groups' captions. + items: Option>, + } +} + +element! { + /// ``: one column of the source, or a field grouped or calculated from others. + pub struct CacheField { + name: String = "name", + caption: String = "caption", + num_fmt_id: u32 = "numFmtId", + /// A calculated field's formula, without `=`. + formula: String = "formula", + /// `false` for a grouping or calculated field. + database_field: bool = "databaseField", + sql_type: i32 = "sqlType", + hierarchy: i32 = "hierarchy", + level: u32 = "level", + } + + { + shared_items: Option, + group: Option, + /// `mpMap` and `extLst`, as written. + other: Vec, + } +} + +element! { + /// ``: a range on a sheet (`reference` and `sheet`), or a defined name or + /// table (`name`); `r:id` for a source in another workbook. + pub struct WorksheetSource { + reference: String = "ref", + name: String = "name", + sheet: String = "sheet", + rel_id: String = "r:id", + } +} + +element! { + /// ``. + pub struct CacheSource { + /// `worksheet`, `external`, `consolidation`, `scenario`. + source_type: String = "type", + connection_id: u32 = "connectionId", + } + + { + worksheet: Option, + /// `consolidation` and `extLst`, as written. + other: Vec, + } +} + +element! { + /// A pivot cache definition (``, 18.10.1.67). Its records part is + /// related from it (`r:id`), which [`crate::Document`] writes. + pub struct PivotCacheDefinition { + invalid: bool = "invalid", + /// `false`: no records are saved, and the cache is refreshed before use. + save_data: bool = "saveData", + refresh_on_load: bool = "refreshOnLoad", + optimize_memory: bool = "optimizeMemory", + enable_refresh: bool = "enableRefresh", + refreshed_by: String = "refreshedBy", + refreshed_date: f64 = "refreshedDate", + background_query: bool = "backgroundQuery", + missing_items_limit: u32 = "missingItemsLimit", + created_version: u32 = "createdVersion", + refreshed_version: u32 = "refreshedVersion", + min_refreshable_version: u32 = "minRefreshableVersion", + record_count: u32 = "recordCount", + upgrade_on_refresh: bool = "upgradeOnRefresh", + } + + { + source: CacheSource, + fields: Vec, + /// `cacheHierarchies`, `kpis`, `tupleCache`, `calculatedItems`, `calculatedMembers`, + /// `dimensions`, `measureGroups`, `maps`, `extLst`, as written. + other: Vec, + } +} + +impl PivotCacheDefinition { + /// A part read with [`crate::ns::CANONICAL`]. `None` when it is not a cache definition, or + /// an item holds OLAP member data this model does not keep. + pub fn read(x: &El) -> Option { + if x.name != "pivotCacheDefinition" { + return None; + } + let source = x.child("cacheSource").map_or_else(CacheSource::default, |s| CacheSource { + worksheet: s.child("worksheetSource").map(WorksheetSource::read_attrs), + other: kids(s, &["worksheetSource"]), + ..CacheSource::read_attrs(s) + }); + let mut fields = Vec::new(); + for f in x.child("cacheFields").into_iter().flat_map(|c| c.children("cacheField")) { + let shared_items = match f.child("sharedItems") { + Some(s) => Some(SharedItems { items: read_items(s)?, ..SharedItems::read_attrs(s) }), + None => None, + }; + let group = match f.child("fieldGroup") { + Some(g) => Some(FieldGroup { + range: g.child("rangePr").map(RangePr::read_attrs), + discrete: g.child("discretePr").map(|d| d.children("x").filter_map(|x| x.attr_u32("v")).collect()), + items: match g.child("groupItems") { + Some(i) => Some(read_items(i)?), + None => None, + }, + ..FieldGroup::read_attrs(g) + }), + None => None, + }; + fields.push(CacheField { shared_items, group, other: kids(f, &["sharedItems", "fieldGroup"]), ..CacheField::read_attrs(f) }); + } + let mut def = PivotCacheDefinition { source, fields, other: kids(x, &["cacheSource", "cacheFields"]), ..PivotCacheDefinition::read_attrs(x) }; + // The records relationship is the writer's. + def.attrs.retain(|(k, _)| k != "r:id"); + Some(def) + } + + /// The part, related to its records by `records_rel` when it has them. + pub fn to_xml(&self, records_rel: Option<&str>) -> Vec { + let mut e = El::new("pivotCacheDefinition"); + if let Some(id) = records_rel { + e.set_attr("r:id", id); + } + self.write_attrs(&mut e); + let mut src = El::new("cacheSource"); + self.source.write_attrs(&mut src); + if let Some(w) = &self.source.worksheet { + let mut we = El::new("worksheetSource"); + w.write_attrs(&mut we); + src.push(we); + } + src.children.extend(self.source.other.iter().cloned().map(Node::El)); + e.push(src); + let mut fields = El::new("cacheFields").with_attr("count", self.fields.len()); + for f in &self.fields { + let mut fe = El::new("cacheField"); + f.write_attrs(&mut fe); + if let Some(s) = &f.shared_items { + let mut se = El::new("sharedItems"); + s.write_attrs(&mut se); + if !s.items.is_empty() { + se.set_attr("count", s.items.len()); + s.items.iter().for_each(|i| se.push(i.to_el())); + } + fe.push(se); + } + if let Some(g) = &f.group { + let mut ge = El::new("fieldGroup"); + g.write_attrs(&mut ge); + if let Some(r) = &g.range { + let mut re = El::new("rangePr"); + r.write_attrs(&mut re); + ge.push(re); + } + if let Some(d) = &g.discrete { + let mut de = El::new("discretePr").with_attr("count", d.len()); + d.iter().for_each(|v| de.push(El::new("x").with_attr("v", v))); + ge.push(de); + } + if let Some(items) = &g.items { + ge.push(items_el("groupItems", items)); + } + fe.push(ge); + } + fe.children.extend(f.other.iter().cloned().map(Node::El)); + fields.push(fe); + } + e.push(fields); + e.children.extend(self.other.iter().cloned().map(Node::El)); + document(e) + } +} + +/// One record of a pivot cache: a value per database field, in field order. +pub type CacheRecord = Vec; + +/// A pivot cache records part (``) for `records`. +pub fn records_xml(records: &[CacheRecord]) -> Vec { + let mut w = craft_xml::Writer::document(); + w.tag("pivotCacheRecords").attr("xmlns", crate::ns::MAIN).attr("xmlns:r", crate::ns::R).attr("count", records.len()).open(); + for r in records { + let mut e = El::new("r"); + r.iter().for_each(|v| e.push(v.to_el())); + w.tree(&e); + } + w.into_bytes() +} + +/// One `` of a records part, read with [`crate::ns::CANONICAL`]; values with OLAP data and +/// unknown elements are left out, so a field may be missing. +pub(crate) fn read_record(r: &El) -> CacheRecord { + r.elements().filter_map(CacheValue::read).collect() +} + +element! { + /// ``: where the PivotTable is (its body: report filters sit above it). + pub struct Location { + first_header_row: u32 = "firstHeaderRow", + first_data_row: u32 = "firstDataRow", + first_data_col: u32 = "firstDataCol", + row_page_count: u32 = "rowPageCount", + col_page_count: u32 = "colPageCount", + } + + { + range: Range, + } +} + +element! { + /// `` of a pivot field: a shared item shown (`x`), or a subtotal (`t`). + pub struct PivotItem { + /// The index of the item in the cache field's shared or group items. + x: u32 = "x", + /// `data` (the default), `default`, `sum`, `countA`, `avg`, ..., `grand`, `blank`. + item_type: String = "t", + /// Hidden (filtered out). + h: bool = "h", + /// Details shown (expanded); `false` is collapsed. + sd: bool = "sd", + /// A caption replacing the item's. + n: String = "n", + s: bool = "s", + f: bool = "f", + m: bool = "m", + c: bool = "c", + d: bool = "d", + e: bool = "e", + } +} + +element! { + /// ``: how one cache field is used (by position: field `i` is cache field `i`). + pub struct PivotField { + name: String = "name", + /// `axisRow`, `axisCol`, `axisPage`, `axisValues`; `None` when the field is on no axis. + axis: String = "axis", + data_field: bool = "dataField", + num_fmt_id: u32 = "numFmtId", + compact: bool = "compact", + outline: bool = "outline", + subtotal_top: bool = "subtotalTop", + show_all: bool = "showAll", + /// `manual` (the default), `ascending`, `descending`. + sort_type: String = "sortType", + default_subtotal: bool = "defaultSubtotal", + multiple_item_selection_allowed: bool = "multipleItemSelectionAllowed", + drag_to_row: bool = "dragToRow", + drag_to_col: bool = "dragToCol", + drag_to_page: bool = "dragToPage", + drag_to_data: bool = "dragToData", + insert_blank_row: bool = "insertBlankRow", + insert_page_break: bool = "insertPageBreak", + } + + { + /// `None` when the field has no ``. + items: Option>, + /// `autoSortScope` and `extLst`, as written. + other: Vec, + } +} + +element! { + /// ``: a report filter. + pub struct PageField { + fld: i32 = "fld", + /// The one item shown; `None` shows all (or those not hidden in the field's items). + item: u32 = "item", + hier: i32 = "hier", + name: String = "name", + cap: String = "cap", + } + + { + ext: Option, + } +} + +element! { + /// ``: a value field. + pub struct DataField { + name: String = "name", + fld: u32 = "fld", + /// `sum` (the default), `count`, `average`, `max`, `min`, `product`, `countNums`, + /// `stdDev`, `stdDevp`, `var`, `varp`. + subtotal: String = "subtotal", + /// `normal`, `difference`, `percent`, `percentDiff`, `runTotal`, `percentOfRow`, + /// `percentOfCol`, `percentOfTotal`, `index`. + show_data_as: String = "showDataAs", + base_field: i32 = "baseField", + base_item: u32 = "baseItem", + num_fmt_id: u32 = "numFmtId", + } + + { + /// `extLst`, as written: `x14:dataField` holds the Excel 2010 show-values-as kinds + /// (`pivotShowAs="rankDescending"`). + ext: Option, + } +} + +element! { + /// ``. + pub struct PivotTableStyleInfo { + name: String = "name", + show_row_headers: bool = "showRowHeaders", + show_col_headers: bool = "showColHeaders", + show_row_stripes: bool = "showRowStripes", + show_col_stripes: bool = "showColStripes", + show_last_column: bool = "showLastColumn", + } +} + +/// The children of a PivotTable definition in schema order (`CT_pivotTableDefinition`). +const TABLE: &[&str] = &[ + "location", + "pivotFields", + "rowFields", + "rowItems", + "colFields", + "colItems", + "pageFields", + "dataFields", + "formats", + "conditionalFormats", + "chartFormats", + "pivotHierarchies", + "pivotTableStyleInfo", + "filters", + "rowHierarchiesUsage", + "colHierarchiesUsage", + "extLst", +]; + +element! { + /// A PivotTable (``, 18.10.1.73) on the cache `cache_id` names. + pub struct PivotTableDefinition { + name: String = "name", + cache_id: u32 = "cacheId", + /// Σ Values is on the rows axis. + data_on_rows: bool = "dataOnRows", + data_position: u32 = "dataPosition", + data_caption: String = "dataCaption", + grand_total_caption: String = "grandTotalCaption", + error_caption: String = "errorCaption", + show_error: bool = "showError", + missing_caption: String = "missingCaption", + show_missing: bool = "showMissing", + apply_number_formats: bool = "applyNumberFormats", + apply_border_formats: bool = "applyBorderFormats", + apply_font_formats: bool = "applyFontFormats", + apply_pattern_formats: bool = "applyPatternFormats", + apply_alignment_formats: bool = "applyAlignmentFormats", + apply_width_height_formats: bool = "applyWidthHeightFormats", + updated_version: u32 = "updatedVersion", + min_refreshable_version: u32 = "minRefreshableVersion", + enable_drill: bool = "enableDrill", + use_auto_formatting: bool = "useAutoFormatting", + /// The grand total row below the report. + row_grand_totals: bool = "rowGrandTotals", + /// The grand total column at its right. + col_grand_totals: bool = "colGrandTotals", + item_print_titles: bool = "itemPrintTitles", + created_version: u32 = "createdVersion", + indent: u32 = "indent", + show_headers: bool = "showHeaders", + compact: bool = "compact", + outline: bool = "outline", + outline_data: bool = "outlineData", + compact_data: bool = "compactData", + grid_drop_zones: bool = "gridDropZones", + multiple_field_filters: bool = "multipleFieldFilters", + row_header_caption: String = "rowHeaderCaption", + col_header_caption: String = "colHeaderCaption", + } + + { + location: Location, + fields: Vec, + /// `rowFields`: cache field indices, `-2` for Σ Values. + row_fields: Vec, + col_fields: Vec, + page_fields: Vec, + data_fields: Vec, + style: Option, + /// `rowItems`, `colItems`, `formats`, `conditionalFormats`, `chartFormats`, + /// `pivotHierarchies`, `filters`, the hierarchy usages and `extLst`, as written. + other: Vec, + } +} + +impl PivotTableDefinition { + /// A part read with [`crate::ns::CANONICAL`]. `None` when it is not a PivotTable definition + /// or has no readable location. + pub fn read(x: &El) -> Option { + if x.name != "pivotTableDefinition" { + return None; + } + let loc = x.child("location")?; + let location = Location { range: Range::parse(&loc.attr("ref")?.replace('$', ""))?, ..Location::read_attrs(loc) }; + let list = |name: &str| -> Vec { + x.child(name).map(|e| e.children("field").filter_map(|f| f.attr("x").and_then(craft_xml::value::to_i32)).collect()).unwrap_or_default() + }; + let fields = x + .child("pivotFields") + .map(|p| { + p.children("pivotField") + .map(|f| PivotField { + items: f.child("items").map(|i| i.children("item").map(PivotItem::read_attrs).collect()), + other: kids(f, &["items"]), + ..PivotField::read_attrs(f) + }) + .collect() + }) + .unwrap_or_default(); + Some(PivotTableDefinition { + location, + fields, + row_fields: list("rowFields"), + col_fields: list("colFields"), + page_fields: x + .child("pageFields") + .map(|p| p.children("pageField").map(|f| PageField { ext: f.child("extLst").cloned(), ..PageField::read_attrs(f) }).collect()) + .unwrap_or_default(), + data_fields: x + .child("dataFields") + .map(|p| p.children("dataField").map(|f| DataField { ext: f.child("extLst").cloned(), ..DataField::read_attrs(f) }).collect()) + .unwrap_or_default(), + style: x.child("pivotTableStyleInfo").map(PivotTableStyleInfo::read_attrs), + other: kids(x, &["location", "pivotFields", "rowFields", "colFields", "pageFields", "dataFields", "pivotTableStyleInfo"]), + ..PivotTableDefinition::read_attrs(x) + }) + } + + /// The part. + pub fn to_xml(&self) -> Vec { + let mut e = El::new("pivotTableDefinition"); + self.write_attrs(&mut e); + let mut kids: Vec = Vec::new(); + let mut loc = El::new("location").with_attr("ref", self.location.range.a1()); + self.location.write_attrs(&mut loc); + kids.push(loc); + if !self.fields.is_empty() { + let mut pf = El::new("pivotFields").with_attr("count", self.fields.len()); + for f in &self.fields { + let mut fe = El::new("pivotField"); + f.write_attrs(&mut fe); + if let Some(items) = &f.items { + let mut ie = El::new("items").with_attr("count", items.len()); + for i in items { + let mut it = El::new("item"); + i.write_attrs(&mut it); + ie.push(it); + } + fe.push(ie); + } + fe.children.extend(f.other.iter().cloned().map(Node::El)); + pf.push(fe); + } + kids.push(pf); + } + for (name, list) in [("rowFields", &self.row_fields), ("colFields", &self.col_fields)] { + if !list.is_empty() { + let mut l = El::new(name).with_attr("count", list.len()); + list.iter().for_each(|x| l.push(El::new("field").with_attr("x", x))); + kids.push(l); + } + } + if !self.page_fields.is_empty() { + let mut l = El::new("pageFields").with_attr("count", self.page_fields.len()); + for p in &self.page_fields { + let mut pe = El::new("pageField"); + p.write_attrs(&mut pe); + pe.children.extend(p.ext.iter().cloned().map(Node::El)); + l.push(pe); + } + kids.push(l); + } + if !self.data_fields.is_empty() { + let mut l = El::new("dataFields").with_attr("count", self.data_fields.len()); + for d in &self.data_fields { + let mut de = El::new("dataField"); + d.write_attrs(&mut de); + de.children.extend(d.ext.iter().cloned().map(Node::El)); + l.push(de); + } + kids.push(l); + } + if let Some(s) = &self.style { + let mut se = El::new("pivotTableStyleInfo"); + s.write_attrs(&mut se); + kids.push(se); + } + kids.extend(self.other.iter().cloned()); + let pos = |n: &str| TABLE.iter().position(|o| *o == n).unwrap_or(TABLE.len()); + kids.sort_by_key(|k| pos(&k.name)); + e.children.extend(kids.into_iter().map(Node::El)); + document(e) + } +} + +fn document(mut root: El) -> Vec { + crate::ns::declare(&mut root); + let mut w = craft_xml::Writer::document(); + w.tree(&root); + w.into_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn values(v: &[CacheValue]) -> SharedItems { + SharedItems::describe(v) + } + + #[test] + fn shared_item_flags_follow_excel() { + use CacheValue::*; + let s = values(&[Number(1.0), Number(2.5)]); + assert_eq!( + (s.contains_semi_mixed_types, s.contains_string, s.contains_number, s.contains_integer, s.min_value, s.max_value), + (Some(false), Some(false), Some(true), None, Some(1.0), Some(2.5)) + ); + let s = values(&[Text("a".into()), Number(3.0), Missing]); + assert_eq!( + (s.contains_semi_mixed_types, s.contains_string, s.contains_blank, s.contains_mixed_types, s.contains_integer), + (None, None, Some(true), Some(true), Some(true)) + ); + // Booleans and errors count as strings. + let s = values(&[Bool(true), Error("#N/A".into())]); + assert_eq!((s.contains_string, s.contains_number, s.contains_mixed_types), (None, None, None)); + let s = values(&[Text("x".repeat(256))]); + assert_eq!(s.long_text, Some(true)); + let s = values(&[Date("2024-03-05T00:00:00".into()), Missing, Date("2023-01-01T00:00:00".into())]); + assert_eq!( + (s.contains_semi_mixed_types, s.contains_non_date, s.contains_date, s.contains_string, s.contains_blank), + (Some(false), Some(false), Some(true), Some(false), Some(true)) + ); + assert_eq!((s.min_date.as_deref(), s.max_date.as_deref()), (Some("2023-01-01T00:00:00"), Some("2024-03-05T00:00:00"))); + } + + #[test] + fn olap_items_do_not_read() { + let x = craft_xml::parse( + br#""#, + ) + .unwrap(); + assert!(PivotCacheDefinition::read(&x).is_none()); + let bad = craft_xml::parse(br#""#).unwrap(); + assert!(PivotCacheDefinition::read(&bad).is_none()); + } + + #[test] + fn a_table_without_location_does_not_read() { + let x = craft_xml::parse(br#""#).unwrap(); + assert!(PivotTableDefinition::read(&x).is_none()); + let x = craft_xml::parse(br#""#).unwrap(); + assert!(PivotTableDefinition::read(&x).is_none()); + } +} diff --git a/crates/xlsx/src/workbook.rs b/crates/xlsx/src/workbook.rs index 09745e3ae..f3ef9bafb 100644 --- a/crates/xlsx/src/workbook.rs +++ b/crates/xlsx/src/workbook.rs @@ -88,6 +88,14 @@ pub struct DefinedName { pub attrs: Vec<(String, String)>, } +/// ``: a pivot cache of the workbook, by the id its PivotTables name it with. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PivotCacheEntry { + pub cache_id: u32, + /// The relationship to the cache definition part. + pub rel_id: String, +} + /// The workbook part. #[derive(Clone, Debug, Default, PartialEq)] pub struct Workbook { @@ -97,6 +105,8 @@ pub struct Workbook { pub defined_names: Vec, /// `activeTab` of the first visible workbook view (of the first view when none is visible). pub active_tab: Option, + /// `pivotCaches`, in order. + pub pivot_caches: Vec, /// The whole part, for everything not lifted out above (calculation settings, views, /// protection, pivot caches, external references). Empty for a binary workbook. pub root: El, @@ -136,6 +146,12 @@ impl Workbook { }); } } + if let Some(caches) = root.child("pivotCaches") { + wb.pivot_caches = caches + .children("pivotCache") + .filter_map(|c| Some(PivotCacheEntry { cache_id: c.attr_u32("cacheId")?, rel_id: c.attr_or_plain("r:id")?.to_string() })) + .collect(); + } if let Some(views) = root.child("bookViews") { let view = views .children("workbookView") @@ -149,8 +165,8 @@ impl Workbook { /// The workbook part's root: `sheets` and `definedNames` regenerated from the fields, the /// date system set in `workbookPr` and the active tab in the view it was read from; - /// everything else in [`Workbook::root`] as kept. Each sheet's `r:id` is written as - /// [`SheetEntry::rel_id`] holds it. + /// `pivotCaches` from [`Workbook::pivot_caches`]; everything else in [`Workbook::root`] as + /// kept. Each sheet's `r:id` is written as [`SheetEntry::rel_id`] holds it. pub fn to_el(&self) -> El { let root = if self.root.name.is_empty() { El::new("workbook") } else { self.root.clone() }; let mut pr = root.child("workbookPr").cloned().unwrap_or_else(|| El::new("workbookPr")); @@ -195,6 +211,16 @@ impl Workbook { ("bookViews", views), ("sheets", Some(sheets)), ("definedNames", (!self.defined_names.is_empty()).then_some(names)), + ( + "pivotCaches", + (!self.pivot_caches.is_empty()).then(|| { + let mut e = El::new("pivotCaches"); + self.pivot_caches + .iter() + .for_each(|c| e.push(El::new("pivotCache").with_attr("cacheId", c.cache_id).with_attr("r:id", &c.rel_id))); + e + }), + ), ]; assemble(&root, WORKBOOK, owned) } diff --git a/crates/xlsx/tests/pivot.rs b/crates/xlsx/tests/pivot.rs new file mode 100644 index 000000000..4d2fd00d5 --- /dev/null +++ b/crates/xlsx/tests/pivot.rs @@ -0,0 +1,221 @@ +//! PivotTables: caches, records and tables read, written back, and made new. + +use craft_xlsx::testing::zip; +use craft_xlsx::{ + CacheItem, CacheValue, DataField, Document, Error, Limits, Location, Package, PivotCacheDefinition, PivotCachePart, PivotField, PivotItem, + PivotTableDefinition, PivotTablePart, Range, SharedItems, records_xml, +}; + +const MAIN: &str = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; +const RT: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + +const CACHE: &str = r#""#; +const RECORDS: &str = r#""#; +const TABLE: &str = r#""#; +/// An OLAP cache: its items carry member tuples, which this crate does not model. +const OLAP: &str = r#""#; +const OLAP_TABLE: &str = r#""#; + +fn rels(items: &[(&str, &str, &str)]) -> String { + let r: String = items.iter().map(|(id, ty, t)| format!(r#""#)).collect(); + format!(r#"{r}"#) +} + +/// A workbook with a pivot cache (with records) whose table relates to it, and an OLAP cache +/// whose table is found by its `cacheId` only. +fn with_pivots() -> Vec { + let book = format!( + r#""# + ); + let sheet = format!( + r#"Region"# + ); + let types = r#""#; + zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", rels(&[("rId1", "officeDocument", "xl/workbook.xml")]).as_bytes()), + ("xl/workbook.xml", book.as_bytes()), + ( + "xl/_rels/workbook.xml.rels", + rels(&[ + ("rId1", "worksheet", "worksheets/sheet1.xml"), + ("rId3", "pivotCacheDefinition", "pivotCache/pivotCacheDefinition1.xml"), + ("rId4", "pivotCacheDefinition", "pivotCache/pivotCacheDefinition2.xml"), + ]) + .as_bytes(), + ), + ("xl/worksheets/sheet1.xml", sheet.as_bytes()), + ( + "xl/worksheets/_rels/sheet1.xml.rels", + rels(&[("rId2", "pivotTable", "../pivotTables/pivotTable1.xml"), ("rId3", "pivotTable", "../pivotTables/pivotTable2.xml")]).as_bytes(), + ), + ("xl/pivotTables/pivotTable1.xml", TABLE.as_bytes()), + ( + "xl/pivotTables/_rels/pivotTable1.xml.rels", + rels(&[("rId1", "pivotCacheDefinition", "../pivotCache/pivotCacheDefinition1.xml")]).as_bytes(), + ), + ("xl/pivotTables/pivotTable2.xml", OLAP_TABLE.as_bytes()), + ("xl/pivotCache/pivotCacheDefinition1.xml", CACHE.as_bytes()), + ("xl/pivotCache/_rels/pivotCacheDefinition1.xml.rels", rels(&[("rId1", "pivotCacheRecords", "pivotCacheRecords1.xml")]).as_bytes()), + ("xl/pivotCache/pivotCacheRecords1.xml", RECORDS.as_bytes()), + ("xl/pivotCache/pivotCacheDefinition2.xml", OLAP.as_bytes()), + ]) +} + +#[test] +fn the_package_reads_caches_records_and_tables() { + let bytes = with_pivots(); + let mut p = Package::open(&bytes, &Limits::default()).unwrap(); + let wb = p.workbook().unwrap(); + assert_eq!(wb.pivot_caches.iter().map(|c| (c.cache_id, c.rel_id.as_str())).collect::>(), [(5, "rId3"), (7, "rId4")]); + let def = p.pivot_cache("xl/pivotCache/pivotCacheDefinition1.xml").unwrap().unwrap(); + let ws = def.source.worksheet.as_ref().unwrap(); + assert_eq!((ws.reference.as_deref(), ws.sheet.as_deref()), (Some("A1:B3"), Some("Data"))); + assert_eq!((def.refresh_on_load, def.record_count, def.attrs.clone()), (Some(true), Some(2), vec![("xr:uid".to_string(), "{C}".to_string())])); + let names: Vec<_> = def.fields.iter().map(|f| f.name.as_deref().unwrap_or("")).collect(); + assert_eq!(names, ["Region", "Day", "Ratio"]); + let region = def.fields[0].shared_items.as_ref().unwrap(); + assert_eq!(region.items[1], CacheItem { value: CacheValue::Text("West".into()), attrs: vec![("u".into(), "1".into())] }); + let group = def.fields[1].group.as_ref().unwrap(); + assert_eq!((group.par, group.base, group.range.as_ref().and_then(|r| r.group_by.as_deref())), (Some(2), Some(1), Some("months"))); + assert_eq!(group.items.as_ref().map(|i| i.len()), Some(3)); + assert_eq!((def.fields[2].formula.as_deref(), def.fields[2].database_field), (Some("Day/2"), Some(false))); + assert_eq!(def.other.len(), 1); + + let mut records = Vec::new(); + p.pivot_records("xl/pivotCache/pivotCacheDefinition1.xml", |r| { + records.push(r); + Ok(()) + }) + .unwrap(); + assert_eq!( + records, + [ + vec![CacheValue::Index(0), CacheValue::Date("2024-01-01T00:00:00".into())], + vec![CacheValue::Index(1), CacheValue::Date("2024-02-01T00:00:00".into())] + ] + ); + // The caller can stop the records. + let stopped = p.pivot_records("xl/pivotCache/pivotCacheDefinition1.xml", |_| Err(Error::TooLarge("enough".into()))); + assert_eq!(stopped, Err(Error::TooLarge("enough".into()))); + + assert!(p.pivot_cache("xl/pivotCache/pivotCacheDefinition2.xml").unwrap().is_none()); + assert!(p.warnings().iter().any(|w| w.contains("pivotCacheDefinition2"))); + + let sheet = p.sheet(&wb.sheets[0], |_| Ok(())).unwrap(); + let tables = p.pivot_tables(&wb, &sheet).unwrap(); + assert_eq!(tables.len(), 2); + let (part, t, cache) = &tables[0]; + assert_eq!((part.as_str(), cache.as_deref()), ("xl/pivotTables/pivotTable1.xml", Some("xl/pivotCache/pivotCacheDefinition1.xml"))); + assert_eq!( + (t.name.as_deref(), t.cache_id, t.row_grand_totals, t.location.range.a1()), + (Some("PivotTable1"), Some(5), Some(false), "D1:E4".into()) + ); + assert_eq!(t.row_fields, [0, -2]); + let items = t.fields[0].items.as_ref().unwrap(); + assert_eq!((items[0].h, items[1].sd, items[2].item_type.as_deref()), (Some(true), Some(false), Some("default"))); + assert_eq!((t.fields[0].sort_type.as_deref(), t.fields[0].axis.as_deref()), (Some("descending"), Some("axisRow"))); + assert_eq!((t.page_fields[0].fld, t.page_fields[0].item), (Some(1), Some(0))); + assert!(t.data_fields[0].ext.as_ref().is_some_and(|e| e.find("x14:dataField").is_some())); + assert_eq!(t.style.as_ref().and_then(|s| s.name.as_deref()), Some("PivotStyleLight16")); + assert_eq!(t.other.iter().map(|e| e.name.as_str()).collect::>(), ["rowItems", "formats"]); + // The second table names its cache by id only. + assert_eq!(tables[1].2.as_deref(), Some("xl/pivotCache/pivotCacheDefinition2.xml")); +} + +fn read(bytes: &[u8]) -> Document { + Document::read(bytes, &Limits::default()).unwrap() +} + +#[test] +fn pivots_round_trip() { + let doc = read(&with_pivots()); + // The OLAP cache and its table are kept parts; the others are typed. + assert_eq!(doc.pivot_caches.len(), 1); + let ws = doc.worksheets[0].as_ref().unwrap(); + assert_eq!(ws.pivot_tables.len(), 1); + let mut kept: Vec<&str> = doc.parts.iter().map(|p| p.name.as_str()).filter(|n| n.contains("pivot")).collect(); + kept.sort(); + assert_eq!(kept, ["xl/pivotCache/pivotCacheDefinition2.xml", "xl/pivotTables/pivotTable2.xml"]); + let c = &doc.pivot_caches[0]; + assert_eq!((c.cache_id, c.records_part.as_str(), c.records.as_deref()), (5, "xl/pivotCache/pivotCacheRecords1.xml", Some(RECORDS.as_bytes()))); + + let bytes = doc.write().unwrap(); + let again = read(&bytes); + assert_eq!(again.pivot_caches, doc.pivot_caches); + assert_eq!(again.worksheets[0].as_ref().unwrap().pivot_tables, ws.pivot_tables); + assert_eq!(again.workbook.pivot_caches, doc.workbook.pivot_caches); + assert_eq!(again.parts.iter().filter(|p| p.name.contains("pivot")).count(), 2); + // The kept table still finds the kept cache by its id. + let mut p = Package::open(&bytes, &Limits::default()).unwrap(); + let wb = p.workbook().unwrap(); + let sheet = p.sheet(&wb.sheets[0], |_| Ok(())).unwrap(); + let tables = p.pivot_tables(&wb, &sheet).unwrap(); + assert_eq!( + tables.iter().map(|t| t.2.clone().unwrap_or_default()).collect::>(), + ["xl/pivotCache/pivotCacheDefinition1.xml", "xl/pivotCache/pivotCacheDefinition2.xml"] + ); + assert_eq!(read(&again.write().unwrap()).pivot_caches, doc.pivot_caches); +} + +#[test] +fn a_new_pivot_table() { + let mut doc = read(&craft_xlsx::testing::workbook(&[("Data", r#"1"#), ("Report", "")], &[])); + let values = [CacheValue::Number(1.0), CacheValue::Number(2.0)]; + let definition = PivotCacheDefinition { + refresh_on_load: Some(true), + source: craft_xlsx::CacheSource { + source_type: Some("worksheet".into()), + worksheet: Some(craft_xlsx::WorksheetSource { reference: Some("A1:A3".into()), sheet: Some("Data".into()), ..Default::default() }), + ..Default::default() + }, + fields: vec![craft_xlsx::CacheField { + name: Some("N".into()), + num_fmt_id: Some(0), + shared_items: Some(SharedItems { items: values.iter().cloned().map(CacheItem::new).collect(), ..SharedItems::describe(&values) }), + ..Default::default() + }], + ..Default::default() + }; + let records = records_xml(&[vec![CacheValue::Index(0)], vec![CacheValue::Index(1)]]); + doc.pivot_caches.push(PivotCachePart { cache_id: 1, definition, records: Some(records), ..Default::default() }); + let table = PivotTableDefinition { + name: Some("P".into()), + cache_id: Some(1), + location: Location { range: Range::parse("A3:B6").unwrap(), first_header_row: Some(1), ..Default::default() }, + fields: vec![PivotField { + axis: Some("axisRow".into()), + data_field: Some(true), + items: Some(vec![PivotItem { x: Some(0), ..Default::default() }, PivotItem { x: Some(1), ..Default::default() }]), + ..Default::default() + }], + row_fields: vec![0], + data_fields: vec![DataField { name: Some("Sum of N".into()), fld: Some(0), ..Default::default() }], + ..Default::default() + }; + let ws = doc.worksheets[1].as_mut().unwrap(); + ws.pivot_tables.push(PivotTablePart { table: table.clone(), ..Default::default() }); + let back = read(&doc.write().unwrap()); + let c = &back.pivot_caches[0]; + assert_eq!((c.part.as_str(), c.records_part.as_str()), ("xl/pivotCache/pivotCacheDefinition1.xml", "xl/pivotCache/pivotCacheRecords1.xml")); + assert_eq!(c.definition, doc.pivot_caches[0].definition); + assert_eq!(c.definition.fields[0].shared_items.as_ref().map(|s| (s.contains_integer, s.min_value)), Some((Some(true), Some(1.0)))); + assert_eq!(back.workbook.pivot_caches.len(), 1); + let t = &back.worksheets[1].as_ref().unwrap().pivot_tables[0]; + assert_eq!((t.part.as_str(), &t.table), ("xl/pivotTables/pivotTable1.xml", &table)); + + // Removing the cache removes its relationship and entry; a table left naming it is refused. + let mut gone = back.clone(); + gone.pivot_caches.clear(); + assert!(matches!(gone.write(), Err(Error::Invalid(_)))); + gone.worksheets[1].as_mut().unwrap().pivot_tables.clear(); + let after = read(&gone.write().unwrap()); + assert!(after.workbook.pivot_caches.is_empty() && after.pivot_caches.is_empty()); + assert!(!after.workbook_rels.list.iter().any(|r| r.kind() == "pivotCacheDefinition")); + assert!(!after.worksheets[1].as_ref().unwrap().part.rels.list.iter().any(|r| r.kind() == "pivotTable")); + + // Two caches may not share an id. + let mut twice = back.clone(); + twice.pivot_caches.push(PivotCachePart { part: String::new(), ..back.pivot_caches[0].clone() }); + assert!(matches!(twice.write(), Err(Error::Invalid(_)))); +}