mirror of
https://github.com/storytold/gridcraft.git
synced 2026-11-18 16:14:57 +00:00
* xlsx/csv: cap secondary reader allocations (#2159) - XML: at most 1M children per element, 16,384 per worksheet row (TooLarge). - Workbook-wide cap of 16M cell elements. - Array-formula ranges indexed by row (budgeted), not scanned per cell. - <cols> writes share a 4M budget. - At most 4096 sheets; duplicate names deduplicated with a HashSet. - Pivot: field lists capped, repeated fields used once, labels borrowed. - CSV: fields past MAX_COLS dropped while reading the record. - At most 4M shared strings. - Relationships looked up by id through a HashMap. * xlsx: close the remaining secondary-reader cap gaps (#2159) - CSV: characters of a field past column 16,384 are not stored; quotes, delimiters and line ends in it are still followed. - Pivot: a field list cut at MAX_FIELDS is reported with a warning. - Sheet names: the prefix shortens once the (n) suffix passes (999), so deduplicated names stay within 31 characters. - Rows dropped for a bad row number still charge their <c> elements against MAX_CELLS. - Test Ctx helper gains the fields upstream added. * #2159: describe MAX_CELLS as the reader's own cap calc::eval::MAX_CELLS has the same value but limits one array result, not a workbook. * #2159: warn on capped pivot cache fields, dedupe pivot names through sets A cache cut at 16,384 fields is now reported, as the table field lists are. Axis filters and calculated fields are deduplicated through a set of lower-cased names instead of a scan per field, which was quadratic in the 16,384 fields allowed; the first of names equal but for case still wins.
This commit is contained in:
1 parent
272ae54101
commit
a3119f57c2
8 files changed
+431
-76
No files matched your search
+17
-2
@@ -233,6 +233,7 @@ struct Import {
|
||||
text: u64,
|
||||
formats: Vec<(&'static str, StyleId)>,
|
||||
spreadsheet_seen: bool,
|
||||
names: crate::read::SheetNames,
|
||||
}
|
||||
|
||||
impl Import {
|
||||
@@ -241,7 +242,20 @@ impl Import {
|
||||
wb.sheets.clear();
|
||||
let mut report = ReadReport::default();
|
||||
report.warn("ODS data-only import: formulas become their saved results and will not recalculate. Source formatting, charts, images and other features are not imported. Save a new .xlsx file; ODS saving is not supported.");
|
||||
Self { wb, report, stack: vec![], sheet: None, row: None, cell: None, cells: 0, merges: 0, text: 0, formats: vec![], spreadsheet_seen: false }
|
||||
Self {
|
||||
wb,
|
||||
report,
|
||||
stack: vec![],
|
||||
sheet: None,
|
||||
row: None,
|
||||
cell: None,
|
||||
cells: 0,
|
||||
merges: 0,
|
||||
text: 0,
|
||||
formats: vec![],
|
||||
spreadsheet_seen: false,
|
||||
names: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse(&mut self, bytes: &[u8]) -> Result<(), IoError> {
|
||||
@@ -317,9 +331,10 @@ impl Import {
|
||||
return Err(IoError::TooLarge("ODS contains more than 1024 sheets".into()));
|
||||
}
|
||||
let original = attr(reader, e, TABLE, b"name")?.unwrap_or_default();
|
||||
let mut name = crate::read::SheetNames::of(&self.wb).unique(original.trim_matches('\''), self.wb.sheets.len());
|
||||
let mut name = self.names.unique(original.trim_matches('\''), self.wb.sheets.len());
|
||||
if self.wb.check_sheet_name(&name, None).is_err() {
|
||||
name = self.wb.next_sheet_name();
|
||||
self.names.insert(&name);
|
||||
}
|
||||
if name != original {
|
||||
self.report.warn("Some ODS sheet names were shortened or renamed to valid, unique XLSX sheet names.");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::{
|
||||
CellRef, Ctx, DateSystem, El, HashMap, IoError, MAX_FIELDS, PivotCalcField, PivotDateGroup, PivotField, PivotFilter, PivotFunc, PivotLayout,
|
||||
PivotShowAs, PivotSort, PivotTable, PivotValue, RangeRef, abs_range, func_caption, func_from, number_to_text, quote_sheet, same,
|
||||
PivotShowAs, PivotSort, PivotTable, PivotValue, RangeRef, abs_range, func_caption, func_from, number_to_text, quote_sheet,
|
||||
};
|
||||
|
||||
// ================================================================ reading
|
||||
@@ -118,8 +118,9 @@ fn read_cache(cx: &mut Ctx<'_>, part: &str, sheet_name: &str) -> Option<CacheIn>
|
||||
};
|
||||
let sys = cx.date_system;
|
||||
let mut fields = Vec::new();
|
||||
let mut truncated = false;
|
||||
if let Some(cf) = x.child("cacheFields") {
|
||||
for f in cf.kids("cacheField").take(MAX_FIELDS) {
|
||||
for f in capped(cf.kids("cacheField"), &mut truncated) {
|
||||
let items = f.child("sharedItems").map(|si| si.children.iter().map(|e| item_label_el(e, sys)).collect()).unwrap_or_default();
|
||||
let group = f.child("fieldGroup").map(|g| {
|
||||
let base = g.attr_u32("base").map(|b| b as usize);
|
||||
@@ -142,6 +143,9 @@ fn read_cache(cx: &mut Ctx<'_>, part: &str, sheet_name: &str) -> Option<CacheIn>
|
||||
});
|
||||
}
|
||||
}
|
||||
if truncated {
|
||||
cx.warn(format!("{part}: cache fields past {MAX_FIELDS} were dropped; its PivotTables may be incomplete"));
|
||||
}
|
||||
if fields.is_empty() {
|
||||
cx.warn("a PivotTable's cache has no fields; kept as plain cells");
|
||||
return None;
|
||||
@@ -231,14 +235,25 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
|
||||
cx.warn(format!("PivotTable \"{}\" has no valid location; kept as plain cells", pt.name));
|
||||
return None;
|
||||
};
|
||||
let pfs: Vec<&El> = x.child("pivotFields").map(|p| p.kids("pivotField").collect()).unwrap_or_default();
|
||||
// Field lists are capped, and a field is used once (Excel puts a field on one axis, once):
|
||||
// each use walks the field's items.
|
||||
// A list cut at the cap is reported, so an incomplete PivotTable can be told apart.
|
||||
let mut truncated = false;
|
||||
let pfs: Vec<&El> = x.child("pivotFields").map(|p| capped(p.kids("pivotField"), &mut truncated)).unwrap_or_default();
|
||||
let pf = |i: usize| pfs.get(i).copied();
|
||||
let idx_list = |name: &str, attr: &str| -> Vec<i64> {
|
||||
x.child(name).map(|e| e.children.iter().filter_map(|f| f.attr_i64(attr)).collect()).unwrap_or_default()
|
||||
let mut used_fields = std::collections::HashSet::new();
|
||||
let mut idx_list = |name: &str, attr: &str| -> Vec<i64> {
|
||||
x.child(name)
|
||||
.map(|e| capped(e.children.iter().filter_map(|f| f.attr_i64(attr)).filter(|&i| i < 0 || used_fields.insert(i)), &mut truncated))
|
||||
.unwrap_or_default()
|
||||
};
|
||||
let row_x = idx_list("rowFields", "x");
|
||||
let col_x = idx_list("colFields", "x");
|
||||
let pages: Vec<&El> = x.child("pageFields").map(|p| p.kids("pageField").collect()).unwrap_or_default();
|
||||
let mut page_fields = std::collections::HashSet::new();
|
||||
let pages: Vec<&El> = x
|
||||
.child("pageFields")
|
||||
.map(|p| capped(p.kids("pageField").filter(|f| f.attr_i64("fld").is_none_or(|i| page_fields.insert(i))), &mut truncated))
|
||||
.unwrap_or_default();
|
||||
|
||||
// Layout from the first axis field (pivotField defaults: compact and outline on).
|
||||
let first_axis = row_x.iter().chain(col_x.iter()).find(|i| **i >= 0).and_then(|i| pf(*i as usize));
|
||||
@@ -268,8 +283,11 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
|
||||
|
||||
let mut filters: Vec<PivotFilter> = Vec::new();
|
||||
|
||||
// Lower-cased names of the fields already filtered on an axis, the first of which wins (as
|
||||
// `same` compares them): scanning the filters for each field would be quadratic.
|
||||
let mut axis_filtered: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||
// (fields, position of Σ Values (`-2`) among them)
|
||||
let axis = |cx: &mut Ctx<'_>, list: &[i64], filters: &mut Vec<PivotFilter>| -> (Vec<PivotField>, Option<usize>) {
|
||||
let mut axis = |cx: &mut Ctx<'_>, list: &[i64], filters: &mut Vec<PivotFilter>| -> (Vec<PivotField>, Option<usize>) {
|
||||
let mut v = Vec::new();
|
||||
let mut values_at = None;
|
||||
for &i in list {
|
||||
@@ -282,7 +300,7 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
|
||||
}
|
||||
let i = i as usize;
|
||||
let Some((name, group)) = field_ref(cx, cache, i) else { continue };
|
||||
let labels = cache.fields.get(i).map(|f| f.labels().to_vec()).unwrap_or_default();
|
||||
let labels: &[String] = cache.fields.get(i).map(|f| f.labels()).unwrap_or(&[]);
|
||||
let mut f = PivotField { source_col: name.clone(), date_group: group, sort: PivotSort::None, collapsed_items: vec![] };
|
||||
if let Some(e) = pf(i) {
|
||||
f.sort = match e.attr("sortType") {
|
||||
@@ -293,11 +311,9 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
|
||||
let items = items_of(e);
|
||||
let lab = |it: &El| it.attr_u32("x").and_then(|k| labels.get(k as usize)).cloned();
|
||||
f.collapsed_items = items.iter().filter(|it| !it.flag("sd", true)).filter_map(|it| lab(it)).collect();
|
||||
if items.iter().any(|it| it.flag("h", false)) {
|
||||
if items.iter().any(|it| it.flag("h", false)) && axis_filtered.insert(name.to_lowercase()) {
|
||||
let shown: Vec<String> = items.iter().filter(|it| !it.flag("h", false)).filter_map(|it| lab(it)).collect();
|
||||
if !filters.iter().any(|ff| same(&ff.source_col, &name)) {
|
||||
filters.push(PivotFilter { source_col: name.clone(), selected: Some(shown) });
|
||||
}
|
||||
filters.push(PivotFilter { source_col: name.clone(), selected: Some(shown) });
|
||||
}
|
||||
}
|
||||
v.push(f);
|
||||
@@ -308,7 +324,7 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
|
||||
for p in &pages {
|
||||
let Some(i) = p.attr_i64("fld").filter(|i| *i >= 0).map(|i| i as usize) else { continue };
|
||||
let Some((name, _)) = field_ref(cx, cache, i) else { continue };
|
||||
let labels = cache.fields.get(i).map(|f| f.labels().to_vec()).unwrap_or_default();
|
||||
let labels: &[String] = cache.fields.get(i).map(|f| f.labels()).unwrap_or(&[]);
|
||||
let selected = if let Some(k) = p.attr_u32("item") {
|
||||
labels.get(k as usize).map(|l| vec![l.clone()])
|
||||
} else {
|
||||
@@ -341,7 +357,8 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
|
||||
|
||||
// Values.
|
||||
if let Some(df) = x.child("dataFields") {
|
||||
for d in df.kids("dataField") {
|
||||
truncated |= df.kids("dataField").nth(MAX_FIELDS).is_some();
|
||||
for d in df.kids("dataField").take(MAX_FIELDS) {
|
||||
let Some(i) = d.attr_u32("fld").map(|i| i as usize) else { continue };
|
||||
let Some(f) = cache.fields.get(i) else { continue };
|
||||
let func = match d.attr("subtotal") {
|
||||
@@ -382,11 +399,13 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
|
||||
}
|
||||
}
|
||||
|
||||
// Calculated fields of the cache (each PivotTable on it can use them).
|
||||
// Calculated fields of the cache (each PivotTable on it can use them); the first of a
|
||||
// lower-cased name wins, looked up in a set rather than by scanning the fields kept so far.
|
||||
let mut calc_names: std::collections::HashSet<String> = pt.calculated_fields.iter().map(|c| c.name.to_lowercase()).collect();
|
||||
for f in &cache.fields {
|
||||
if let Some(formula) = &f.formula
|
||||
&& !f.name.trim().is_empty()
|
||||
&& !pt.calculated_fields.iter().any(|c| same(&c.name, &f.name))
|
||||
&& calc_names.insert(f.name.to_lowercase())
|
||||
{
|
||||
let formula = formula.trim();
|
||||
let formula = if formula.starts_with('=') { formula.to_string() } else { format!("={formula}") };
|
||||
@@ -394,8 +413,21 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
|
||||
}
|
||||
}
|
||||
|
||||
if truncated {
|
||||
cx.warn(format!("PivotTable \"{}\": field lists past {MAX_FIELDS} entries were dropped; the table is incomplete", pt.name));
|
||||
}
|
||||
pt.anchor = loc.start;
|
||||
let above = if pages.is_empty() { 0 } else { pages.len() as u32 + 1 };
|
||||
pt.last_range = Some(RangeRef::new(CellRef::new(loc.start.row.saturating_sub(above), loc.start.col), loc.end));
|
||||
Some(pt)
|
||||
}
|
||||
|
||||
/// The first [`MAX_FIELDS`] items; sets `truncated` when there were more.
|
||||
fn capped<T>(items: impl Iterator<Item = T>, truncated: &mut bool) -> Vec<T> {
|
||||
let mut v: Vec<T> = items.take(MAX_FIELDS.saturating_add(1)).collect();
|
||||
if v.len() > MAX_FIELDS {
|
||||
v.truncate(MAX_FIELDS);
|
||||
*truncated = true;
|
||||
}
|
||||
v
|
||||
}
|
||||
+69
-44
@@ -46,6 +46,57 @@ pub struct Ctx<'a> {
|
||||
pub known_function: Option<&'a dyn Fn(&str) -> bool>,
|
||||
/// The code of the workbook's Python cells, from its Python part.
|
||||
pub python: crate::python_part::ScriptsIn,
|
||||
/// Cell elements read so far, across every sheet (capped by `sheet_read::MAX_CELLS`).
|
||||
pub cells: u64,
|
||||
}
|
||||
|
||||
/// The most worksheets read from one workbook (as the XLS reader allows).
|
||||
const MAX_SHEETS: usize = 4096;
|
||||
/// The most shared strings read from one workbook (as the XLS reader allows): each is an
|
||||
/// `Arc<str>` of its own, so a hundred million tiny `<si/>` would ask for gigabytes.
|
||||
const MAX_STRINGS: usize = 4_000_000;
|
||||
|
||||
/// Sheet names taken while reading, compared as `Workbook::sheet_index` compares them (ASCII case
|
||||
/// folded), with the next `(n)` suffix to try for each base, so renaming duplicates stays linear.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct SheetNames {
|
||||
taken: HashSet<String>,
|
||||
next_suffix: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
impl SheetNames {
|
||||
/// Records `name` as taken.
|
||||
pub(crate) fn insert(&mut self, name: &str) {
|
||||
self.taken.insert(name.to_ascii_lowercase());
|
||||
}
|
||||
|
||||
/// A valid name for the `i`-th sheet based on `name`, not yet taken; it is recorded as taken.
|
||||
pub(crate) fn unique(&mut self, name: &str, i: usize) -> String {
|
||||
let mut base: String = name.chars().filter(|c| !matches!(c, ':' | '\\' | '/' | '?' | '*' | '[' | ']')).take(31).collect();
|
||||
if base.trim().is_empty() {
|
||||
base = format!("Sheet{}", i + 1);
|
||||
}
|
||||
if self.taken.insert(base.to_ascii_lowercase()) {
|
||||
return base;
|
||||
}
|
||||
let key: String = base.chars().take(25).collect::<String>().to_ascii_lowercase();
|
||||
let mut n = self.next_suffix.get(&key).copied().unwrap_or(2);
|
||||
loop {
|
||||
// 25 characters of the base, fewer once the suffix grows past ` (999)`, so the name
|
||||
// stays within 31 characters however many duplicates there are.
|
||||
let suffix = format!(" ({n})");
|
||||
let short: String = base.chars().take(31usize.saturating_sub(suffix.len()).min(25)).collect();
|
||||
let candidate = format!("{short}{suffix}");
|
||||
n = n.saturating_add(1);
|
||||
if self.taken.insert(candidate.to_ascii_lowercase()) {
|
||||
self.next_suffix.insert(key, n);
|
||||
return candidate;
|
||||
}
|
||||
if n == usize::MAX {
|
||||
return base;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Ctx<'_> {
|
||||
@@ -120,6 +171,9 @@ fn read_shared_strings(cx: &mut Ctx<'_>, wb_rels: &[Rel]) -> Result<(), IoError>
|
||||
let mut phonetics = Vec::new();
|
||||
let mut dropped = 0usize;
|
||||
let res = xml::parse_streaming(&b, &["si"], &mut |si| {
|
||||
if sst.len() >= MAX_STRINGS {
|
||||
return Err(IoError::TooLarge(format!("the workbook has more than {MAX_STRINGS} shared strings")));
|
||||
}
|
||||
let text = xml::rich_text(&si);
|
||||
let raw = si.kids("rPh").count();
|
||||
let (runs, props) = xml::phonetic_data(&si, &text);
|
||||
@@ -130,8 +184,10 @@ fn read_shared_strings(cx: &mut Ctx<'_>, wb_rels: &[Rel]) -> Result<(), IoError>
|
||||
sst.push(Arc::from(text));
|
||||
Ok(())
|
||||
});
|
||||
if let Err(e) = res {
|
||||
cx.warn(format!("shared strings are damaged ({e}); kept {} strings", sst.len()));
|
||||
match res {
|
||||
Ok(_) => {}
|
||||
Err(IoError::TooLarge(m)) => return Err(IoError::TooLarge(m)),
|
||||
Err(e) => cx.warn(format!("shared strings are damaged ({e}); kept {} strings", sst.len())),
|
||||
}
|
||||
if dropped > 0 {
|
||||
cx.warn(format!("shared strings: {dropped} phonetic runs were dropped (bad sb/eb or no reading)"));
|
||||
@@ -267,6 +323,7 @@ pub fn read_xlsx<'a>(bytes: &'a [u8], opts: crate::ReadOptions<'a>) -> Result<(W
|
||||
shared_formula_bytes: 0,
|
||||
known_function,
|
||||
python: Default::default(),
|
||||
cells: 0,
|
||||
};
|
||||
let root_rels = cx.pkg.rels("")?;
|
||||
let wb_part = find_rel(&root_rels, "officeDocument").map(|r| r.target.clone()).unwrap_or_else(|| "xl/workbook.xml".into());
|
||||
@@ -318,11 +375,19 @@ pub fn read_xlsx<'a>(bytes: &'a [u8], opts: crate::ReadOptions<'a>) -> Result<(W
|
||||
// Legacy formula cells per model sheet, rewritten once names and tables are known.
|
||||
let mut legacy_formulas: Vec<(usize, Vec<gridcraft_core::CellRef>)> = vec![];
|
||||
let mut pivot_caches = crate::pivot::CachesIn::new(&wb_xml, &wb_rels);
|
||||
let sheet_els: Vec<El> = wb_xml.child("sheets").map(|s| s.kids("sheet").cloned().collect()).unwrap_or_default();
|
||||
let sheet_els: Vec<&El> = wb_xml.child("sheets").map(|s| s.kids("sheet").collect()).unwrap_or_default();
|
||||
if sheet_els.len() > MAX_SHEETS {
|
||||
return Err(IoError::TooLarge(format!("the workbook has more than {MAX_SHEETS} sheets")));
|
||||
}
|
||||
let mut rel_by_id: HashMap<&str, &Rel> = HashMap::with_capacity(wb_rels.len());
|
||||
for r in &wb_rels {
|
||||
// The first of duplicate ids wins, as a linear search would find it.
|
||||
rel_by_id.entry(r.id.as_str()).or_insert(r);
|
||||
}
|
||||
let mut names = SheetNames::default();
|
||||
for (i, se) in sheet_els.iter().enumerate() {
|
||||
let name = se.attr("name").unwrap_or("").to_string();
|
||||
let rel = se.attr("id").and_then(|id| wb_rels.iter().find(|r| r.id == id)).cloned();
|
||||
let rel = se.attr("id").and_then(|id| rel_by_id.get(id).copied()).cloned();
|
||||
let Some(rel) = rel else {
|
||||
cx.warn(format!("sheet \"{name}\" has no part; skipped"));
|
||||
file_to_model.push(None);
|
||||
@@ -476,46 +541,6 @@ pub fn read_xlsx<'a>(bytes: &'a [u8], opts: crate::ReadOptions<'a>) -> Result<(W
|
||||
Ok((wb, cx.report))
|
||||
}
|
||||
|
||||
/// The sheet names an import has assigned so far, for repairing repeated ones.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct SheetNames {
|
||||
/// Assigned names, ASCII-lowercased.
|
||||
taken: HashSet<String>,
|
||||
/// For each lowercased stem, the next suffix to try: a file full of repeated names would
|
||||
/// otherwise retry every occupied suffix for every sheet.
|
||||
next: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
impl SheetNames {
|
||||
pub(crate) fn of(wb: &Workbook) -> Self {
|
||||
SheetNames { taken: wb.sheets.iter().map(|s| s.name.to_ascii_lowercase()).collect(), next: HashMap::new() }
|
||||
}
|
||||
|
||||
/// A valid name for the i-th sheet, unique among the names assigned so far, which it joins.
|
||||
pub(crate) fn unique(&mut self, name: &str, i: usize) -> String {
|
||||
let mut base: String = name.chars().filter(|c| !matches!(c, ':' | '\\' | '/' | '?' | '*' | '[' | ']')).take(31).collect();
|
||||
if base.trim().is_empty() {
|
||||
base = format!("Sheet{}", i + 1);
|
||||
}
|
||||
let name = if self.taken.contains(&base.to_ascii_lowercase()) {
|
||||
let stem: String = base.chars().take(25).collect();
|
||||
let next = self.next.entry(stem.to_ascii_lowercase()).or_insert(2);
|
||||
let found = (*next..).find(|n| !self.taken.contains(&format!("{stem} ({n})").to_ascii_lowercase()));
|
||||
match found {
|
||||
Some(n) => {
|
||||
*next = n + 1;
|
||||
format!("{stem} ({n})")
|
||||
}
|
||||
None => base,
|
||||
}
|
||||
} else {
|
||||
base
|
||||
};
|
||||
self.taken.insert(name.to_ascii_lowercase());
|
||||
name
|
||||
}
|
||||
}
|
||||
|
||||
/// Strips a sheet qualifier (`'My Sheet'!$A$1` → `$A$1`).
|
||||
fn strip_sheet(s: &str) -> &str {
|
||||
match s.rfind('!') {
|
||||
|
||||
@@ -134,6 +134,9 @@ fn settle_python_spills_past_budget(st: &RowState, sheet: &mut Sheet) {
|
||||
}
|
||||
}
|
||||
|
||||
/// The most cell elements read from one workbook, across its sheets: the reader's own cap.
|
||||
/// (`calc::eval::MAX_CELLS` has the same value but limits one array result, not a workbook.)
|
||||
pub(crate) const MAX_CELLS: u64 = 16_000_000;
|
||||
/// Target and tooltip bytes kept for hyperlinks of one sheet, over all cells: the same URL is
|
||||
/// copied for every cell of its range, so length × cells must stay bounded whatever the file.
|
||||
const MAX_HYPERLINK_BYTES: u64 = 8 * 1024 * 1024;
|
||||
@@ -191,12 +194,10 @@ pub fn read_sheet(cx: &mut Ctx<'_>, part: &str, name: &str) -> Result<(Sheet, cr
|
||||
let rels = cx.pkg.rels(part)?;
|
||||
let mut sheet = Sheet::new(name);
|
||||
let mut st = RowState::default();
|
||||
let res = xml::parse_streaming(&bytes, &["row"], &mut |row| {
|
||||
read_row(cx, &mut sheet, &mut st, &row);
|
||||
Ok(())
|
||||
});
|
||||
let res = xml::parse_streaming(&bytes, &["row"], &mut |row| read_row(cx, &mut sheet, &mut st, &row));
|
||||
let root = match res {
|
||||
Ok(r) => r,
|
||||
Err(IoError::TooLarge(m)) => return Err(IoError::TooLarge(m)),
|
||||
Err(e) => {
|
||||
cx.warn(format!("sheet \"{name}\" is damaged ({e}); kept {} rows", st.rows));
|
||||
El::default()
|
||||
@@ -208,6 +209,12 @@ pub fn read_sheet(cx: &mut Ctx<'_>, part: &str, name: &str) -> Result<(Sheet, cr
|
||||
settle_python_spills_past_budget(&st, &mut sheet);
|
||||
drop(bytes);
|
||||
|
||||
// Relationships by id: hyperlinks, drawings and tables each look one up.
|
||||
let mut rel_by_id: HashMap<&str, &Rel> = HashMap::with_capacity(rels.len());
|
||||
for r in &rels {
|
||||
// The first of duplicate ids wins, as a linear search would find it.
|
||||
rel_by_id.entry(r.id.as_str()).or_insert(r);
|
||||
}
|
||||
let mut fit_to_page = false;
|
||||
let mut link_budget = LinkBudget::default();
|
||||
let mut cf_kept_cells = 0u64;
|
||||
@@ -245,7 +252,7 @@ pub fn read_sheet(cx: &mut Ctx<'_>, part: &str, name: &str) -> Result<(Sheet, cr
|
||||
}
|
||||
}
|
||||
}
|
||||
"hyperlinks" => read_hyperlinks(cx, name, e, &rels, &mut sheet, &mut link_budget),
|
||||
"hyperlinks" => read_hyperlinks(cx, name, e, &rel_by_id, &mut sheet, &mut link_budget),
|
||||
"autoFilter" => sheet.autofilter = read_autofilter(cx, e),
|
||||
"printOptions" => {
|
||||
sheet.print.gridlines = e.flag("gridLines", false);
|
||||
@@ -319,14 +326,14 @@ pub fn read_sheet(cx: &mut Ctx<'_>, part: &str, name: &str) -> Result<(Sheet, cr
|
||||
});
|
||||
}
|
||||
"drawing" => {
|
||||
if let Some(rel) = e.attr("id").and_then(|id| rels.iter().find(|r| r.id == id)) {
|
||||
if let Some(rel) = e.attr("id").and_then(|id| rel_by_id.get(id).copied()) {
|
||||
let target = rel.target.clone();
|
||||
crate::drawing::read_drawing(cx, &target, &mut sheet)?;
|
||||
}
|
||||
}
|
||||
"tableParts" => {
|
||||
for tp in e.kids("tablePart") {
|
||||
if let Some(rel) = tp.attr("id").and_then(|id| rels.iter().find(|r| r.id == id)) {
|
||||
if let Some(rel) = tp.attr("id").and_then(|id| rel_by_id.get(id).copied()) {
|
||||
let target = rel.target.clone();
|
||||
if let Some(x) = cx.optional_xml(&target)?
|
||||
&& let Some(mut t) = read_table(cx, &x)
|
||||
@@ -496,19 +503,30 @@ fn read_cols(cx: &Ctx<'_>, e: &El, sheet: &mut Sheet, remaining: &mut u64) -> Re
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_row(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, row: &El) {
|
||||
/// Charges `n` cell elements against the workbook's [`MAX_CELLS`].
|
||||
fn charge_cells(cx: &mut Ctx<'_>, n: u64) -> Result<(), IoError> {
|
||||
cx.cells = cx.cells.saturating_add(n);
|
||||
if cx.cells > MAX_CELLS {
|
||||
return Err(IoError::TooLarge(format!("the workbook has more than {MAX_CELLS} cells")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One `<row>`. Every `<c>` in it counts against [`MAX_CELLS`], also in a row that is dropped
|
||||
/// for a bad row number: one worksheet part read for many sheets can't parse cells uncounted.
|
||||
fn read_row(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, row: &El) -> Result<(), IoError> {
|
||||
st.rows += 1;
|
||||
let r = match row.attr_u32("r") {
|
||||
Some(r) if r >= 1 => r - 1,
|
||||
Some(_) => {
|
||||
st.dropped += 1;
|
||||
return;
|
||||
return charge_cells(cx, row.kids("c").count() as u64);
|
||||
}
|
||||
None => st.next_row,
|
||||
};
|
||||
if r >= MAX_ROWS {
|
||||
st.dropped += 1;
|
||||
return;
|
||||
return charge_cells(cx, row.kids("c").count() as u64);
|
||||
}
|
||||
st.next_row = r + 1;
|
||||
let mut info = LineInfo::default();
|
||||
@@ -532,6 +550,7 @@ fn read_row(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, row: &El) {
|
||||
}
|
||||
let mut next_col = 0u32;
|
||||
for c in row.kids("c") {
|
||||
charge_cells(cx, 1)?;
|
||||
let pos = match c.attr("r") {
|
||||
Some(a) => match CellRef::parse(a.trim()) {
|
||||
Some(p) => p,
|
||||
@@ -549,6 +568,7 @@ fn read_row(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, row: &El) {
|
||||
next_col = pos.col + 1;
|
||||
read_cell(cx, sheet, st, c, pos);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_cell(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, c: &El, pos: CellRef) {
|
||||
@@ -929,6 +949,57 @@ mod tests {
|
||||
assert_eq!(parse_iso_datetime("2024-13-01", DateSystem::D1900), None);
|
||||
}
|
||||
|
||||
fn ctx(pkg: crate::package::Package<'_>) -> Ctx<'_> {
|
||||
Ctx {
|
||||
pkg,
|
||||
report: Default::default(),
|
||||
sst: vec![],
|
||||
phonetics: vec![],
|
||||
xf_map: vec![],
|
||||
dxfs: vec![],
|
||||
palette: vec![],
|
||||
date_system: DateSystem::D1900,
|
||||
styles: Default::default(),
|
||||
persons: Default::default(),
|
||||
next_id: 1,
|
||||
num_fmts: Default::default(),
|
||||
cell_pictures: vec![],
|
||||
cf_ext_ids: vec![],
|
||||
known_function: None,
|
||||
python: Default::default(),
|
||||
cells: 0,
|
||||
shared_formula_bytes: 0,
|
||||
legacy_formulas: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
/// #2159: the workbook-wide cell cap refuses the open once passed, across sheets.
|
||||
#[test]
|
||||
fn cells_past_the_workbook_cap_are_refused() {
|
||||
let bytes = crate::tests::minimal(r#"<sheetData><row r="1"><c r="A1"><v>1</v></c><c r="B1"><v>2</v></c></row></sheetData>"#, &[], "", "");
|
||||
let pkg = crate::package::Package::open(&bytes).unwrap();
|
||||
let mut cx = ctx(pkg);
|
||||
cx.cells = MAX_CELLS - 2;
|
||||
let (sheet, _) = read_sheet(&mut cx, "xl/worksheets/sheet1.xml", "Data").unwrap();
|
||||
assert_eq!(sheet.cells.len(), 2);
|
||||
assert!(matches!(read_sheet(&mut cx, "xl/worksheets/sheet1.xml", "Data"), Err(IoError::TooLarge(_))));
|
||||
}
|
||||
|
||||
/// #2159: the cells of a row dropped for its row number still count against the cap, so a
|
||||
/// part read for many sheets can't parse cells uncounted.
|
||||
#[test]
|
||||
fn cells_of_dropped_rows_count_against_the_cap() {
|
||||
let rows = r#"<sheetData><row r="0"><c><v>1</v></c><c><v>2</v></c></row><row r="2000000"><c><v>3</v></c></row></sheetData>"#;
|
||||
let bytes = crate::tests::minimal(rows, &[], "", "");
|
||||
let pkg = crate::package::Package::open(&bytes).unwrap();
|
||||
let mut cx = ctx(pkg);
|
||||
let (sheet, _) = read_sheet(&mut cx, "xl/worksheets/sheet1.xml", "Data").unwrap();
|
||||
assert!(sheet.cells.is_empty(), "both rows are dropped");
|
||||
assert_eq!(cx.cells, 3);
|
||||
cx.cells = MAX_CELLS - 2;
|
||||
assert!(matches!(read_sheet(&mut cx, "xl/worksheets/sheet1.xml", "Data"), Err(IoError::TooLarge(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sqref_keeps_more_than_ten_thousand_ranges() {
|
||||
let sq = (1..=12_000).map(|r| format!("A{r}")).collect::<Vec<_>>().join(" ");
|
||||
|
||||
@@ -4,12 +4,12 @@ use super::{
|
||||
from_file, truncate_chars, xml,
|
||||
};
|
||||
|
||||
/// `budget` counts the cells and bytes this sheet's hyperlinks have copied so far, across
|
||||
/// `<hyperlinks>` blocks.
|
||||
pub(super) fn read_hyperlinks(cx: &mut Ctx<'_>, name: &str, e: &El, rels: &[Rel], sheet: &mut Sheet, budget: &mut LinkBudget) {
|
||||
/// `rels` maps relationship ids to relationships (the first of duplicate ids). `budget` counts the
|
||||
/// cells and bytes this sheet's hyperlinks have copied so far, across `<hyperlinks>` blocks.
|
||||
pub(super) fn read_hyperlinks(cx: &mut Ctx<'_>, name: &str, e: &El, rels: &HashMap<&str, &Rel>, sheet: &mut Sheet, budget: &mut LinkBudget) {
|
||||
for h in e.kids("hyperlink") {
|
||||
let Some(range) = h.attr("ref").and_then(RangeRef::parse) else { continue };
|
||||
let rel_target = h.attr("id").and_then(|id| rels.iter().find(|r| r.id == id)).map(|r| r.target.as_str()).unwrap_or("");
|
||||
let rel_target = h.attr("id").and_then(|id| rels.get(id)).map(|r| r.target.as_str()).unwrap_or("");
|
||||
let location = h.attr("location").filter(|l| !l.is_empty());
|
||||
// Each covered cell gets its own copy of the link: refuse oversized targets before copying.
|
||||
if rel_target.len().saturating_add(location.map_or(0, str::len)) > MAX_HYPERLINK_TARGET {
|
||||
|
||||
@@ -360,6 +360,43 @@ fn the_cap_counts_the_bytes_the_cells_hold_and_accepted_exports_keep_every_posit
|
||||
assert_eq!(tabs.len(), 14);
|
||||
}
|
||||
|
||||
/// #2159: a record keeps at most one field per sheet column while it is read, not after; the rest
|
||||
/// of the line is still read through, so the next line starts the next record.
|
||||
#[test]
|
||||
fn fields_past_the_last_column_are_not_collected() {
|
||||
let src = format!("{}last\nnext,row\n", "x,".repeat(100_000));
|
||||
let mut records = vec![];
|
||||
parse_records(&src, ',', '"', MAX_COLS as usize, |rec| {
|
||||
records.push(rec);
|
||||
true
|
||||
});
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0].len(), 16_384);
|
||||
assert_eq!(records[1], vec!["next".to_string(), "row".to_string()]);
|
||||
// The same through the reader, and a quoted delimiter past the cap doesn't split the line.
|
||||
let src = format!("{}\"a,\nb\",z\nnext\n", "x,".repeat(20_000));
|
||||
let wb = read_csv(src.as_bytes(), &CsvOptions { delimiter: b',', ..CsvOptions::default() }).unwrap();
|
||||
assert_eq!(v(&wb, 0, 16_383), Value::text("x"));
|
||||
assert_eq!(v(&wb, 1, 0), Value::text("next"));
|
||||
assert_eq!(wb.sheet(0).unwrap().cells.len(), 16_385);
|
||||
}
|
||||
|
||||
/// #2159: a long field past the last column is skipped character by character, not collected
|
||||
/// and then thrown away; a quote inside it is a plain character, as in a kept field.
|
||||
#[test]
|
||||
fn a_long_field_past_the_last_column_is_not_stored() {
|
||||
let src = format!("{}{}a\"b,{}\nnext,row\n", "x,".repeat(16_384), "y".repeat(1_000_000), "z".repeat(1_000));
|
||||
let mut records = vec![];
|
||||
parse_records(&src, ',', '"', MAX_COLS as usize, |rec| {
|
||||
records.push(rec);
|
||||
true
|
||||
});
|
||||
assert_eq!(records.len(), 2, "the quote didn't open a quoted field across the line end");
|
||||
assert_eq!(records[0].len(), 16_384);
|
||||
assert!(records[0].iter().all(|f| f == "x"));
|
||||
assert_eq!(records[1], vec!["next".to_string(), "row".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_stops_collecting_fields_at_max_cols() {
|
||||
let extra = 50;
|
||||
|
||||
@@ -280,6 +280,119 @@ fn hostile_formula_text_does_not_overflow_the_stack() {
|
||||
assert_eq!(a_value, gridcraft_core::Value::Number(3.0));
|
||||
}
|
||||
|
||||
/// A one-sheet-per-entry workbook: `names` are the sheet names, every sheet sharing one part.
|
||||
fn many_sheets(names: &[String]) -> Vec<u8> {
|
||||
let sheets: String = names.iter().enumerate().map(|(i, n)| format!(r#"<sheet name="{n}" sheetId="{}" r:id="rId1"/>"#, i + 1)).collect();
|
||||
let wb = format!(
|
||||
r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets>{sheets}</sheets></workbook>"#
|
||||
);
|
||||
let rels = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>"#;
|
||||
let sheet = r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData/></worksheet>"#;
|
||||
make_zip(&[
|
||||
("[Content_Types].xml", super::CT.as_bytes()),
|
||||
("_rels/.rels", super::ROOT_RELS.as_bytes()),
|
||||
("xl/workbook.xml", wb.as_bytes()),
|
||||
("xl/_rels/workbook.xml.rels", rels.as_bytes()),
|
||||
("xl/worksheets/sheet1.xml", sheet.as_bytes()),
|
||||
])
|
||||
}
|
||||
|
||||
/// #2159: a row of endless `<c>` is not built in memory: the part's kept-element cap
|
||||
/// ([`crate::xml::MAX_XML_NODES`]) stops it, and the sheet is reported as damaged.
|
||||
#[test]
|
||||
fn a_row_wider_than_the_grid_is_refused() {
|
||||
let row = |n: usize| format!(r#"<sheetData><row r="1">{}</row></sheetData>"#, "<c/>".repeat(n));
|
||||
let (wb, rep) = read_xlsx(&minimal(&row(crate::xml::MAX_XML_NODES + 1), &[], "", "")).unwrap();
|
||||
assert!(wb.sheet(0).unwrap().cells.is_empty());
|
||||
assert!(rep.warnings.iter().any(|w| w.contains("damaged")), "{:?}", rep.warnings);
|
||||
assert!(read_xlsx(&minimal(&row(16_384), &[], "", "")).is_ok());
|
||||
// Every other part has the cap too (a merge list of more than a million entries).
|
||||
let merges = format!("<sheetData/><mergeCells>{}</mergeCells>", r#"<mergeCell ref="A1:B2"/>"#.repeat(crate::xml::MAX_XML_NODES + 1));
|
||||
let (wb, rep) = read_xlsx(&minimal(&merges, &[], "", "")).unwrap();
|
||||
assert!(wb.sheet(0).unwrap().merges.is_empty());
|
||||
assert!(rep.warnings.iter().any(|w| w.contains("damaged")), "{:?}", rep.warnings);
|
||||
}
|
||||
|
||||
/// #2159: overlapping `<col min=1 max=16384>` entries share one budget of column writes.
|
||||
#[test]
|
||||
fn overlapping_column_entries_have_a_budget() {
|
||||
let parts = [("xl/styles.xml", r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"/>"#)];
|
||||
let rels = r#"<Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/>"#;
|
||||
let cols = format!("<cols>{}</cols><sheetData/>", r#"<col min="1" max="16384" width="20" customWidth="1"/>"#.repeat(300));
|
||||
assert!(matches!(read_xlsx(&minimal(&cols, &parts, rels, "")), Err(IoError::TooLarge(_))));
|
||||
// A normal list reads with no warning.
|
||||
let (wb, rep) = read_xlsx(&minimal(r#"<cols><col min="2" max="4" width="20" customWidth="1"/></cols><sheetData/>"#, &parts, rels, "")).unwrap();
|
||||
assert_eq!(wb.sheet(0).unwrap().cols.len(), 3);
|
||||
assert!(rep.warnings.is_empty(), "{:?}", rep.warnings);
|
||||
}
|
||||
|
||||
/// #2159: the sheet count is capped, and renaming duplicate sheet names stays linear.
|
||||
#[test]
|
||||
fn sheet_count_is_capped_and_duplicate_names_are_renamed() {
|
||||
let names: Vec<String> = (0..4097).map(|i| format!("S{i}")).collect();
|
||||
assert!(matches!(read_xlsx(&many_sheets(&names)), Err(IoError::TooLarge(_))));
|
||||
let dups: Vec<String> = (0..300).map(|i| if i % 2 == 0 { "a".to_string() } else { "A".to_string() }).collect();
|
||||
let (wb, _) = read_xlsx(&many_sheets(&dups)).unwrap();
|
||||
assert_eq!(wb.sheets.len(), 300);
|
||||
assert_eq!(wb.sheets[0].name, "a");
|
||||
assert_eq!(wb.sheets[1].name, "A (2)");
|
||||
assert_eq!(wb.sheets[299].name, "A (300)");
|
||||
let distinct: std::collections::HashSet<String> = wb.sheets.iter().map(|s| s.name.to_ascii_lowercase()).collect();
|
||||
assert_eq!(distinct.len(), 300);
|
||||
// The name set alone: thousands of duplicates, with a suffixed name already taken.
|
||||
let mut taken = crate::read::SheetNames::default();
|
||||
assert_eq!(taken.unique("x (3)", 0), "x (3)");
|
||||
let got: Vec<String> = (0..4096).map(|i| taken.unique("X", i)).collect();
|
||||
assert_eq!(got[0], "X");
|
||||
assert_eq!(got[1], "X (2)");
|
||||
assert_eq!(got[2], "X (4)");
|
||||
assert_eq!(got.iter().map(|n| n.to_ascii_lowercase()).collect::<std::collections::HashSet<_>>().len(), 4096);
|
||||
assert_eq!(taken.unique("", 9), "Sheet10");
|
||||
// A 25-character name repeated past ` (999)` shortens its prefix so every name fits 31.
|
||||
let mut taken = crate::read::SheetNames::default();
|
||||
let base = "N".repeat(25);
|
||||
let got: Vec<String> = (0..4096).map(|i| taken.unique(&base, i)).collect();
|
||||
assert!(got.iter().all(|n| n.chars().count() <= 31), "{:?}", got.iter().find(|n| n.chars().count() > 31));
|
||||
assert_eq!(got[998], format!("{base} (999)"));
|
||||
assert_eq!(got[999], format!("{} (1000)", "N".repeat(24)));
|
||||
assert_eq!(got.iter().map(|n| n.to_ascii_lowercase()).collect::<std::collections::HashSet<_>>().len(), 4096);
|
||||
}
|
||||
|
||||
/// #2159: the shared-string count is capped (each string is an allocation of its own).
|
||||
#[test]
|
||||
fn too_many_shared_strings_are_refused() {
|
||||
let sst = |n: usize| format!(r#"<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">{}</sst>"#, "<si/>".repeat(n));
|
||||
let rels = r#"<Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" Target="sharedStrings.xml"/>"#;
|
||||
let big = sst(4_000_001);
|
||||
assert!(matches!(read_xlsx(&minimal("<sheetData/>", &[("xl/sharedStrings.xml", &big)], rels, "")), Err(IoError::TooLarge(_))));
|
||||
let small = sst(3);
|
||||
assert!(read_xlsx(&minimal("<sheetData/>", &[("xl/sharedStrings.xml", &small)], rels, "")).is_ok());
|
||||
}
|
||||
|
||||
/// #2159: hyperlinks find their relationship by id (a map, not a scan per hyperlink); the first
|
||||
/// of duplicate ids wins, as before.
|
||||
#[test]
|
||||
fn hyperlinks_resolve_relationships_by_id() {
|
||||
let n = 5_000;
|
||||
let mut rels = String::from(r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">"#);
|
||||
for i in 0..n {
|
||||
rels.push_str(&format!(
|
||||
r#"<Relationship Id="h{i}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink" Target="https://example.com/{i}" TargetMode="External"/>"#
|
||||
));
|
||||
}
|
||||
rels.push_str(r#"<Relationship Id="h0" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink" Target="https://example.com/second" TargetMode="External"/>"#);
|
||||
rels.push_str("</Relationships>");
|
||||
let links: String = (0..n).map(|i| format!(r#"<hyperlink ref="A{}" r:id="h{i}"/>"#, i + 1)).collect();
|
||||
let body = format!(r#"<sheetData/><hyperlinks>{links}<hyperlink ref="B1" r:id="missing"/></hyperlinks>"#);
|
||||
let (wb, _) = read_xlsx(&minimal(&body, &[("xl/worksheets/_rels/sheet1.xml.rels", &rels)], "", "")).unwrap();
|
||||
let s = wb.sheet(0).unwrap();
|
||||
assert_eq!(s.hyperlinks.len(), n);
|
||||
let link = |a: &str| s.hyperlinks.get(&gridcraft_core::CellRef::parse(a).unwrap()).map(|h| h.target.clone());
|
||||
assert_eq!(link("A1").as_deref(), Some("https://example.com/0"));
|
||||
assert_eq!(link("A5000").as_deref(), Some("https://example.com/4999"));
|
||||
assert_eq!(link("B1"), None);
|
||||
}
|
||||
|
||||
/// #2173: a hyperlink is copied into every cell it covers, so oversized targets are dropped,
|
||||
/// ScreenTips are cut to 255 characters and the cell cap holds across `<hyperlinks>` blocks.
|
||||
#[test]
|
||||
|
||||
@@ -545,6 +545,68 @@ fn hostile_pivot_parts_warn_not_panic() {
|
||||
assert!(report.warnings.iter().any(|w| w.contains("index")), "{:?}", report.warnings);
|
||||
}
|
||||
|
||||
/// #2159: a field repeated on the axes or as a page field is used once (each use walks the
|
||||
/// field's items and copied its labels), and the lists are capped.
|
||||
#[test]
|
||||
fn repeated_pivot_fields_are_used_once() {
|
||||
let items: String = (0..2_000).map(|_| r#"<item x="0" sd="0"/>"#).collect();
|
||||
let rows = r#"<field x="0"/>"#.repeat(20_000);
|
||||
let pages = r#"<pageField fld="0"/>"#.repeat(20_000);
|
||||
let table = format!(
|
||||
r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="1"><pivotField axis="axisRow"><items>{items}</items></pivotField></pivotFields><rowFields>{rows}<field x="-2"/></rowFields><colFields>{rows}</colFields><pageFields>{pages}</pageFields></pivotTableDefinition>"#
|
||||
);
|
||||
let (wb, report) = read_xlsx(&with_pivot(&table, Some(GOOD_CACHE))).unwrap();
|
||||
let pt = &wb.sheet(0).unwrap().pivots[0];
|
||||
assert_eq!(pt.rows.len(), 1, "{:?}", report.warnings);
|
||||
assert!(pt.columns.is_empty());
|
||||
assert_eq!(pt.rows[0].collapsed_items.len(), 2_000);
|
||||
assert_eq!(pt.filters.iter().filter(|f| f.source_col == "K").count(), 1);
|
||||
// One page field: the table's range starts two rows above its location.
|
||||
assert_eq!(pt.last_range, Some(rr("D3:F9")));
|
||||
}
|
||||
|
||||
/// #2159: a field list cut at the 16,384-entry cap is reported, so an incomplete PivotTable can be
|
||||
/// told apart from a complete one.
|
||||
#[test]
|
||||
fn capped_pivot_field_lists_warn() {
|
||||
let fields = r#"<pivotField/>"#.repeat(16_385);
|
||||
let table = format!(
|
||||
r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="16385">{fields}</pivotFields></pivotTableDefinition>"#
|
||||
);
|
||||
let (_, report) = read_xlsx(&with_pivot(&table, Some(GOOD_CACHE))).unwrap();
|
||||
assert!(report.warnings.iter().any(|w| w.contains("field lists past 16384 entries")), "{:?}", report.warnings);
|
||||
let table = r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="1"><pivotField/></pivotFields></pivotTableDefinition>"#;
|
||||
let (_, report) = read_xlsx(&with_pivot(table, Some(GOOD_CACHE))).unwrap();
|
||||
assert!(!report.warnings.iter().any(|w| w.contains("field lists past")), "{:?}", report.warnings);
|
||||
}
|
||||
|
||||
/// #2159: a cache cut at 16,384 fields is reported too, as the table field lists are.
|
||||
#[test]
|
||||
fn capped_cache_fields_warn() {
|
||||
let fields = r#"<cacheField name="F"><sharedItems/></cacheField>"#.repeat(16_385);
|
||||
let cache = format!(
|
||||
r#"<pivotCacheDefinition xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><cacheSource type="worksheet"><worksheetSource ref="A1:B3"/></cacheSource><cacheFields count="16385">{fields}</cacheFields></pivotCacheDefinition>"#
|
||||
);
|
||||
let table = r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="1"><pivotField/></pivotFields></pivotTableDefinition>"#;
|
||||
let (_, report) = read_xlsx(&with_pivot(table, Some(&cache))).unwrap();
|
||||
assert!(report.warnings.iter().any(|w| w.contains("cache fields past 16384")), "{:?}", report.warnings);
|
||||
let (_, report) = read_xlsx(&with_pivot(table, Some(GOOD_CACHE))).unwrap();
|
||||
assert!(!report.warnings.iter().any(|w| w.contains("cache fields past")), "{:?}", report.warnings);
|
||||
}
|
||||
|
||||
/// #2159: axis filters and calculated fields are deduplicated through a set of lower-cased names;
|
||||
/// the first of names equal but for case still wins, as with the scan it replaced.
|
||||
#[test]
|
||||
fn axis_filters_and_calculated_fields_dedupe_ignoring_case() {
|
||||
let cache = r#"<pivotCacheDefinition xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><cacheSource type="worksheet"><worksheetSource ref="A1:B3"/></cacheSource><cacheFields count="4"><cacheField name="K"><sharedItems><s v="a"/><s v="b"/></sharedItems></cacheField><cacheField name="k"><sharedItems><s v="a"/><s v="b"/></sharedItems></cacheField><cacheField name="Price" formula="1" databaseField="0"/><cacheField name="PRICE" formula="2" databaseField="0"/></cacheFields></pivotCacheDefinition>"#;
|
||||
let table = r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="2"><pivotField axis="axisRow"><items><item x="0"/><item x="1" h="1"/></items></pivotField><pivotField axis="axisCol"><items><item x="0" h="1"/><item x="1"/></items></pivotField></pivotFields><rowFields><field x="0"/></rowFields><colFields><field x="1"/></colFields></pivotTableDefinition>"#;
|
||||
let (wb, report) = read_xlsx(&with_pivot(table, Some(cache))).unwrap();
|
||||
let pt = &wb.sheet(0).unwrap().pivots[0];
|
||||
assert_eq!((pt.rows.len(), pt.columns.len()), (1, 1), "{:?}", report.warnings);
|
||||
assert_eq!(pt.filters, vec![PivotFilter { source_col: "K".into(), selected: Some(vec!["a".into()]) }]);
|
||||
assert_eq!(pt.calculated_fields, vec![PivotCalcField { name: "Price".into(), formula: "=1".into() }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_found_by_workbook_cache_id() {
|
||||
let rels_extra = r#"<Relationship Id="rId9" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/pivotCacheDefinition" Target="pivotCache/pivotCacheDefinition1.xml"/>"#;
|
||||
|
||||
Reference in new issue
Block a user