xlsx/csv: cap secondary reader allocations (#2159) (#2199)

* xlsx/csv: cap secondary reader allocations (#2159)
  
  - XML: at most 1M children per element, 16,384 per worksheet row (TooLarge).
  - Workbook-wide cap of 16M cell elements.
  - Array-formula ranges indexed by row (budgeted), not scanned per cell.
  - <cols> writes share a 4M budget.
  - At most 4096 sheets; duplicate names deduplicated with a HashSet.
  - Pivot: field lists capped, repeated fields used once, labels borrowed.
  - CSV: fields past MAX_COLS dropped while reading the record.
  - At most 4M shared strings.
  - Relationships looked up by id through a HashMap.

* xlsx: close the remaining secondary-reader cap gaps (#2159)
  
  - CSV: characters of a field past column 16,384 are not stored; quotes,
    delimiters and line ends in it are still followed.
  - Pivot: a field list cut at MAX_FIELDS is reported with a warning.
  - Sheet names: the prefix shortens once the (n) suffix passes (999), so
    deduplicated names stay within 31 characters.
  - Rows dropped for a bad row number still charge their <c> elements
    against MAX_CELLS.
  - Test Ctx helper gains the fields upstream added.

* #2159: describe MAX_CELLS as the reader's own cap
  
  calc::eval::MAX_CELLS has the same value but limits one array result,
  not a workbook.

* #2159: warn on capped pivot cache fields, dedupe pivot names through sets
  
  A cache cut at 16,384 fields is now reported, as the table field lists
  are. Axis filters and calculated fields are deduplicated through a set
  of lower-cased names instead of a scan per field, which was quadratic in
  the 16,384 fields allowed; the first of names equal but for case still
  wins.
This commit is contained in:
Michael Higgins authored and GitHub committed 2026-11-18 16:36:37 +07:00
1 parent 272ae54101
commit a3119f57c2
8 files changed
+431 -76

No files matched your search

+17 -2
View File
@@ -233,6 +233,7 @@ struct Import {
text: u64,
formats: Vec<(&'static str, StyleId)>,
spreadsheet_seen: bool,
names: crate::read::SheetNames,
}
impl Import {
@@ -241,7 +242,20 @@ impl Import {
wb.sheets.clear();
let mut report = ReadReport::default();
report.warn("ODS data-only import: formulas become their saved results and will not recalculate. Source formatting, charts, images and other features are not imported. Save a new .xlsx file; ODS saving is not supported.");
Self { wb, report, stack: vec![], sheet: None, row: None, cell: None, cells: 0, merges: 0, text: 0, formats: vec![], spreadsheet_seen: false }
Self {
wb,
report,
stack: vec![],
sheet: None,
row: None,
cell: None,
cells: 0,
merges: 0,
text: 0,
formats: vec![],
spreadsheet_seen: false,
names: Default::default(),
}
}
fn parse(&mut self, bytes: &[u8]) -> Result<(), IoError> {
@@ -317,9 +331,10 @@ impl Import {
return Err(IoError::TooLarge("ODS contains more than 1024 sheets".into()));
}
let original = attr(reader, e, TABLE, b"name")?.unwrap_or_default();
let mut name = crate::read::SheetNames::of(&self.wb).unique(original.trim_matches('\''), self.wb.sheets.len());
let mut name = self.names.unique(original.trim_matches('\''), self.wb.sheets.len());
if self.wb.check_sheet_name(&name, None).is_err() {
name = self.wb.next_sheet_name();
self.names.insert(&name);
}
if name != original {
self.report.warn("Some ODS sheet names were shortened or renamed to valid, unique XLSX sheet names.");
+48 -16
View File
@@ -1,6 +1,6 @@
use super::{
CellRef, Ctx, DateSystem, El, HashMap, IoError, MAX_FIELDS, PivotCalcField, PivotDateGroup, PivotField, PivotFilter, PivotFunc, PivotLayout,
PivotShowAs, PivotSort, PivotTable, PivotValue, RangeRef, abs_range, func_caption, func_from, number_to_text, quote_sheet, same,
PivotShowAs, PivotSort, PivotTable, PivotValue, RangeRef, abs_range, func_caption, func_from, number_to_text, quote_sheet,
};
// ================================================================ reading
@@ -118,8 +118,9 @@ fn read_cache(cx: &mut Ctx<'_>, part: &str, sheet_name: &str) -> Option<CacheIn>
};
let sys = cx.date_system;
let mut fields = Vec::new();
let mut truncated = false;
if let Some(cf) = x.child("cacheFields") {
for f in cf.kids("cacheField").take(MAX_FIELDS) {
for f in capped(cf.kids("cacheField"), &mut truncated) {
let items = f.child("sharedItems").map(|si| si.children.iter().map(|e| item_label_el(e, sys)).collect()).unwrap_or_default();
let group = f.child("fieldGroup").map(|g| {
let base = g.attr_u32("base").map(|b| b as usize);
@@ -142,6 +143,9 @@ fn read_cache(cx: &mut Ctx<'_>, part: &str, sheet_name: &str) -> Option<CacheIn>
});
}
}
if truncated {
cx.warn(format!("{part}: cache fields past {MAX_FIELDS} were dropped; its PivotTables may be incomplete"));
}
if fields.is_empty() {
cx.warn("a PivotTable's cache has no fields; kept as plain cells");
return None;
@@ -231,14 +235,25 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
cx.warn(format!("PivotTable \"{}\" has no valid location; kept as plain cells", pt.name));
return None;
};
let pfs: Vec<&El> = x.child("pivotFields").map(|p| p.kids("pivotField").collect()).unwrap_or_default();
// Field lists are capped, and a field is used once (Excel puts a field on one axis, once):
// each use walks the field's items.
// A list cut at the cap is reported, so an incomplete PivotTable can be told apart.
let mut truncated = false;
let pfs: Vec<&El> = x.child("pivotFields").map(|p| capped(p.kids("pivotField"), &mut truncated)).unwrap_or_default();
let pf = |i: usize| pfs.get(i).copied();
let idx_list = |name: &str, attr: &str| -> Vec<i64> {
x.child(name).map(|e| e.children.iter().filter_map(|f| f.attr_i64(attr)).collect()).unwrap_or_default()
let mut used_fields = std::collections::HashSet::new();
let mut idx_list = |name: &str, attr: &str| -> Vec<i64> {
x.child(name)
.map(|e| capped(e.children.iter().filter_map(|f| f.attr_i64(attr)).filter(|&i| i < 0 || used_fields.insert(i)), &mut truncated))
.unwrap_or_default()
};
let row_x = idx_list("rowFields", "x");
let col_x = idx_list("colFields", "x");
let pages: Vec<&El> = x.child("pageFields").map(|p| p.kids("pageField").collect()).unwrap_or_default();
let mut page_fields = std::collections::HashSet::new();
let pages: Vec<&El> = x
.child("pageFields")
.map(|p| capped(p.kids("pageField").filter(|f| f.attr_i64("fld").is_none_or(|i| page_fields.insert(i))), &mut truncated))
.unwrap_or_default();
// Layout from the first axis field (pivotField defaults: compact and outline on).
let first_axis = row_x.iter().chain(col_x.iter()).find(|i| **i >= 0).and_then(|i| pf(*i as usize));
@@ -268,8 +283,11 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
let mut filters: Vec<PivotFilter> = Vec::new();
// Lower-cased names of the fields already filtered on an axis, the first of which wins (as
// `same` compares them): scanning the filters for each field would be quadratic.
let mut axis_filtered: std::collections::HashSet<String> = std::collections::HashSet::new();
// (fields, position of Σ Values (`-2`) among them)
let axis = |cx: &mut Ctx<'_>, list: &[i64], filters: &mut Vec<PivotFilter>| -> (Vec<PivotField>, Option<usize>) {
let mut axis = |cx: &mut Ctx<'_>, list: &[i64], filters: &mut Vec<PivotFilter>| -> (Vec<PivotField>, Option<usize>) {
let mut v = Vec::new();
let mut values_at = None;
for &i in list {
@@ -282,7 +300,7 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
}
let i = i as usize;
let Some((name, group)) = field_ref(cx, cache, i) else { continue };
let labels = cache.fields.get(i).map(|f| f.labels().to_vec()).unwrap_or_default();
let labels: &[String] = cache.fields.get(i).map(|f| f.labels()).unwrap_or(&[]);
let mut f = PivotField { source_col: name.clone(), date_group: group, sort: PivotSort::None, collapsed_items: vec![] };
if let Some(e) = pf(i) {
f.sort = match e.attr("sortType") {
@@ -293,11 +311,9 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
let items = items_of(e);
let lab = |it: &El| it.attr_u32("x").and_then(|k| labels.get(k as usize)).cloned();
f.collapsed_items = items.iter().filter(|it| !it.flag("sd", true)).filter_map(|it| lab(it)).collect();
if items.iter().any(|it| it.flag("h", false)) {
if items.iter().any(|it| it.flag("h", false)) && axis_filtered.insert(name.to_lowercase()) {
let shown: Vec<String> = items.iter().filter(|it| !it.flag("h", false)).filter_map(|it| lab(it)).collect();
if !filters.iter().any(|ff| same(&ff.source_col, &name)) {
filters.push(PivotFilter { source_col: name.clone(), selected: Some(shown) });
}
filters.push(PivotFilter { source_col: name.clone(), selected: Some(shown) });
}
}
v.push(f);
@@ -308,7 +324,7 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
for p in &pages {
let Some(i) = p.attr_i64("fld").filter(|i| *i >= 0).map(|i| i as usize) else { continue };
let Some((name, _)) = field_ref(cx, cache, i) else { continue };
let labels = cache.fields.get(i).map(|f| f.labels().to_vec()).unwrap_or_default();
let labels: &[String] = cache.fields.get(i).map(|f| f.labels()).unwrap_or(&[]);
let selected = if let Some(k) = p.attr_u32("item") {
labels.get(k as usize).map(|l| vec![l.clone()])
} else {
@@ -341,7 +357,8 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
// Values.
if let Some(df) = x.child("dataFields") {
for d in df.kids("dataField") {
truncated |= df.kids("dataField").nth(MAX_FIELDS).is_some();
for d in df.kids("dataField").take(MAX_FIELDS) {
let Some(i) = d.attr_u32("fld").map(|i| i as usize) else { continue };
let Some(f) = cache.fields.get(i) else { continue };
let func = match d.attr("subtotal") {
@@ -382,11 +399,13 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
}
}
// Calculated fields of the cache (each PivotTable on it can use them).
// Calculated fields of the cache (each PivotTable on it can use them); the first of a
// lower-cased name wins, looked up in a set rather than by scanning the fields kept so far.
let mut calc_names: std::collections::HashSet<String> = pt.calculated_fields.iter().map(|c| c.name.to_lowercase()).collect();
for f in &cache.fields {
if let Some(formula) = &f.formula
&& !f.name.trim().is_empty()
&& !pt.calculated_fields.iter().any(|c| same(&c.name, &f.name))
&& calc_names.insert(f.name.to_lowercase())
{
let formula = formula.trim();
let formula = if formula.starts_with('=') { formula.to_string() } else { format!("={formula}") };
@@ -394,8 +413,21 @@ fn read_table(cx: &mut Ctx<'_>, x: &El, cache: &CacheIn, id: u32) -> Option<Pivo
}
}
if truncated {
cx.warn(format!("PivotTable \"{}\": field lists past {MAX_FIELDS} entries were dropped; the table is incomplete", pt.name));
}
pt.anchor = loc.start;
let above = if pages.is_empty() { 0 } else { pages.len() as u32 + 1 };
pt.last_range = Some(RangeRef::new(CellRef::new(loc.start.row.saturating_sub(above), loc.start.col), loc.end));
Some(pt)
}
/// The first [`MAX_FIELDS`] items; sets `truncated` when there were more.
fn capped<T>(items: impl Iterator<Item = T>, truncated: &mut bool) -> Vec<T> {
let mut v: Vec<T> = items.take(MAX_FIELDS.saturating_add(1)).collect();
if v.len() > MAX_FIELDS {
v.truncate(MAX_FIELDS);
*truncated = true;
}
v
}
+69 -44
View File
@@ -46,6 +46,57 @@ pub struct Ctx<'a> {
pub known_function: Option<&'a dyn Fn(&str) -> bool>,
/// The code of the workbook's Python cells, from its Python part.
pub python: crate::python_part::ScriptsIn,
/// Cell elements read so far, across every sheet (capped by `sheet_read::MAX_CELLS`).
pub cells: u64,
}
/// The most worksheets read from one workbook (as the XLS reader allows).
const MAX_SHEETS: usize = 4096;
/// The most shared strings read from one workbook (as the XLS reader allows): each is an
/// `Arc<str>` of its own, so a hundred million tiny `<si/>` would ask for gigabytes.
const MAX_STRINGS: usize = 4_000_000;
/// Sheet names taken while reading, compared as `Workbook::sheet_index` compares them (ASCII case
/// folded), with the next `(n)` suffix to try for each base, so renaming duplicates stays linear.
#[derive(Default)]
pub(crate) struct SheetNames {
taken: HashSet<String>,
next_suffix: HashMap<String, usize>,
}
impl SheetNames {
/// Records `name` as taken.
pub(crate) fn insert(&mut self, name: &str) {
self.taken.insert(name.to_ascii_lowercase());
}
/// A valid name for the `i`-th sheet based on `name`, not yet taken; it is recorded as taken.
pub(crate) fn unique(&mut self, name: &str, i: usize) -> String {
let mut base: String = name.chars().filter(|c| !matches!(c, ':' | '\\' | '/' | '?' | '*' | '[' | ']')).take(31).collect();
if base.trim().is_empty() {
base = format!("Sheet{}", i + 1);
}
if self.taken.insert(base.to_ascii_lowercase()) {
return base;
}
let key: String = base.chars().take(25).collect::<String>().to_ascii_lowercase();
let mut n = self.next_suffix.get(&key).copied().unwrap_or(2);
loop {
// 25 characters of the base, fewer once the suffix grows past ` (999)`, so the name
// stays within 31 characters however many duplicates there are.
let suffix = format!(" ({n})");
let short: String = base.chars().take(31usize.saturating_sub(suffix.len()).min(25)).collect();
let candidate = format!("{short}{suffix}");
n = n.saturating_add(1);
if self.taken.insert(candidate.to_ascii_lowercase()) {
self.next_suffix.insert(key, n);
return candidate;
}
if n == usize::MAX {
return base;
}
}
}
}
impl Ctx<'_> {
@@ -120,6 +171,9 @@ fn read_shared_strings(cx: &mut Ctx<'_>, wb_rels: &[Rel]) -> Result<(), IoError>
let mut phonetics = Vec::new();
let mut dropped = 0usize;
let res = xml::parse_streaming(&b, &["si"], &mut |si| {
if sst.len() >= MAX_STRINGS {
return Err(IoError::TooLarge(format!("the workbook has more than {MAX_STRINGS} shared strings")));
}
let text = xml::rich_text(&si);
let raw = si.kids("rPh").count();
let (runs, props) = xml::phonetic_data(&si, &text);
@@ -130,8 +184,10 @@ fn read_shared_strings(cx: &mut Ctx<'_>, wb_rels: &[Rel]) -> Result<(), IoError>
sst.push(Arc::from(text));
Ok(())
});
if let Err(e) = res {
cx.warn(format!("shared strings are damaged ({e}); kept {} strings", sst.len()));
match res {
Ok(_) => {}
Err(IoError::TooLarge(m)) => return Err(IoError::TooLarge(m)),
Err(e) => cx.warn(format!("shared strings are damaged ({e}); kept {} strings", sst.len())),
}
if dropped > 0 {
cx.warn(format!("shared strings: {dropped} phonetic runs were dropped (bad sb/eb or no reading)"));
@@ -267,6 +323,7 @@ pub fn read_xlsx<'a>(bytes: &'a [u8], opts: crate::ReadOptions<'a>) -> Result<(W
shared_formula_bytes: 0,
known_function,
python: Default::default(),
cells: 0,
};
let root_rels = cx.pkg.rels("")?;
let wb_part = find_rel(&root_rels, "officeDocument").map(|r| r.target.clone()).unwrap_or_else(|| "xl/workbook.xml".into());
@@ -318,11 +375,19 @@ pub fn read_xlsx<'a>(bytes: &'a [u8], opts: crate::ReadOptions<'a>) -> Result<(W
// Legacy formula cells per model sheet, rewritten once names and tables are known.
let mut legacy_formulas: Vec<(usize, Vec<gridcraft_core::CellRef>)> = vec![];
let mut pivot_caches = crate::pivot::CachesIn::new(&wb_xml, &wb_rels);
let sheet_els: Vec<El> = wb_xml.child("sheets").map(|s| s.kids("sheet").cloned().collect()).unwrap_or_default();
let sheet_els: Vec<&El> = wb_xml.child("sheets").map(|s| s.kids("sheet").collect()).unwrap_or_default();
if sheet_els.len() > MAX_SHEETS {
return Err(IoError::TooLarge(format!("the workbook has more than {MAX_SHEETS} sheets")));
}
let mut rel_by_id: HashMap<&str, &Rel> = HashMap::with_capacity(wb_rels.len());
for r in &wb_rels {
// The first of duplicate ids wins, as a linear search would find it.
rel_by_id.entry(r.id.as_str()).or_insert(r);
}
let mut names = SheetNames::default();
for (i, se) in sheet_els.iter().enumerate() {
let name = se.attr("name").unwrap_or("").to_string();
let rel = se.attr("id").and_then(|id| wb_rels.iter().find(|r| r.id == id)).cloned();
let rel = se.attr("id").and_then(|id| rel_by_id.get(id).copied()).cloned();
let Some(rel) = rel else {
cx.warn(format!("sheet \"{name}\" has no part; skipped"));
file_to_model.push(None);
@@ -476,46 +541,6 @@ pub fn read_xlsx<'a>(bytes: &'a [u8], opts: crate::ReadOptions<'a>) -> Result<(W
Ok((wb, cx.report))
}
/// The sheet names an import has assigned so far, for repairing repeated ones.
#[derive(Default)]
pub(crate) struct SheetNames {
/// Assigned names, ASCII-lowercased.
taken: HashSet<String>,
/// For each lowercased stem, the next suffix to try: a file full of repeated names would
/// otherwise retry every occupied suffix for every sheet.
next: HashMap<String, u64>,
}
impl SheetNames {
pub(crate) fn of(wb: &Workbook) -> Self {
SheetNames { taken: wb.sheets.iter().map(|s| s.name.to_ascii_lowercase()).collect(), next: HashMap::new() }
}
/// A valid name for the i-th sheet, unique among the names assigned so far, which it joins.
pub(crate) fn unique(&mut self, name: &str, i: usize) -> String {
let mut base: String = name.chars().filter(|c| !matches!(c, ':' | '\\' | '/' | '?' | '*' | '[' | ']')).take(31).collect();
if base.trim().is_empty() {
base = format!("Sheet{}", i + 1);
}
let name = if self.taken.contains(&base.to_ascii_lowercase()) {
let stem: String = base.chars().take(25).collect();
let next = self.next.entry(stem.to_ascii_lowercase()).or_insert(2);
let found = (*next..).find(|n| !self.taken.contains(&format!("{stem} ({n})").to_ascii_lowercase()));
match found {
Some(n) => {
*next = n + 1;
format!("{stem} ({n})")
}
None => base,
}
} else {
base
};
self.taken.insert(name.to_ascii_lowercase());
name
}
}
/// Strips a sheet qualifier (`'My Sheet'!$A$1` → `$A$1`).
fn strip_sheet(s: &str) -> &str {
match s.rfind('!') {
+81 -10
View File
@@ -134,6 +134,9 @@ fn settle_python_spills_past_budget(st: &RowState, sheet: &mut Sheet) {
}
}
/// The most cell elements read from one workbook, across its sheets: the reader's own cap.
/// (`calc::eval::MAX_CELLS` has the same value but limits one array result, not a workbook.)
pub(crate) const MAX_CELLS: u64 = 16_000_000;
/// Target and tooltip bytes kept for hyperlinks of one sheet, over all cells: the same URL is
/// copied for every cell of its range, so length × cells must stay bounded whatever the file.
const MAX_HYPERLINK_BYTES: u64 = 8 * 1024 * 1024;
@@ -191,12 +194,10 @@ pub fn read_sheet(cx: &mut Ctx<'_>, part: &str, name: &str) -> Result<(Sheet, cr
let rels = cx.pkg.rels(part)?;
let mut sheet = Sheet::new(name);
let mut st = RowState::default();
let res = xml::parse_streaming(&bytes, &["row"], &mut |row| {
read_row(cx, &mut sheet, &mut st, &row);
Ok(())
});
let res = xml::parse_streaming(&bytes, &["row"], &mut |row| read_row(cx, &mut sheet, &mut st, &row));
let root = match res {
Ok(r) => r,
Err(IoError::TooLarge(m)) => return Err(IoError::TooLarge(m)),
Err(e) => {
cx.warn(format!("sheet \"{name}\" is damaged ({e}); kept {} rows", st.rows));
El::default()
@@ -208,6 +209,12 @@ pub fn read_sheet(cx: &mut Ctx<'_>, part: &str, name: &str) -> Result<(Sheet, cr
settle_python_spills_past_budget(&st, &mut sheet);
drop(bytes);
// Relationships by id: hyperlinks, drawings and tables each look one up.
let mut rel_by_id: HashMap<&str, &Rel> = HashMap::with_capacity(rels.len());
for r in &rels {
// The first of duplicate ids wins, as a linear search would find it.
rel_by_id.entry(r.id.as_str()).or_insert(r);
}
let mut fit_to_page = false;
let mut link_budget = LinkBudget::default();
let mut cf_kept_cells = 0u64;
@@ -245,7 +252,7 @@ pub fn read_sheet(cx: &mut Ctx<'_>, part: &str, name: &str) -> Result<(Sheet, cr
}
}
}
"hyperlinks" => read_hyperlinks(cx, name, e, &rels, &mut sheet, &mut link_budget),
"hyperlinks" => read_hyperlinks(cx, name, e, &rel_by_id, &mut sheet, &mut link_budget),
"autoFilter" => sheet.autofilter = read_autofilter(cx, e),
"printOptions" => {
sheet.print.gridlines = e.flag("gridLines", false);
@@ -319,14 +326,14 @@ pub fn read_sheet(cx: &mut Ctx<'_>, part: &str, name: &str) -> Result<(Sheet, cr
});
}
"drawing" => {
if let Some(rel) = e.attr("id").and_then(|id| rels.iter().find(|r| r.id == id)) {
if let Some(rel) = e.attr("id").and_then(|id| rel_by_id.get(id).copied()) {
let target = rel.target.clone();
crate::drawing::read_drawing(cx, &target, &mut sheet)?;
}
}
"tableParts" => {
for tp in e.kids("tablePart") {
if let Some(rel) = tp.attr("id").and_then(|id| rels.iter().find(|r| r.id == id)) {
if let Some(rel) = tp.attr("id").and_then(|id| rel_by_id.get(id).copied()) {
let target = rel.target.clone();
if let Some(x) = cx.optional_xml(&target)?
&& let Some(mut t) = read_table(cx, &x)
@@ -496,19 +503,30 @@ fn read_cols(cx: &Ctx<'_>, e: &El, sheet: &mut Sheet, remaining: &mut u64) -> Re
Ok(())
}
fn read_row(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, row: &El) {
/// Charges `n` cell elements against the workbook's [`MAX_CELLS`].
fn charge_cells(cx: &mut Ctx<'_>, n: u64) -> Result<(), IoError> {
cx.cells = cx.cells.saturating_add(n);
if cx.cells > MAX_CELLS {
return Err(IoError::TooLarge(format!("the workbook has more than {MAX_CELLS} cells")));
}
Ok(())
}
/// One `<row>`. Every `<c>` in it counts against [`MAX_CELLS`], also in a row that is dropped
/// for a bad row number: one worksheet part read for many sheets can't parse cells uncounted.
fn read_row(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, row: &El) -> Result<(), IoError> {
st.rows += 1;
let r = match row.attr_u32("r") {
Some(r) if r >= 1 => r - 1,
Some(_) => {
st.dropped += 1;
return;
return charge_cells(cx, row.kids("c").count() as u64);
}
None => st.next_row,
};
if r >= MAX_ROWS {
st.dropped += 1;
return;
return charge_cells(cx, row.kids("c").count() as u64);
}
st.next_row = r + 1;
let mut info = LineInfo::default();
@@ -532,6 +550,7 @@ fn read_row(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, row: &El) {
}
let mut next_col = 0u32;
for c in row.kids("c") {
charge_cells(cx, 1)?;
let pos = match c.attr("r") {
Some(a) => match CellRef::parse(a.trim()) {
Some(p) => p,
@@ -549,6 +568,7 @@ fn read_row(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, row: &El) {
next_col = pos.col + 1;
read_cell(cx, sheet, st, c, pos);
}
Ok(())
}
fn read_cell(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, c: &El, pos: CellRef) {
@@ -929,6 +949,57 @@ mod tests {
assert_eq!(parse_iso_datetime("2024-13-01", DateSystem::D1900), None);
}
fn ctx(pkg: crate::package::Package<'_>) -> Ctx<'_> {
Ctx {
pkg,
report: Default::default(),
sst: vec![],
phonetics: vec![],
xf_map: vec![],
dxfs: vec![],
palette: vec![],
date_system: DateSystem::D1900,
styles: Default::default(),
persons: Default::default(),
next_id: 1,
num_fmts: Default::default(),
cell_pictures: vec![],
cf_ext_ids: vec![],
known_function: None,
python: Default::default(),
cells: 0,
shared_formula_bytes: 0,
legacy_formulas: vec![],
}
}
/// #2159: the workbook-wide cell cap refuses the open once passed, across sheets.
#[test]
fn cells_past_the_workbook_cap_are_refused() {
let bytes = crate::tests::minimal(r#"<sheetData><row r="1"><c r="A1"><v>1</v></c><c r="B1"><v>2</v></c></row></sheetData>"#, &[], "", "");
let pkg = crate::package::Package::open(&bytes).unwrap();
let mut cx = ctx(pkg);
cx.cells = MAX_CELLS - 2;
let (sheet, _) = read_sheet(&mut cx, "xl/worksheets/sheet1.xml", "Data").unwrap();
assert_eq!(sheet.cells.len(), 2);
assert!(matches!(read_sheet(&mut cx, "xl/worksheets/sheet1.xml", "Data"), Err(IoError::TooLarge(_))));
}
/// #2159: the cells of a row dropped for its row number still count against the cap, so a
/// part read for many sheets can't parse cells uncounted.
#[test]
fn cells_of_dropped_rows_count_against_the_cap() {
let rows = r#"<sheetData><row r="0"><c><v>1</v></c><c><v>2</v></c></row><row r="2000000"><c><v>3</v></c></row></sheetData>"#;
let bytes = crate::tests::minimal(rows, &[], "", "");
let pkg = crate::package::Package::open(&bytes).unwrap();
let mut cx = ctx(pkg);
let (sheet, _) = read_sheet(&mut cx, "xl/worksheets/sheet1.xml", "Data").unwrap();
assert!(sheet.cells.is_empty(), "both rows are dropped");
assert_eq!(cx.cells, 3);
cx.cells = MAX_CELLS - 2;
assert!(matches!(read_sheet(&mut cx, "xl/worksheets/sheet1.xml", "Data"), Err(IoError::TooLarge(_))));
}
#[test]
fn sqref_keeps_more_than_ten_thousand_ranges() {
let sq = (1..=12_000).map(|r| format!("A{r}")).collect::<Vec<_>>().join(" ");
+4 -4
View File
@@ -4,12 +4,12 @@ use super::{
from_file, truncate_chars, xml,
};
/// `budget` counts the cells and bytes this sheet's hyperlinks have copied so far, across
/// `<hyperlinks>` blocks.
pub(super) fn read_hyperlinks(cx: &mut Ctx<'_>, name: &str, e: &El, rels: &[Rel], sheet: &mut Sheet, budget: &mut LinkBudget) {
/// `rels` maps relationship ids to relationships (the first of duplicate ids). `budget` counts the
/// cells and bytes this sheet's hyperlinks have copied so far, across `<hyperlinks>` blocks.
pub(super) fn read_hyperlinks(cx: &mut Ctx<'_>, name: &str, e: &El, rels: &HashMap<&str, &Rel>, sheet: &mut Sheet, budget: &mut LinkBudget) {
for h in e.kids("hyperlink") {
let Some(range) = h.attr("ref").and_then(RangeRef::parse) else { continue };
let rel_target = h.attr("id").and_then(|id| rels.iter().find(|r| r.id == id)).map(|r| r.target.as_str()).unwrap_or("");
let rel_target = h.attr("id").and_then(|id| rels.get(id)).map(|r| r.target.as_str()).unwrap_or("");
let location = h.attr("location").filter(|l| !l.is_empty());
// Each covered cell gets its own copy of the link: refuse oversized targets before copying.
if rel_target.len().saturating_add(location.map_or(0, str::len)) > MAX_HYPERLINK_TARGET {
+37
View File
@@ -360,6 +360,43 @@ fn the_cap_counts_the_bytes_the_cells_hold_and_accepted_exports_keep_every_posit
assert_eq!(tabs.len(), 14);
}
/// #2159: a record keeps at most one field per sheet column while it is read, not after; the rest
/// of the line is still read through, so the next line starts the next record.
#[test]
fn fields_past_the_last_column_are_not_collected() {
let src = format!("{}last\nnext,row\n", "x,".repeat(100_000));
let mut records = vec![];
parse_records(&src, ',', '"', MAX_COLS as usize, |rec| {
records.push(rec);
true
});
assert_eq!(records.len(), 2);
assert_eq!(records[0].len(), 16_384);
assert_eq!(records[1], vec!["next".to_string(), "row".to_string()]);
// The same through the reader, and a quoted delimiter past the cap doesn't split the line.
let src = format!("{}\"a,\nb\",z\nnext\n", "x,".repeat(20_000));
let wb = read_csv(src.as_bytes(), &CsvOptions { delimiter: b',', ..CsvOptions::default() }).unwrap();
assert_eq!(v(&wb, 0, 16_383), Value::text("x"));
assert_eq!(v(&wb, 1, 0), Value::text("next"));
assert_eq!(wb.sheet(0).unwrap().cells.len(), 16_385);
}
/// #2159: a long field past the last column is skipped character by character, not collected
/// and then thrown away; a quote inside it is a plain character, as in a kept field.
#[test]
fn a_long_field_past_the_last_column_is_not_stored() {
let src = format!("{}{}a\"b,{}\nnext,row\n", "x,".repeat(16_384), "y".repeat(1_000_000), "z".repeat(1_000));
let mut records = vec![];
parse_records(&src, ',', '"', MAX_COLS as usize, |rec| {
records.push(rec);
true
});
assert_eq!(records.len(), 2, "the quote didn't open a quoted field across the line end");
assert_eq!(records[0].len(), 16_384);
assert!(records[0].iter().all(|f| f == "x"));
assert_eq!(records[1], vec!["next".to_string(), "row".to_string()]);
}
#[test]
fn parser_stops_collecting_fields_at_max_cols() {
let extra = 50;
+113
View File
@@ -280,6 +280,119 @@ fn hostile_formula_text_does_not_overflow_the_stack() {
assert_eq!(a_value, gridcraft_core::Value::Number(3.0));
}
/// A one-sheet-per-entry workbook: `names` are the sheet names, every sheet sharing one part.
fn many_sheets(names: &[String]) -> Vec<u8> {
let sheets: String = names.iter().enumerate().map(|(i, n)| format!(r#"<sheet name="{n}" sheetId="{}" r:id="rId1"/>"#, i + 1)).collect();
let wb = format!(
r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets>{sheets}</sheets></workbook>"#
);
let rels = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>"#;
let sheet = r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData/></worksheet>"#;
make_zip(&[
("[Content_Types].xml", super::CT.as_bytes()),
("_rels/.rels", super::ROOT_RELS.as_bytes()),
("xl/workbook.xml", wb.as_bytes()),
("xl/_rels/workbook.xml.rels", rels.as_bytes()),
("xl/worksheets/sheet1.xml", sheet.as_bytes()),
])
}
/// #2159: a row of endless `<c>` is not built in memory: the part's kept-element cap
/// ([`crate::xml::MAX_XML_NODES`]) stops it, and the sheet is reported as damaged.
#[test]
fn a_row_wider_than_the_grid_is_refused() {
let row = |n: usize| format!(r#"<sheetData><row r="1">{}</row></sheetData>"#, "<c/>".repeat(n));
let (wb, rep) = read_xlsx(&minimal(&row(crate::xml::MAX_XML_NODES + 1), &[], "", "")).unwrap();
assert!(wb.sheet(0).unwrap().cells.is_empty());
assert!(rep.warnings.iter().any(|w| w.contains("damaged")), "{:?}", rep.warnings);
assert!(read_xlsx(&minimal(&row(16_384), &[], "", "")).is_ok());
// Every other part has the cap too (a merge list of more than a million entries).
let merges = format!("<sheetData/><mergeCells>{}</mergeCells>", r#"<mergeCell ref="A1:B2"/>"#.repeat(crate::xml::MAX_XML_NODES + 1));
let (wb, rep) = read_xlsx(&minimal(&merges, &[], "", "")).unwrap();
assert!(wb.sheet(0).unwrap().merges.is_empty());
assert!(rep.warnings.iter().any(|w| w.contains("damaged")), "{:?}", rep.warnings);
}
/// #2159: overlapping `<col min=1 max=16384>` entries share one budget of column writes.
#[test]
fn overlapping_column_entries_have_a_budget() {
let parts = [("xl/styles.xml", r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"/>"#)];
let rels = r#"<Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/>"#;
let cols = format!("<cols>{}</cols><sheetData/>", r#"<col min="1" max="16384" width="20" customWidth="1"/>"#.repeat(300));
assert!(matches!(read_xlsx(&minimal(&cols, &parts, rels, "")), Err(IoError::TooLarge(_))));
// A normal list reads with no warning.
let (wb, rep) = read_xlsx(&minimal(r#"<cols><col min="2" max="4" width="20" customWidth="1"/></cols><sheetData/>"#, &parts, rels, "")).unwrap();
assert_eq!(wb.sheet(0).unwrap().cols.len(), 3);
assert!(rep.warnings.is_empty(), "{:?}", rep.warnings);
}
/// #2159: the sheet count is capped, and renaming duplicate sheet names stays linear.
#[test]
fn sheet_count_is_capped_and_duplicate_names_are_renamed() {
let names: Vec<String> = (0..4097).map(|i| format!("S{i}")).collect();
assert!(matches!(read_xlsx(&many_sheets(&names)), Err(IoError::TooLarge(_))));
let dups: Vec<String> = (0..300).map(|i| if i % 2 == 0 { "a".to_string() } else { "A".to_string() }).collect();
let (wb, _) = read_xlsx(&many_sheets(&dups)).unwrap();
assert_eq!(wb.sheets.len(), 300);
assert_eq!(wb.sheets[0].name, "a");
assert_eq!(wb.sheets[1].name, "A (2)");
assert_eq!(wb.sheets[299].name, "A (300)");
let distinct: std::collections::HashSet<String> = wb.sheets.iter().map(|s| s.name.to_ascii_lowercase()).collect();
assert_eq!(distinct.len(), 300);
// The name set alone: thousands of duplicates, with a suffixed name already taken.
let mut taken = crate::read::SheetNames::default();
assert_eq!(taken.unique("x (3)", 0), "x (3)");
let got: Vec<String> = (0..4096).map(|i| taken.unique("X", i)).collect();
assert_eq!(got[0], "X");
assert_eq!(got[1], "X (2)");
assert_eq!(got[2], "X (4)");
assert_eq!(got.iter().map(|n| n.to_ascii_lowercase()).collect::<std::collections::HashSet<_>>().len(), 4096);
assert_eq!(taken.unique("", 9), "Sheet10");
// A 25-character name repeated past ` (999)` shortens its prefix so every name fits 31.
let mut taken = crate::read::SheetNames::default();
let base = "N".repeat(25);
let got: Vec<String> = (0..4096).map(|i| taken.unique(&base, i)).collect();
assert!(got.iter().all(|n| n.chars().count() <= 31), "{:?}", got.iter().find(|n| n.chars().count() > 31));
assert_eq!(got[998], format!("{base} (999)"));
assert_eq!(got[999], format!("{} (1000)", "N".repeat(24)));
assert_eq!(got.iter().map(|n| n.to_ascii_lowercase()).collect::<std::collections::HashSet<_>>().len(), 4096);
}
/// #2159: the shared-string count is capped (each string is an allocation of its own).
#[test]
fn too_many_shared_strings_are_refused() {
let sst = |n: usize| format!(r#"<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">{}</sst>"#, "<si/>".repeat(n));
let rels = r#"<Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" Target="sharedStrings.xml"/>"#;
let big = sst(4_000_001);
assert!(matches!(read_xlsx(&minimal("<sheetData/>", &[("xl/sharedStrings.xml", &big)], rels, "")), Err(IoError::TooLarge(_))));
let small = sst(3);
assert!(read_xlsx(&minimal("<sheetData/>", &[("xl/sharedStrings.xml", &small)], rels, "")).is_ok());
}
/// #2159: hyperlinks find their relationship by id (a map, not a scan per hyperlink); the first
/// of duplicate ids wins, as before.
#[test]
fn hyperlinks_resolve_relationships_by_id() {
let n = 5_000;
let mut rels = String::from(r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">"#);
for i in 0..n {
rels.push_str(&format!(
r#"<Relationship Id="h{i}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink" Target="https://example.com/{i}" TargetMode="External"/>"#
));
}
rels.push_str(r#"<Relationship Id="h0" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink" Target="https://example.com/second" TargetMode="External"/>"#);
rels.push_str("</Relationships>");
let links: String = (0..n).map(|i| format!(r#"<hyperlink ref="A{}" r:id="h{i}"/>"#, i + 1)).collect();
let body = format!(r#"<sheetData/><hyperlinks>{links}<hyperlink ref="B1" r:id="missing"/></hyperlinks>"#);
let (wb, _) = read_xlsx(&minimal(&body, &[("xl/worksheets/_rels/sheet1.xml.rels", &rels)], "", "")).unwrap();
let s = wb.sheet(0).unwrap();
assert_eq!(s.hyperlinks.len(), n);
let link = |a: &str| s.hyperlinks.get(&gridcraft_core::CellRef::parse(a).unwrap()).map(|h| h.target.clone());
assert_eq!(link("A1").as_deref(), Some("https://example.com/0"));
assert_eq!(link("A5000").as_deref(), Some("https://example.com/4999"));
assert_eq!(link("B1"), None);
}
/// #2173: a hyperlink is copied into every cell it covers, so oversized targets are dropped,
/// ScreenTips are cut to 255 characters and the cell cap holds across `<hyperlinks>` blocks.
#[test]
+62
View File
@@ -545,6 +545,68 @@ fn hostile_pivot_parts_warn_not_panic() {
assert!(report.warnings.iter().any(|w| w.contains("index")), "{:?}", report.warnings);
}
/// #2159: a field repeated on the axes or as a page field is used once (each use walks the
/// field's items and copied its labels), and the lists are capped.
#[test]
fn repeated_pivot_fields_are_used_once() {
let items: String = (0..2_000).map(|_| r#"<item x="0" sd="0"/>"#).collect();
let rows = r#"<field x="0"/>"#.repeat(20_000);
let pages = r#"<pageField fld="0"/>"#.repeat(20_000);
let table = format!(
r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="1"><pivotField axis="axisRow"><items>{items}</items></pivotField></pivotFields><rowFields>{rows}<field x="-2"/></rowFields><colFields>{rows}</colFields><pageFields>{pages}</pageFields></pivotTableDefinition>"#
);
let (wb, report) = read_xlsx(&with_pivot(&table, Some(GOOD_CACHE))).unwrap();
let pt = &wb.sheet(0).unwrap().pivots[0];
assert_eq!(pt.rows.len(), 1, "{:?}", report.warnings);
assert!(pt.columns.is_empty());
assert_eq!(pt.rows[0].collapsed_items.len(), 2_000);
assert_eq!(pt.filters.iter().filter(|f| f.source_col == "K").count(), 1);
// One page field: the table's range starts two rows above its location.
assert_eq!(pt.last_range, Some(rr("D3:F9")));
}
/// #2159: a field list cut at the 16,384-entry cap is reported, so an incomplete PivotTable can be
/// told apart from a complete one.
#[test]
fn capped_pivot_field_lists_warn() {
let fields = r#"<pivotField/>"#.repeat(16_385);
let table = format!(
r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="16385">{fields}</pivotFields></pivotTableDefinition>"#
);
let (_, report) = read_xlsx(&with_pivot(&table, Some(GOOD_CACHE))).unwrap();
assert!(report.warnings.iter().any(|w| w.contains("field lists past 16384 entries")), "{:?}", report.warnings);
let table = r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="1"><pivotField/></pivotFields></pivotTableDefinition>"#;
let (_, report) = read_xlsx(&with_pivot(table, Some(GOOD_CACHE))).unwrap();
assert!(!report.warnings.iter().any(|w| w.contains("field lists past")), "{:?}", report.warnings);
}
/// #2159: a cache cut at 16,384 fields is reported too, as the table field lists are.
#[test]
fn capped_cache_fields_warn() {
let fields = r#"<cacheField name="F"><sharedItems/></cacheField>"#.repeat(16_385);
let cache = format!(
r#"<pivotCacheDefinition xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><cacheSource type="worksheet"><worksheetSource ref="A1:B3"/></cacheSource><cacheFields count="16385">{fields}</cacheFields></pivotCacheDefinition>"#
);
let table = r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="1"><pivotField/></pivotFields></pivotTableDefinition>"#;
let (_, report) = read_xlsx(&with_pivot(table, Some(&cache))).unwrap();
assert!(report.warnings.iter().any(|w| w.contains("cache fields past 16384")), "{:?}", report.warnings);
let (_, report) = read_xlsx(&with_pivot(table, Some(GOOD_CACHE))).unwrap();
assert!(!report.warnings.iter().any(|w| w.contains("cache fields past")), "{:?}", report.warnings);
}
/// #2159: axis filters and calculated fields are deduplicated through a set of lower-cased names;
/// the first of names equal but for case still wins, as with the scan it replaced.
#[test]
fn axis_filters_and_calculated_fields_dedupe_ignoring_case() {
let cache = r#"<pivotCacheDefinition xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><cacheSource type="worksheet"><worksheetSource ref="A1:B3"/></cacheSource><cacheFields count="4"><cacheField name="K"><sharedItems><s v="a"/><s v="b"/></sharedItems></cacheField><cacheField name="k"><sharedItems><s v="a"/><s v="b"/></sharedItems></cacheField><cacheField name="Price" formula="1" databaseField="0"/><cacheField name="PRICE" formula="2" databaseField="0"/></cacheFields></pivotCacheDefinition>"#;
let table = r#"<pivotTableDefinition name="P" cacheId="1"><location ref="D5:F9"/><pivotFields count="2"><pivotField axis="axisRow"><items><item x="0"/><item x="1" h="1"/></items></pivotField><pivotField axis="axisCol"><items><item x="0" h="1"/><item x="1"/></items></pivotField></pivotFields><rowFields><field x="0"/></rowFields><colFields><field x="1"/></colFields></pivotTableDefinition>"#;
let (wb, report) = read_xlsx(&with_pivot(table, Some(cache))).unwrap();
let pt = &wb.sheet(0).unwrap().pivots[0];
assert_eq!((pt.rows.len(), pt.columns.len()), (1, 1), "{:?}", report.warnings);
assert_eq!(pt.filters, vec![PivotFilter { source_col: "K".into(), selected: Some(vec!["a".into()]) }]);
assert_eq!(pt.calculated_fields, vec![PivotCalcField { name: "Price".into(), formula: "=1".into() }]);
}
#[test]
fn cache_found_by_workbook_cache_id() {
let rels_extra = r#"<Relationship Id="rId9" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/pivotCacheDefinition" Target="pivotCache/pivotCacheDefinition1.xml"/>"#;