mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-17 15:55:14 +00:00
craft-ods: add an option to cut text past max_text short instead of refusing (#1482)
This commit is contained in:
1 parent
b8eaa8e2f3
commit
e2583a4a58
5 files changed
+194
-43
No files matched your search
@@ -35,3 +35,9 @@ expanded; blank padding past the grid's edge is clipped, data past it is an erro
|
||||
`craft-xml`'s depth and attribute limits. At most 1024 tables.
|
||||
|
||||
`Options::only_sheet` keeps one sheet's cells and lists the others by name.
|
||||
|
||||
`Options::truncate_text` keeps the cells read before the text passes `max_text`, leaves out
|
||||
every cell after that point and sets `Workbook::text_truncated`, so the caller can warn instead
|
||||
of failing. Rows and cells after the cell that passes it are not read. The one-cell limit still
|
||||
applies to every cell read, that cell included; the cell and merged-range limits count the cells
|
||||
kept.
|
||||
+118
-43
@@ -187,15 +187,17 @@ impl Import<'_> {
|
||||
let name = self.attr(attrs, "table:name")?.unwrap_or_default();
|
||||
self.sheet = Some(SheetDraft { sheet: Sheet { name, ..Sheet::default() }, depth, row: 0, keep });
|
||||
}
|
||||
// Once the text is cut short, nothing after it is kept: its rows and cells are not read.
|
||||
Tag::Row
|
||||
if self
|
||||
.sheet
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.keep && self.stack.get(s.depth + 1..).is_some_and(|p| p.iter().all(|t| *t == Tag::Rows))) =>
|
||||
if !self.wb.text_truncated
|
||||
&& self
|
||||
.sheet
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.keep && self.stack.get(s.depth + 1..).is_some_and(|p| p.iter().all(|t| *t == Tag::Rows))) =>
|
||||
{
|
||||
self.row = Some(RowDraft { depth, repeat: self.count(attrs, "table:number-rows-repeated", u32::MAX)?, col: 0, cells: vec![] });
|
||||
}
|
||||
Tag::Cell | Tag::Covered if self.row.as_ref().is_some_and(|r| depth == r.depth + 1) => {
|
||||
Tag::Cell | Tag::Covered if !self.wb.text_truncated && self.row.as_ref().is_some_and(|r| depth == r.depth + 1) => {
|
||||
let value_type = if self.attr(attrs, "calcext:value-type")?.as_deref() == Some("error") {
|
||||
Some(ValueType::Error)
|
||||
} else {
|
||||
@@ -320,6 +322,24 @@ impl Import<'_> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// With [`Options::truncate_text`], how many of `copies` copies of `each` bytes of text fit
|
||||
/// in what is left of `max_text`; fewer than `copies` marks the workbook truncated, and none
|
||||
/// fit once it is.
|
||||
fn fit(&mut self, copies: u32, each: u64) -> u32 {
|
||||
if self.wb.text_truncated {
|
||||
return 0;
|
||||
}
|
||||
if each == 0 {
|
||||
return copies;
|
||||
}
|
||||
let room = self.options.max_text.saturating_sub(self.text) / each;
|
||||
if room < u64::from(copies) {
|
||||
self.wb.text_truncated = true;
|
||||
return u32::try_from(room).unwrap_or(0);
|
||||
}
|
||||
copies
|
||||
}
|
||||
|
||||
fn charge_text(&mut self, bytes: u64) -> Result<(), Error> {
|
||||
self.text = self.text.saturating_add(bytes);
|
||||
if self.text > self.options.max_text {
|
||||
@@ -342,9 +362,13 @@ impl Import<'_> {
|
||||
row.col = end;
|
||||
if kept {
|
||||
let len = draft.cell.text.len() + draft.cell.raw.as_ref().map_or(0, String::len);
|
||||
self.charge_text(u64::from(draft.repeat).saturating_mul(len as u64))?;
|
||||
let repeat = if self.options.truncate_text { self.fit(draft.repeat, len as u64) } else { draft.repeat };
|
||||
if repeat == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
self.charge_text(u64::from(repeat).saturating_mul(len as u64))?;
|
||||
if let Some(row) = self.row.as_mut() {
|
||||
row.cells.push(RepeatedCell { col: start, repeat: draft.repeat, width, height, cell: draft.cell });
|
||||
row.cells.push(RepeatedCell { col: start, repeat, width, height, cell: draft.cell });
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -360,44 +384,71 @@ impl Import<'_> {
|
||||
};
|
||||
let first_row = sheet.row;
|
||||
sheet.row = end;
|
||||
for entry in row.cells {
|
||||
let count = u64::from(entry.repeat) * u64::from(row.repeat);
|
||||
if !entry.cell.is_blank() {
|
||||
self.cells = self.cells.saturating_add(count);
|
||||
let len = entry.cell.text.len() + entry.cell.raw.as_ref().map_or(0, String::len);
|
||||
// The first row's copies were charged with the cell.
|
||||
self.text = self.text.saturating_add(count.saturating_sub(u64::from(entry.repeat)).saturating_mul(len as u64));
|
||||
// The first copy of the row was charged cell by cell. With truncation, keep the further
|
||||
// copies whose text fits, then the cells of the next copy that fit.
|
||||
let row_repeat = row.repeat;
|
||||
let mut partial = vec![];
|
||||
let repeat = if self.options.truncate_text && row.repeat > 1 && !self.wb.text_truncated {
|
||||
let each = row.cells.iter().map(|e| u64::from(e.repeat).saturating_mul(charged(&e.cell))).fold(0u64, u64::saturating_add);
|
||||
let full = 1 + self.fit(row.repeat - 1, each);
|
||||
if full < row.repeat {
|
||||
let room = self.options.max_text.saturating_sub(self.text).saturating_sub(each.saturating_mul(u64::from(full - 1)));
|
||||
partial = cells_that_fit(&row.cells, room);
|
||||
}
|
||||
let merged = entry.width > 1 || entry.height > 1;
|
||||
if merged {
|
||||
self.merges = self.merges.saturating_add(count);
|
||||
full
|
||||
} else if self.options.truncate_text && self.wb.text_truncated {
|
||||
1
|
||||
} else {
|
||||
row.repeat
|
||||
};
|
||||
let kept_end = first_row.saturating_add(repeat).min(end);
|
||||
// The first copy's text was charged in `finish_cell`; the partial copy's was not.
|
||||
for (cells, rows, first_charged) in [(row.cells, first_row..kept_end, true), (partial, kept_end..kept_end + 1, false)] {
|
||||
for entry in cells {
|
||||
self.place(&entry, rows.clone(), first_charged, end, row_repeat)?;
|
||||
}
|
||||
if self.cells > self.options.max_cells || self.merges > self.options.max_merges || self.text > self.options.max_text {
|
||||
return Err(Error::TooLarge(format!(
|
||||
"more than {} cells, {} merged ranges or {} MiB of text",
|
||||
self.options.max_cells,
|
||||
self.options.max_merges,
|
||||
self.options.max_text >> 20
|
||||
)));
|
||||
}
|
||||
let last_col = entry.col.checked_add(entry.repeat - 1).and_then(|c| c.checked_add(entry.width - 1));
|
||||
let last_row = end.checked_add(entry.height - 1);
|
||||
if last_col.is_none_or(|c| c >= MAX_COLS) || last_row.is_none_or(|r| r > MAX_ROWS) {
|
||||
return Err(Error::TooLarge("a merged cell reaches outside the grid".into()));
|
||||
}
|
||||
if merged && ((entry.repeat > 1 && entry.width > 1) || (row.repeat > 1 && entry.height > 1)) {
|
||||
return Err(Error::Invalid("repeated cells have overlapping merges".into()));
|
||||
}
|
||||
let Some(sheet) = self.sheet.as_mut() else { return Ok(()) };
|
||||
for r in first_row..end {
|
||||
for c in entry.col..entry.col + entry.repeat {
|
||||
let at = CellRef::new(r, c);
|
||||
if !entry.cell.is_blank() {
|
||||
sheet.sheet.cells.insert(at, entry.cell.clone());
|
||||
}
|
||||
if merged {
|
||||
sheet.sheet.merges.push(Range::new(at, CellRef::new(r + entry.height - 1, c + entry.width - 1)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Counts, checks and stores one cell entry over `rows`.
|
||||
fn place(&mut self, entry: &RepeatedCell, rows: std::ops::Range<u32>, first_charged: bool, end: u32, row_repeat: u32) -> Result<(), Error> {
|
||||
let count = u64::from(entry.repeat) * u64::from(rows.end.saturating_sub(rows.start));
|
||||
if !entry.cell.is_blank() {
|
||||
self.cells = self.cells.saturating_add(count);
|
||||
// The first row's copies were charged with the cell.
|
||||
let copies = if first_charged { count.saturating_sub(u64::from(entry.repeat)) } else { count };
|
||||
self.text = self.text.saturating_add(copies.saturating_mul(charged(&entry.cell)));
|
||||
}
|
||||
let merged = entry.width > 1 || entry.height > 1;
|
||||
if merged {
|
||||
self.merges = self.merges.saturating_add(count);
|
||||
}
|
||||
if self.cells > self.options.max_cells || self.merges > self.options.max_merges || self.text > self.options.max_text {
|
||||
return Err(Error::TooLarge(format!(
|
||||
"more than {} cells, {} merged ranges or {} MiB of text",
|
||||
self.options.max_cells,
|
||||
self.options.max_merges,
|
||||
self.options.max_text >> 20
|
||||
)));
|
||||
}
|
||||
let last_col = entry.col.checked_add(entry.repeat - 1).and_then(|c| c.checked_add(entry.width - 1));
|
||||
let last_row = end.checked_add(entry.height - 1);
|
||||
if last_col.is_none_or(|c| c >= MAX_COLS) || last_row.is_none_or(|r| r > MAX_ROWS) {
|
||||
return Err(Error::TooLarge("a merged cell reaches outside the grid".into()));
|
||||
}
|
||||
if merged && ((entry.repeat > 1 && entry.width > 1) || (row_repeat > 1 && entry.height > 1)) {
|
||||
return Err(Error::Invalid("repeated cells have overlapping merges".into()));
|
||||
}
|
||||
let Some(sheet) = self.sheet.as_mut() else { return Ok(()) };
|
||||
for r in rows {
|
||||
for c in entry.col..entry.col + entry.repeat {
|
||||
let at = CellRef::new(r, c);
|
||||
if !entry.cell.is_blank() {
|
||||
sheet.sheet.cells.insert(at, entry.cell.clone());
|
||||
}
|
||||
if merged {
|
||||
sheet.sheet.merges.push(Range::new(at, CellRef::new(r + entry.height - 1, c + entry.width - 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -405,6 +456,30 @@ impl Import<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes of text a kept cell is charged: its displayed text and its value attribute.
|
||||
fn charged(cell: &Cell) -> u64 {
|
||||
if cell.is_blank() { 0 } else { (cell.text.len() + cell.raw.as_ref().map_or(0, String::len)) as u64 }
|
||||
}
|
||||
|
||||
/// The leading entries of a row copy whose text fits in `room` bytes, the last one with only
|
||||
/// the repeats that fit.
|
||||
fn cells_that_fit(cells: &[RepeatedCell], mut room: u64) -> Vec<RepeatedCell> {
|
||||
let mut out = vec![];
|
||||
for e in cells {
|
||||
let len = charged(&e.cell);
|
||||
let n = room.checked_div(len).map_or(e.repeat, |fit| u32::try_from(fit).unwrap_or(u32::MAX).min(e.repeat));
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
room -= u64::from(n) * len;
|
||||
out.push(RepeatedCell { col: e.col, repeat: n, width: e.width, height: e.height, cell: e.cell.clone() });
|
||||
if n < e.repeat {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn push_text(out: &mut String, text: &str, max: usize) -> Result<(), Error> {
|
||||
if out.len().saturating_add(text.len()) > max {
|
||||
return Err(Error::TooLarge(format!("a cell's text is longer than {max} bytes")));
|
||||
|
||||
@@ -73,6 +73,11 @@ pub struct Options {
|
||||
pub max_merges: u64,
|
||||
/// Bytes of cell text, repeats expanded, over all sheets.
|
||||
pub max_text: u64,
|
||||
/// When the text would pass `max_text`, keep the cells read before that point and stop
|
||||
/// keeping cells, with [`Workbook::text_truncated`] set, instead of refusing the document.
|
||||
/// Rows and cells after the cell that passes it are not read. `max_cell_text` still applies
|
||||
/// to every cell read, that cell included; `max_cells` and `max_merges` count the cells kept.
|
||||
pub truncate_text: bool,
|
||||
/// Bytes of one cell's text, and of one attribute value.
|
||||
pub max_cell_text: usize,
|
||||
/// Keep the cells of this sheet only (by position); the others are listed by name, empty.
|
||||
@@ -86,6 +91,7 @@ impl Default for Options {
|
||||
max_cells: 1_000_000,
|
||||
max_merges: 100_000,
|
||||
max_text: 64 << 20,
|
||||
truncate_text: false,
|
||||
max_cell_text: 1 << 20,
|
||||
only_sheet: None,
|
||||
}
|
||||
|
||||
@@ -105,6 +105,9 @@ pub struct Sheet {
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Workbook {
|
||||
pub sheets: Vec<Sheet>,
|
||||
/// [`Options::truncate_text`](crate::Options::truncate_text) cut the text short: the cells
|
||||
/// after the point where it passed `max_text` were left out.
|
||||
pub text_truncated: bool,
|
||||
}
|
||||
|
||||
fn digits(s: &str) -> Option<&str> {
|
||||
|
||||
@@ -240,6 +240,67 @@ fn expansion_bounds_stop_small_repeat_bombs() {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_past_the_limit_can_be_cut_short_instead_of_refused() {
|
||||
let cut = Options { truncate_text: true, ..Options::default() };
|
||||
// A repeated row past 64 MiB keeps the copies that fit.
|
||||
let cell = format!(r#"<table:table-cell office:value-type="string" office:string-value="{}"/>"#, "y".repeat(1024));
|
||||
let rows =
|
||||
ods(&format!(r#"<table:table table:name="Data"><table:table-row table:number-rows-repeated="65537">{cell}</table:table-row></table:table>"#));
|
||||
assert!(matches!(open(&rows), Err(Error::TooLarge(_))));
|
||||
let wb = read(&rows, &cut).unwrap();
|
||||
assert!(wb.text_truncated);
|
||||
assert_eq!(wb.sheets[0].cells.len(), 65_536);
|
||||
assert!(wb.sheets[0].cells.contains_key(&CellRef::new(65_535, 0)));
|
||||
assert!(
|
||||
!open(&ods(&format!(r#"<table:table table:name="Data"><table:table-row>{cell}</table:table-row></table:table>"#))).unwrap().text_truncated
|
||||
);
|
||||
// Repeated columns keep the copies that fit, and nothing after them is kept: not a later
|
||||
// cell that would fit, not the row's further copies, not later rows or sheets.
|
||||
let small = Options { max_text: 10, ..cut };
|
||||
let bytes = ods(r#"
|
||||
<table:table table:name="A">
|
||||
<table:table-row table:number-rows-repeated="2"><table:table-cell table:number-columns-repeated="3" office:value-type="string" office:string-value="abcd"/><table:table-cell office:value-type="string" office:string-value="z"/><table:table-cell table:number-columns-spanned="2"/></table:table-row>
|
||||
<table:table-row><table:table-cell office:value-type="float" office:value="1"/></table:table-row>
|
||||
</table:table>
|
||||
<table:table table:name="B"><table:table-row><table:table-cell table:formula="of:=1"/></table:table-row></table:table>"#);
|
||||
assert!(matches!(read(&bytes, &Options { max_text: 10, ..Options::default() }), Err(Error::TooLarge(_))));
|
||||
let wb = read(&bytes, &small).unwrap();
|
||||
assert!(wb.text_truncated);
|
||||
assert_eq!(wb.sheets.iter().map(|s| s.name.as_str()).collect::<Vec<_>>(), ["A", "B"]);
|
||||
assert_eq!(wb.sheets[0].cells.keys().copied().collect::<Vec<_>>(), [CellRef::new(0, 0), CellRef::new(0, 1)]);
|
||||
assert_eq!(shown(&wb.sheets[0], CellRef::new(0, 1)), text("abcd"));
|
||||
assert!(wb.sheets[0].merges.is_empty() && wb.sheets[1].cells.is_empty());
|
||||
// A repeated row keeps the cells of its next copy that fit.
|
||||
let bytes = ods(
|
||||
r#"<table:table table:name="A"><table:table-row table:number-rows-repeated="3"><table:table-cell office:value-type="string" office:string-value="abcd"/><table:table-cell office:value-type="string" office:string-value="efgh"/></table:table-row></table:table>"#,
|
||||
);
|
||||
let wb = read(&bytes, &Options { max_text: 12, ..cut }).unwrap();
|
||||
assert!(wb.text_truncated);
|
||||
assert_eq!(wb.sheets[0].cells.keys().copied().collect::<Vec<_>>(), [CellRef::new(0, 0), CellRef::new(0, 1), CellRef::new(1, 0)]);
|
||||
// Cells after the cut are not read, so their own size limit does not refuse the document.
|
||||
let small = Options { max_text: 4, max_cell_text: 10, ..cut };
|
||||
let bytes = ods(r#"<table:table table:name="A">
|
||||
<table:table-row><table:table-cell office:value-type="string" office:string-value="abcd"/><table:table-cell office:value-type="string" office:string-value="x"/><table:table-cell office:value-type="string" office:string-value="eleven char"/></table:table-row>
|
||||
<table:table-row><table:table-cell><text:p>more than ten bytes</text:p></table:table-cell></table:table-row>
|
||||
</table:table>"#);
|
||||
assert!(matches!(read(&bytes, &Options { truncate_text: false, ..small }), Err(Error::TooLarge(_))));
|
||||
assert_eq!(read(&bytes, &small).unwrap().sheets[0].cells.keys().copied().collect::<Vec<_>>(), [CellRef::new(0, 0)]);
|
||||
// The cell that passes the limit is read whole first, so its own size limit still applies.
|
||||
let over = row(r#"<table:table-cell office:value-type="string" office:string-value="eleven char"/>"#);
|
||||
assert!(matches!(read(&over, &small), Err(Error::TooLarge(_))));
|
||||
// The other limits refuse the cells kept, and do not count the cells left out.
|
||||
let few = Options { max_cells: 1, ..cut };
|
||||
let after = row(
|
||||
r#"<table:table-cell office:value-type="string" office:string-value="ab"/><table:table-cell table:number-columns-repeated="2" office:value-type="float" office:value="1"/>"#,
|
||||
);
|
||||
assert!(read(&after, &Options { max_text: 1, ..few }).unwrap().sheets[0].cells.is_empty());
|
||||
assert!(matches!(
|
||||
read(&row(r#"<table:table-cell table:number-columns-repeated="2" office:value-type="float" office:value="1"/>"#), &few),
|
||||
Err(Error::TooLarge(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_result_never_reads_as_its_placeholder_zero() {
|
||||
let wb = open(&row(
|
||||
|
||||
Reference in new issue
Block a user