From 44c663aea20433afb65a9f731ed7067365d16d47 Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:45:24 -0400 Subject: [PATCH 01/24] Model: Python cells, and the #PYTHON!, #BUSY!, #TIMEOUT! and #BLOCKED! errors Cell gains `py: Option>`, read with a serde default and not written for ordinary cells, so older native files load and ordinary cells serialize as before. PyCode holds the code, the return type (an Excel value or a Python object), the last result (value, type name, preview, whether it converted to cells) and a stale flag for manual calculation. The cell's `value` stays the cached result, so formulas, display and saving read a Python cell like any other. is_blank, CellStore::has and input_text (which returns the code) take the code into account; Cell::has_content and clear_content cover value, formula and code together. The model reads and writes the PY function's syntax as Microsoft documents it, PY("chunk"&"chunk",0|1), with string literals of at most 255 characters and the return type 0 for an Excel value, 1 for a Python object. The reader takes the `_xlfn._xlws.` prefix (and `_xlfn.` or `_xlws.` alone, in any letter case); to_file_formula writes it. (An xlsx file keeps the code in its Python part instead; the xlsx commit adds it.) The size limit is Excel's 8,192 characters for that formula, not a count of code characters: doubled quotes and the "&" between literals count, so code with many quotes fits in fewer characters (8,079 characters of plain code fit exactly). The model also finds the xl("...") literals in code, skipping comments and other strings; a plain literal may escape a quote or a backslash. A rewritten reference that holds the literal's quote (a sheet name that needs quotes, 'Q4 2026'!B2) takes the other quote as the literal's delimiter, or an escaped quote when it holds both, so it never ends the string early. Edits: cell.set over a Python cell, Clear Contents, Clear All and the cells a merge empties remove the code; Clear Formats keeps it. Copy and paste (All, Formulas), Fill, sort, cut-move, sheet copy and undo carry it; a cut to another sheet first qualifies its xl() references with the source sheet, as a moved formula's references are, so they keep reading the same cells. Paste Values and Paste Link do not carry it, and paste operations do not combine numbers with a Python cell. Fill copies Python cells instead of extending them as a series. cell.set with =PY(...) is refused with a message that points to python.set, and Text to Columns keeps a piece that reads as =PY(...) as text instead of clearing it. Goal Seek refuses a Python cell as its changing cell, and Solver as a variable cell, as both refuse formulas: the number written there would replace the code. The sheet-protection check compares the code, so a locked Python cell cannot be changed. cell.get reports {python: code, output, typeName, preview, stale}. The xl() literals follow reference-moving edits: rewrite_all_formulas (row and column insert and delete, cell insert and delete, cut-move), sheet rename and sheet delete adjust each literal like a formula reference, and replace it only when it changed. Errors: CellError gains Python, Busy, Timeout and Blocked, and ALL has 14 entries, so the lexer reads them as literals. ERROR.TYPE gives 11 for #BLOCKED!, the number Excel returns. Microsoft documents no ERROR.TYPE number for #PYTHON!, #BUSY! or #TIMEOUT!; they take 19, 20 and 21, after the numbers Excel is known to use. The xlsx writer caches the four as #VALUE!, as it already does #SPILL! and #CALC!, because they are not error codes of the file format. Tests cover the =PY refusal, removal by entry and clearing, undo, copy, paste values, fill and sort, xl() rewrites through row insert, sheet rename (also to names that need quotes, with one quote and with both), cut-move (also to another sheet) and column delete, Goal Seek and Solver refusing Python cells, Text to Columns keeping a =PY piece, native JSON round trips, the new errors, the prefixes, escaped quotes in literals, and the encoded size limit at 8,192 characters exactly and with doubled quotes. --- crates/core/src/value.rs | 27 +- crates/engine/src/cmd/analysis.rs | 4 +- crates/engine/src/cmd/data.rs | 11 +- crates/engine/src/cmd/edit.rs | 104 +++++- crates/engine/src/cmd/extra.rs | 1 + crates/engine/src/cmd/forecast.rs | 2 +- crates/engine/src/cmd/format.rs | 3 +- crates/engine/src/cmd/formulas.rs | 2 +- crates/engine/src/cmd/insert.rs | 7 +- crates/engine/src/cmd/inspect.rs | 9 +- crates/engine/src/cmd/mod.rs | 11 +- crates/engine/src/cmd/pivot.rs | 7 +- crates/engine/src/cmd/sheet.rs | 2 + crates/engine/src/cmd/solver.rs | 7 +- crates/engine/src/cmd/whatif.rs | 17 +- crates/engine/src/fill.rs | 17 +- crates/engine/src/io.rs | 20 +- crates/engine/src/pivot.rs | 5 +- crates/engine/src/tests.rs | 1 + crates/engine/src/tests/python_cells.rs | 229 +++++++++++++ crates/model/src/cell.rs | 33 +- crates/model/src/lib.rs | 2 + crates/model/src/python.rs | 437 ++++++++++++++++++++++++ crates/model/src/sheet.rs | 4 +- crates/model/src/store.rs | 2 +- crates/xlsx/src/csv.rs | 2 +- crates/xlsx/src/sheet_read.rs | 2 +- crates/xlsx/src/sheet_write.rs | 6 +- crates/xlsx/src/tests/locale_numfmt.rs | 2 +- crates/xlsx/src/tests/phonetic_tests.rs | 2 +- crates/xlsx/src/tests/pivot_tests.rs | 2 +- crates/xlsx/src/tests/roundtrip.rs | 4 +- crates/xlsx/src/xls/mod.rs | 10 +- 33 files changed, 913 insertions(+), 81 deletions(-) create mode 100644 crates/engine/src/tests/python_cells.rs create mode 100644 crates/model/src/python.rs diff --git a/crates/core/src/value.rs b/crates/core/src/value.rs index c531de4dc..459372d31 100644 --- a/crates/core/src/value.rs +++ b/crates/core/src/value.rs @@ -21,10 +21,18 @@ pub enum CellError { Calc, /// Circular reference that could not be resolved (shown as 0 by Excel; we keep it explicit). Circ, + /// A Python cell's code raised an error, or Python stopped. + Python, + /// A Python cell is waiting to run. + Busy, + /// A Python cell ran longer than the time limit. + Timeout, + /// Python can't run here: the workbook's content isn't enabled, or this build has no Python. + Blocked, } impl CellError { - pub const ALL: [CellError; 10] = [ + pub const ALL: [CellError; 14] = [ CellError::Null, CellError::Div0, CellError::Value, @@ -35,6 +43,10 @@ impl CellError { CellError::GettingData, CellError::Spill, CellError::Calc, + CellError::Python, + CellError::Busy, + CellError::Timeout, + CellError::Blocked, ]; pub fn as_str(&self) -> &'static str { match self { @@ -49,9 +61,16 @@ impl CellError { CellError::Spill => "#SPILL!", CellError::Calc => "#CALC!", CellError::Circ => "#CIRC!", + CellError::Python => "#PYTHON!", + CellError::Busy => "#BUSY!", + CellError::Timeout => "#TIMEOUT!", + CellError::Blocked => "#BLOCKED!", } } - /// `ERROR.TYPE` number. + /// `ERROR.TYPE` number. Microsoft documents 1 to 8; 9 (#SPILL!), 11 (#BLOCKED!) and 14 (#CALC!) + /// are the numbers Excel returns. Microsoft documents no number for #PYTHON!, #BUSY! or + /// #TIMEOUT!: 19, 20 and 21 are GridCraft's own choice, after the numbers Excel is known to use, + /// and may change if Excel's numbers become known. pub fn code(&self) -> u32 { match self { CellError::Null => 1, @@ -65,6 +84,10 @@ impl CellError { CellError::Spill => 9, CellError::Calc => 14, CellError::Circ => 15, + CellError::Blocked => 11, + CellError::Python => 19, + CellError::Busy => 20, + CellError::Timeout => 21, } } pub fn parse(s: &str) -> Option { diff --git a/crates/engine/src/cmd/analysis.rs b/crates/engine/src/cmd/analysis.rs index 6225b4934..ffcbccf94 100644 --- a/crates/engine/src/cmd/analysis.rs +++ b/crates/engine/src/cmd/analysis.rs @@ -1124,13 +1124,13 @@ fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result let sh = cx.sheet_mut(si)?; for c in &occupied { let style = sh.style_id(*c); - sh.set_cell(*c, Cell { value: Value::Empty, formula: None, style }); + sh.set_cell(*c, Cell { value: Value::Empty, formula: None, style, py: None }); } let mut touched = occupied.clone(); for ((r, c), v) in &g.cells { let cell = CellRef::new(at.row + r, at.col + c); let style = pct_styles.get(&cell).copied().unwrap_or_else(|| sh.style_id(cell)); - sh.set_cell(cell, Cell { value: v.clone(), formula: None, style }); + sh.set_cell(cell, Cell { value: v.clone(), formula: None, style, py: None }); touched.push(cell); } for c in touched { diff --git a/crates/engine/src/cmd/data.rs b/crates/engine/src/cmd/data.rs index fcab9db71..0c255a689 100644 --- a/crates/engine/src/cmd/data.rs +++ b/crates/engine/src/cmd/data.rs @@ -976,7 +976,16 @@ fn text_to_columns(s: &mut Session, p: &Json) -> Result { } } for (c, t) in out { - let cell = super::edit::input_to_cell(&t, sheet, c, &mut cx.wb).unwrap_or(None); + let cell = match super::edit::input_to_cell(&t, sheet, c, &mut cx.wb) { + Ok(cell) => cell, + // `=PY(…)` doesn't make a Python cell (that takes python.set): the piece is kept + // as text rather than lost. + Err(_) if gridcraft_model::python::is_py_formula(&t) => { + let style = cx.wb.sheet(sheet).map(|sh| sh.style_id(c)).unwrap_or_default(); + Some(Cell { value: Value::text(t.as_str()), formula: None, style, py: None }) + } + Err(_) => None, + }; let shm = cx.sheet_mut(sheet)?; match cell { Some(x) => shm.set_cell(c, x), diff --git a/crates/engine/src/cmd/edit.rs b/crates/engine/src/cmd/edit.rs index 28ce338d6..64b26b022 100644 --- a/crates/engine/src/cmd/edit.rs +++ b/crates/engine/src/cmd/edit.rs @@ -118,24 +118,30 @@ pub fn specs() -> Vec { // ---------------------------------------------------------------- entry +/// Typing `=PY(` doesn't make a Python cell; this says what does. +pub(crate) const PY_FORMULA: &str = "Python cells are made with the python.set command (the code as `code`), not by typing =PY(…)."; + /// What typing `input` into a cell produces (constant or formula, plus an automatic number /// format). Inherits the destination's cell, row or column style, including its number /// format, before parsing. Errors when the formula can't be parsed. pub(crate) fn input_to_cell(input: &str, sheet: usize, at: CellRef, wb: &mut gridcraft_model::Workbook) -> Result> { let style = wb.sheet(sheet).map(|sh| sh.style_id(at)).unwrap_or_default(); if input.is_empty() { - let c = Cell { value: Value::Empty, formula: None, style }; + let c = Cell { value: Value::Empty, formula: None, style, py: None }; return Ok(if c.is_blank() { None } else { Some(c) }); } let fmt_is_text = wb.styles.get(style).num_fmt.as_str() == "@"; if fmt_is_text { - return Ok(Some(Cell { value: Value::text(input), formula: None, style })); + return Ok(Some(Cell { value: Value::text(input), formula: None, style, py: None })); } let is_formula = input.starts_with('=') || (input.len() > 1 && (input.starts_with('+') || input.starts_with('-')) && gridcraft_core::parse::parse_number_text(input).is_none() && input.chars().nth(1).is_some_and(|c| c.is_ascii_alphabetic() || c == '(')); + if is_formula && gridcraft_model::python::is_py_formula(input) { + return Err(EngineError::Other(PY_FORMULA.into())); + } if is_formula { let body = input.strip_prefix('=').unwrap_or(input); let expr = wb.name_call_case(parse_typed_formula(body)?); @@ -187,7 +193,7 @@ pub(crate) fn input_to_cell(input: &str, sheet: usize, at: CellRef, wb: &mut gri if input.contains('\n') && !wb.styles.get(style).align.wrap { style = wb.styles.derive(style, |s| s.align.wrap = true); } - Ok(Some(Cell { value: parsed.value, formula: None, style })) + Ok(Some(Cell { value: parsed.value, formula: None, style, py: None })) } /// Parses a formula as it is typed (without the `=`). Excel closes missing parentheses for you, @@ -255,7 +261,7 @@ fn cell_set(s: &mut Session, p: &Json) -> Result { let sh = cx.sheet_mut(sheet)?; // The array takes the whole range: other cells in it lose their contents. for c in range.iter().skip(1).take(1_000_000) { - if let Some(old) = sh.cells.get(c).filter(|x| !x.value.is_empty() || x.formula.is_some()) { + if let Some(old) = sh.cells.get(c).filter(|x| x.has_content()) { let style = old.style; sh.set_cell(c, Cell { style, ..Cell::default() }); } @@ -327,10 +333,14 @@ fn range_set_values(s: &mut Session, p: &Json) -> Result { value: Value::number(n.as_f64().unwrap_or(0.0)), formula: None, style: cx.wb.sheet(sheet).map(|sh| sh.style_id(at)).unwrap_or_default(), + py: None, + }), + Json::Bool(b) => Some(Cell { + value: Value::Bool(*b), + formula: None, + style: cx.wb.sheet(sheet).map(|sh| sh.style_id(at)).unwrap_or_default(), + py: None, }), - Json::Bool(b) => { - Some(Cell { value: Value::Bool(*b), formula: None, style: cx.wb.sheet(sheet).map(|sh| sh.style_id(at)).unwrap_or_default() }) - } // Text as given: no date or number reading, and a leading `'` is part of the // text rather than the marker that makes it text. A formula is still a // formula, which is what the ticket asks for and what `=` means in a cell. @@ -340,7 +350,7 @@ fn range_set_values(s: &mut Session, p: &Json) -> Result { if text.contains('\n') && !cx.wb.styles.get(style).align.wrap { style = cx.wb.styles.derive(style, |s| s.align.wrap = true); } - Some(Cell { value: Value::Text(text.as_str().into()), formula: None, style }) + Some(Cell { value: Value::Text(text.as_str().into()), formula: None, style, py: None }) } _ => input_to_cell(&input, sheet, at, &mut cx.wb)?, }; @@ -743,8 +753,8 @@ fn go_to_special(s: &mut Session, p: &Json) -> Result { let v = sheet.value(c); let ok = match kind { "blanks" => v.is_empty(), - "constants" => cell.is_some_and(|x| x.formula.is_none() && !x.value.is_empty()), - "formulas" => cell.is_some_and(|x| x.formula.is_some()), + "constants" => cell.is_some_and(|x| x.formula.is_none() && x.py.is_none() && !x.value.is_empty()), + "formulas" => cell.is_some_and(|x| x.formula.is_some() || x.py.is_some()), "comments" | "notes" => sheet.comments.contains_key(&c), "errors" => v.is_error(), "numbers" => v.is_number(), @@ -920,7 +930,7 @@ fn slide_together(wb: &gridcraft_model::Workbook, sheet: usize, areas: &[RangeRe for c in cells { let Some(to) = c.offset(dr, dc) else { continue }; let style = src.cell(c).map(|x| x.style).unwrap_or_default(); - dst.set_cell(to, Cell { value: src.value(c), formula: None, style }); + dst.set_cell(to, Cell { value: src.value(c), formula: None, style, py: None }); if let Some(picture) = src.cell_pictures.get(&c) { dst.cell_pictures.insert(to, picture.clone()); } @@ -1183,7 +1193,11 @@ fn paste_special(s: &mut Session, p: &Json) -> Result { }; let src_val = src_sheet.value(sc); let src_picture = if cut_move { moved_pictures.get(&sc) } else { src_sheet.cell_pictures.get(&sc) }.cloned(); - if skip_blanks && src_picture.is_none() && src_val.is_empty() && src_cell.as_ref().is_none_or(|x| x.formula.is_none()) { + if skip_blanks + && src_picture.is_none() + && src_val.is_empty() + && src_cell.as_ref().is_none_or(|x| x.formula.is_none() && x.py.is_none()) + { continue; } let old = cx.wb.sheet(dest_sheet).and_then(|sh| sh.cell(dest)).cloned().unwrap_or_default(); @@ -1218,6 +1232,30 @@ fn paste_special(s: &mut Session, p: &Json) -> Result { gridcraft_formula::adjust::shift_relative(e, dest.row as i64 - sc.row as i64, dest.col as i64 - sc.col as i64); Some(std::sync::Arc::new(Formula { dynamic: f.dynamic, ..Formula::from_expr(moved) })) }; + // A Python cell's code travels with it. Cut to another sheet, it keeps + // reading the cells its `xl()` references named on its original sheet, as + // a moved formula does: they are qualified with that sheet before the + // workbook-wide move adjustment below. + let moved_py = |cell: &Cell| -> Option> { + let py = cell.py.clone()?; + if !cut_move || dest_sheet == clip.sheet { + return Some(py); + } + let qualified = gridcraft_model::python::rewrite_xl(&py.code, |reference| { + let e = gridcraft_formula::parse(reference).ok()?.map(&mut |x| match x { + gridcraft_formula::Expr::Ref(mut r) if r.sheet == gridcraft_formula::SheetSel::Current => { + r.sheet = gridcraft_formula::SheetSel::Named(src_sheet_name.clone()); + gridcraft_formula::Expr::Ref(r) + } + other => other, + }); + Some(gridcraft_formula::print(&e)) + }); + Some(match qualified { + Some(code) => std::sync::Arc::new(gridcraft_model::PyCode { code, ..(*py).clone() }), + None => py, + }) + }; if link { let sheet_prefix = if dest_sheet == clip.sheet && same_doc { String::new() @@ -1226,6 +1264,7 @@ fn paste_special(s: &mut Session, p: &Json) -> Result { }; new.formula = Some(std::sync::Arc::new(Formula::new(&format!("={sheet_prefix}{}", sc.a1())))); new.value = Value::Empty; + new.py = None; new_picture = None; } else { let sc_cell = src_cell.clone().unwrap_or_default(); @@ -1234,6 +1273,7 @@ fn paste_special(s: &mut Session, p: &Json) -> Result { new_picture = src_picture.clone(); new.formula = shift(&sc_cell); new.value = if new.formula.is_some() { Value::Empty } else { sc_cell.value.clone() }; + new.py = moved_py(&sc_cell); let st = map_style(sc_cell.style, &mut cx.wb); new.style = if what == What::AllExceptBorders { let borders = cx.wb.styles.get(old.style).borders; @@ -1246,6 +1286,7 @@ fn paste_special(s: &mut Session, p: &Json) -> Result { new_picture = src_picture.clone(); new.formula = shift(&sc_cell); new.value = if new.formula.is_some() { Value::Empty } else { sc_cell.value.clone() }; + new.py = moved_py(&sc_cell); if what == What::FormulasAndNumberFormats { let nf = src_wb.styles.get(sc_cell.style).num_fmt.clone(); new.style = cx.wb.styles.derive(old.style, |s| s.num_fmt = nf); @@ -1254,6 +1295,7 @@ fn paste_special(s: &mut Session, p: &Json) -> Result { What::Values | What::ValuesAndNumberFormats => { new_picture = src_picture.clone(); new.formula = None; + new.py = None; new.value = src_val.clone(); if what == What::ValuesAndNumberFormats { let nf = src_wb.styles.get(sc_cell.style).num_fmt.clone(); @@ -1267,6 +1309,7 @@ fn paste_special(s: &mut Session, p: &Json) -> Result { if op != "none" && matches!(what, What::All | What::Values | What::ValuesAndNumberFormats | What::Formulas) && new.formula.is_none() + && new.py.is_none() { if old_picture.is_some() || new_picture.is_some() { new_picture = None; @@ -1443,6 +1486,42 @@ pub(crate) fn rewrite_all_formulas(wb: &mut gridcraft_model::Workbook, target: & n.formula = gridcraft_formula::print(&new); } } + rewrite_python_refs(wb, &|host, expr| gridcraft_formula::adjust::adjust(expr, host, target, e)); +} + +/// Rewrites the `xl("…")` literals in every Python cell with `f`, which maps a reference held on +/// sheet `host` to its new form. Every reference-moving edit (insert and delete of lines or cells, +/// cut-move, sheet rename and delete) runs it, as it adjusts formulas. A literal is only replaced +/// when the reference it names changed, and text that doesn't parse is kept. +pub(crate) fn rewrite_python_refs(wb: &mut gridcraft_model::Workbook, f: &impl Fn(&str, gridcraft_formula::Expr) -> gridcraft_formula::Expr) { + for si in 0..wb.sheets.len() { + let Some(sh) = wb.sheets.get(si) else { continue }; + let host = sh.name.clone(); + let mut new_code: Vec<(CellRef, String)> = Vec::new(); + for (c, cell) in sh.cells.iter() { + let Some(py) = &cell.py else { continue }; + let changed = gridcraft_model::python::rewrite_xl(&py.code, |reference| { + let expr = gridcraft_formula::parse(reference).ok()?; + let before = gridcraft_formula::print(&expr); + let after = gridcraft_formula::print(&f(&host, expr)); + (after != before).then_some(after) + }); + if let Some(code) = changed { + new_code.push((c, code)); + } + } + if new_code.is_empty() { + continue; + } + let Some(sh) = wb.sheet_mut(si) else { continue }; + for (c, code) in new_code { + if let Some(cell) = sh.cells.get_mut(c) + && let Some(py) = cell.py.as_mut() + { + std::sync::Arc::make_mut(py).code = code; + } + } + } } // ---------------------------------------------------------------- clear @@ -1481,6 +1560,7 @@ fn clear(s: &mut Session, p: &Json, what: &str) -> Result { for (_, cell) in cells.iter_mut() { let checkbox = cx.wb.styles.get(cell.style).num_fmt.as_str() == "checkbox"; cell.formula = None; + cell.py = None; if checkbox && uncheck_only { cell.value = Value::Bool(false); } else { diff --git a/crates/engine/src/cmd/extra.rs b/crates/engine/src/cmd/extra.rs index 60ffa102b..364b81ed7 100644 --- a/crates/engine/src/cmd/extra.rs +++ b/crates/engine/src/cmd/extra.rs @@ -1211,6 +1211,7 @@ fn toggle_checkbox(s: &mut Session, p: &Json) -> Result { let sh = cx.sheet_mut(si)?; if let Some(cell) = sh.cells.get_mut(c) && cell.formula.is_none() + && cell.py.is_none() { // An empty checkbox cell counts as FALSE. let next = match cell.value { diff --git a/crates/engine/src/cmd/forecast.rs b/crates/engine/src/cmd/forecast.rs index b7472fd22..c45c1179f 100644 --- a/crates/engine/src/cmd/forecast.rs +++ b/crates/engine/src/cmd/forecast.rs @@ -315,7 +315,7 @@ fn forecast_sheet(s: &mut Session, p: &Json) -> Result { let vstyle = cx.wb.styles.derive(StyleId::DEFAULT, |st| st.num_fmt = vfmt.clone()); let mut sheet = Sheet::new(sheet_name.clone()); let put = |sheet: &mut Sheet, row: u32, col: u32, v: Value, style: StyleId| { - sheet.set_cell(CellRef::new(row, col), Cell { value: v, formula: None, style }); + sheet.set_cell(CellRef::new(row, col), Cell { value: v, formula: None, style, py: None }); }; let formula = |sheet: &mut Sheet, row: u32, col: u32, text: String, style: StyleId| { let mut cell = Cell::formula(Formula::new(&text)); diff --git a/crates/engine/src/cmd/format.rs b/crates/engine/src/cmd/format.rs index 10dd2989d..30efbf5a6 100644 --- a/crates/engine/src/cmd/format.rs +++ b/crates/engine/src/cmd/format.rs @@ -665,8 +665,7 @@ fn merge(s: &mut Session, p: &Json, how: &str) -> Result { let others: Vec = sh.cells.iter_range(*r).map(|(c, _)| c).filter(|c| *c != r.start).collect(); for c in others { if let Some(mut cell) = sh.cells.get(c).cloned() { - cell.value = gridcraft_core::Value::Empty; - cell.formula = None; + cell.clear_content(); sh.set_cell(c, cell); } touched.push((sheet, c)); diff --git a/crates/engine/src/cmd/formulas.rs b/crates/engine/src/cmd/formulas.rs index 88c4cc76c..edeefb196 100644 --- a/crates/engine/src/cmd/formulas.rs +++ b/crates/engine/src/cmd/formulas.rs @@ -623,7 +623,7 @@ fn paste_name(s: &mut Session, p: &Json) -> Result { let sh = cx.sheet_mut(sheet)?; let style = sh.style_id(c); // The second column holds the refers-to text as a text value, with no formula. - sh.set_cell(c, Cell { value: Value::text(text.as_str()), formula: None, style }); + sh.set_cell(c, Cell { value: Value::text(text.as_str()), formula: None, style, py: None }); cx.touch(sheet, c); } } diff --git a/crates/engine/src/cmd/insert.rs b/crates/engine/src/cmd/insert.rs index 8274dfeed..a37084523 100644 --- a/crates/engine/src/cmd/insert.rs +++ b/crates/engine/src/cmd/insert.rs @@ -445,7 +445,10 @@ fn write_totals(sh: &mut Sheet, ti: usize) { let c = CellRef::new(row, t.range.start.col + i as u32); if let Some(code) = col.totals.subtotal_code() { let f = format!("=SUBTOTAL({code},{}[{}])", t.name, col.name); - sh.set_cell(c, Cell { formula: Some(std::sync::Arc::new(Formula::new(&f))), value: gridcraft_core::Value::Empty, style: sh.style_id(c) }); + sh.set_cell( + c, + Cell { formula: Some(std::sync::Arc::new(Formula::new(&f))), value: gridcraft_core::Value::Empty, style: sh.style_id(c), py: None }, + ); } else if let Some(l) = &col.totals_label { sh.set_value(c, gridcraft_core::Value::text(l.as_str())); } else { @@ -1260,7 +1263,7 @@ fn insert_link(s: &mut Session, p: &Json) -> Result { }); let sh = cx.sheet_mut(sheet)?; if let Some(t) = text.clone().or_else(|| sh.value(at).is_empty().then(|| target.clone())) { - sh.set_cell(at, Cell { value: gridcraft_core::Value::text(t.as_str()), formula: None, style: link_style }); + sh.set_cell(at, Cell { value: gridcraft_core::Value::text(t.as_str()), formula: None, style: link_style, py: None }); } else { sh.set_style(at, link_style); } diff --git a/crates/engine/src/cmd/inspect.rs b/crates/engine/src/cmd/inspect.rs index f9d5f9dc3..06a2c08b0 100644 --- a/crates/engine/src/cmd/inspect.rs +++ b/crates/engine/src/cmd/inspect.rs @@ -9,7 +9,7 @@ pub fn specs() -> Vec { vec![ cmd!(query "document.inspect", "Inspect Workbook", [], None, "{} → sheets, selection, names, tables, charts, dirty, trust (local|fromFile), history", has_doc, inspect), cmd!(query "sheet.read", "Read Range", [], None, "{range?: \"A1:D10\" (default: used range), sheet?, formulas?: bool, formatted?: bool} → rows of values", has_doc, read), - cmd!(query "cell.get", "Get Cell", [], None, "{cell?, sheet?} → value, formula, display text, style, comment, link", has_doc, cell_get), + cmd!(query "cell.get", "Get Cell", [], None, "{cell?, sheet?} → value, formula, display text, style, comment, link, python {code, output, typeName, preview, stale}", has_doc, cell_get), cmd!(query "selection.stats", "Selection Statistics", [], None, "{} → count, sum, average, min, max", has_doc, sel_stats), cmd!(query "history.list", "Undo History", [], None, "{} → undo/redo labels, grouping (an undo group is open)", has_doc, history), cmd!(query "app.commands", "List Commands", [], None, "{search?} → ids, labels, params", always, list_commands), @@ -137,6 +137,13 @@ fn cell_get(s: &mut Session, p: &Json) -> Result { "merge": sh.merge_at(c).map(|m| m.a1()), "comment": sh.comments.get(&c), "hyperlink": sh.hyperlinks.get(&c), + "python": cell.and_then(|x| x.py.as_deref()).map(|py| json!({ + "code": py.code, + "output": if py.object { "object" } else { "value" }, + "typeName": py.output.as_ref().map(|o| o.type_name.as_str()), + "preview": py.output.as_ref().map(|o| o.preview.as_str()), + "stale": py.stale, + })), })) } diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index ef6714eea..2f1becd23 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -476,13 +476,18 @@ fn check_arrays(old: &Workbook, new: &Workbook) -> Result<()> { Ok(()) } -/// Two cells hold the same value or formula (formats aside). +/// Two cells hold the same value, formula or Python code (formats aside; a Python cell's result +/// is not something the person edits). fn same_content(a: Option<&gridcraft_model::Cell>, b: Option<&gridcraft_model::Cell>) -> bool { - let empty = |c: Option<&gridcraft_model::Cell>| c.is_none_or(|c| c.formula.is_none() && c.value.is_empty()); + let empty = |c: Option<&gridcraft_model::Cell>| c.is_none_or(|c| !c.has_content()); match (a, b) { (Some(a), Some(b)) => match (&a.formula, &b.formula) { (Some(x), Some(y)) => x.text == y.text && x.array == y.array, - (None, None) => a.value == b.value, + (None, None) => match (&a.py, &b.py) { + (Some(x), Some(y)) => x.code == y.code && x.object == y.object, + (None, None) => a.value == b.value, + _ => false, + }, _ => false, }, _ => empty(a) && empty(b), diff --git a/crates/engine/src/cmd/pivot.rs b/crates/engine/src/cmd/pivot.rs index 164ae4449..2cf76a6ae 100644 --- a/crates/engine/src/cmd/pivot.rs +++ b/crates/engine/src/cmd/pivot.rs @@ -812,7 +812,10 @@ fn show_details(s: &mut Session, p: &Json) -> Result { } } for (c, h) in src.headers.iter().enumerate() { - sh.set_cell(CellRef::new(0, c as u32), Cell { value: gridcraft_core::Value::text(h.as_str()), formula: None, style: StyleId::default() }); + sh.set_cell( + CellRef::new(0, c as u32), + Cell { value: gridcraft_core::Value::text(h.as_str()), formula: None, style: StyleId::default(), py: None }, + ); } for (r, i) in records.iter().enumerate() { let Some(rec) = src.rows.get(*i) else { continue }; @@ -821,7 +824,7 @@ fn show_details(s: &mut Session, p: &Json) -> Result { if v.is_empty() && style == StyleId::default() { continue; } - sh.set_cell(CellRef::new(r as u32 + 1, c as u32), Cell { value: v.clone(), formula: None, style }); + sh.set_cell(CellRef::new(r as u32 + 1, c as u32), Cell { value: v.clone(), formula: None, style, py: None }); } } let name = sh.name.clone(); diff --git a/crates/engine/src/cmd/sheet.rs b/crates/engine/src/cmd/sheet.rs index ea615cb83..44a3c9d60 100644 --- a/crates/engine/src/cmd/sheet.rs +++ b/crates/engine/src/cmd/sheet.rs @@ -790,6 +790,7 @@ fn delete_sheet(s: &mut Session, p: &Json) -> Result { } } } + super::edit::rewrite_python_refs(&mut cx.wb, &|_, e| gridcraft_formula::adjust::delete_sheet(e, &name, &order)); cx.wb.names.retain(|n| n.scope != Some(i)); for n in cx.wb.names.iter_mut() { if let Some(sc) = n.scope.as_mut() @@ -867,6 +868,7 @@ fn rename_sheet(s: &mut Session, p: &Json) -> Result { n.formula = gridcraft_formula::print(&gridcraft_formula::adjust::rename_sheet(e, &old, &name)); } } + super::edit::rewrite_python_refs(&mut cx.wb, &|_, e| gridcraft_formula::adjust::rename_sheet(e, &old, &name)); cx.structural = true; Ok(Json::Null) }) diff --git a/crates/engine/src/cmd/solver.rs b/crates/engine/src/cmd/solver.rs index 21dbc1926..a8337ed13 100644 --- a/crates/engine/src/cmd/solver.rs +++ b/crates/engine/src/cmd/solver.rs @@ -331,7 +331,8 @@ fn parse_model(s: &Session, p: &Json) -> Result { } let mut seen = std::collections::HashSet::new(); vars.retain(|c| seen.insert(*c)); - if vars.iter().any(|c| sh.cell(*c).is_some_and(|c| c.formula.is_some())) { + // Python cells count as formulas: a solution written over one would lose its code. + if vars.iter().any(|c| sh.cell(*c).is_some_and(|c| c.formula.is_some() || c.py.is_some())) { return Err(EngineError::Other("Variable cells must hold constants, not formulas.".into())); } let mut cons = Vec::new(); @@ -1899,7 +1900,7 @@ fn write_reports(cx: &mut Ctx, at: usize, built: Vec<(String, Lines)>) -> Vec Result { let sh = cx.sheet_mut(sheet)?; for (c, v) in model.vars.iter().zip(x) { let style = sh.style_id(*c); - sh.set_cell(*c, Cell { value: Value::number(*v), formula: None, style }); + sh.set_cell(*c, Cell { value: Value::number(*v), formula: None, style, py: None }); } for c in &model.vars { cx.touch(sheet + count, *c); diff --git a/crates/engine/src/cmd/whatif.rs b/crates/engine/src/cmd/whatif.rs index 53b0dc3cc..2cc9c88bb 100644 --- a/crates/engine/src/cmd/whatif.rs +++ b/crates/engine/src/cmd/whatif.rs @@ -201,7 +201,7 @@ impl Trial { for (si, c, v) in sets { if let Some(sh) = wb.sheet_mut(*si) { let style = sh.style_id(*c); - sh.set_cell(*c, Cell { value: v.clone(), formula: None, style }); + sh.set_cell(*c, Cell { value: v.clone(), formula: None, style, py: None }); keys.push((*si, *c)); } } @@ -317,7 +317,8 @@ fn goal_seek(s: &mut Session, p: &Json) -> Result { if !has_formula(ssi, set) { return Err(EngineError::Other("Cell must contain a formula.".into())); } - if has_formula(csi, chg) { + // A Python cell is a formula to Excel (`=PY(…)`): writing a number over it would lose its code. + if has_formula(csi, chg) || d.wb.sheet(csi).and_then(|sh| sh.cell(chg)).is_some_and(|c| c.py.is_some()) { return Err(EngineError::Other("Cell must contain a value.".into())); } let x0 = d.wb.sheet(csi).map(|sh| sh.value(chg)).and_then(|v| v.as_f64()).unwrap_or(0.0); @@ -330,7 +331,7 @@ fn goal_seek(s: &mut Session, p: &Json) -> Result { edit(s, |cx| { let sh = cx.sheet_mut(csi)?; let style = sh.style_id(chg); - sh.set_cell(chg, Cell { value: Value::number(r.x), formula: None, style }); + sh.set_cell(chg, Cell { value: Value::number(r.x), formula: None, style, py: None }); cx.touch(csi, chg); Ok(()) })?; @@ -493,7 +494,7 @@ fn scenario_show(s: &mut Session, p: &Json) -> Result { let cell = match v.as_f64() { Some(n) => { let style = cx.wb.sheet(si).map(|sh| sh.style_id(*c)).unwrap_or_default(); - Some(Cell { value: Value::Number(n), formula: None, style }) + Some(Cell { value: Value::Number(n), formula: None, style, py: None }) } None => super::edit::input_to_cell(&super::edit::json_to_input(v), si, *c, &mut cx.wb)?, }; @@ -578,7 +579,7 @@ fn scenario_summary(s: &mut Session, p: &Json) -> Result { let bold = cx.wb.styles.derive(StyleId::DEFAULT, |st| st.font.bold = true); let mut sheet = Sheet::new(name.clone()); let put = |sheet: &mut Sheet, row: u32, col: u32, v: Value, style: StyleId| { - sheet.set_cell(CellRef::new(row, col), Cell { value: v, formula: None, style }); + sheet.set_cell(CellRef::new(row, col), Cell { value: v, formula: None, style, py: None }); }; put(&mut sheet, 1, 1, Value::text("Scenario Summary"), bold); put(&mut sheet, 2, 2, Value::text("Current Values:"), bold); @@ -701,7 +702,7 @@ fn data_table(s: &mut Session, p: &Json) -> Result { for (c, v) in out { let sh = cx.sheet_mut(si)?; let style = sh.style_id(c); - sh.set_cell(c, Cell { value: v, formula: None, style }); + sh.set_cell(c, Cell { value: v, formula: None, style, py: None }); cx.touch(si, c); } Ok(()) @@ -895,7 +896,7 @@ fn advanced_filter(s: &mut Session, p: &Json) -> Result { // The full output footprint was checked above. let to = CellRef::new(dest.row + i as u32, dest.col + c - list.start.col); let style = sh.style_id(src); - (to, Cell { value: sh.value(src), formula: None, style }) + (to, Cell { value: sh.value(src), formula: None, style, py: None }) }) }) .collect(); @@ -1122,7 +1123,7 @@ fn consolidate(s: &mut Session, p: &Json) -> Result { if v.is_empty() && style == StyleId::DEFAULT { sh.remove_cell(c); } else { - sh.set_cell(c, Cell { value: v, formula: None, style }); + sh.set_cell(c, Cell { value: v, formula: None, style, py: None }); } cx.touch(dsi, c); } diff --git a/crates/engine/src/fill.rs b/crates/engine/src/fill.rs index b12700e4f..644a17860 100644 --- a/crates/engine/src/fill.rs +++ b/crates/engine/src/fill.rs @@ -171,10 +171,12 @@ pub fn fill(cx: &mut Ctx, sheet: usize, src: RangeRef, target: RangeRef, mode: F cell.formula = Some(Arc::new(Formula::from_expr(shifted))); cell.value = Value::Empty; } - } else if let Some(v) = match per_cell.get(src_i) { - Some(p) => p.value(k as i64 / src_cells.len() as i64 + 1, forward, 1), - None => pattern.value(step, forward, src_cells.len()), - } { + } else if cell.py.is_none() + && let Some(v) = match per_cell.get(src_i) { + Some(p) => p.value(k as i64 / src_cells.len() as i64 + 1, forward, 1), + None => pattern.value(step, forward, src_cells.len()), + } + { cell.value = v; } cell.style = style; @@ -296,7 +298,8 @@ impl Pattern { fn detect(cells: &[(CellRef, Option)], custom: &[Vec], sys: gridcraft_core::DateSystem, fmt: &str) -> Pattern { let vals: Vec = cells.iter().map(|(_, c)| c.as_ref().map(|c| c.value.clone()).unwrap_or_default()).collect(); - if cells.iter().any(|(_, c)| c.as_ref().is_some_and(|c| c.formula.is_some())) { + // Formulas and Python cells copy. + if cells.iter().any(|(_, c)| c.as_ref().is_some_and(|c| c.formula.is_some() || c.py.is_some())) { return Pattern::Copy; } // Numbers. @@ -587,7 +590,7 @@ pub fn series(cx: &mut Ctx, sheet: usize, r: RangeRef, rows: bool, kind: &str, s break; } let c = if rows { CellRef::new(lane, r.start.col + i) } else { CellRef::new(r.start.row + i, lane) }; - writes.push((c, Cell { value: Value::number(round15(v)), formula: None, style })); + writes.push((c, Cell { value: Value::number(round15(v)), formula: None, style, py: None })); } } let sh = cx.sheet_mut(sheet)?; @@ -646,7 +649,7 @@ pub fn flash_fill(cx: &mut Ctx, sheet: usize, at: CellRef) -> Result { } let sh = cx.sheet_mut(sheet)?; for (c, out) in &writes { - sh.set_cell(*c, Cell { value: Value::text(out.as_str()), formula: None, style }); + sh.set_cell(*c, Cell { value: Value::text(out.as_str()), formula: None, style, py: None }); } for (c, _) in writes { cx.touch(sheet, c); diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 77467b210..4ad9f3d83 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -1196,17 +1196,17 @@ mod tests { (styled("yyyy-mm-dd"), styled("$#,##0.00"), styled("0.0%"), styled("# ?/?"), styled("@"), styled("0.00"), styled("#,##0")); let sh = wb.sheet_mut(0).unwrap(); let cells = [ - ("A1", Cell { value: Value::Number(46023.0), formula: None, style: date }), - ("B1", Cell { value: Value::Number(1234.5), formula: None, style: money }), - ("C1", Cell { value: Value::Number(0.125), formula: None, style: pct }), - ("D1", Cell { value: Value::Number(1.5), formula: None, style: frac }), - ("E1", Cell { value: Value::text("007"), formula: None, style: text }), - ("F1", Cell { value: Value::Error(CellError::NA), formula: None, style: money }), - ("G1", Cell { value: Value::Bool(true), formula: None, style: StyleId::DEFAULT }), + ("A1", Cell { value: Value::Number(46023.0), formula: None, style: date, py: None }), + ("B1", Cell { value: Value::Number(1234.5), formula: None, style: money, py: None }), + ("C1", Cell { value: Value::Number(0.125), formula: None, style: pct, py: None }), + ("D1", Cell { value: Value::Number(1.5), formula: None, style: frac, py: None }), + ("E1", Cell { value: Value::text("007"), formula: None, style: text, py: None }), + ("F1", Cell { value: Value::Error(CellError::NA), formula: None, style: money, py: None }), + ("G1", Cell { value: Value::Bool(true), formula: None, style: StyleId::DEFAULT, py: None }), ("A2", Cell { value: Value::Number(46024.0), style: date, ..Cell::formula(Formula::new("A1+1")) }), - ("B2", Cell { value: Value::Number(12345678.9), formula: None, style: fixed }), - ("C2", Cell { value: Value::Number(-0.25), formula: None, style: StyleId::DEFAULT }), - ("D2", Cell { value: Value::Number(1234567.0), formula: None, style: thousands }), + ("B2", Cell { value: Value::Number(12345678.9), formula: None, style: fixed, py: None }), + ("C2", Cell { value: Value::Number(-0.25), formula: None, style: StyleId::DEFAULT, py: None }), + ("D2", Cell { value: Value::Number(1234567.0), formula: None, style: thousands, py: None }), ]; for (a, c) in cells { sh.set_cell(CellRef::parse(a).unwrap(), c); diff --git a/crates/engine/src/pivot.rs b/crates/engine/src/pivot.rs index a074f5729..e541199d9 100644 --- a/crates/engine/src/pivot.rs +++ b/crates/engine/src/pivot.rs @@ -1518,7 +1518,10 @@ pub fn refresh(wb: &mut Workbook, sheet: usize, idx: usize, replace: bool) -> Re if oc.value.is_empty() && id == StyleId::default() { continue; } - cells.push((CellRef::new(origin.row + ri as u32, origin.col + ci as u32), Cell { value: oc.value.clone(), formula: None, style: id })); + cells.push(( + CellRef::new(origin.row + ri as u32, origin.col + ci as u32), + Cell { value: oc.value.clone(), formula: None, style: id, py: None }, + )); } } // Column widths that fit the labels. diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 13fbb9b3a..c9d10e36e 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5093,6 +5093,7 @@ fn a_script_cannot_switch_windows() { assert_eq!(v(&s, "A1"), Value::number(2.0)); } +mod python_cells; mod seams; #[test] diff --git a/crates/engine/src/tests/python_cells.rs b/crates/engine/src/tests/python_cells.rs new file mode 100644 index 000000000..91a1fed77 --- /dev/null +++ b/crates/engine/src/tests/python_cells.rs @@ -0,0 +1,229 @@ +//! Python cells under edits: what keeps, moves, rewrites or removes their code. + +use std::sync::Arc; + +use gridcraft_core::{CellRef, Value}; +use gridcraft_model::{Cell, PyCode}; +use serde_json::json; + +use super::s; +use crate::Session; + +/// Makes `cell` on sheet `sheet` a Python cell with `code` and a cached value. +fn plant(s: &mut Session, sheet: usize, cell: &str, code: &str) { + let d = s.doc_mut().unwrap(); + let mut wb = (*d.wb).clone(); + let sh = wb.sheet_mut(sheet).unwrap(); + let at = CellRef::parse(cell).unwrap(); + let style = sh.style_id(at); + sh.set_cell(at, Cell { value: Value::Number(1.0), formula: None, style, py: Some(Arc::new(PyCode::new(code))) }); + d.wb = Arc::new(wb); +} + +fn code(s: &Session, sheet: usize, cell: &str) -> Option { + let d = s.doc().unwrap(); + d.wb.sheet(sheet).unwrap().cell(CellRef::parse(cell).unwrap()).and_then(|c| c.py.as_ref()).map(|p| p.code.clone()) +} + +#[test] +fn typing_py_formulas_points_to_python_set() { + let mut s = s(); + let e = s.execute("cell.set", json!({"cell": "A1", "input": "=PY(\"1+1\",0)"})).unwrap_err(); + assert!(e.to_string().contains("python.set"), "{e}"); + assert!(s.execute("cell.set", json!({"cell": "A1", "input": "=py"})).is_err()); + // Other functions starting with PY are not affected. + s.execute("cell.set", json!({"cell": "A2", "input": "=PI()"})).unwrap(); +} + +#[test] +fn entering_or_clearing_a_python_cell_removes_its_code() { + let mut s = s(); + plant(&mut s, 0, "A1", "1 + 1"); + plant(&mut s, 0, "A2", "2 + 2"); + plant(&mut s, 0, "A3", "3 + 3"); + let got = s.execute("cell.get", json!({"cell": "A1"})).unwrap(); + assert_eq!(got["input"], "1 + 1"); + assert_eq!(got["python"]["code"], "1 + 1"); + assert_eq!(got["python"]["output"], "value"); + s.execute("cell.set", json!({"cell": "A1", "input": "hello"})).unwrap(); + assert_eq!(code(&s, 0, "A1"), None); + s.execute("edit.clearFormats", json!({"range": "A2"})).unwrap(); + assert_eq!(code(&s, 0, "A2").as_deref(), Some("2 + 2"), "clearing formats keeps the code"); + s.execute("edit.clearContents", json!({"range": "A2"})).unwrap(); + assert_eq!(code(&s, 0, "A2"), None); + s.execute("edit.clearAll", json!({"range": "A3"})).unwrap(); + assert_eq!(code(&s, 0, "A3"), None); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(code(&s, 0, "A3").as_deref(), Some("3 + 3"), "undo brings the code back"); +} + +#[test] +fn copy_fill_and_sort_carry_the_code_and_paste_values_drops_it() { + let mut s = s(); + plant(&mut s, 0, "A1", "xl('B1') * 2"); + s.execute("edit.copy", json!({"range": "A1"})).unwrap(); + s.execute("edit.paste", json!({"at": "C1"})).unwrap(); + assert_eq!(code(&s, 0, "C1").as_deref(), Some("xl('B1') * 2"), "the code is copied as it is"); + s.execute("edit.pasteSpecial", json!({"at": "D1", "what": "values"})).unwrap(); + assert_eq!(code(&s, 0, "D1"), None); + assert_eq!(super::v(&s, "D1"), Value::Number(1.0), "the cached value is pasted"); + s.execute("edit.fillDown", json!({"range": "A1:A3"})).unwrap(); + assert_eq!(code(&s, 0, "A3").as_deref(), Some("xl('B1') * 2")); + // Sorting moves Python cells with their rows. + s.execute("range.setValues", json!({"range": "E1", "values": [[3], [1], [2]]})).unwrap(); + plant(&mut s, 0, "F1", "'three'"); + plant(&mut s, 0, "F2", "'one'"); + s.execute("data.sort", json!({"range": "E1:F3", "keys": [{"column": "E"}]})).unwrap(); + assert_eq!(code(&s, 0, "F1").as_deref(), Some("'one'")); + assert_eq!(code(&s, 0, "F3").as_deref(), Some("'three'")); +} + +#[test] +fn xl_references_follow_inserts_moves_and_renames() { + let mut s = s(); + s.execute("home.insertSheet", json!({})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 0})).unwrap(); + let other = s.doc().unwrap().wb.sheet(1).unwrap().name.clone(); + plant(&mut s, 0, "C5", &format!("a = xl(\"A1:A3\")\nb = xl('{other}!B2', headers=True)\nc = xl(dynamic)\n# xl(\"A1\") stays in comments")); + s.execute("home.insertRows", json!({"rows": "2:2"})).unwrap(); + let c = code(&s, 0, "C6").unwrap(); + assert!(c.starts_with("a = xl(\"A1:A4\")\n"), "{c}"); + assert!(c.contains(&format!("xl('{other}!B2', headers=True)")), "{c}"); + assert!(c.contains("# xl(\"A1\") stays"), "{c}"); + s.execute("sheet.rename", json!({"sheet": 1, "name": "Data"})).unwrap(); + assert!(code(&s, 0, "C6").unwrap().contains("xl('Data!B2', headers=True)")); + // Cutting the cells a reference names moves the reference with them. + s.execute("edit.cut", json!({"range": "A1:A4"})).unwrap(); + s.execute("edit.paste", json!({"at": "H1"})).unwrap(); + assert!(code(&s, 0, "C6").unwrap().starts_with("a = xl(\"H1:H4\")"), "{:?}", code(&s, 0, "C6")); + // Deleting the referenced cells leaves #REF!, which fails when the code runs. + // The selection is elsewhere, so only the named column goes. + s.execute("selection.set", json!({"range": "A1"})).unwrap(); + s.execute("home.deleteColumns", json!({"cols": "H:H"})).unwrap(); + assert!(code(&s, 0, "C6").unwrap().starts_with("a = xl(\"#REF!\")"), "{:?}", code(&s, 0, "C6")); +} + +#[test] +fn a_rename_to_a_name_that_needs_quotes_rewrites_single_quoted_references() { + let mut s = s(); + s.execute("home.insertSheet", json!({})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 0})).unwrap(); + let other = s.doc().unwrap().wb.sheet(1).unwrap().name.clone(); + plant(&mut s, 0, "A1", &format!("a = xl('{other}!B2')\nb = xl(\"{other}!C3\")")); + s.execute("sheet.rename", json!({"sheet": 1, "name": "Q4 2026"})).unwrap(); + assert_eq!(code(&s, 0, "A1").unwrap(), "a = xl(\"'Q4 2026'!B2\")\nb = xl(\"'Q4 2026'!C3\")"); + // A name with both quotes in it: the literal's own quote is escaped. + s.execute("sheet.rename", json!({"sheet": 1, "name": "Bob's \"Q4\""})).unwrap(); + let c = code(&s, 0, "A1").unwrap(); + assert_eq!(c, "a = xl(\"'Bob''s \\\"Q4\\\"'!B2\")\nb = xl(\"'Bob''s \\\"Q4\\\"'!C3\")"); + let refs: Vec> = gridcraft_model::python::xl_calls(&c).into_iter().map(|x| x.reference).collect(); + assert_eq!(refs, [Some("'Bob''s \"Q4\"'!B2".to_string()), Some("'Bob''s \"Q4\"'!C3".to_string())]); + // And back to a plain name. + s.execute("sheet.rename", json!({"sheet": 1, "name": "Data"})).unwrap(); + assert_eq!(code(&s, 0, "A1").unwrap(), "a = xl(\"Data!B2\")\nb = xl(\"Data!C3\")"); +} + +#[test] +fn a_cut_to_another_sheet_keeps_references_on_the_source_sheet() { + let mut s = s(); + s.execute("home.insertSheet", json!({})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 0})).unwrap(); + let (src, dest) = { + let wb = &s.doc().unwrap().wb; + (wb.sheet(0).unwrap().name.clone(), wb.sheet(1).unwrap().name.clone()) + }; + plant(&mut s, 0, "A1", "a = xl(\"B5\")\nb = xl('A2')"); + plant(&mut s, 0, "A2", "1"); + s.execute("edit.cut", json!({"range": "A1:A2"})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 1})).unwrap(); + s.execute("edit.paste", json!({"at": "C1"})).unwrap(); + // B5 stays on the source sheet; A2 moved with the cut, so it follows to the destination. + assert_eq!(code(&s, 1, "C1").unwrap(), format!("a = xl(\"{src}!B5\")\nb = xl('{dest}!C2')")); + assert_eq!(code(&s, 0, "A1"), None); + // Within one sheet nothing is qualified. + s.execute("edit.cut", json!({"range": "C1"})).unwrap(); + s.execute("edit.paste", json!({"at": "E1"})).unwrap(); + assert_eq!(code(&s, 1, "E1").unwrap(), format!("a = xl(\"{src}!B5\")\nb = xl('{dest}!C2')")); + // A copy to another sheet still carries the code as it is. + plant(&mut s, 0, "A9", "xl(\"B5\")"); + s.execute("sheet.activate", json!({"sheet": 0})).unwrap(); + s.execute("edit.copy", json!({"range": "A9"})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 1})).unwrap(); + s.execute("edit.paste", json!({"at": "A9"})).unwrap(); + assert_eq!(code(&s, 1, "A9").as_deref(), Some("xl(\"B5\")")); +} + +#[test] +fn goal_seek_and_solver_refuse_to_change_a_python_cell() { + let mut s = s(); + plant(&mut s, 0, "A1", "2"); + s.execute("cell.set", json!({"cell": "B1", "input": "=A1*2"})).unwrap(); + let e = s.execute("data.goalSeek", json!({"set": "B1", "to": 10, "changing": "A1"})).unwrap_err(); + assert!(e.to_string().contains("must contain a value"), "{e}"); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("2"), "the code is kept"); + // The set cell must be a formula: a Python cell isn't one this build can recalculate. + s.execute("cell.set", json!({"cell": "C1", "input": "3"})).unwrap(); + assert!(s.execute("data.goalSeek", json!({"set": "A1", "to": 10, "changing": "C1"})).is_err()); + let e = s.execute("data.solver", json!({"objective": "B1", "goal": "max", "variables": "A1"})).unwrap_err(); + assert!(e.to_string().contains("constants"), "{e}"); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("2")); +} + +#[test] +fn text_to_columns_keeps_a_py_piece_as_text() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "'=PY(\"x\");7"})).unwrap(); + assert_eq!(super::v(&s, "A1"), Value::text("=PY(\"x\");7")); + s.execute("data.textToColumns", json!({"range": "A1", "delimiters": [";"]})).unwrap(); + // The default text qualifier takes the quotes away, as in Excel. + assert_eq!(super::v(&s, "A1"), Value::text("=PY(x)"), "the piece is kept, not cleared"); + assert_eq!(code(&s, 0, "A1"), None, "and it is not a Python cell"); + assert_eq!(super::v(&s, "B1"), Value::Number(7.0)); +} + +#[test] +fn python_cells_survive_native_json() { + let mut s = s(); + plant(&mut s, 0, "B2", "x = 1\nx"); + let wb = s.doc().unwrap().wb.clone(); + let json = serde_json::to_string(&*wb).unwrap(); + let back: gridcraft_model::Workbook = serde_json::from_str(&json).unwrap(); + let cell = back.sheet(0).unwrap().cell(CellRef::parse("B2").unwrap()).unwrap(); + assert_eq!(cell.py.as_ref().unwrap().code, "x = 1\nx"); + // Cells written before Python existed have no `py` field. + let plain: Cell = serde_json::from_value(json!({"value": {"t": "Number", "v": 2.0}, "formula": null, "style": 0})).unwrap(); + assert!(plain.py.is_none()); + assert!(!serde_json::to_string(&plain).unwrap().contains("py"), "no field is written for ordinary cells"); +} + +#[test] +fn new_errors_parse_and_have_error_type_numbers() { + use gridcraft_core::CellError; + for (text, e) in + [("#PYTHON!", CellError::Python), ("#BUSY!", CellError::Busy), ("#TIMEOUT!", CellError::Timeout), ("#BLOCKED!", CellError::Blocked)] + { + assert_eq!(CellError::parse(text), Some(e)); + assert_eq!(e.as_str(), text); + } + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "=ERROR.TYPE(#BLOCKED!)"})).unwrap(); + assert_eq!(super::v(&s, "A1"), Value::Number(11.0)); + s.execute("cell.set", json!({"cell": "A2", "input": "=ISERROR(#PYTHON!)"})).unwrap(); + assert_eq!(super::v(&s, "A2"), Value::Bool(true)); +} + +#[test] +fn go_to_special_counts_a_python_cell_as_a_formula_not_a_constant() { + let mut s = s(); + plant(&mut s, 0, "A1", "1 + 1"); + s.execute("cell.set", json!({"cell": "A2", "input": "=1+1"})).unwrap(); + s.execute("cell.set", json!({"cell": "A3", "input": "7"})).unwrap(); + s.execute("selection.set", json!({"range": "A1"})).unwrap(); + let r = s.execute("edit.goToSpecial", json!({"kind": "formulas"})).unwrap(); + assert_eq!(r["count"], 2, "{r}"); + assert_eq!(r["selection"], "A1:A2", "{r}"); + s.execute("selection.set", json!({"range": "A1"})).unwrap(); + let r = s.execute("edit.goToSpecial", json!({"kind": "constants"})).unwrap(); + assert_eq!(r["count"], 1, "{r}"); + assert_eq!(r["selection"], "A3", "{r}"); +} diff --git a/crates/model/src/cell.rs b/crates/model/src/cell.rs index d4466365f..58df4ebeb 100644 --- a/crates/model/src/cell.rs +++ b/crates/model/src/cell.rs @@ -6,6 +6,7 @@ use gridcraft_core::{RangeRef, Value}; use gridcraft_formula::Expr; use serde::{Deserialize, Serialize}; +use crate::python::PyCode; use crate::style::StyleId; /// A parsed formula. `text` is the canonical formula text without the leading `=`. @@ -60,25 +61,45 @@ pub struct CellPicture { #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Cell { - /// Constant value, or the cached result of the formula. + /// Constant value, or the cached result of the formula or Python code. pub value: Value, pub formula: Option>, pub style: StyleId, + /// Python code, for a Python cell (which has no formula). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub py: Option>, } impl Cell { pub fn value(v: Value) -> Cell { - Cell { value: v, formula: None, style: StyleId::DEFAULT } + Cell { value: v, formula: None, style: StyleId::DEFAULT, py: None } } pub fn formula(f: Formula) -> Cell { - Cell { value: Value::Empty, formula: Some(Arc::new(f)), style: StyleId::DEFAULT } + Cell { value: Value::Empty, formula: Some(Arc::new(f)), style: StyleId::DEFAULT, py: None } } - /// No value, no formula and the default style: storing it is pointless. + /// A value cell with this style. + pub fn styled(v: Value, style: StyleId) -> Cell { + Cell { value: v, formula: None, style, py: None } + } + /// No value, no formula, no Python code and the default style: storing it is pointless. pub fn is_blank(&self) -> bool { - self.value.is_empty() && self.formula.is_none() && self.style == StyleId::DEFAULT + self.value.is_empty() && self.formula.is_none() && self.style == StyleId::DEFAULT && self.py.is_none() } - /// What the formula bar shows. + /// Whether the cell holds content (a value, a formula or Python code), formats aside. + pub fn has_content(&self) -> bool { + !self.value.is_empty() || self.formula.is_some() || self.py.is_some() + } + /// Removes the content (value, formula and Python code), keeping the style. + pub fn clear_content(&mut self) { + self.value = Value::Empty; + self.formula = None; + self.py = None; + } + /// What the formula bar shows: a formula, a Python cell's code, or the value. pub fn input_text(&self) -> String { + if let Some(py) = &self.py { + return py.code.clone(); + } if let Some(f) = &self.formula { return format!("={}", f.text); } diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs index 071de683c..300fdc66a 100644 --- a/crates/model/src/lib.rs +++ b/crates/model/src/lib.rs @@ -10,6 +10,7 @@ mod cellmap; pub mod features; pub mod password; pub mod preserved; +pub mod python; pub mod sheet; pub mod store; pub mod style; @@ -19,6 +20,7 @@ pub use cell::{Cell, CellPicture, Formula}; pub use features::*; pub use password::PasswordHash; pub use preserved::{Preserved, PreservedPart, PreservedRel, PreservedSheet, PreservedXml}; +pub use python::{PyCode, PyOutput}; pub use sheet::{DEFAULT_COL_WIDTH, DEFAULT_ROW_HEIGHT, LineIndex, Sheet, SheetViewMode, Visibility, points_to_chars}; pub use store::CellStore; pub use style::{ diff --git a/crates/model/src/python.rs b/crates/model/src/python.rs new file mode 100644 index 000000000..abd4c9414 --- /dev/null +++ b/crates/model/src/python.rs @@ -0,0 +1,437 @@ +//! Python cells: the code a cell holds, its last result, and the text forms around it (the +//! `PY(…)` formula files store it as, and the `xl("…")` references inside it). + +use gridcraft_core::{CellError, Value}; +use serde::{Deserialize, Serialize}; + +/// Excel's limit on the characters of one formula. A Python cell's code must fit in it once it is +/// encoded as a file formula ([`PyCode::to_file_formula`]): quotes are doubled and every 255 +/// characters add `"&"`, so the limit on the code itself depends on what the code holds. +pub const MAX_FORMULA_CHARS: usize = 8_192; + +/// The prefix files put before `PY(`, as Excel writes it ([MS-XLSX]: `_xlfn._xlws.PY`). +pub const FILE_PREFIX: &str = "_xlfn._xlws."; + +/// Characters in one string literal of a `PY(…)` formula (Excel's limit for a literal). +const CHUNK_CHARS: usize = 255; + +/// A Python cell. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct PyCode { + pub code: String, + /// Show the result as a Python object (true) or convert it to cell values (false). + pub object: bool, + /// The last result, when the code ran without an error. + pub output: Option, + /// The cell's inputs changed while calculation was manual: its value is out of date until + /// the workbook is calculated. + pub stale: bool, +} + +/// A Python result kept on the cell. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct PyOutput { + /// The result as cells (an array spills), or its preview as text when it has no cell form. + pub value: Value, + /// Python's name for the result's type. + pub type_name: String, + /// The result's `repr`, shortened. + pub preview: String, + /// Whether `value` is the result converted to cells. + pub convertible: bool, +} + +impl PyCode { + pub fn new(code: &str) -> PyCode { + PyCode { code: normalize_code(code), ..PyCode::default() } + } + + /// What the cell shows for its result: cell values (an array spills from the cell), or, for a + /// Python object, its type name. `None` before the code has run. + pub fn result_value(&self) -> Option { + let out = self.output.as_ref()?; + let scalar = !matches!(out.value, Value::Array(_)) && out.convertible; + Some(if self.object && !scalar { Value::text(out.type_name.as_str()) } else { out.value.clone() }) + } + + /// The cell as a formula (without `=`): `PY("…"&"…",0)`, with the code in string literals of + /// at most 255 characters and the return type Excel documents: 0 for an Excel value, 1 for a + /// Python object. + pub fn to_formula(&self) -> String { + let chunks = chunk(&self.code, CHUNK_CHARS); + let lits: Vec = chunks.iter().map(|c| format!("\"{}\"", c.replace('"', "\"\""))).collect(); + let lits = if lits.is_empty() { "\"\"".to_string() } else { lits.join("&") }; + format!("PY({lits},{})", u8::from(self.object)) + } + + /// The formula a file stores the cell as: [`PyCode::to_formula`] with Excel's + /// `_xlfn._xlws.` prefix, so Excel reads it as its own Python function. + pub fn to_file_formula(&self) -> String { + format!("{FILE_PREFIX}{}", self.to_formula()) + } + + /// Whether the code fits in a file formula of at most [`MAX_FORMULA_CHARS`] characters. + pub fn fits(&self) -> bool { + // Code longer than the limit can't fit; this also bounds the work for hostile input. + self.code.chars().nth(MAX_FORMULA_CHARS).is_none() && self.to_file_formula().chars().nth(MAX_FORMULA_CHARS).is_none() + } + + /// Reads a `PY(…)` formula (with or without `=`, and with or without the `_xlfn.` and + /// `_xlws.` prefixes): string literals joined by `&`, optionally followed by `,0` or `,1`. + /// `None` for anything else. + pub fn from_formula(text: &str) -> Option { + let body = py_body(text); + let rest = body.get(..3).filter(|h| h.eq_ignore_ascii_case("PY(")).and_then(|_| body.get(3..))?; + let mut chars = rest.chars().peekable(); + let mut code = String::new(); + loop { + skip_space(&mut chars); + if chars.next()? != '"' { + return None; + } + loop { + match chars.next()? { + '"' if chars.peek() == Some(&'"') => { + chars.next(); + code.push('"'); + } + '"' => break, + c => code.push(c), + } + } + skip_space(&mut chars); + match chars.next()? { + '&' => continue, + ',' => { + skip_space(&mut chars); + let object = match chars.next()? { + '0' => false, + '1' => true, + _ => return None, + }; + skip_space(&mut chars); + return (chars.next()? == ')' && chars.all(char::is_whitespace)).then(|| PyCode { object, ..PyCode::new(&code) }); + } + ')' => return chars.all(char::is_whitespace).then(|| PyCode::new(&code)), + _ => return None, + } + } + } +} + +/// `text` without a leading `=`, spaces and the `_xlfn.` and `_xlws.` prefixes files put before +/// newer functions (in any letter case). +fn py_body(text: &str) -> &str { + let mut body = text.trim().trim_start_matches('=').trim_start(); + for prefix in ["_xlfn.", "_xlws."] { + if body.get(..prefix.len()).is_some_and(|h| h.eq_ignore_ascii_case(prefix)) { + body = body.get(prefix.len()..).unwrap_or_default(); + } + } + body +} + +fn skip_space(chars: &mut std::iter::Peekable>) { + while chars.peek().is_some_and(|c| c.is_whitespace()) { + chars.next(); + } +} + +/// Line breaks as `\n`. +pub fn normalize_code(code: &str) -> String { + code.replace("\r\n", "\n").replace('\r', "\n") +} + +/// `s` in pieces of at most `n` characters. +fn chunk(s: &str, n: usize) -> Vec { + let chars: Vec = s.chars().collect(); + chars.chunks(n.max(1)).map(|c| c.iter().collect()).collect() +} + +/// Whether `text` (a formula, with or without `=`) is a `PY(` call: such formulas are Python cells, +/// set with `python.set`. +pub fn is_py_formula(text: &str) -> bool { + let body = py_body(text); + body.get(..3).is_some_and(|h| h.eq_ignore_ascii_case("PY(")) || body.eq_ignore_ascii_case("PY") +} + +/// An `xl()` call in Python code. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct XlCall { + /// The reference text, when the first argument is a plain string literal. + pub reference: Option, + /// Byte range of the literal's contents (between the quotes) in the code. + pub span: Option>, +} + +/// The `xl(…)` calls in `code`, skipping comments and other string literals. A call whose first +/// argument isn't a plain string literal has no reference: the cells it reads are only known when +/// it runs. +pub fn xl_calls(code: &str) -> Vec { + let bytes = code.as_bytes(); + let mut out = Vec::new(); + let mut i = 0usize; + while let Some(&b) = bytes.get(i) { + match b { + b'#' => { + // A comment runs to the end of the line. + while bytes.get(i).is_some_and(|c| *c != b'\n') { + i += 1; + } + } + b'"' | b'\'' => i = skip_string(bytes, i), + b'x' if bytes.get(i + 1) == Some(&b'l') && (i == 0 || bytes.get(i - 1).is_some_and(|p| !is_ident(*p) && *p != b'.')) => { + let mut j = i + 2; + if bytes.get(j).is_some_and(|c| is_ident(*c)) { + i = j; + continue; + } + while bytes.get(j).is_some_and(|c| *c == b' ' || *c == b'\t') { + j += 1; + } + if bytes.get(j) != Some(&b'(') { + i = j; + continue; + } + j += 1; + while bytes.get(j).is_some_and(|c| c.is_ascii_whitespace()) { + j += 1; + } + let call = match bytes.get(j) { + Some(&q @ (b'"' | b'\'')) => { + let start = j + 1; + let end = skip_string(bytes, j); + // A plain literal: closed on its own line, not part of an expression + // (`xl("A" + n)`), not a triple-quoted string, and escaping nothing but a + // quote or a backslash (`\'`, `\"`, `\\`), which a sheet name may need. + let close = end.saturating_sub(1); + let mut k = end; + while bytes.get(k).is_some_and(|c| *c == b' ' || *c == b'\t') { + k += 1; + } + let plain = close > start + && bytes.get(close) == Some(&q) + && bytes.get(start) != Some(&q) + && matches!(bytes.get(k), Some(b')' | b',')); + i = end; + match code.get(start..close).filter(|_| plain).and_then(unescape) { + Some(t) => XlCall { reference: Some(t), span: Some(start..close) }, + None => XlCall { reference: None, span: None }, + } + } + _ => { + i = j; + XlCall { reference: None, span: None } + } + }; + out.push(call); + } + _ => i += 1, + } + } + out +} + +fn is_ident(b: u8) -> bool { + b.is_ascii_alphanumeric() || b == b'_' || b >= 0x80 +} + +/// The index just past the string literal starting at `at` (single, double or triple quoted). +fn skip_string(bytes: &[u8], at: usize) -> usize { + let Some(&q) = bytes.get(at) else { return at.saturating_add(1) }; + let triple = bytes.get(at + 1) == Some(&q) && bytes.get(at + 2) == Some(&q); + let mut i = if triple { at + 3 } else { at + 1 }; + while let Some(&c) = bytes.get(i) { + if c == b'\\' { + i += 2; + continue; + } + if c == q { + if !triple { + return i + 1; + } + if bytes.get(i + 1) == Some(&q) && bytes.get(i + 2) == Some(&q) { + return i + 3; + } + } else if c == b'\n' && !triple { + return i; + } + i += 1; + } + i +} + +/// The text of a plain literal's contents: `\'`, `\"` and `\\` stand for the character after the +/// backslash. `None` for any other escape or a line break, which make the literal not plain. +fn unescape(raw: &str) -> Option { + let mut out = String::with_capacity(raw.len()); + let mut chars = raw.chars(); + while let Some(c) = chars.next() { + match c { + '\n' => return None, + '\\' => match chars.next()? { + e @ ('\\' | '\'' | '"') => out.push(e), + _ => return None, + }, + c => out.push(c), + } + } + Some(out) +} + +/// `text` as a plain literal that was delimited by `quote`: its delimiter (`quote`, or the other +/// quote when `text` holds `quote` but not the other one, so `xl('Sheet2!B2')` becomes +/// `xl("'Q4 2026'!B2")` on a rename) and its contents, with backslashes escaped, and the +/// delimiter too when `text` holds both quotes. `None` for a line break, which no literal on +/// one line holds. +fn literal(text: &str, quote: char) -> Option<(char, String)> { + if text.contains('\n') { + return None; + } + let other = if quote == '\'' { '"' } else { '\'' }; + let delim = if text.contains(quote) && !text.contains(other) { other } else { quote }; + let mut body = String::with_capacity(text.len()); + for c in text.chars() { + if c == '\\' || c == delim { + body.push('\\'); + } + body.push(c); + } + Some((delim, body)) +} + +/// Rewrites the reference of every `xl("…")` literal in `code` with `f`; `None` from `f` leaves +/// it. Returns the new code when anything changed. A new reference holding the literal's quote +/// (a sheet name that needs quotes, `'Q4 2026'!B2`) gets the other quote as its delimiter, or an +/// escaped quote when it holds both, so it never ends the string early. +pub fn rewrite_xl(code: &str, mut f: impl FnMut(&str) -> Option) -> Option { + let mut out = String::with_capacity(code.len()); + let mut last = 0usize; + let mut changed = false; + for call in xl_calls(code) { + let (Some(reference), Some(span)) = (call.reference, call.span) else { continue }; + let Some(new) = f(&reference).filter(|n| *n != reference) else { continue }; + let open = span.start.saturating_sub(1); + let Some(quote) = code.get(open..span.start).and_then(|q| q.chars().next()) else { continue }; + let Some((delim, body)) = literal(&new, quote) else { continue }; + // The literal with its quotes: the delimiter may change. + out.push_str(code.get(last..open).unwrap_or_default()); + out.push(delim); + out.push_str(&body); + out.push(delim); + last = span.end.saturating_add(1); + changed = true; + } + if !changed { + return None; + } + out.push_str(code.get(last..).unwrap_or_default()); + Some(out) +} + +/// The value a Python cell shows when it can't run: `#BLOCKED!`. +pub const BLOCKED: Value = Value::Error(CellError::Blocked); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn formula_round_trip_with_chunks_and_quotes() { + let code = format!("s = \"quoted\"\n{}\nlen(s)", "x".repeat(600)); + let py = PyCode { object: true, ..PyCode::new(&code) }; + let f = py.to_formula(); + assert!(f.starts_with("PY(\"s = \"\"quoted\"\"\n")); + assert!(f.ends_with(",1)")); + assert_eq!(f.matches("\"&\"").count(), 2, "{f}"); + let back = PyCode::from_formula(&format!("={f}")).unwrap(); + assert_eq!(back.code, code); + assert!(back.object); + assert_eq!(PyCode::from_formula("_xlfn.PY(\"1+1\")").unwrap().code, "1+1"); + // Excel writes `_xlfn._xlws.PY`; the reader takes it in any letter case. + assert_eq!(PyCode::from_formula("=_xlfn._xlws.PY(\"1+1\",1)").unwrap().code, "1+1"); + assert_eq!(PyCode::from_formula("_XLFN._XLWS.py(\"2\",0)").unwrap().code, "2"); + assert_eq!(PyCode::from_formula("_xlws.PY(\"3\",0)").unwrap().code, "3"); + assert!(is_py_formula("=_xlfn._xlws.PY(\"1\",0)")); + assert_eq!(py.to_file_formula(), format!("_xlfn._xlws.{f}")); + assert_eq!(PyCode::from_formula(&py.to_file_formula()).unwrap(), PyCode::from_formula(&f).unwrap()); + assert_eq!(PyCode::from_formula(" py( \"a\" & \"b\" , 0 ) ").unwrap().code, "ab"); + assert_eq!(PyCode::new("a\r\nb").code, "a\nb"); + for bad in ["PY(A1)", "PY(\"a\",2)", "PY(\"a\"", "PY(\"a\")+1", "SUM(1)", "PY(\"a\"&B1)", "PY(", ""] { + assert!(PyCode::from_formula(bad).is_none(), "{bad}"); + } + assert!(is_py_formula("=PY(\"1\")") && is_py_formula("=py") && !is_py_formula("=PYTHON(1)")); + } + + #[test] + fn the_size_limit_counts_the_encoded_formula() { + // Plain code: the limit is the formula's, so it is a little under 8,192 characters of code. + // `_xlfn._xlws.PY(` + 31 literals of 255 + 30 `&` + `&"…"` + `,0)`: 8,079 characters of + // code give a formula of exactly 8,192. + let at = PyCode::new(&"y".repeat(8_079)); + assert_eq!(at.to_file_formula().chars().count(), MAX_FORMULA_CHARS); + assert!(at.fits()); + assert!(!PyCode::new(&"y".repeat(8_080)).fits()); + assert_eq!(PyCode::from_formula(&at.to_file_formula()).unwrap().code, at.code); + // Quotes are doubled in the file, so 7,000 characters with 1,200 of them don't fit. + let quoted = format!("{}{}", "\"".repeat(1_200), "y".repeat(5_800)); + assert!(PyCode::new("ed).to_file_formula().chars().count() > MAX_FORMULA_CHARS); + assert!(!PyCode::new("ed).fits()); + // Hostile sizes are refused without encoding them. + assert!(!PyCode::new(&"y".repeat(4_000_000)).fits()); + assert!(PyCode::new("").fits()); + } + + #[test] + fn result_values() { + let arr = gridcraft_core::Array::new(2, 1, vec![1.0.into(), 2.0.into()]).map(Value::from).unwrap(); + let out = PyOutput { value: arr.clone(), type_name: "list".into(), preview: "[1, 2]".into(), convertible: true }; + let mut py = PyCode { output: Some(out), ..PyCode::new("[1, 2]") }; + assert_eq!(py.result_value(), Some(arr)); + py.object = true; + assert_eq!(py.result_value(), Some(Value::text("list"))); + py.output = Some(PyOutput { value: Value::Number(3.0), type_name: "int".into(), preview: "3".into(), convertible: true }); + assert_eq!(py.result_value(), Some(Value::Number(3.0)), "scalars show as values in object mode too"); + py.output = None; + assert_eq!(py.result_value(), None); + } + + #[test] + fn finds_xl_literals() { + let code = "a = xl(\"A1:B2\", headers=True)\n# xl('C1') in a comment\ns = \"xl('D1')\"\nb = xl( 'Sheet2!E5' )\nc = xl(name)\nd = df.xl('F1')\nxlx('G1')\ne = xl(\"A\" + \"1\")"; + let calls = xl_calls(code); + let refs: Vec> = calls.iter().map(|c| c.reference.as_deref()).collect(); + assert_eq!(refs, vec![Some("A1:B2"), Some("Sheet2!E5"), None, None]); + let new = rewrite_xl(code, |r| (r == "A1:B2").then(|| "A2:B3".to_string())).unwrap(); + assert!(new.starts_with("a = xl(\"A2:B3\", headers=True)")); + assert_eq!(new.len(), code.len()); + assert_eq!(rewrite_xl(code, |_| None), None); + // A replacement can't break out of its string: a quote in it changes the delimiter, or is + // escaped when it holds both quotes, and the result reads back as the same reference. + assert_eq!(rewrite_xl("xl(\"A1\")", |_| Some("'Q'!A1".into())), Some("xl(\"'Q'!A1\")".into())); + assert_eq!(rewrite_xl("xl('Sheet2!B2', headers=True)", |_| Some("'Q4 2026'!B2".into())), Some("xl(\"'Q4 2026'!B2\", headers=True)".into())); + assert_eq!(rewrite_xl("xl(\"A1\")", |_| Some("'Q\"4'!A1".into())), Some("xl(\"'Q\\\"4'!A1\")".into())); + assert_eq!(rewrite_xl("xl('A1')", |_| Some("'Q\"4'!A1".into())), Some("xl('\\'Q\"4\\'!A1')".into())); + for (delim, new) in [("'", "'Q4 2026'!B2"), ("'", "'Q\"4'!A1"), ("\"", "'Q\"4'!A1"), ("\"", "'a\\b'!A1"), ("'", "B2")] { + let code = format!("x = xl({delim}A1{delim}) + 1"); + let out = rewrite_xl(&code, |_| Some(new.to_string())).unwrap(); + assert_eq!(xl_calls(&out)[0].reference.as_deref(), Some(new), "{out}"); + assert!(out.ends_with(") + 1"), "{out}"); + } + assert_eq!(rewrite_xl("xl('A1')", |_| Some("A\n1".into())), None, "a line break can't go in a literal"); + // Escaped quotes and backslashes read as the characters; any other escape is not plain. + assert_eq!(xl_calls("xl('Bob\\'s!A1')")[0].reference.as_deref(), Some("Bob's!A1")); + assert_eq!(xl_calls("xl(\"A\\\\1\")")[0].reference.as_deref(), Some("A\\1")); + assert_eq!(xl_calls("xl(\"A\\n1\")")[0].reference, None); + assert_eq!(xl_calls("xl(\"A1\\\")")[0].reference, None, "an escaped closing quote leaves the literal open"); + // Unclosed and odd input doesn't loop or panic. + for odd in ["xl(", "xl(\"", "xl('''A1''')", "x", "xl", "\"unterminated", "xl ( \"A1\" )", "é xl('A1')"] { + let _ = xl_calls(odd); + let _ = rewrite_xl(odd, |_| Some("B2".into())); + } + assert_eq!(xl_calls("xl ( \"A1\" )")[0].reference.as_deref(), Some("A1")); + assert_eq!(xl_calls("xl('''A1''')")[0].reference, None); + } +} diff --git a/crates/model/src/sheet.rs b/crates/model/src/sheet.rs index abc4731dc..6bbb331f4 100644 --- a/crates/model/src/sheet.rs +++ b/crates/model/src/sheet.rs @@ -259,7 +259,7 @@ impl Sheet { } pub fn set_value(&mut self, c: CellRef, v: Value) { let style = self.style_id(c); - self.set_cell(c, Cell { value: v, formula: None, style }); + self.set_cell(c, Cell { value: v, formula: None, style, py: None }); } /// Replaces content, removing any picture at this address. Style-only edits use set_style. pub fn set_cell(&mut self, c: CellRef, cell: Cell) { @@ -278,7 +278,7 @@ impl Sheet { /// Pictures retain a scalar fallback for calculation and unsupported consumers. pub fn set_picture(&mut self, c: CellRef, picture: Arc) { let style = self.style_id(c); - self.cells.set(c, Cell { value: Value::Error(gridcraft_core::CellError::Value), formula: None, style }); + self.cells.set(c, Cell { value: Value::Error(gridcraft_core::CellError::Value), formula: None, style, py: None }); self.cell_pictures.insert(c, picture); } pub fn input_text(&self, c: CellRef) -> String { diff --git a/crates/model/src/store.rs b/crates/model/src/store.rs index 531273ce9..001b7cf60 100644 --- a/crates/model/src/store.rs +++ b/crates/model/src/store.rs @@ -14,7 +14,7 @@ use crate::cell::Cell; const BAND: u32 = 64; fn has_value(cell: &Cell) -> bool { - !cell.value.is_empty() || cell.formula.is_some() + cell.has_content() } #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] diff --git a/crates/xlsx/src/csv.rs b/crates/xlsx/src/csv.rs index dfc694ea6..65ef1d17b 100644 --- a/crates/xlsx/src/csv.rs +++ b/crates/xlsx/src/csv.rs @@ -288,7 +288,7 @@ pub fn read_csv(bytes: &[u8], opts: &CsvOptions) -> Result { }, None => StyleId::DEFAULT, }; - Cell { value: p.value, formula: None, style } + Cell { value: p.value, formula: None, style, py: None } } else { Cell::value(Value::Text(Arc::from(f))) }; diff --git a/crates/xlsx/src/sheet_read.rs b/crates/xlsx/src/sheet_read.rs index aadd1141e..bec67bd1a 100644 --- a/crates/xlsx/src/sheet_read.rs +++ b/crates/xlsx/src/sheet_read.rs @@ -516,7 +516,7 @@ fn read_cell(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, c: &El, pos let dynamic = c.attr("cm").is_some(); let formula = c.child("f").and_then(|f| read_formula(cx, st, f, pos, dynamic)); let value = if formula.is_none() && in_dynamic(st, pos) { Value::Empty } else { value }; - sheet.cells.set(pos, Cell { value, formula: formula.map(Arc::new), style }); + sheet.cells.set(pos, Cell { value, formula: formula.map(Arc::new), style, py: None }); } fn read_formula(cx: &mut Ctx<'_>, st: &mut RowState, f: &El, pos: CellRef, dynamic: bool) -> Option { diff --git a/crates/xlsx/src/sheet_write.rs b/crates/xlsx/src/sheet_write.rs index 52dc979e6..9251edaa4 100644 --- a/crates/xlsx/src/sheet_write.rs +++ b/crates/xlsx/src/sheet_write.rs @@ -698,8 +698,10 @@ fn write_cell( let value = match value { Value::Error(CellError::Circ) => Value::Number(0.0), // #SPILL! and #CALC! are not error codes of the file format (Excel refuses a file that - // caches them); Excel itself caches them as #VALUE!. - Value::Error(CellError::Spill | CellError::Calc) => Value::Error(CellError::Value), + // caches them); Excel itself caches them as #VALUE!. The Python errors are the same kind. + Value::Error(CellError::Spill | CellError::Calc | CellError::Python | CellError::Busy | CellError::Timeout | CellError::Blocked) => { + Value::Error(CellError::Value) + } v => v, }; // A formula read with dynamic-array metadata keeps it, even where the shape analysis would diff --git a/crates/xlsx/src/tests/locale_numfmt.rs b/crates/xlsx/src/tests/locale_numfmt.rs index 3259a3657..749d832ad 100644 --- a/crates/xlsx/src/tests/locale_numfmt.rs +++ b/crates/xlsx/src/tests/locale_numfmt.rs @@ -25,7 +25,7 @@ fn round_trip(code: &str) -> (u32, String, String) { let mut wb = Workbook::new(); let style = wb.styles.derive(StyleId::DEFAULT, |s| s.num_fmt = NumFmt::new(code)); let mut sheet = Sheet::new("Sheet1"); - sheet.cells.set(CellRef::new(0, 0), Cell { value: Value::Number(-0.14), formula: None, style }); + sheet.cells.set(CellRef::new(0, 0), Cell { value: Value::Number(-0.14), formula: None, style, py: None }); wb.sheets = vec![Arc::new(sheet)]; let bytes = write_xlsx(&wb).unwrap(); let xml = styles_xml(&bytes); diff --git a/crates/xlsx/src/tests/phonetic_tests.rs b/crates/xlsx/src/tests/phonetic_tests.rs index 37f37ed2b..43b30b6ba 100644 --- a/crates/xlsx/src/tests/phonetic_tests.rs +++ b/crates/xlsx/src/tests/phonetic_tests.rs @@ -33,7 +33,7 @@ fn part(bytes: &[u8], name: &str) -> String { } fn text_cell(sheet: &mut gridcraft_model::Sheet, a: &str, text: &str) { - sheet.cells.set(at(a), Cell { value: Value::text(text), formula: None, style: StyleId::DEFAULT }); + sheet.cells.set(at(a), Cell { value: Value::text(text), formula: None, style: StyleId::DEFAULT, py: None }); } #[test] diff --git a/crates/xlsx/src/tests/pivot_tests.rs b/crates/xlsx/src/tests/pivot_tests.rs index b9123e432..4f2c89dbd 100644 --- a/crates/xlsx/src/tests/pivot_tests.rs +++ b/crates/xlsx/src/tests/pivot_tests.rs @@ -52,7 +52,7 @@ fn workbook() -> Workbook { let r = r as u32 + 1; data.cells.set(CellRef::new(r, 0), Cell::value(Value::text(*a))); data.cells.set(CellRef::new(r, 1), Cell::value(Value::text(*b))); - data.cells.set(CellRef::new(r, 2), Cell { value: Value::number(*d), formula: None, style: date_style }); + data.cells.set(CellRef::new(r, 2), Cell { value: Value::number(*d), formula: None, style: date_style, py: None }); data.cells.set(CellRef::new(r, 3), Cell::value(Value::number(*u))); data.cells.set(CellRef::new(r, 4), Cell::value(Value::number(*s))); data.cells.set(CellRef::new(r, 5), Cell::value(Value::text(*ch))); diff --git a/crates/xlsx/src/tests/roundtrip.rs b/crates/xlsx/src/tests/roundtrip.rs index 14fc784f9..fe01f284e 100644 --- a/crates/xlsx/src/tests/roundtrip.rs +++ b/crates/xlsx/src/tests/roundtrip.rs @@ -64,7 +64,7 @@ fn sample() -> Workbook { }; let mut s = Sheet::new("Data & Stuff"); - let set = |s: &mut Sheet, a: &str, v: Value, style: StyleId| s.cells.set(at(a), Cell { value: v, formula: None, style }); + let set = |s: &mut Sheet, a: &str, v: Value, style: StyleId| s.cells.set(at(a), Cell { value: v, formula: None, style, py: None }); set(&mut s, "A1", Value::text("Region"), bold); set(&mut s, "B1", Value::text("Amount"), bold); set(&mut s, "C1", Value::text(" spaced\nline _x0041_ \u{1} "), StyleId::DEFAULT); @@ -839,7 +839,7 @@ fn writing_repairs_invalid_models() { table_type: None, connection_id: None, }); - s.cells.set(at("B2"), Cell { value: Value::Number(1.0), formula: None, style: StyleId(9999) }); + s.cells.set(at("B2"), Cell { value: Value::Number(1.0), formula: None, style: StyleId(9999), py: None }); let (back, _) = read_xlsx(&write_xlsx(&wb).unwrap()).unwrap(); let s = back.sheet(0).unwrap(); assert_eq!(s.visibility, Visibility::Visible, "at least one sheet stays visible"); diff --git a/crates/xlsx/src/xls/mod.rs b/crates/xlsx/src/xls/mod.rs index ca3ec9eb3..90ecf629d 100644 --- a/crates/xlsx/src/xls/mod.rs +++ b/crates/xlsx/src/xls/mod.rs @@ -454,7 +454,7 @@ impl SheetState<'_> { return Ok(Some(pos)); } budget.cell()?; - self.sheet.cells.set(pos, Cell { value, formula: None, style }); + self.sheet.cells.set(pos, Cell { value, formula: None, style, py: None }); Ok(Some(pos)) } @@ -472,7 +472,7 @@ impl SheetState<'_> { if let Some(cell) = self.sheet.cells.get_mut(pos) { cell.formula = Some(Arc::new(f)); } else { - self.sheet.cells.set(pos, Cell { value: Value::Empty, formula: Some(Arc::new(f)), style: StyleId::DEFAULT }); + self.sheet.cells.set(pos, Cell { value: Value::Empty, formula: Some(Arc::new(f)), style: StyleId::DEFAULT, py: None }); } } Err(Unsupported(why)) => self.unsupported(why), @@ -531,7 +531,7 @@ fn blank_array_followers(st: &mut SheetState<'_>, pending: &[Pending]) { let filled = st.sheet.cell(p.anchor).and_then(|c| c.formula.as_ref()).and_then(|f| f.array).is_some_and(|r| r.contains(p.pos)); if p.pos != p.anchor && filled { let style = st.sheet.style_id(p.pos); - st.sheet.cells.set(p.pos, Cell { value: Value::Empty, formula: None, style }); + st.sheet.cells.set(p.pos, Cell { value: Value::Empty, formula: None, style, py: None }); } } } @@ -677,7 +677,7 @@ fn sheet_record(rec: &Record<'_>, st: &mut SheetState<'_>, budget: &mut Budget, if let Some(cell) = st.sheet.cells.get_mut(pos) { cell.value = Value::from(s); } else { - st.sheet.cells.set(pos, Cell { value: Value::from(s), formula: None, style }); + st.sheet.cells.set(pos, Cell { value: Value::from(s), formula: None, style, py: None }); } } } @@ -832,7 +832,7 @@ fn formula_cell(c: &mut Cursor<'_, '_>, st: &mut SheetState<'_>, budget: &mut Bu let Some(pos) = st.pos(r, col) else { return Ok(()) }; budget.cell()?; let style = st.style(x); - st.sheet.cells.set(pos, Cell { value, formula: None, style }); + st.sheet.cells.set(pos, Cell { value, formula: None, style, py: None }); st.last_formula = Some(pos); if string_result { st.pending_string = Some((pos, style)); From c6c54c57e61da673fe26d44cd424e256f8ab87f3 Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:47:09 -0400 Subject: [PATCH 02/24] Engine: python.set makes a Python cell; this build stores code and doesn't run it python.set {cell?, code, output?: object|value} makes the cell (the active cell by default) a Python cell, keeping its style and taking a formula and its spill away. `output` is the return type Microsoft documents for PY(): an Excel value (the default) or a Python object; without it a Python cell keeps the one it had. The code must fit in Excel's 8,192-character formula once written as PY(...), quotes doubled and literals joined, so the command checks the encoded formula rather than a character count. A table's header row refuses Python (it holds the column's name, as Excel keeps formulas out of headers), and a locked cell on a protected sheet refuses it through the usual protection check. Nothing runs Python here. Unchanged code keeps its cached result and value; new or changed code shows #BLOCKED!. One undo step takes the cell back. Sheet::clear_spill removes the spilled values inside a range with work bounded by the values stored, not by the range's size. In the interface, F2, a double click or the formula bar on a Python cell edits its code, and a changed text is committed through python.set, which keeps the return type; through cell.set it would have replaced the Python cell with an ordinary cell holding the code as text. Typing over a Python cell replaces it, as typing over a formula does, and committing the code unchanged writes nothing. Deleting all the code clears the cell through cell.set, as an empty entry clears any other cell; code of spaces is still code. Tests: the code and return type in cell.get, #BLOCKED! for new code, cached results kept for unchanged code, the active-cell default, undo, refused params (missing or non-text code, a bad cell, a bad return type), the encoded limit at 8,079 and 8,080 characters and with doubled quotes, a table header, a protected sheet, and a formula's spill taken away. A UI test edits a Python cell's code with F2 and Enter, commits it unchanged, and types over it; another clears a Python cell by deleting all its code, keeps code of spaces, and undoes the clearing. --- crates/engine/src/cmd/mod.rs | 2 + crates/engine/src/cmd/python.rs | 96 ++++++++++++++++ crates/engine/src/tests.rs | 1 + crates/engine/src/tests/python_commands.rs | 91 ++++++++++++++++ crates/model/src/sheet.rs | 13 +++ crates/ui-egui/src/editor.rs | 5 + crates/ui-egui/src/lib.rs | 44 +++++++- crates/ui-egui/tests/it/main.rs | 1 + crates/ui-egui/tests/it/python_cell_edit.rs | 115 ++++++++++++++++++++ 9 files changed, 362 insertions(+), 6 deletions(-) create mode 100644 crates/engine/src/cmd/python.rs create mode 100644 crates/engine/src/tests/python_commands.rs create mode 100644 crates/ui-egui/tests/it/python_cell_edit.rs diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 2f1becd23..2bd2a13bb 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -14,6 +14,7 @@ pub mod insert; pub mod inspect; pub mod pivot; pub mod print; +pub mod python; pub mod review; pub mod sheet; pub mod slicer; @@ -147,6 +148,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(spelling::specs()); v.extend(solver::specs()); v.extend(pivot::specs()); + v.extend(python::specs()); v.extend(slicer::specs()); v.extend(whatif::specs()); v.extend(forecast::specs()); diff --git a/crates/engine/src/cmd/python.rs b/crates/engine/src/cmd/python.rs new file mode 100644 index 000000000..29288e64e --- /dev/null +++ b/crates/engine/src/cmd/python.rs @@ -0,0 +1,96 @@ +//! Python in cells: making and editing Python cells. This build stores the code and the last +//! result a file cached; it doesn't run Python, so a cell with new or changed code shows +//! `#BLOCKED!`. + +use std::sync::Arc; + +use gridcraft_core::{CellRef, Value}; +use gridcraft_model::{Cell, PyCode, python::MAX_FORMULA_CHARS}; +use serde_json::{Value as Json, json}; + +use super::{CommandSpec, bad, cell_param_on, cmd, edit, has_doc, str_param, target_sheet}; +use crate::{EngineError, Result, Session}; + +pub fn specs() -> Vec { + vec![cmd!( + "python.set", + "Set Python Code", + [], + None, + "{cell?, code, output?: object|value} makes the cell (default: the active cell) a Python cell with this code; the code must fit in Excel's 8,192-character formula once written as PY(…). `output` is the return type: `value` (an Excel value, the default) or `object` (a Python object)", + has_doc, + set + )] +} + +fn target_cell(s: &Session, p: &Json) -> Result<(usize, CellRef)> { + let sheet = target_sheet(s, p)?; + let at = match cell_param_on(s, p, "cell", sheet)? { + Some(c) => c, + None => s.doc()?.selection.active, + }; + Ok((sheet, at)) +} + +fn existing(s: &Session, sheet: usize, at: CellRef) -> Result> { + Ok(s.doc()?.wb.sheet(sheet).and_then(|sh| sh.cell(at)).and_then(|c| c.py.as_deref()).cloned()) +} + +fn output_mode(p: &Json, key: &str) -> Result> { + match str_param(p, key) { + None => Ok(None), + Some(m) if m.eq_ignore_ascii_case("object") => Ok(Some(true)), + Some(m) if m.eq_ignore_ascii_case("value") => Ok(Some(false)), + Some(m) => Err(bad("python.set", format!("`{key}` is `object` or `value`, not `{m}`"))), + } +} + +fn set(s: &mut Session, p: &Json) -> Result { + let (sheet, at) = target_cell(s, p)?; + let code = match p.get("code") { + Some(Json::String(c)) => c.as_str(), + Some(_) => return Err(bad("python.set", "`code` must be text")), + None => return Err(bad("python.set", "missing `code`")), + }; + let mode = output_mode(p, "output")?; + let old = existing(s, sheet, at)?; + let object = mode.or(old.as_ref().map(|o| o.object)).unwrap_or(false); + let mut py = PyCode { object, ..PyCode::new(code) }; + if !py.fits() { + return Err(bad( + "python.set", + format!("the code doesn't fit in one cell: written as PY(…) it would be longer than Excel's {MAX_FORMULA_CHARS}-character formula limit"), + )); + } + // A table header is the column's name, never a formula or code (as in Excel). + let header = + s.doc()?.wb.sheet(sheet).is_some_and(|sh| sh.tables.iter().any(|t| t.header_row && t.range.start.row == at.row && t.range.contains(at))); + if header { + return Err(EngineError::Other(format!("{} is a table header; Python cells can't go in a table's header row", at.a1()))); + } + // Unchanged code keeps its result (and its value); new code has none until it runs. + let keep = old.filter(|o| o.code == py.code); + let value = match &keep { + Some(o) => { + py.output = o.output.clone(); + py.stale = o.stale; + None + } + None => Some(gridcraft_model::python::BLOCKED), + }; + edit(s, |cx| { + let sh = cx.sheet_mut(sheet)?; + let style = sh.style_id(at); + let value = value.unwrap_or_else(|| sh.cell(at).map(|c| c.value.clone()).unwrap_or(Value::Empty)); + // A formula's spill (or the old result's) goes with what the cell held. + if let Some(r) = sh.spill_ranges.remove(&at) { + sh.clear_spill(r); + } + sh.set_cell(at, Cell { value, formula: None, style, py: Some(Arc::new(py)) }); + cx.touch(sheet, at); + Ok(()) + })?; + let d = s.doc()?; + let shown = d.wb.sheet(sheet).map(|sh| sh.value(at)).unwrap_or_default(); + Ok(json!({"cell": at.a1(), "value": crate::cmd::inspect::value_json(&shown)})) +} diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index c9d10e36e..47eada0b4 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5094,6 +5094,7 @@ fn a_script_cannot_switch_windows() { } mod python_cells; +mod python_commands; mod seams; #[test] diff --git a/crates/engine/src/tests/python_commands.rs b/crates/engine/src/tests/python_commands.rs new file mode 100644 index 000000000..6d0bfac42 --- /dev/null +++ b/crates/engine/src/tests/python_commands.rs @@ -0,0 +1,91 @@ +//! The python.* commands. + +use gridcraft_core::{CellError, Value}; +use serde_json::json; + +use super::{s, v}; + +fn set(s: &mut crate::Session, cell: &str, code: &str) -> serde_json::Value { + s.execute("python.set", json!({"cell": cell, "code": code})).unwrap() +} + +#[test] +fn set_stores_the_code_and_blocks_the_cell_until_it_can_run() { + let mut s = s(); + let r = set(&mut s, "B1", "xl('A1') + 1"); + assert_eq!(r["cell"], "B1"); + assert_eq!(v(&s, "B1"), Value::Error(CellError::Blocked)); + let got = s.execute("cell.get", json!({"cell": "B1"})).unwrap(); + assert_eq!(got["python"]["code"], "xl('A1') + 1"); + assert_eq!(got["python"]["output"], "value"); + assert_eq!(got["input"], "xl('A1') + 1"); + // The active cell by default. + s.execute("selection.set", json!({"cell": "D4"})).unwrap(); + s.execute("python.set", json!({"code": "2"})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "D4"})).unwrap()["python"]["code"], "2"); + // One undo step takes the cell back. + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.execute("cell.get", json!({"cell": "D4"})).unwrap()["python"].is_null()); +} + +#[test] +fn unchanged_code_keeps_its_cached_result() { + let mut s = s(); + set(&mut s, "A1", "21 * 2"); + // As a file would have cached it. + { + let d = s.doc_mut().unwrap(); + let wb = std::sync::Arc::make_mut(&mut d.wb); + let sh = wb.sheet_mut(0).unwrap(); + let cell = sh.cells.get_mut(gridcraft_core::CellRef::new(0, 0)).unwrap(); + let py = std::sync::Arc::make_mut(cell.py.as_mut().unwrap()); + py.output = Some(gridcraft_model::PyOutput { value: Value::Number(42.0), type_name: "int".into(), preview: "42".into(), convertible: true }); + cell.value = Value::Number(42.0); + } + set(&mut s, "A1", "21 * 2"); + assert_eq!(v(&s, "A1"), Value::Number(42.0), "same code, same result"); + s.execute("python.set", json!({"cell": "A1", "code": "21 * 2", "output": "object"})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["python"]["output"], "object"); + set(&mut s, "A1", "21 * 3"); + assert_eq!(v(&s, "A1"), Value::Error(CellError::Blocked), "new code has no result yet"); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["python"]["output"], "object", "the return type stays"); +} + +#[test] +fn set_refuses_bad_params_long_code_headers_and_protected_cells() { + let mut s = s(); + for p in [ + json!({"cell": "A1"}), + json!({"cell": "A1", "code": 5}), + json!({"cell": "not a cell", "code": "1"}), + json!({"cell": "A1", "code": "1", "output": "sideways"}), + ] { + assert!(s.execute("python.set", p.clone()).is_err(), "{p}"); + } + // The limit is the stored PY(…) formula's 8,192 characters. + assert!(s.execute("python.set", json!({"cell": "A1", "code": "y".repeat(8_080)})).is_err()); + s.execute("python.set", json!({"cell": "A1", "code": "y".repeat(8_079)})).unwrap(); + let quoted = format!("{}{}", "\"".repeat(1_200), "y".repeat(5_800)); + let e = s.execute("python.set", json!({"cell": "A2", "code": quoted})).unwrap_err(); + assert!(e.to_string().contains("8192") || e.to_string().contains("8,192"), "{e}"); + // A table header is the column's name. + s.execute("range.setValues", json!({"range": "C1", "values": [["Name", "Score"], ["a", 1]]})).unwrap(); + s.execute("insert.table", json!({"range": "C1:D2", "headers": true})).unwrap(); + assert!(s.execute("python.set", json!({"cell": "D1", "code": "1"})).is_err()); + assert_eq!(v(&s, "D1"), Value::text("Score")); + set(&mut s, "D2", "1"); + // A locked cell on a protected sheet refuses it. + s.execute("review.protectSheet", json!({})).unwrap(); + let e = s.execute("python.set", json!({"cell": "F1", "code": "1"})).unwrap_err(); + assert!(e.to_string().contains("protected"), "{e}"); +} + +#[test] +fn set_over_a_spilling_formula_takes_the_spill_away() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "=SEQUENCE(3)"})).unwrap(); + assert_eq!(v(&s, "A3"), Value::Number(3.0)); + set(&mut s, "A1", "[1, 2, 3]"); + assert_eq!(v(&s, "A3"), Value::Empty); + assert!(s.doc().unwrap().wb.sheet(0).unwrap().spill_ranges.is_empty()); +} diff --git a/crates/model/src/sheet.rs b/crates/model/src/sheet.rs index 6bbb331f4..433451277 100644 --- a/crates/model/src/sheet.rs +++ b/crates/model/src/sheet.rs @@ -202,6 +202,19 @@ impl Sheet { self.spill_ranges.iter().find(|(anchor, range)| **anchor != c && range.contains(c)).map(|(anchor, _)| *anchor) } + /// Removes the spilled values inside `r`. The work is bounded by the values stored, not by + /// the size of `r`, so a range from a file that claims the whole sheet costs no more than the + /// values it holds. + pub fn clear_spill(&mut self, r: RangeRef) { + if r.count() <= self.spill.len() as u64 { + for c in r.iter() { + self.spill.remove(&c); + } + } else { + self.spill.retain(|c, _| !r.contains(*c)); + } + } + /// Displayed value of a cell: its own value, or a spilled value. pub fn value(&self, c: CellRef) -> Value { match self.cells.get(c) { diff --git a/crates/ui-egui/src/editor.rs b/crates/ui-egui/src/editor.rs index 847afe3ae..8fe3bc7d2 100644 --- a/crates/ui-egui/src/editor.rs +++ b/crates/ui-egui/src/editor.rs @@ -47,6 +47,10 @@ pub struct EditState { /// The cell's text when an F2-style edit began. Committing it unchanged leaves the cell alone /// (no undo step, the workbook stays clean). `None` for an edit started by typing. pub original: Option, + /// The edit began on a Python cell's existing content (F2, a double click or the formula + /// bar), so the text is the cell's code: a change goes in as new code through `python.set`. + /// `false` for an edit started by typing, which replaces the cell as it does a formula. + pub code_edit: bool, } impl EditState { @@ -75,6 +79,7 @@ impl EditState { caret_to_end: true, completion: None, original, + code_edit: false, } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 55b3bd740..8cfccb68f 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1146,6 +1146,7 @@ impl SheetApp { None => (current.clone(), false), }; let mut ed = editor::EditState::new(d.uid, d.wb.active_sheet(), at, text, replace, from_formula_bar); + ed.code_edit = !replace && sh.cell(at).is_some_and(|c| c.py.is_some()); ed.window = self.session.active_window(); ed.locale = locale; if replace && self.ui.autocomplete && !ed.is_formula() { @@ -1207,12 +1208,17 @@ impl SheetApp { _ => ed.text.clone(), }; // F2 then Enter (or a file command) on an untouched draft: nothing to write. - if !fill_selection && !array && ed.original.as_ref() == Some(&text) { + if (ed.code_edit || (!fill_selection && !array)) && ed.original.as_ref() == Some(&text) { if dr != 0 || dc != 0 { self.move_after_enter(dr, dc); } return true; } + // All the code deleted: the cell is cleared through cell.set, as an empty entry clears any + // other cell. Code of spaces is still code. + if ed.code_edit && !text.is_empty() { + return self.commit_code_edit(ed, text, dr, dc); + } let text = match self.session.active().map(|d| formula_locale::canonical(&text, ed.locale, &d.wb, ed.sheet, ed.cell)) { Some(Ok(text)) => text, Some(Err(error)) => { @@ -1250,11 +1256,9 @@ impl SheetApp { let unchanged = !fill_selection && !array && text.is_empty() - && self - .session - .active() - .and_then(|d| d.wb.sheet(ed.sheet)) - .is_some_and(|sh| sh.cell(ed.cell).is_none_or(|c| c.formula.is_none() && c.value == gridcraft_engine::core::Value::Empty)); + && self.session.active().and_then(|d| d.wb.sheet(ed.sheet)).is_some_and(|sh| { + sh.cell(ed.cell).is_none_or(|c| c.formula.is_none() && c.py.is_none() && c.value == gridcraft_engine::core::Value::Empty) + }); let r = if unchanged { Ok(json!({})) } else if fill_selection { @@ -1279,6 +1283,34 @@ impl SheetApp { } } + /// Commits a changed edit of a Python cell's code through `python.set`, which keeps the cell's + /// return type; `cell.set` would replace the Python cell with an ordinary one holding the code + /// as text. When the command refuses the code, its message shows and the edit stays open. + fn commit_code_edit(&mut self, ed: editor::EditState, code: String, dr: i64, dc: i64) -> bool { + // Point mode can leave another sheet active; the edit belongs to the sheet it started on. + let r = if self.session.active().is_some_and(|d| d.wb.active_sheet() != ed.sheet) { + self.session.run("sheet.activate", json!({"sheet": ed.sheet})) + } else { + Ok(Json::Null) + }; + let r = r.and_then(|_| self.session.run("python.set", json!({"sheet": ed.sheet, "cell": ed.cell.a1(), "code": code}))); + match r { + Ok(_) => { + if dr != 0 || dc != 0 { + self.move_after_enter(dr, dc); + } + self.after_engine(); + true + } + Err(e) => { + self.message = Some(("GridCraft".into(), clean_error(&e))); + self.editor = Some(ed); + self.session.mode = gridcraft_engine::Mode::Edit; + false + } + } + } + /// Commits the edit with an arrow key. The arrow is an ordinary move, so it ends a run of Tabs /// instead of continuing it (Right) or going back to its column like Enter (Down), as in Excel. pub fn commit_edit_arrow(&mut self, dr: i64, dc: i64) -> bool { diff --git a/crates/ui-egui/tests/it/main.rs b/crates/ui-egui/tests/it/main.rs index 53625015e..09abd1097 100644 --- a/crates/ui-egui/tests/it/main.rs +++ b/crates/ui-egui/tests/it/main.rs @@ -73,6 +73,7 @@ mod pivot_show_details; mod pivot_show_details_menu; mod popup_dismissal; mod protect_password; +mod python_cell_edit; mod ribbon_layout; mod right_to_left; mod scroll_position; diff --git a/crates/ui-egui/tests/it/python_cell_edit.rs b/crates/ui-egui/tests/it/python_cell_edit.rs new file mode 100644 index 000000000..0afa43185 --- /dev/null +++ b/crates/ui-egui/tests/it/python_cell_edit.rs @@ -0,0 +1,115 @@ +//! Editing a Python cell's content (F2, the formula bar) edits its code; typing over it replaces +//! it, as typing over a formula does. + +use egui::{Event, Key, Modifiers}; +use gridcraft_engine::Session; +use gridcraft_engine::core::{CellRef, Value}; +use gridcraft_ui_egui::SheetApp; +use serde_json::json; + +type Harness = egui_kittest::Harness<'static, SheetApp>; + +fn harness() -> Harness { + let mut session = Session::new(); + session.new_workbook(); + session.execute("python.set", json!({"cell": "A1", "code": "1 + 1", "output": "object"})).unwrap(); + session.execute("selection.set", json!({"range": "A1"})).unwrap(); + crate::common::harness_with(crate::common::builder().with_size(egui::vec2(1200.0, 800.0)), crate::common::app(session), 4) +} + +fn text(h: &mut Harness, text: &str) { + h.input_mut().events.push(Event::Text(text.into())); + h.run_steps(2); +} + +fn key(h: &mut Harness, key: Key) { + for pressed in [true, false] { + h.input_mut().events.push(Event::Key { key, physical_key: None, pressed, repeat: false, modifiers: Modifiers::NONE }); + } + h.run_steps(2); +} + +/// A1's code and return type (true: a Python object), or `None` when it isn't a Python cell. +fn python(h: &Harness) -> Option<(String, bool)> { + let sh = h.state().session.active().unwrap().wb.active().unwrap(); + sh.cell(CellRef::parse("A1").unwrap()).and_then(|c| c.py.as_deref()).map(|p| (p.code.clone(), p.object)) +} + +fn select_a1(h: &mut Harness) { + h.state_mut().session.execute("selection.set", json!({"range": "A1"})).unwrap(); + h.run_steps(2); +} + +#[test] +fn f2_edits_a_python_cells_code_and_typing_replaces_it() { + let mut h = harness(); + let undo_depth = |h: &Harness| h.state().session.active().unwrap().undo.len(); + let before = undo_depth(&h); + + // F2 and Enter without a change writes nothing. + key(&mut h, Key::F2); + assert_eq!(h.state().editor.as_ref().map(|e| e.text.as_str()), Some("1 + 1")); + key(&mut h, Key::Enter); + assert!(h.state().editor.is_none()); + assert_eq!(undo_depth(&h), before, "an unchanged edit is no undo step"); + assert_eq!(python(&h), Some(("1 + 1".into(), true))); + + // A change is new code for the same Python cell, which keeps its return type. + select_a1(&mut h); + key(&mut h, Key::F2); + text(&mut h, " + 2"); + key(&mut h, Key::Enter); + assert!(h.state().editor.is_none(), "{:?}", h.state().message); + assert_eq!(python(&h), Some(("1 + 1 + 2".into(), true)), "the cell stays a Python cell with the new code"); + + // Typing over the cell replaces it. + select_a1(&mut h); + text(&mut h, "5"); + key(&mut h, Key::Enter); + assert_eq!(python(&h), None); + assert_eq!(h.state().session.active().unwrap().wb.active().unwrap().value(CellRef::parse("A1").unwrap()), Value::Number(5.0)); +} + +#[test] +fn deleting_all_the_code_clears_the_cell() { + let mut h = harness(); + let a1 = |h: &Harness| h.state().session.active().unwrap().wb.active().unwrap().value(CellRef::parse("A1").unwrap()); + // Code of spaces is still code, for the same Python cell. + key(&mut h, Key::F2); + h.state_mut().editor.as_mut().unwrap().text = " ".into(); + assert!(h.state_mut().commit_edit(0, 0, false, false)); + assert_eq!(python(&h), Some((" ".into(), true))); + // No code at all clears the cell, as an empty entry clears any other cell. + select_a1(&mut h); + key(&mut h, Key::F2); + h.state_mut().editor.as_mut().unwrap().text.clear(); + assert!(h.state_mut().commit_edit(0, 0, false, false)); + assert!(h.state().editor.is_none()); + assert_eq!(python(&h), None, "the cell is no longer a Python cell"); + assert_eq!(a1(&h), Value::Empty); + h.state_mut().run("edit.undo", json!({})).unwrap(); + assert_eq!(python(&h), Some((" ".into(), true)), "one undo step brings the code back"); +} + +#[test] +fn deleting_all_the_code_clears_a_python_cell_with_no_cached_value() { + let mut h = harness(); + // A Python cell from a file that cached no value (the value is empty, the code is not). + { + let d = h.state_mut().session.doc_mut().unwrap(); + let mut wb = (*d.wb).clone(); + let i = wb.active_sheet(); + let sh = wb.sheet_mut(i).unwrap(); + let at = CellRef::parse("A1").unwrap(); + let mut cell = sh.cell(at).cloned().unwrap(); + cell.value = Value::Empty; + sh.set_cell(at, cell); + d.wb = std::sync::Arc::new(wb); + } + h.run_steps(2); + key(&mut h, Key::F2); + h.state_mut().editor.as_mut().unwrap().text.clear(); + assert!(h.state_mut().commit_edit(0, 0, false, false)); + assert!(h.state().editor.is_none()); + assert_eq!(python(&h), None, "the code is removed, not kept as an unchanged empty cell"); +} From fcbbfa51424c7d38c87d2dc940b2c500aa317fe3 Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:54:01 -0400 Subject: [PATCH 03/24] XLSX: Python cells as _xlfn._xlws.PY() formulas, code in the Python part, cached values; opening limits A Python cell is written to xlsx as [MS-XLSX] 2.3.10 specifies: its formula is _xlfn._xlws.PY(i,0|1), where i is the zero-based index of the cell's script in the workbook's Python part and 0|1 the return type, 0 for an Excel value and 1 for a Python object. The part (2.1.23) is xl/python.xml, content type application/vnd.ms-excel.python+xml, related from the workbook part with http://schemas.microsoft.com/office/2023/09/relationships/Python; its root python holds pythonScripts, one pythonScript per Python cell in the order the cells are written, with the code in code. The code keeps its xl() references as text, so the formula has no further arguments. The cell keeps its cached value. A spilling result is written like a dynamic array (cm="1", t="array" ref=...) with the spilled values cached, so other applications show the values (and #NAME? if they recalculate). The reader turns a PY( formula with or without the _xlfn. and _xlws. prefixes into a Python cell. An index names a script of the Python part, and each %Px% in the script becomes the text of the formula's argument x (2.3.10.4). The string-literal form, PY("chunk"&"chunk",0|1), still reads, from the formula's own text so line breaks in the code survive. A formula naming a script the part doesn't hold keeps only its cached value, with a warning: no calculation can evaluate it. The reader keeps the cached spilled values as the cell's result, so a workbook from Excel shows its arrays without running anything. Python errors are cached as #VALUE!, as #SPILL! and #CALC! already are. Native JSON keeps Cell.py whole: code, return type, the last result and the exact error. Calc lays out a Python cell's kept result: Python cells are nodes of the dependency graph that read nothing, and a pass takes a dirty Python cell's result instead of evaluating it, so its array spills, is blocked (#SPILL!) and unblocked, and feeds its dependents exactly like a formula's. An array that is malformed or has more than 1,000,000 cells (only a crafted file holds one) shows #SPILL!. A dirty Python cell is pending in the pass, as a dirty formula is, so a formula, a range read or a cell-by-cell range walk that reaches it before its turn lays it out first rather than reading its old value. Opening a file can't make the app loop over a claimed size: the reader keeps a Python cell's spill range only when it has at most 1,000,000 cells (a range claiming the whole sheet is dropped with its values, as a formula's spilled values are), and io::adopt_python rebuilds results for at most 4,000,000 cells over the whole workbook, with a warning for the cells it skips. The Python part is read up to 64 MB and 1,000,000 scripts, a script is kept up to one character past what a cell may hold, and the cells take at most 64 MB of code from the part in all, since several formulas can name one script. io::enforce_python_limits drops the code of a cell whose code doesn't fit in 8,192 characters as PY(...) (the cell keeps its cached value, with a warning) and a kept result that isn't a well-formed array within the cap, each with its spill range and spilled values, so the sheet and a save no longer show the old spill. Formats that keep PY() as an ordinary formula (xls, xlsb, ods) are adopted the same way. Opening a file runs nothing. The command-sequence fuzzer sets Python cells (code with xl() references, quotes, line breaks, and code at and over the size limit), and its xlsx round-trip check allows the Python errors becoming #VALUE!, which is what the file format stores. Tests: the written formula naming its script, the Python part with its content type and relationship, return types, a spilling table saved as a dynamic array, Python errors as #VALUE!, the round trip of values, code, long code with quotes and line breaks and the error, and no Python part without Python cells; a workbook laid out as the format specifies (an index, a parameter named as %P2%) read as Python cells with their spilled values and saved back with its code in a Python part; the string-literal form; a formula naming a missing script, with and without a Python part; 20,000 cells naming one long script against the code budget; the part's reader and writer (escaping, odd %P tokens, a long script, a repeated parameter); a blocker in the spill area giving #SPILL! and clearing it spilling again; the native JSON round trip; a spill range claiming the whole sheet; the workbook-wide budget; hostile JSON (code over the limit, a result claiming far more cells than it holds); a malformed result that never spills; PY() formulas at 8,079 and 8,080 characters of code. Calc: formulas, MAX, SUM, COUNTA and SUMPRODUCT read a dirty Python cell whose array is blocked before its turn, in a full and an incremental recalculation, and see the array once it is unblocked; enforce_python_limits removes the spill of dropped code and of a dropped result, and keeps a sound one. --- crates/calc/src/recalc.rs | 80 +++- crates/calc/src/tests.rs | 60 +++ crates/engine/src/io.rs | 132 ++++++ crates/engine/src/tests.rs | 1 + crates/engine/src/tests/python_files.rs | 441 ++++++++++++++++++ .../engine/tests/it/command_sequence_fuzz.rs | 29 +- crates/model/src/python.rs | 18 +- crates/xlsx/src/lib.rs | 1 + crates/xlsx/src/python_part.rs | 318 +++++++++++++ crates/xlsx/src/read.rs | 8 + crates/xlsx/src/sheet_read.rs | 45 +- crates/xlsx/src/sheet_write.rs | 23 +- crates/xlsx/src/write.rs | 9 + 13 files changed, 1133 insertions(+), 32 deletions(-) create mode 100644 crates/engine/src/tests/python_files.rs create mode 100644 crates/xlsx/src/python_part.rs diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 4cbbcc8b4..e1b1e89cf 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -436,6 +436,10 @@ struct PassHost<'a> { stack_top: usize, /// Whether an evaluation ran past the stack budget since the innermost formula started. out_of_stack: bool, + /// The kept results of the Python cells in the pass: a Python cell isn't evaluated, its + /// result is laid out (an array spills from the cell, as a formula's does). Each one is in + /// `pending` until it is laid out, so a formula that reads it first gets its new value. + python: HashMap, } impl<'a> PassHost<'a> { @@ -468,8 +472,17 @@ impl<'a> PassHost<'a> { taint: false, stack_top: 0, out_of_stack: false, + python: HashMap::default(), } } + + /// Adds the dirty Python cells of the pass and their kept results: like the formulas, they + /// are pending until laid out, so every read path calculates them before reading them. + fn with_python(mut self, python: HashMap) -> Self { + self.pending.extend(python.keys().copied()); + self.python = python; + self + } } /// Whether an array result at `k` can't spill: it is in a table, or its area runs off the sheet, @@ -483,7 +496,10 @@ fn spill_blocked(host: &PassHost<'_>, (sheet, anchor): Key, a: &Array) -> bool { let own = sh.spill_ranges.get(&anchor); let spilled_over = |c: CellRef| { !own.is_some_and(|o| o.contains(c)) - && sh.spill_ranges.iter().any(|(o, r)| *o != anchor && r.contains(c) && !host.exprs.contains_key(&(sheet, *o))) + && sh.spill_ranges.iter().any(|(o, r)| { + // A formula or Python cell recalculated in this pass lays its array out again. + *o != anchor && r.contains(c) && !host.exprs.contains_key(&(sheet, *o)) && !host.python.contains_key(&(sheet, *o)) + }) }; end.row >= gridcraft_core::MAX_ROWS || end.col >= gridcraft_core::MAX_COLS @@ -560,6 +576,9 @@ impl PassHost<'_> { } return v; } + if let Some(result) = self.python.get(&k).cloned() { + return self.python_result(k, result); + } let Some(expr) = self.exprs.get(&k) else { return self.wb.sheet(k.0).map(|s| s.value(k.1)).unwrap_or_default(); }; @@ -677,6 +696,36 @@ impl PassHost<'_> { v } + /// A Python cell's kept result as its value: an array spills from the cell unless it is + /// blocked, malformed or larger than [`gridcraft_model::python::MAX_RESULT_CELLS`] (only a + /// crafted file holds one), which shows `#SPILL!`. + fn python_result(&mut self, k: Key, result: Value) -> Value { + self.pending.remove(&k); + let v = match result { + Value::Array(a) => { + let sane = a.rows >= 1 + && a.cols >= 1 + && a.rows.checked_mul(a.cols).is_some_and(|n| n == a.data.len() && n <= gridcraft_model::python::MAX_RESULT_CELLS); + if sane { + self.spills.insert(k, a.clone()); + a.data.first().cloned().unwrap_or_default() + } else { + Value::Error(CellError::Spill) + } + } + v => v, + }; + let v = match self.spills.get(&k) { + Some(a) if spill_blocked(self, k, a) => { + self.spills.remove(&k); + Value::Error(CellError::Spill) + } + _ => v, + }; + self.results.insert(k, v.clone()); + v + } + /// A formula's previous value for iterative calculation: from the last iteration, else the /// value it shows. Blank, and the `#CIRC!` of a loop calculated with iteration off, are 0. fn previous(&self, k: Key) -> Value { @@ -817,12 +866,12 @@ impl Host for PassHost<'_> { let (h, w) = (range.height() as usize, range.width() as usize); let mut out = vec![Value::Empty; h * w]; let Some(sh) = self.wb.sheet(sheet) else { return out }; - // Sparse walk: only stored cells; formula cells that are (or were) dirty go through - // `compute`, constants are copied straight from the store. + // Sparse walk: only stored cells; formula and Python cells that are (or were) dirty go + // through `compute`, constants are copied straight from the store. let mut formulas: Vec<(usize, CellRef)> = Vec::new(); for (c, cell) in sh.cells.iter_range(range) { let i = (c.row - range.start.row) as usize * w + (c.col - range.start.col) as usize; - if cell.formula.is_some() { + if cell.formula.is_some() || cell.py.is_some() { formulas.push((i, c)); } else if let Some(slot) = out.get_mut(i) { *slot = cell.value.clone(); @@ -888,11 +937,11 @@ impl Host for PassHost<'_> { fn visit_range(&mut self, sheet: usize, range: RangeRef, f: &mut dyn FnMut(&Value) -> bool) -> bool { let wb = self.wb; let Some(sh) = wb.sheet(sheet) else { return false }; - // Formula cells first, in row-major order like `range_values`: reading one calculates it - // and the pass remembers the value, so the walk below gets the same value again without - // this method holding any. + // Formula and Python cells first, in row-major order like `range_values`: reading one + // calculates it (a Python cell lays out its result) and the pass remembers the value, so + // the walk below gets the same value again without this method holding any. for (c, cell) in sh.cells.iter_range(range) { - if cell.formula.is_some() { + if cell.formula.is_some() || cell.py.is_some() { let _ = self.cell_value(sheet, c); } } @@ -902,7 +951,7 @@ impl Host for PassHost<'_> { return false; } for (c, cell) in sh.cells.iter_range(range) { - let go = if cell.formula.is_some() { f(&self.cell_value(sheet, c)) } else { f(&cell.value) }; + let go = if cell.formula.is_some() || cell.py.is_some() { f(&self.cell_value(sheet, c)) } else { f(&cell.value) }; if !go { break; } @@ -953,6 +1002,10 @@ impl Calc { if cell.value == Value::Error(CellError::Circ) { circ.push((si, c)); } + } else if cell.py.is_some() { + // A Python cell lays out its kept result, so it is calculated like a formula + // that reads nothing: its spill is blocked and unblocked like a formula's. + self.graph.insert((si, c), Node { areas: Vec::new(), dynamic: false }); } } } @@ -1074,11 +1127,14 @@ impl Calc { let mut cleared: Vec<(usize, RangeRef)> = Vec::new(); for &k in changed { let formula = wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.formula.clone()); + let python = wb.sheet(k.0).and_then(|s| s.cell(k.1)).is_some_and(|c| c.py.is_some()); match formula.and_then(|f| f.expr()) { Some(e) => { let (areas, dynamic) = precedents(wb, k.0, &e); self.graph.insert(k, Node { areas, dynamic }); } + // See `rebuild`: its spill is laid out again from its result when it is calculated. + None if python => self.graph.insert(k, Node { areas: Vec::new(), dynamic: false }), None => { self.graph.remove(k); // A formula replaced by a constant (or cleared) takes its spilled values with it. @@ -1206,11 +1262,15 @@ impl Calc { exprs.insert(*k, e); } } + let python: HashMap = dirty + .iter() + .filter_map(|k| wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.py.as_ref()).and_then(|p| p.result_value()).map(|v| (*k, v))) + .collect(); let now = self.now(); let mut rng = self.rng; let tp = prof_now(); let (results, spills, cycles) = { - let mut host = PassHost::new(wb, &exprs, &mut rng, now); + let mut host = PassHost::new(wb, &exprs, &mut rng, now).with_python(python); host.iterate = wb.calc.iterative; for k in &dirty { host.calculate(*k); diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 5b9e90085..1f68a58fc 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2274,3 +2274,63 @@ fn loan_schedule_residue_keeps_excels_sign() { } assert_eq!(t.get("P7"), Value::Text("".into())); } + +/// Makes `at` a Python cell showing `cached` (its value before this calculation), with `result` +/// as its kept result, and returns the cell's key for `Calc::cells_changed`. +fn plant_python(t: &mut T, at: &str, cached: Value, result: Value) -> (usize, CellRef) { + use gridcraft_model::python::{PyCode, PyOutput}; + let output = PyOutput { value: result, type_name: "list".into(), preview: String::new(), convertible: true }; + let py = PyCode { output: Some(output), ..PyCode::new("[5, 6]") }; + let cell = Cell { value: cached, formula: None, style: gridcraft_model::StyleId::default(), py: Some(std::sync::Arc::new(py)) }; + t.wb.sheet_mut(0).unwrap().cells.set(c(at), cell); + (0, c(at)) +} + +fn column(values: &[f64]) -> Value { + let data = values.iter().map(|v| Value::Number(*v)).collect(); + Value::Array(std::sync::Arc::new(gridcraft_core::Array::new(values.len(), 1, data).unwrap())) +} + +#[test] +fn formulas_read_a_dirty_python_cell_after_it_is_laid_out() { + // B1's result can't spill (B2 holds a value), so it is #SPILL!. A1, A2 and A3 come before + // it in the pass; each must lay it out first rather than read its old value, 1. + let build = |t: &mut T| { + t.set("B2", "10"); + t.set("B3", "100"); + t.set("A1", "=B1"); + t.set("A2", "=MAX(B1:B3)"); + t.set("A3", "=SUM(B1:B3)"); + t.set("A4", "=COUNTA(B1:B3)"); + t.set("A5", "=SUMPRODUCT(--ISERROR(B1:B3))"); + }; + let check = |t: &T, how: &str| { + assert_eq!(t.get("B1"), Value::Error(CellError::Spill), "{how}"); + assert_eq!(t.get("A1"), Value::Error(CellError::Spill), "{how}: a reference"); + assert_eq!(t.get("A2"), Value::Error(CellError::Spill), "{how}: a range read"); + assert_eq!(t.get("A3"), Value::Error(CellError::Spill), "{how}: a range walked cell by cell"); + assert_eq!(t.num("A4"), 3.0, "{how}"); + assert_eq!(t.num("A5"), 1.0, "{how}: the error is seen in the range"); + }; + + // A full recalculation. + let mut t = T::new(); + build(&mut t); + plant_python(&mut t, "B1", Value::Number(1.0), column(&[5.0, 6.0])); + t.calc.recalc_all(&mut t.wb); + check(&t, "full"); + + // An incremental one: B1 becomes a Python cell after its readers. + let mut t = T::new(); + t.set("B1", "1"); + build(&mut t); + assert_eq!((t.num("A1"), t.num("A3")), (1.0, 111.0)); + let k = plant_python(&mut t, "B1", Value::Number(1.0), column(&[5.0, 6.0])); + t.calc.cells_changed(&mut t.wb, &[k]); + check(&t, "incremental"); + + // Unblocked, the readers see the new array. + t.set("B2", ""); + assert_eq!((t.num("B1"), t.num("B2")), (5.0, 6.0)); + assert_eq!((t.num("A1"), t.num("A2"), t.num("A3")), (5.0, 100.0, 111.0)); +} diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 4ad9f3d83..a65139dd6 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -119,7 +119,139 @@ fn decrypt_message(name: &str, e: gridcraft_xlsx::DecryptError) -> String { } /// Parses bytes that aren't encrypted. +/// Parses bytes that aren't encrypted, and adopts the Python cells they hold (see +/// [`adopt_python`]) within the limits of [`enforce_python_limits`]. fn open_plain(name: &str, bytes: &[u8], csv_as_text: bool) -> Result<(Workbook, Vec)> { + let (mut wb, mut warnings) = read_plain(name, bytes, csv_as_text)?; + warnings.extend(adopt_python(&mut wb)); + warnings.extend(enforce_python_limits(&mut wb)); + Ok((wb, warnings)) +} + +/// The most result cells [`adopt_python`] rebuilds from a file's cached spills, over all its +/// Python cells (each one is also limited to [`gridcraft_model::python::MAX_RESULT_CELLS`]). +pub const MAX_ADOPTED_RESULT_CELLS: u64 = 4_000_000; + +/// Python cells from a file: a cell whose formula is `PY("…",0|1)` (a format that kept it as a +/// formula) becomes a Python cell, and a Python cell whose spilled values the file cached gets +/// them back as its result, so it shows them until it runs again. Results are rebuilt for at most +/// [`MAX_ADOPTED_RESULT_CELLS`] cells in all; the rest keep only their own cached value. Returns +/// the warnings. +pub fn adopt_python(wb: &mut Workbook) -> Vec { + let mut budget = MAX_ADOPTED_RESULT_CELLS; + let mut skipped = 0usize; + for si in 0..wb.sheets.len() { + let Some(sh) = wb.sheets.get(si) else { continue }; + let formulas: Vec<(CellRef, gridcraft_model::PyCode)> = sh + .cells + .iter() + .filter_map(|(c, cell)| cell.formula.as_ref().and_then(|f| gridcraft_model::PyCode::from_formula(&f.text)).map(|p| (c, p))) + .collect(); + let max = gridcraft_model::python::MAX_RESULT_CELLS as u64; + let spills: Vec<(CellRef, RangeRef)> = sh + .spill_ranges + .iter() + .filter(|(a, r)| r.count() <= max && sh.cell(**a).and_then(|c| c.py.as_ref()).is_some_and(|p| p.output.is_none())) + .map(|(a, r)| (*a, *r)) + .collect(); + if formulas.is_empty() && spills.is_empty() { + continue; + } + let Some(sh) = wb.sheet_mut(si) else { continue }; + for (c, py) in formulas { + if let Some(cell) = sh.cells.get_mut(c) { + cell.formula = None; + cell.py = Some(std::sync::Arc::new(py)); + } + } + for (anchor, range) in spills { + let Some(left) = budget.checked_sub(range.count()) else { + skipped += 1; + continue; + }; + budget = left; + let data: Vec = + range.iter().map(|c| if c == anchor { sh.cell(c).map(|x| x.value.clone()).unwrap_or_default() } else { sh.value(c) }).collect(); + let Some(array) = gridcraft_core::Array::new(range.height() as usize, range.width() as usize, data) else { continue }; + if let Some(py) = sh.cells.get_mut(anchor).and_then(|x| x.py.as_mut()) { + std::sync::Arc::make_mut(py).output = + Some(gridcraft_model::PyOutput { value: array.into(), type_name: "list".into(), preview: String::new(), convertible: true }); + } + } + } + if skipped == 0 { + return Vec::new(); + } + vec![format!( + "The file's Python results are larger than {MAX_ADOPTED_RESULT_CELLS} cells in all, so {skipped} Python cell(s) show only their first value until they run again." + )] +} + +/// A file is somebody else's work: a Python cell whose code doesn't fit in Excel's +/// 8,192-character formula keeps only its cached value, and a kept result that is not a +/// well-formed array of at most [`gridcraft_model::python::MAX_RESULT_CELLS`] cells is dropped +/// (the cell keeps its value). Returns the warnings. +pub fn enforce_python_limits(wb: &mut Workbook) -> Vec { + let mut long = Vec::new(); + for si in 0..wb.sheets.len() { + let Some(sh) = wb.sheets.get(si) else { continue }; + let mut over = Vec::new(); + let mut bad_output = Vec::new(); + for (c, cell) in sh.cells.iter() { + let Some(py) = &cell.py else { continue }; + if !py.fits() { + long.push(format!("{}!{}", sh.name, c.a1())); + over.push(c); + } else if py.output.as_ref().is_some_and(|o| !sane_python_output(&o.value)) { + bad_output.push(c); + } + } + if over.is_empty() && bad_output.is_empty() { + continue; + } + let dropped: Vec = over.iter().chain(&bad_output).copied().collect(); + let Some(sh) = wb.sheet_mut(si) else { continue }; + for c in over { + if let Some(cell) = sh.cells.get_mut(c) { + cell.py = None; + } + } + for c in bad_output { + if let Some(py) = sh.cells.get_mut(c).and_then(|x| x.py.as_mut()) { + std::sync::Arc::make_mut(py).output = None; + } + } + // The spill of a dropped result (or of code dropped to its value) goes with it, so the + // sheet and a save no longer show the old spilled values. + for c in dropped { + if let Some(r) = sh.spill_ranges.remove(&c) { + sh.clear_spill(r); + } + } + } + let Some(first) = long.first() else { return vec![] }; + let more = if long.len() > 1 { format!(" (and {} more)", long.len() - 1) } else { String::new() }; + vec![format!( + "A Python cell's code is longer than Excel's {}-character formula limit, so {first}{more} kept only its last calculated value.", + gridcraft_model::python::MAX_FORMULA_CHARS + )] +} + +/// A Python result a file may hold: a scalar, or an array whose size matches its data and has at +/// most [`gridcraft_model::python::MAX_RESULT_CELLS`] cells. +fn sane_python_output(v: &gridcraft_core::Value) -> bool { + match v { + gridcraft_core::Value::Array(a) => { + a.rows >= 1 + && a.cols >= 1 + && a.rows.checked_mul(a.cols).is_some_and(|n| n == a.data.len() && n <= gridcraft_model::python::MAX_RESULT_CELLS) + } + _ => true, + } +} + +/// Parses bytes that aren't encrypted, as they are. +fn read_plain(name: &str, bytes: &[u8], csv_as_text: bool) -> Result<(Workbook, Vec)> { let kind = FileKind::from_path(name); let sniffed = gridcraft_xlsx::sniff(bytes); if sniffed == gridcraft_xlsx::Format::Xls { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 47eada0b4..c0adf6002 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5095,6 +5095,7 @@ fn a_script_cannot_switch_windows() { mod python_cells; mod python_commands; +mod python_files; mod seams; #[test] diff --git a/crates/engine/src/tests/python_files.rs b/crates/engine/src/tests/python_files.rs new file mode 100644 index 000000000..00b597445 --- /dev/null +++ b/crates/engine/src/tests/python_files.rs @@ -0,0 +1,441 @@ +//! Python cells in files: native JSON, and xlsx as `_xlfn._xlws.PY(index,0|1)` with the code in +//! the Python part ([MS-XLSX] 2.1.23, 2.3.10) and cached values. Nothing here runs Python: results +//! are cached ones, as a file would hold them. + +use std::io::{Read, Write}; +use std::sync::Arc; + +use gridcraft_core::{Array, CellError, CellRef, RangeRef, Value}; +use gridcraft_model::{Cell, PyCode, PyOutput, Workbook}; +use serde_json::json; + +use super::{s, v}; +use crate::{Session, Trust}; + +fn py(s: &mut Session, cell: &str, code: &str) { + s.execute("python.set", json!({"cell": cell, "code": code})).unwrap(); +} + +/// Gives a Python cell a result, as running it (or a file's cache) would. +fn cache(s: &mut Session, cell: &str, value: Value, type_name: &str) { + let d = s.doc_mut().unwrap(); + let wb = Arc::make_mut(&mut d.wb); + let sh = wb.sheet_mut(0).unwrap(); + let c = sh.cells.get_mut(CellRef::parse(cell).unwrap()).unwrap(); + let p = Arc::make_mut(c.py.as_mut().unwrap()); + p.output = Some(PyOutput { value, type_name: type_name.into(), preview: String::new(), convertible: true }); +} + +fn array(rows: usize, cols: usize, data: Vec) -> Value { + Value::Array(Arc::new(Array::new(rows, cols, data).unwrap())) +} + +fn save(s: &mut Session, format: &str) -> Vec { + let r = s.execute("file.saveBytes", json!({"format": format})).unwrap(); + crate::io::base64_decode(r["base64"].as_str().unwrap()).unwrap() +} + +fn open(s: &mut Session, name: &str, bytes: &[u8]) { + s.execute("file.open", json!({"name": name, "base64": crate::io::base64_encode(bytes)})).unwrap(); +} + +fn part(bytes: &[u8], name: &str) -> String { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut xml = String::new(); + z.by_name(name).unwrap().read_to_string(&mut xml).unwrap(); + xml +} + +fn sheet_xml(bytes: &[u8]) -> String { + part(bytes, "xl/worksheets/sheet1.xml") +} + +/// The package with `edit` applied to each XML part (by name), and `add` added. +fn with_parts(bytes: &[u8], edit: impl Fn(&str, &str) -> String, add: &[(&str, &str)]) -> Vec { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut out = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for i in 0..z.len() { + let mut f = z.by_index(i).unwrap(); + let name = f.name().to_string(); + let mut data = Vec::new(); + f.read_to_end(&mut data).unwrap(); + if let Ok(text) = std::str::from_utf8(&data) { + data = edit(&name, text).into_bytes(); + } + out.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); + out.write_all(&data).unwrap(); + } + for (name, text) in add { + out.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + out.write_all(text.as_bytes()).unwrap(); + } + out.finish().unwrap().into_inner() +} + +/// The package with `edit` applied to the first sheet's XML. +fn with_sheet_xml(bytes: &[u8], edit: impl Fn(&str) -> String) -> Vec { + with_parts(bytes, |name, text| if name == "xl/worksheets/sheet1.xml" { edit(text) } else { text.to_string() }, &[]) +} + +/// `xml` with its `` replaced by `data`. +fn replace_sheet_data(xml: &str, data: &str) -> String { + match (xml.find(""), xml.find("")) { + (Some(a), Some(b), _) => format!("{}{data}{}", &xml[..a], &xml[b + "".len()..]), + (_, _, Some(a)) => format!("{}{data}{}", &xml[..a], &xml[a + "".len()..]), + _ => panic!("no sheetData in {xml}"), + } +} + +/// An empty workbook's package with `data` as its first sheet's cells and, when `scripts` is +/// given, a Python part holding them (the relationship and the content type included), as +/// [MS-XLSX] 2.1.23 describes it. +fn excel_shaped(data: &str, scripts: Option<&[&str]>) -> Vec { + let mut s = s(); + let plain = save(&mut s, "xlsx"); + let Some(scripts) = scripts else { return with_sheet_xml(&plain, |xml| replace_sheet_data(xml, data)) }; + let codes: String = scripts.iter().map(|c| format!("{c}")).collect(); + let python = format!( + "\n{codes}" + ); + const REL: &str = + ""; + const TYPE: &str = ""; + let edit = |name: &str, text: &str| match name { + "xl/worksheets/sheet1.xml" => replace_sheet_data(text, data), + "xl/_rels/workbook.xml.rels" => text.replace("", &format!("{REL}")), + "[Content_Types].xml" => text.replace("", &format!("{TYPE}")), + _ => text.to_string(), + }; + with_parts(&plain, edit, &[("xl/python/python1.xml", &python)]) +} + +fn code(s: &Session, cell: &str) -> Option { + s.doc().unwrap().wb.active().unwrap().cell(CellRef::parse(cell).unwrap()).and_then(|c| c.py.as_deref()).cloned() +} + +/// A workbook with a value, a formula over a Python cell, a spilling table, an object, long code +/// with quotes and line breaks, and an error. +fn build() -> (Session, String) { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "21"})).unwrap(); + py(&mut s, "B1", "xl('A1') * 2"); + cache(&mut s, "B1", Value::Number(42.0), "int"); + s.execute("cell.set", json!({"cell": "A5", "input": "=B1+1"})).unwrap(); + py(&mut s, "C1", "[[1, 'two'], [3, \"four\"]]"); + cache(&mut s, "C1", array(2, 2, vec![1.0.into(), Value::text("two"), 3.0.into(), Value::text("four")]), "list"); + s.execute("python.set", json!({"cell": "E1", "code": "[1, 2]", "output": "object"})).unwrap(); + cache(&mut s, "E1", array(2, 1, vec![1.0.into(), 2.0.into()]), "list"); + let long = format!("# {}\ntext = \"say \\\"hi\\\"\"\nlen(text)", "x".repeat(7_900)); + assert!(PyCode::new(&long).fits()); + py(&mut s, "F1", &long); + cache(&mut s, "F1", Value::Number(8.0), "int"); + py(&mut s, "G1", "1/0"); + { + let d = s.doc_mut().unwrap(); + let sh = Arc::make_mut(&mut d.wb).sheet_mut(0).unwrap(); + sh.cells.get_mut(CellRef::parse("G1").unwrap()).unwrap().value = Value::Error(CellError::Python); + } + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "B1"), Value::Number(42.0)); + assert_eq!(v(&s, "A5"), Value::Number(43.0)); + assert_eq!(v(&s, "D2"), Value::text("four")); + assert_eq!(v(&s, "E1"), Value::text("list"), "a Python object shows its type"); + assert_eq!(v(&s, "E2"), Value::Empty, "and doesn't spill"); + (s, long) +} + +#[test] +fn xlsx_stores_the_code_in_the_python_part_with_cached_values() { + let (mut s, long) = build(); + let bytes = save(&mut s, "xlsx"); + let xml = sheet_xml(&bytes); + // The formula names the cell's script by index, in the order the cells are written: B1, C1, + // E1, F1, G1. + assert!(xml.contains("_xlfn._xlws.PY(0,0)42"), "{xml}"); + assert!(xml.contains(">_xlfn._xlws.PY(2,1)"), "the Python object cell has return type 1: {xml}"); + assert!(!xml.contains("xl('A1')"), "the code is not in the sheet"); + let python = part(&bytes, "xl/python.xml"); + assert!( + python.contains("xl('A1') * 2"), + "{python}" + ); + assert_eq!(python.matches("").count(), 5); + assert!( + part(&bytes, "[Content_Types].xml").contains("") + ); + assert!( + part(&bytes, "xl/_rels/workbook.xml.rels") + .contains("Type=\"http://schemas.microsoft.com/office/2023/09/relationships/Python\" Target=\"python.xml\"") + ); + assert!(xml.contains("t=\"array\" ref=\"C1:D2\""), "the spill is saved like a dynamic array's: {xml}"); + assert!(xml.contains("#VALUE!") && !xml.contains("#PYTHON!"), "Python errors are cached as #VALUE!"); + + open(&mut s, "book.xlsx", &bytes); + assert_eq!(s.doc().unwrap().trust, Trust::FromFile); + // Cached values show, spills included, and the code is back. + assert_eq!(v(&s, "B1"), Value::Number(42.0)); + assert_eq!(v(&s, "A5"), Value::Number(43.0)); + assert_eq!( + (v(&s, "C1"), v(&s, "D1"), v(&s, "C2"), v(&s, "D2")), + (Value::Number(1.0), Value::text("two"), Value::Number(3.0), Value::text("four")) + ); + assert_eq!(code(&s, "B1").unwrap().code, "xl('A1') * 2"); + assert!(code(&s, "E1").unwrap().object); + assert_eq!(code(&s, "F1").unwrap().code, long, "long code with quotes and line breaks survives"); + assert_eq!(v(&s, "G1"), Value::Error(CellError::Value)); + // A blocker typed into the spill area gives #SPILL!; clearing it spills again, without + // running anything. + s.execute("cell.set", json!({"cell": "D2", "input": "x"})).unwrap(); + assert_eq!(v(&s, "C1"), Value::Error(CellError::Spill)); + s.execute("edit.clearContents", json!({"range": "D2"})).unwrap(); + assert_eq!(v(&s, "D2"), Value::text("four")); + // Saved again, the file holds the same formulas and scripts. + let again = save(&mut s, "xlsx"); + assert!(sheet_xml(&again).contains("_xlfn._xlws.PY(0,0)")); + assert_eq!(part(&again, "xl/python.xml"), python); + // A workbook without Python cells has no Python part. + let mut plain = super::s(); + assert!(!save(&mut plain, "xlsx").windows(10).any(|w| w == b"python.xml")); +} + +#[test] +fn excel_authored_python_cells_are_read_as_python() { + // A workbook as [MS-XLSX] 2.3.10 describes it: `_xlfn._xlws.PY(index,0|1,params…)` naming a + // script of the Python part, whose code names the parameters as `%Px%`, with a spilled result + // cached. + let data = "_xlfn._xlws.PY(0,0)1\ + _xlfn._xlws.PY(1,1,C1:C3)DataFrame\ + 2"; + let bytes = excel_shaped(data, Some(&["[1, 2]", "df = xl(\"%P2%\", headers=True)\ndf.describe()"])); + let mut s = s(); + open(&mut s, "excel.xlsx", &bytes); + let a1 = code(&s, "A1").expect("an Excel Python cell is a Python cell"); + assert_eq!(a1.code, "[1, 2]"); + assert!(!a1.object); + assert_eq!((v(&s, "A1"), v(&s, "A2")), (Value::Number(1.0), Value::Number(2.0)), "its spilled values are kept"); + let cell = s.doc().unwrap().wb.active().unwrap().cell(CellRef::new(0, 0)).unwrap().clone(); + assert!(cell.formula.is_none()); + let b1 = code(&s, "B1").unwrap(); + assert_eq!(b1.code, "df = xl(\"C1:C3\", headers=True)\ndf.describe()", "a parameter is put back as its reference"); + assert!(b1.object); + assert_eq!(v(&s, "B1"), Value::text("DataFrame")); + // Saved by GridCraft, the code goes back into a Python part, the reference as text. + let again = save(&mut s, "xlsx"); + assert!(part(&again, "xl/python.xml").contains("df = xl(\"C1:C3\", headers=True)\ndf.describe()")); + // The string-literal form (other applications, files GridCraft wrote before) still reads. + let literal = "_xlfn._xlws.PY(\"1 + 1\",1)2"; + open(&mut s, "literal.xlsx", &excel_shaped(literal, None)); + assert_eq!(code(&s, "A1").map(|p| (p.code, p.object)), Some(("1 + 1".to_string(), true))); +} + +#[test] +fn a_py_formula_naming_a_missing_script_keeps_only_its_value() { + let data = "_xlfn._xlws.PY(0,0)5_xlfn._xlws.PY(3,0)6"; + for (name, bytes) in [("one.xlsx", excel_shaped(data, Some(&["5"]))), ("none.xlsx", excel_shaped(data, None))] { + let (wb, warnings) = crate::io::open_bytes(name, &bytes).unwrap(); + let sh = wb.sheet(0).unwrap(); + let b1 = sh.cell(CellRef::parse("B1").unwrap()).unwrap(); + assert!(b1.py.is_none() && b1.formula.is_none(), "{name}: no code to keep, and no formula #NAME? would replace"); + assert_eq!(b1.value, Value::Number(6.0), "{name}"); + assert!(warnings.iter().any(|w| w.contains("Python")), "{name}: {warnings:?}"); + } + let (wb, _) = crate::io::open_bytes("one.xlsx", &excel_shaped(data, Some(&["5"]))).unwrap(); + assert_eq!(wb.sheet(0).unwrap().cell(CellRef::parse("A1").unwrap()).unwrap().py.as_ref().map(|p| p.code.as_str()), Some("5")); +} + +#[test] +fn many_cells_naming_one_long_script_take_from_one_budget() { + // 20,000 cells name one script of 8,000 characters: 160 MB of code if each took a copy. + let long = "x".repeat(8_000); + let mut data = String::from(""); + for r in 1..=20_000 { + data.push_str(&format!("_xlfn._xlws.PY(0,0)1")); + } + data.push_str(""); + let (wb, warnings) = crate::io::open_bytes("many.xlsx", &excel_shaped(&data, Some(&[&long]))).unwrap(); + let sh = wb.sheet(0).unwrap(); + let with_code = sh.cells.iter().filter(|(_, c)| c.py.is_some()).count(); + assert_eq!(with_code, 64 * 1024 * 1024 / 8_000); + assert!(warnings.iter().any(|w| w.contains("Python")), "{warnings:?}"); + assert_eq!(sh.value(CellRef::parse("A20000").unwrap()), Value::Number(1.0), "the others keep their values"); +} + +#[test] +fn native_json_keeps_python_cells_and_their_results() { + let (mut s, long) = build(); + let bytes = save(&mut s, "json"); + open(&mut s, "book.json", &bytes); + assert_eq!(s.doc().unwrap().trust, Trust::FromFile); + assert_eq!(v(&s, "B1"), Value::Number(42.0)); + assert_eq!(v(&s, "D2"), Value::text("four")); + assert_eq!(v(&s, "G1"), Value::Error(CellError::Python), "the native format keeps the exact error"); + assert_eq!(code(&s, "F1").unwrap().code, long); + let e1 = code(&s, "E1").unwrap(); + assert!(e1.object); + assert_eq!(e1.output.unwrap().type_name, "list"); +} + +#[test] +fn a_spill_range_claiming_the_whole_sheet_is_dropped_on_open() { + let (mut s, _) = build(); + let bytes = save(&mut s, "xlsx"); + let evil = with_sheet_xml(&bytes, |xml| xml.replace("ref=\"C1:D2\"", "ref=\"C1:XFD1048576\"")); + assert_ne!(evil, bytes); + // Walking the claimed range would take hours; the test finishing is the check. + open(&mut s, "evil.xlsx", &evil); + let sh = s.doc().unwrap().wb.active().unwrap(); + assert!(sh.spill_ranges.values().all(|r| r.count() <= gridcraft_model::python::MAX_RESULT_CELLS as u64), "{:?}", sh.spill_ranges); + // Edits afterwards don't walk the claimed range either. + s.execute("cell.set", json!({"cell": "H9", "input": "1"})).unwrap(); + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "C1"), Value::Number(1.0), "the anchor keeps its cached value"); +} + +#[test] +fn rebuilt_results_have_a_budget_across_the_workbook() { + // Five Python cells each claiming a 1,000 x 1,000 spill: more than the budget in all. + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + for i in 0..5u32 { + let anchor = CellRef::new(0, i * 1_000); + sh.set_cell(anchor, Cell { py: Some(Arc::new(PyCode::new("x"))), ..Cell::value(Value::Number(1.0)) }); + sh.spill_ranges.insert(anchor, RangeRef::new(anchor, CellRef::new(999, i * 1_000 + 999))); + } + let warnings = crate::io::adopt_python(&mut wb); + let sh = wb.sheet(0).unwrap(); + let rebuilt = sh.cells.iter().filter(|(_, c)| c.py.as_ref().is_some_and(|p| p.output.is_some())).count(); + assert_eq!(rebuilt as u64, crate::io::MAX_ADOPTED_RESULT_CELLS / 1_000_000); + assert!(warnings.iter().any(|w| w.contains("Python")), "{warnings:?}"); +} + +#[test] +fn hostile_python_content_in_files_is_capped() { + let (mut s, _) = build(); + let bytes = save(&mut s, "json"); + // Make the Python cells hostile: code over the limit, and a result whose shape claims far + // more cells than it holds (the array fields are public, so a crafted file can say anything). + let mut wb: Workbook = serde_json::from_slice(&bytes).unwrap(); + let sh = wb.sheet_mut(0).unwrap(); + let b1 = sh.cells.get_mut(CellRef::parse("B1").unwrap()).unwrap(); + Arc::make_mut(b1.py.as_mut().unwrap()).code = "y".repeat(9_000); + let c1 = sh.cells.get_mut(CellRef::parse("C1").unwrap()).unwrap(); + let out = Arc::make_mut(c1.py.as_mut().unwrap()).output.as_mut().unwrap(); + out.value = Value::Array(Arc::new(Array { rows: 100_000, cols: 100_000, data: vec![Value::Number(1.0)] })); + let bytes = serde_json::to_vec(&wb).unwrap(); + let (wb, warnings) = crate::io::open_bytes("evil.json", &bytes).unwrap(); + let sh = wb.sheet(0).unwrap(); + let b1 = sh.cell(CellRef::parse("B1").unwrap()).unwrap(); + assert!(b1.py.is_none(), "code over the formula limit is dropped"); + assert_eq!(b1.value, Value::Number(42.0), "the cached value stays"); + assert!(warnings.iter().any(|w| w.contains("Python cell")), "{warnings:?}"); + let c1 = sh.cell(CellRef::parse("C1").unwrap()).unwrap(); + assert!(c1.py.as_ref().unwrap().output.is_none(), "a malformed result is dropped"); + open(&mut s, "evil.json", &bytes); + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "C1"), Value::Number(1.0)); + + // PY formulas from other formats become Python cells; non-literal PY calls stay formulas. + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + sh.set_cell(CellRef::new(0, 0), Cell::formula(gridcraft_model::Formula::new("=PY(\"1+\"&\"1\",1)"))); + sh.set_cell(CellRef::new(1, 0), Cell::formula(gridcraft_model::Formula::new("=PY(A1,0)"))); + sh.set_cell(CellRef::new(2, 0), Cell::formula(gridcraft_model::Formula::new("=_xlfn._xlws.PY(\"2\",0)"))); + crate::io::adopt_python(&mut wb); + let sh = wb.sheet(0).unwrap(); + let a1 = sh.cell(CellRef::new(0, 0)).unwrap(); + assert_eq!(a1.py.as_ref().map(|p| (p.code.as_str(), p.object)), Some(("1+1", true))); + assert!(a1.formula.is_none()); + assert!(sh.cell(CellRef::new(1, 0)).unwrap().py.is_none()); + assert_eq!(sh.cell(CellRef::new(2, 0)).unwrap().py.as_ref().map(|p| p.code.as_str()), Some("2")); +} + +#[test] +fn a_malformed_result_never_spills() { + let mut s = s(); + py(&mut s, "A1", "x"); + let bad = Array { rows: 4_000_000_000, cols: 4_000_000_000, data: vec![] }; + cache(&mut s, "A1", Value::Array(Arc::new(bad)), "list"); + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "A1"), Value::Error(CellError::Spill)); + assert!(s.doc().unwrap().wb.active().unwrap().spill.is_empty()); +} + +#[test] +fn py_formulas_over_the_size_limit_keep_only_their_value() { + let lits = |n: usize| { + let mut v = vec![format!("\"{}\"", "x".repeat(255)); n / 255]; + if !n.is_multiple_of(255) { + v.push(format!("\"{}\"", "x".repeat(n % 255))); + } + v.join("&") + }; + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + let cell = |text: String| Cell { value: Value::text("cached"), ..Cell::formula(gridcraft_model::Formula::new(&text)) }; + sh.set_cell(CellRef::new(0, 0), cell(format!("_xlfn._xlws.PY({},0)", lits(8_080)))); + sh.set_cell(CellRef::new(1, 0), cell(format!("_xlfn._xlws.PY({},0)", lits(8_079)))); + let xlsx = gridcraft_xlsx::write_xlsx(&wb).unwrap(); + let (wb, warnings) = crate::io::open_bytes("long.xlsx", &xlsx).unwrap(); + let sh = wb.sheet(0).unwrap(); + let over = sh.cell(CellRef::new(0, 0)).unwrap(); + assert!(over.py.is_none() && over.formula.is_none(), "8,080 characters of code is over the limit"); + assert_eq!(over.value, Value::text("cached")); + let within = sh.cell(CellRef::new(1, 0)).unwrap(); + assert_eq!(within.py.as_ref().map(|p| p.code.chars().count()), Some(8_079)); + assert!(warnings.iter().any(|w| w.contains("A1")), "{warnings:?}"); +} + +#[test] +fn a_dropped_python_result_takes_its_spill_with_it() { + // A1's code is over the limit and B1's result is malformed; each had spilled down to row 3, + // and C1's well-formed result keeps its spill. + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + let out = |value: Value| Some(PyOutput { value, type_name: "list".into(), preview: String::new(), convertible: true }); + let col = array(3, 1, vec![Value::Number(7.0), Value::Number(8.0), Value::Number(9.0)]); + let bad = Value::Array(Arc::new(Array { rows: 3, cols: 1, data: vec![Value::Number(7.0)] })); + for (at, code, value) in [("A1", "y".repeat(9_000), col.clone()), ("B1", "b".into(), bad), ("C1", "c".into(), col)] { + let at = CellRef::parse(at).unwrap(); + let py = PyCode { output: out(value), ..PyCode::new(&code) }; + sh.set_cell(at, Cell { value: Value::Number(7.0), formula: None, style: Default::default(), py: Some(Arc::new(py)) }); + let r = RangeRef::new(at, CellRef::new(2, at.col)); + sh.spill_ranges.insert(at, r); + for c in r.iter().skip(1) { + sh.spill.insert(c, Value::Number(8.0)); + } + } + let warnings = crate::io::enforce_python_limits(&mut wb); + assert!(warnings.iter().any(|w| w.contains("A1")), "{warnings:?}"); + let sh = wb.sheet(0).unwrap(); + for (anchor, below) in [("A1", "A2"), ("B1", "B3")] { + assert!(!sh.spill_ranges.contains_key(&CellRef::parse(anchor).unwrap()), "{anchor}'s spill range stays"); + assert_eq!(sh.value(CellRef::parse(below).unwrap()), Value::Empty, "{below} still shows {anchor}'s old spill"); + assert_eq!(sh.value(CellRef::parse(anchor).unwrap()), Value::Number(7.0), "{anchor} keeps its value"); + } + assert_eq!(sh.value(CellRef::parse("C2").unwrap()), Value::Number(8.0), "a sound result keeps its spill"); + // A save writes no value for the dropped spills. + let xml = sheet_xml(&gridcraft_xlsx::write_xlsx(&wb).unwrap()); + assert!(!xml.contains("r=\"A2\"") && !xml.contains("r=\"B3\""), "{xml}"); + assert!(xml.contains("r=\"C2\""), "{xml}"); +} + +#[test] +fn a_formula_entered_in_a_python_spill_takes_the_area_as_a_formula_would() { + // A1's cached array spills over A1:A3; =SEQUENCE(2) entered in A2 is not blocked by that old + // spill (the Python cell is laid out again in the same pass), so it spills and A1 is blocked. + let mut s = s(); + py(&mut s, "A1", "[1, 2, 3]"); + cache(&mut s, "A1", array(3, 1, vec![10.0.into(), 20.0.into(), 30.0.into()]), "list"); + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "A3"), Value::Number(30.0)); + s.execute("cell.set", json!({"cell": "A2", "input": "=SEQUENCE(2)"})).unwrap(); + assert_eq!((v(&s, "A2"), v(&s, "A3")), (Value::Number(1.0), Value::Number(2.0))); + assert_eq!(v(&s, "A1"), Value::Error(CellError::Spill)); + let wb = &s.doc().unwrap().wb; + let mut fresh = (**wb).clone(); + gridcraft_calc::Calc::new().recalc_all(&mut fresh); + for c in ["A1", "A2", "A3"] { + let c = CellRef::parse(c).unwrap(); + assert_eq!(wb.active().unwrap().value(c), fresh.active().unwrap().value(c), "{c}"); + } +} diff --git a/crates/engine/tests/it/command_sequence_fuzz.rs b/crates/engine/tests/it/command_sequence_fuzz.rs index 21a856c75..3c241995e 100644 --- a/crates/engine/tests/it/command_sequence_fuzz.rs +++ b/crates/engine/tests/it/command_sequence_fuzz.rs @@ -32,7 +32,7 @@ use std::collections::BTreeSet; use std::sync::Arc; -use gridcraft_core::{CellRef, MAX_COLS, MAX_ROWS, RangeRef, Value, col_to_letters}; +use gridcraft_core::{CellError, CellRef, MAX_COLS, MAX_ROWS, RangeRef, Value, col_to_letters}; use gridcraft_engine::calc::Calc; use gridcraft_engine::model::{CalcMode, Formula, Sheet, Workbook}; use gridcraft_engine::{EngineError, Session, command_specs, find_command, io}; @@ -433,7 +433,12 @@ fn value_diff(a: &Workbook, b: &Workbook, what: &str) -> Option { } for c in keys { let (va, vb) = (x.value(c), y.value(c)); - if !same_value(&va, &vb) { + // A file caches the Python errors as #VALUE! (they aren't error codes of the file + // format), in a Python cell and in the formulas that read one; reading it back runs + // nothing. + let python_error = matches!(va, Value::Error(CellError::Python | CellError::Busy | CellError::Timeout | CellError::Blocked)) + && vb == Value::Error(CellError::Value); + if !same_value(&va, &vb) && !python_error { let f = x.cell(c).and_then(|c| c.formula.as_ref()).map(|f| format!(" (formula ={})", f.text)).unwrap_or_default(); return Some(format!("{}!{c}{f}: {va:?} in memory, {vb:?} after {what}", x.name)); } @@ -785,6 +790,7 @@ const CURATED: &[(&str, usize)] = &[ ("formulas.defineName", 2), ("formulas.deleteName", 1), ("edit.replace", 1), + ("python.set", 3), ]; fn generate(rng: &mut Rng, s: &Session) -> Step { @@ -818,6 +824,25 @@ fn curated_params(rng: &mut Rng, wb: &Workbook, id: &str) -> Json { p } "range.setValues" => json!({"range": near_cell(rng).a1(), "values": values_2d(rng, wb)}), + "python.set" => { + // Code with xl() references (which follow structural edits), quotes and line breaks; + // sometimes long enough to meet the formula limit. + let mut code = match rng.below(5) { + 0 => format!("xl(\"{}\") + 1", near_range(rng).a1()), + 1 => format!("a = xl('{}')\nb = \"q\\\"uote\"\na", near_cell(rng).a1()), + 2 => "[[1, 2], [3, 4]]".to_string(), + 3 => String::new(), + _ => "\"".repeat(rng.int(1, 5000) as usize), + }; + if rng.chance(5) { + code.push_str(&"x".repeat(8_100)); + } + let mut p = json!({"cell": near_cell(rng).a1(), "code": code}); + if rng.chance(30) { + p["output"] = json!(if rng.chance(50) { "object" } else { "value" }); + } + p + } "range.fill" => json!({"range": near_range(rng).a1(), "input": input(rng, wb)}), "selection.set" => { let mut p = json!({"range": range(rng)}); diff --git a/crates/model/src/python.rs b/crates/model/src/python.rs index abd4c9414..c8a993915 100644 --- a/crates/model/src/python.rs +++ b/crates/model/src/python.rs @@ -1,14 +1,19 @@ //! Python cells: the code a cell holds, its last result, and the text forms around it (the -//! `PY(…)` formula files store it as, and the `xl("…")` references inside it). +//! `PY("…",0|1)` formula of the function's syntax, and the `xl("…")` references inside it). An +//! xlsx file keeps the code in its Python part instead (see the xlsx crate). use gridcraft_core::{CellError, Value}; use serde::{Deserialize, Serialize}; /// Excel's limit on the characters of one formula. A Python cell's code must fit in it once it is -/// encoded as a file formula ([`PyCode::to_file_formula`]): quotes are doubled and every 255 -/// characters add `"&"`, so the limit on the code itself depends on what the code holds. +/// written as the PY formula of the function's syntax ([`PyCode::to_file_formula`]): quotes are +/// doubled and every 255 characters add `"&"`, so the limit on the code itself depends on what the +/// code holds. pub const MAX_FORMULA_CHARS: usize = 8_192; +/// The most cells a Python result may have (a crafted file can claim any size). +pub const MAX_RESULT_CELLS: usize = 1_000_000; + /// The prefix files put before `PY(`, as Excel writes it ([MS-XLSX]: `_xlfn._xlws.PY`). pub const FILE_PREFIX: &str = "_xlfn._xlws."; @@ -66,13 +71,14 @@ impl PyCode { format!("PY({lits},{})", u8::from(self.object)) } - /// The formula a file stores the cell as: [`PyCode::to_formula`] with Excel's - /// `_xlfn._xlws.` prefix, so Excel reads it as its own Python function. + /// [`PyCode::to_formula`] with Excel's `_xlfn._xlws.` prefix: the string-literal form + /// [`PyCode::from_formula`] reads from files that hold it, and what the size limit measures. + /// An xlsx file names the code by index instead ([MS-XLSX] 2.3.10.2). pub fn to_file_formula(&self) -> String { format!("{FILE_PREFIX}{}", self.to_formula()) } - /// Whether the code fits in a file formula of at most [`MAX_FORMULA_CHARS`] characters. + /// Whether the code fits in a PY formula of at most [`MAX_FORMULA_CHARS`] characters. pub fn fits(&self) -> bool { // Code longer than the limit can't fit; this also bounds the work for hostile input. self.code.chars().nth(MAX_FORMULA_CHARS).is_none() && self.to_file_formula().chars().nth(MAX_FORMULA_CHARS).is_none() diff --git a/crates/xlsx/src/lib.rs b/crates/xlsx/src/lib.rs index 650b22820..b687c35dc 100644 --- a/crates/xlsx/src/lib.rs +++ b/crates/xlsx/src/lib.rs @@ -24,6 +24,7 @@ mod ods; mod package; mod passthrough; mod pivot; +mod python_part; mod read; mod richdata; mod sheet_read; diff --git a/crates/xlsx/src/python_part.rs b/crates/xlsx/src/python_part.rs new file mode 100644 index 000000000..11793f9fc --- /dev/null +++ b/crates/xlsx/src/python_part.rs @@ -0,0 +1,318 @@ +//! The Python part ([MS-XLSX] 2.1.23, 2.3.10): where a workbook keeps the code of its Python cells. +//! +//! A Python cell's formula is `_xlfn._xlws.PY(index, return_type, …)` ([MS-XLSX] 2.3.10.2): the +//! first argument is the zero-based index of a `pythonScript` element in this part, whose `code` +//! is the cell's Python code, and the second is 0 (an Excel value) or 1 (a Python object). Further +//! arguments are references the code names as `%Px%`, `x` being the argument's zero-based index +//! ([MS-XLSX] 2.3.10.4). The part's root is `python` ([MS-XLSX] 2.4.102, CT_Python 2.6.249), with +//! `pythonScripts` (2.6.253) holding the `pythonScript` elements (2.6.252). +//! +//! GridCraft writes one script per Python cell, in the order the cells are written, with the code +//! as it is (its `xl()` references stay text in the code, so no further arguments). Reading +//! resolves the index and puts each `%Px%` back as the text of argument `x`. A formula that names +//! no script the part has keeps only its cached value. Hostile input is bounded: the part's size, +//! the number of scripts, each script's length, and the code all cells take from the part +//! together. + +use gridcraft_formula::Expr; +use gridcraft_model::PyCode; +use gridcraft_model::python::{FILE_PREFIX, MAX_FORMULA_CHARS}; + +use crate::IoError; +use crate::read::Ctx; +use crate::xml::{self, El}; + +/// Namespace of the part. +pub const NS: &str = "http://schemas.microsoft.com/office/spreadsheetml/2023/python"; +/// Relationship type from the workbook part. +pub const REL: &str = "http://schemas.microsoft.com/office/2023/09/relationships/Python"; +/// Last segment of [`REL`], as [`crate::package::Rel::kind`] holds it. +const REL_KIND: &str = "Python"; +/// Content type of the part. +pub const CONTENT_TYPE: &str = "application/vnd.ms-excel.python+xml"; +/// Part name we write (the format names none; the relationship finds it). +pub const PART: &str = "xl/python.xml"; + +/// Largest Python part read. +const MAX_PART_BYTES: u64 = 64 * 1024 * 1024; +/// Most scripts read from the part. +const MAX_SCRIPTS: usize = 1_000_000; +/// Most characters kept of one script: one more than a cell's code may have, so a longer script +/// still fails [`PyCode::fits`] and its cell keeps only its value, with the usual warning. +const MAX_SCRIPT_CHARS: usize = MAX_FORMULA_CHARS + 1; +/// Most bytes of code all the cells of a workbook take from the part together. Several formulas +/// may name one script, so without it a small part could fill memory. +pub const MAX_CODE_BYTES: usize = 64 * 1024 * 1024; + +/// The scripts read from a workbook's Python part, and what the cells took from them. +#[derive(Default)] +pub struct ScriptsIn { + scripts: Vec, + /// Bytes of code the cells may still take ([`MAX_CODE_BYTES`] in all). + budget: usize, + /// Python cells that kept only their value: no such script, or over the budget. + missing: usize, +} + +impl ScriptsIn { + /// The warning for the Python cells that kept only their value, if any. + pub fn warning(&self) -> Option { + (self.missing > 0).then(|| { + format!( + "{} Python cell(s) name code the file's Python part doesn't hold (or holds too much of); they kept only their values", + self.missing + ) + }) + } +} + +/// Reads the workbook's Python part, if it has one. A damaged part is a warning: its cells keep +/// their values. +pub fn read(cx: &mut Ctx<'_>, rels: &[crate::package::Rel]) -> Result { + let mut out = ScriptsIn { budget: MAX_CODE_BYTES, ..ScriptsIn::default() }; + let Some(rel) = rels.iter().find(|r| !r.external && r.kind.eq_ignore_ascii_case(REL_KIND)) else { return Ok(out) }; + let bytes = match cx.pkg.read_limited(&rel.target, MAX_PART_BYTES) { + Ok(Some(b)) => b, + Ok(None) => { + cx.warn(format!("missing part {}", rel.target)); + return Ok(out); + } + Err(IoError::TooLarge(m)) => { + cx.warn(format!("the Python part is too large ({m}); Python cells kept only their values")); + return Ok(out); + } + Err(e) => return Err(e), + }; + let mut over = 0usize; + let parsed = xml::parse_streaming(&bytes, &["pythonScript"], &mut |el: El| { + if out.scripts.len() < MAX_SCRIPTS { + out.scripts.push(script_code(&el)); + } else { + over += 1; + } + Ok(()) + }); + match parsed { + Ok(root) if root.name != "python" => { + cx.warn(format!("the Python part has the root <{}>; Python cells kept only their values", root.name)); + out.scripts.clear(); + } + Ok(_) => {} + Err(e) => cx.warn(format!("the Python part is damaged ({e}); kept {} scripts", out.scripts.len())), + } + if over > 0 { + cx.warn(format!("the Python part holds more than {MAX_SCRIPTS} scripts; {over} were not read")); + } + Ok(out) +} + +/// A `pythonScript` element's code (ST_Xstring), with `\n` line breaks, cut after +/// [`MAX_SCRIPT_CHARS`] characters. +fn script_code(el: &El) -> String { + let text = el.child("code").map(|c| c.text.as_str()).unwrap_or_default(); + let text = match text.char_indices().nth(MAX_SCRIPT_CHARS) { + Some((at, _)) => text.get(..at).unwrap_or_default(), + None => text, + }; + gridcraft_model::python::normalize_code(&xml::decode_xstring(text)) +} + +/// `PY(index, 0|1, params…)`, as a file's formula holds it: the index, whether the return type is +/// a Python object, and the parameters. `None` for anything else (the code in a string literal +/// is [`PyCode::from_formula`]'s form). +fn py_call(expr: &Expr) -> Option<(usize, bool, &[Expr])> { + let Expr::Call(name, args) = expr else { return None }; + let base = name.trim_start_matches("_XLFN.").trim_start_matches("_XLWS."); + if !base.eq_ignore_ascii_case("PY") { + return None; + } + let index = match args.first()? { + Expr::Number(n) if *n >= 0.0 && n.fract() == 0.0 && *n < MAX_SCRIPTS as f64 => *n as usize, + _ => return None, + }; + let object = match args.get(1)? { + Expr::Number(n) if *n == 0.0 => false, + Expr::Number(n) if *n == 1.0 => true, + _ => return None, + }; + Some((index, object, args.get(2..).unwrap_or_default())) +} + +/// What a file formula is to Python. +#[derive(Debug, PartialEq)] +pub enum PyFormula { + /// Not a `PY(…)` call: an ordinary formula. + Not, + /// A Python cell with this code. + Code(PyCode), + /// A `PY(index, …)` call naming a script the part doesn't hold (or over the budget): the + /// cell keeps only its cached value, since no calculation can evaluate the formula. + Missing, +} + +/// What a file formula makes: a Python cell from the Python part when it names a script, or from +/// string literals ([`PyCode::from_formula`]). `formula` is the parsed formula, `raw` the +/// formula's own text, which keeps line breaks in literal code. +pub fn cell_code(scripts: &mut ScriptsIn, formula: &gridcraft_model::Formula, raw: &str) -> PyFormula { + // The cheap test first: most formulas are not `PY(` calls. + if !gridcraft_model::python::is_py_formula(&formula.text) { + return PyFormula::Not; + } + let expr = formula.expr_arc(); + let Some((index, object, params)) = expr.as_deref().and_then(py_call) else { + return PyCode::from_formula(raw).map_or(PyFormula::Not, PyFormula::Code); + }; + let code = scripts.scripts.get(index).map(|code| with_params(code, params)); + match code.filter(|c| c.len() <= scripts.budget) { + Some(code) => { + scripts.budget -= code.len(); + PyFormula::Code(PyCode { object, ..PyCode::new(&code) }) + } + None => { + scripts.missing += 1; + PyFormula::Missing + } + } +} + +/// `code` with every `%Px%` (`x` at least 2, naming one of `params`) replaced by the parameter's +/// text. Stops growing past [`MAX_SCRIPT_CHARS`] characters: such code can't be a cell's. +fn with_params(code: &str, params: &[Expr]) -> String { + if params.is_empty() || !code.contains("%P") { + return code.to_string(); + } + let texts: Vec = params.iter().map(gridcraft_formula::print).collect(); + let mut out = String::with_capacity(code.len()); + let mut chars = 0usize; + let mut push = |out: &mut String, piece: &str| { + out.push_str(piece); + chars = chars.saturating_add(piece.chars().count()); + chars > MAX_SCRIPT_CHARS + }; + let mut rest = code; + while let Some(at) = rest.find("%P") { + if push(&mut out, rest.get(..at).unwrap_or_default()) { + return out; + } + let after = rest.get(at + 2..).unwrap_or_default(); + let digits = after.bytes().take_while(u8::is_ascii_digit).count(); + let param = (1..=6) + .contains(&digits) + .then(|| after.get(..digits).and_then(|d| d.parse::().ok())) + .flatten() + .filter(|_| after.as_bytes().get(digits) == Some(&b'%')) + .and_then(|x| x.checked_sub(2)) + .and_then(|x| texts.get(x)); + let full = match param { + Some(text) => { + rest = after.get(digits + 1..).unwrap_or_default(); + push(&mut out, text) + } + None => { + rest = after; + push(&mut out, "%P") + } + }; + if full { + return out; + } + } + out.push_str(rest); + out +} + +/// The formula a Python cell is saved as: `_xlfn._xlws.PY(index,0|1)`, its code being script +/// `index` of the part ([`write`]). +pub fn file_formula(index: usize, object: bool) -> String { + format!("{FILE_PREFIX}PY({index},{})", u8::from(object)) +} + +/// The Python part holding `scripts`, in order. Only written when there is at least one: the +/// format requires one `pythonScript` in `pythonScripts`. +pub fn write(scripts: &[String]) -> String { + let mut xml = format!("{}", crate::write::XML_DECL); + for code in scripts { + xml.push_str(""); + xml.push_str(&xml::esc(&xml::encode_xstring(code))); + xml.push_str(""); + } + xml.push_str(""); + xml +} + +#[cfg(test)] +mod tests { + use super::*; + + fn formula(text: &str) -> gridcraft_model::Formula { + gridcraft_model::Formula::new(&crate::fmla::from_file(text)) + } + + fn scripts(list: &[&str]) -> ScriptsIn { + ScriptsIn { scripts: list.iter().map(|s| s.to_string()).collect(), budget: MAX_CODE_BYTES, missing: 0 } + } + + #[test] + fn reads_the_index_form_and_parameters() { + let mut s = scripts(&["1 + 1", "df = xl(\"%P2%\", headers=True)\nxl('%P3%') + %P9% + %P1%"]); + let code = |s: &mut ScriptsIn, text: &str| match cell_code(s, &formula(text), text) { + PyFormula::Code(py) => Some(py), + _ => None, + }; + let a = code(&mut s, "_xlfn._xlws.PY(0,1)").unwrap(); + assert_eq!((a.code.as_str(), a.object), ("1 + 1", true)); + let b = code(&mut s, "_xlfn._xlws.PY(1,0,A1:B3,Sheet2!$C$4)").unwrap(); + assert_eq!(b.code, "df = xl(\"A1:B3\", headers=True)\nxl('Sheet2!$C$4') + %P9% + %P1%"); + assert!(!b.object); + // The literal form still reads (other applications, older files). + assert_eq!(code(&mut s, "_xlfn._xlws.PY(\"x = 1\",0)").unwrap().code, "x = 1"); + // A script the part doesn't hold: the cell keeps its value, and is counted for the warning. + let gone = "_xlfn._xlws.PY(7,0)"; + assert_eq!(cell_code(&mut s, &formula(gone), gone), PyFormula::Missing); + assert!(s.warning().is_some()); + for not in ["_xlfn._xlws.PY(-1,0)", "_xlfn._xlws.PY(0.5,0)", "_xlfn._xlws.PY(0,2)", "PY(A1,0)", "SUM(0,1)", "PYTHON_TYPE(A1)"] { + assert_eq!(cell_code(&mut s, &formula(not), not), PyFormula::Not, "{not}"); + } + } + + #[test] + fn hostile_scripts_are_bounded() { + // Many formulas naming one script take from one budget. + let code = "x".repeat(1_000); + let mut s = ScriptsIn { scripts: vec![code], budget: 2_500, missing: 0 }; + let f = formula("_xlfn._xlws.PY(0,0)"); + let taken = (0..5).filter(|_| matches!(cell_code(&mut s, &f, ""), PyFormula::Code(_))).count(); + assert_eq!(taken, 2); + assert_eq!(s.missing, 3); + // A parameter repeated many times stops growing the code just past a cell's limit. + let many = "%P2%".repeat(5_000); + let out = with_params(&many, &[Expr::Name("A_VERY_LONG_NAME_FOR_A_RANGE".into())]); + assert!((MAX_SCRIPT_CHARS..MAX_SCRIPT_CHARS + 40).contains(&out.chars().count()), "{}", out.len()); + assert!(!PyCode::new(&out).fits()); + // Odd tokens stay as they are. + assert_eq!(with_params("%P%P2%%P99999999%%P", &[Expr::Number(1.0)]), "%P1%P99999999%%P"); + // A long script is cut just past the limit, so it fails the cell's check. + let el = El { + name: "pythonScript".into(), + children: vec![El { name: "code".into(), text: "y".repeat(20_000), ..El::default() }], + ..El::default() + }; + assert_eq!(script_code(&el).chars().count(), MAX_SCRIPT_CHARS); + } + + #[test] + fn written_scripts_read_back() { + let codes = vec!["a = 1\nb = \"two\" & 'three' ".to_string(), "_x0041_ \u{1}".to_string(), String::new()]; + let xml = write(&codes); + assert!(xml.contains(NS) && xml.contains("a = 1\nb = \"two\" & 'three' <tag>"), "{xml}"); + let mut read = Vec::new(); + let root = xml::parse_streaming(xml.as_bytes(), &["pythonScript"], &mut |el| { + read.push(script_code(&el)); + Ok(()) + }) + .unwrap_or_default(); + assert_eq!(root.name, "python"); + assert_eq!(read, codes); + assert_eq!(file_formula(3, true), "_xlfn._xlws.PY(3,1)"); + } +} diff --git a/crates/xlsx/src/read.rs b/crates/xlsx/src/read.rs index 55ef6757d..d7b9240e2 100644 --- a/crates/xlsx/src/read.rs +++ b/crates/xlsx/src/read.rs @@ -32,6 +32,8 @@ pub struct Ctx<'a> { pub cell_pictures: Vec>>, /// x14 conditional-format ids of the current sheet's data bars, with the index of the rule in `cond_formats`. pub cf_ext_ids: Vec<(String, usize)>, + /// The code of the workbook's Python cells, from its Python part. + pub python: crate::python_part::ScriptsIn, } impl Ctx<'_> { @@ -113,6 +115,7 @@ pub fn read_xlsx(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { num_fmts: Default::default(), cell_pictures: vec![], cf_ext_ids: vec![], + python: Default::default(), }; let root_rels = cx.pkg.rels("")?; let wb_part = find_rel(&root_rels, "officeDocument").map(|r| r.target.clone()).unwrap_or_else(|| "xl/workbook.xml".into()); @@ -140,6 +143,8 @@ pub fn read_xlsx(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { vec![] } }; + // The code of Python cells, which their formulas name by index. + cx.python = crate::python_part::read(&mut cx, &wb_rels)?; let mut wb = Workbook::new(); wb.sheets.clear(); @@ -394,6 +399,9 @@ pub fn read_xlsx(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { } } wb.preserved = kept.workbook(&mut cx, &wb_part, &wb_xml, &file_to_model); + if let Some(w) = cx.python.warning() { + cx.warn(w); + } // Calls of LAMBDA names are spelled like the names (the parser upper-cases them). wb.apply_name_call_case(); wb.styles = cx.styles; diff --git a/crates/xlsx/src/sheet_read.rs b/crates/xlsx/src/sheet_read.rs index bec67bd1a..5945a9669 100644 --- a/crates/xlsx/src/sheet_read.rs +++ b/crates/xlsx/src/sheet_read.rs @@ -41,16 +41,17 @@ struct RowState { rows: u64, } -/// Whether `pos` lies in a known dynamic range that isn't its anchor: the cell's cached value -/// gives way to the formula that will fill it. Charges every scanned range against one work -/// budget, then gives up consulting the ranges at all: the worst kind of file may then import -/// a few cached spill values, but it never stalls the whole import. -fn in_dynamic(st: &mut RowState, pos: CellRef) -> bool { +/// The known dynamic range `pos` lies in, when it isn't the range's anchor: the cell's cached +/// value gives way to the formula that will fill it (or, for a Python cell, becomes its +/// result). Charges every scanned range against one work budget, then gives up consulting the +/// ranges at all: the worst kind of file may then import a few cached spill values, but it +/// never stalls the whole import. +fn dynamic_range(st: &mut RowState, pos: CellRef) -> Option { if st.dynamic_checks > MAX_DYNAMIC_CHECKS || st.dynamic.is_empty() { - return false; + return None; } let ranges = st.dynamic.len(); - let hit = st.dynamic.iter().any(|r| r.contains(pos) && r.start != pos); + let hit = st.dynamic.iter().find(|r| r.contains(pos) && r.start != pos).copied(); st.dynamic_checks = st.dynamic_checks.saturating_add(ranges as u64); hit } @@ -515,8 +516,34 @@ fn read_cell(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, c: &El, pos } let dynamic = c.attr("cm").is_some(); let formula = c.child("f").and_then(|f| read_formula(cx, st, f, pos, dynamic)); - let value = if formula.is_none() && in_dynamic(st, pos) { Value::Empty } else { value }; - sheet.cells.set(pos, Cell { value, formula: formula.map(Arc::new), style, py: None }); + // A `PY(index,0|1)` formula is a Python cell whose code is a script of the Python part. The + // older `PY("…",0|1)` form is read from the formula's own text: the formula parser would fold + // line breaks in the code. A formula naming a script the part lacks keeps only its value. + let py = match (&formula, c.child("f")) { + (Some(fm), Some(f)) => crate::python_part::cell_code(&mut cx.python, fm, &f.text), + _ => crate::python_part::PyFormula::Not, + }; + let (formula, py) = match py { + crate::python_part::PyFormula::Not => (formula, None), + crate::python_part::PyFormula::Code(py) => (None, Some(Arc::new(py))), + crate::python_part::PyFormula::Missing => (None, None), + }; + let spilled_from = if formula.is_none() && py.is_none() { dynamic_range(st, pos) } else { None }; + let value = match spilled_from { + // A Python cell's spilled values are kept: they are its result until it runs again. A + // range larger than a result may be is a crafted file's: its values are dropped, like a + // formula's spilled values, and no spill is recorded. + Some(r) if r.count() <= gridcraft_model::python::MAX_RESULT_CELLS as u64 && sheet.cells.get(r.start).is_some_and(|a| a.py.is_some()) => { + if !value.is_empty() { + sheet.spill.insert(pos, value); + } + sheet.spill_ranges.insert(r.start, r); + Value::Empty + } + Some(_) => Value::Empty, + None => value, + }; + sheet.cells.set(pos, Cell { value, formula: formula.map(Arc::new), style, py }); } fn read_formula(cx: &mut Ctx<'_>, st: &mut RowState, f: &El, pos: CellRef, dynamic: bool) -> Option { diff --git a/crates/xlsx/src/sheet_write.rs b/crates/xlsx/src/sheet_write.rs index 9251edaa4..5203adc1b 100644 --- a/crates/xlsx/src/sheet_write.rs +++ b/crates/xlsx/src/sheet_write.rs @@ -119,7 +119,9 @@ pub fn write_sheet(wb: &Workbook, si: usize, selected: bool, out: &mut Out) -> ( let dynamic: BTreeMap = sheet .spill_ranges .iter() - .filter(|(a, r)| !r.is_single() && sheet.cells.get(**a).is_some_and(|c| c.formula.as_ref().is_some_and(|f| f.array.is_none()))) + .filter(|(a, r)| { + !r.is_single() && sheet.cells.get(**a).is_some_and(|c| c.py.is_some() || c.formula.as_ref().is_some_and(|f| f.array.is_none())) + }) .map(|(a, r)| (*a, *r)) .collect(); let mut extra: BTreeMap = BTreeMap::new(); @@ -686,6 +688,17 @@ fn write_cell( return; } let formula = if ov.is_some() { None } else { cell.and_then(|x| x.formula.as_ref()) }; + // A Python cell is saved as [MS-XLSX] 2.3.10 specifies, `_xlfn._xlws.PY(index,0|1)` with its + // code as script `index` of the Python part, and with its cached value; an application without + // Python shows the value (and #NAME? if it recalculates). + let py_text = if ov.is_some() || formula.is_some() { + None + } else { + cell.and_then(|x| x.py.as_ref()).map(|p| { + out.python_scripts.push(p.code.clone()); + crate::python_part::file_formula(out.python_scripts.len().saturating_sub(1), p.object) + }) + }; let value: Value = match (ov, extra) { (Some(t), _) => Value::text(t.as_str()), (None, Some(v)) => v.clone(), @@ -709,11 +722,12 @@ fn write_cell( let own = formula.filter(|f| f.dynamic && f.array.is_none()).map(|_| RangeRef::cell(c)); let dynamic = dynamic.or(own.as_ref()); // A non-spilling formula may still need array evaluation to avoid implicit intersection. - if dynamic.is_some() && formula.is_some_and(|f| f.array.is_none()) { + if dynamic.is_some() && (formula.is_some_and(|f| f.array.is_none()) || py_text.is_some()) { s.push_str(" cm=\"1\""); out.dynamic_arrays = true; } - if let Some(f) = formula { + let text = formula.map(|f| esc(&crate::fmla::to_file(f))).or_else(|| py_text.map(|t| esc(&t))); + if let Some(text) = text { let t = match &value { Value::Text(_) => " t=\"str\"", Value::Bool(_) => " t=\"b\"", @@ -722,8 +736,7 @@ fn write_cell( }; s.push_str(t); s.push('>'); - let text = esc(&crate::fmla::to_file(f)); - match (f.array, dynamic) { + match (formula.and_then(|f| f.array), dynamic) { (Some(r), _) => { let _ = write!(s, "{text}", r.a1()); } diff --git a/crates/xlsx/src/write.rs b/crates/xlsx/src/write.rs index b96df35c7..41ab3e8a3 100644 --- a/crates/xlsx/src/write.rs +++ b/crates/xlsx/src/write.rs @@ -128,6 +128,8 @@ pub struct Out { pub cell_pictures: crate::richdata::PicturesOut, /// Kept (pass-through) parts written so far. pub kept: crate::passthrough::KeptOut, + /// The code of the Python cells written so far, in order: script `i` of the Python part. + pub python_scripts: Vec, } impl Out { @@ -237,6 +239,7 @@ pub fn write_xlsx_named(wb: &Workbook, kind: XlsxKind, book_name: Option<&str>) needs_calc: false, cell_pictures: Default::default(), kept: Default::default(), + python_scripts: vec![], }; out.kept.reserve_vml_blocks(wb); out.default_type("rels", "application/vnd.openxmlformats-package.relationships+xml"); @@ -399,6 +402,12 @@ pub fn write_xlsx_named(wb: &Workbook, kind: XlsxKind, book_name: Option<&str>) pivots.finish(wb, &mut out, &|code: &str| crate::tables::builtin_id(code).or_else(|| fmt_ids.get(code).copied())); out.part("xl/styles.xml", Some("application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"), styles.into_bytes()); wb_rels.add("styles", "styles.xml"); + if !out.python_scripts.is_empty() { + use crate::python_part as py; + let part = py::write(&out.python_scripts); + out.part(py::PART, Some(py::CONTENT_TYPE), part.into_bytes()); + wb_rels.add_uri(py::REL, "python.xml"); + } if wb.styles.iter().any(|(_, st)| st.num_fmt.as_str() == "checkbox") { use crate::feature_bags as fpb; out.part(fpb::PART, Some(fpb::CONTENT_TYPE), fpb::checkbox_part().into_bytes()); From 1d7206ebdd105aa768b97a336aaf9c631a7ef4c7 Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:55:01 -0400 Subject: [PATCH 04/24] Docs: Python cells as data (architecture, file formats, functions, roadmap) architecture.md gains a Python cells section: the model type, the text forms and the size limit, the xlsx Python part, calc laying out kept results as graph nodes, and the limits on open. file-format-parity.md gains a Python cells table (xlsx as [MS-XLSX] 2.3.10 specifies, the formula naming its code in the Python part, with cached values and spills; other formats; GridCraft JSON; the limits on open; and rich-value object results as not kept). function-parity.md marks the four Python errors done with their ERROR.TYPE numbers and adds a Python in cells row (stored, not run). ROADMAP.md records the progress. The return types are written as "0 or 1" so no table cell holds an unescaped pipe. architecture.md's "Last updated" label loses the space inside its bold markers. --- ROADMAP.md | 4 +++- docs/architecture.md | 22 +++++++++++++++++++++- docs/file-format-parity.md | 13 ++++++++++++- docs/function-parity.md | 6 ++++-- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 6b9b0fce7..0e94e97bb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ **Stage: alpha** · next: beta, ~25 points of "ready for real work" and ~160–240 h away -> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** minor (AutoSum over a selected row puts its total to the right, and fills a selection's empty last row and column) · **Target:** Microsoft Excel (Microsoft 365) +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** minor (Python cells stored and kept in files; earlier: AutoSum over a selected row puts its total to the right, and fills a selection's empty last row and column) · **Target:** Microsoft Excel (Microsoft 365) GridCraft aims at full Microsoft Excel parity, and at being better in places: faster, open (XLSX is the native format), fully scriptable by agents (MCP, control channel, CLI), and on the web. @@ -135,6 +135,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | What landed | |---|---| +| 2026-10-25 | Python cells as data: `python.set` stores code in a cell, Python cells copy, move, fill and undo like formulas, and `.xlsx` reads and writes Python cells as [MS-XLSX] specifies (`_xlfn._xlws.PY(index,0 or 1)`, the code in the workbook's Python part) with cached values and spills, so such workbooks open with their results; nothing runs Python (new code shows `#BLOCKED!`), and opening a file limits code size, result size and spill ranges | | 2026-10-25 | AutoSum over a selection does what Excel does: a selected row gets one total to its right (in its last cell if that is empty), not a one-cell total under each cell; a block whose last column is empty gets each row's total there, one whose last row is empty each column's, and with both the corner gets the total of the block; otherwise totals go under each column as before | | 2026-10-24 | Go To lists the named ranges of every sheet (`Sheet2!Name` for a name scoped to another sheet; hidden names, constants and formulas left out) and the references entered before, a double-click goes to an entry, and the Reference box takes several names or references separated by commas (`Price, Type`, `B14:C22,F19:G30`) | | 2026-10-24 | Status bar: a right-click opens a menu that sets which selection statistics show (Average, Count and Sum by default; Numerical Count, Minimum and Maximum are new), kept with the preferences (`view.statusBar`) | @@ -242,6 +243,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | Change | Summary | |---|---|---| +| 2026-10-25 | minor | Progress log: Python cells as data | | 2026-10-25 | minor | Progress log: AutoSum over a selected row, and a selection's empty last row and column | | 2026-10-23 | minor | Progress log: accounting negatives in number text | | 2026-10-23 | minor | Progress log: INDEX with one index on a 2-D reference is #REF! | diff --git a/docs/architecture.md b/docs/architecture.md index c92e08e20..203004573 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -> **Last reviewed:** 2026-10-10 · ** Last updated:** 2026-10-24 · **Change:** minor (engine building blocks for multi-command actions) · **Target:** Microsoft Excel (Microsoft 365) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-25 · **Change:** minor (Python cells as data; earlier: engine building blocks for multi-command actions) · **Target:** Microsoft Excel (Microsoft 365) How GridCraft is built today. Rules for contributors are in [`AGENTS.md`](../AGENTS.md). @@ -125,6 +125,25 @@ Actions made of many commands (macros, a debugger that runs one step per call) u under (a hidden workbook-scope name of either shape; a visible one is the person's own), and `formulas.createFromSelection` skips such labels, unless `force: true` is passed. +## Python cells + +A Python cell is a `Cell` whose `py` holds a `model::PyCode`: the code, the return type (an Excel +value or a Python object, the `0` or `1` of Excel's `PY(code, return_type)`), the last result and a +stale flag. Its `value` is the cached result, so formulas, display and saving read it like any +other cell. `model::python` has the text forms: `PY("…"&"…",n)` with literals of at most 255 +characters (the function's syntax, which some files hold), the `_xlfn._xlws.` prefix, the +8,192-character limit measured on that formula (`PyCode::fits`), and the `xl("…")` literals in +code, which structural edits rewrite like formula references. + +Calc treats a Python cell as a graph node that reads nothing: a pass takes its kept result instead +of evaluating it, so an array result spills, is blocked and unblocked, and reaches its dependents +through the same code as a formula's. The xlsx reader and writer map Python cells to and from +`_xlfn._xlws.PY(index,n)` formulas with cached values and the workbook's Python part, which holds +the code (`xlsx::python_part`, [MS-XLSX] 2.3.10); `io::adopt_python` and `io::enforce_python_limits` +bound what a file can make the app hold (result size, spill ranges, total rebuilt cells, code +size). This build runs no Python: `python.set` stores code, and new or changed code shows +`#BLOCKED!`. + ## Agent control - Desktop control channel: `gridcraft --control 7979`, JSON lines on localhost @@ -150,6 +169,7 @@ rpm, tarball), FreeBSD, and web (WASM; WebGPU or WebGL2). Release workflows in | Date | Change | Summary | |---|---|---| +| 2026-10-25 | minor | Python cells as data: `model::PyCode`, the file forms, calc laying out kept results, and the limits on open | | 2026-10-24 | minor | Engine building blocks for multi-command actions: undo groups, recording buffer, document trust, chunked hidden-name storage | | 2026-10-14 | minor | Document Undo's retained-cell-storage budget and latest-step guarantee | | 2026-10-10 | major | Created from the code on main: crates, layers, data flow, agent control, known gaps | diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index cf567f8bd..4e0252ee4 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File-format parity -> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-21 · **Change:** minor (.xls dialog sheets are skipped instead of read as worksheets; `.xls` import reads each sheet's display options from WINDOW2: right-to-left, gridlines, headings, zeros, formulas; XLSX text boxes and shapes keep their paragraphs' right-to-left direction on save; CSV/TSV/TXT can open with every field kept as text via `file.open {csvAsText}`; formulas over Excel's 8,192 characters, and `%` or call chains past the nesting limit, are parse errors instead of stack overflows when a file is opened; XLSX PivotTables read and write `rowGrandTotals`/`colGrandTotals` the way Excel does, so a table saved with only one grand total opens with the right one; HTML export keeps right-to-left; CSV and text import detects likely Windows-1256 Arabic when invalid UTF-8 has Arabic byte runs; other invalid UTF-8 remains Windows-1252; HTML export places pictures and charts, as inline SVG, at their anchors) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** minor (Python cells in `.xlsx` as [MS-XLSX] specifies, `_xlfn._xlws.PY(…)` naming code in the Python part, with cached values, and in GridCraft JSON, with limits on open; earlier: .xls dialog sheets are skipped instead of read as worksheets; `.xls` import reads each sheet's display options from WINDOW2: right-to-left, gridlines, headings, zeros, formulas; XLSX text boxes and shapes keep their paragraphs' right-to-left direction on save; CSV/TSV/TXT can open with every field kept as text via `file.open {csvAsText}`; formulas over Excel's 8,192 characters, and `%` or call chains past the nesting limit, are parse errors instead of stack overflows when a file is opened; XLSX PivotTables read and write `rowGrandTotals`/`colGrandTotals` the way Excel does, so a table saved with only one grand total opens with the right one; HTML export keeps right-to-left; CSV and text import detects likely Windows-1256 Arabic when invalid UTF-8 has Arabic byte runs; other invalid UTF-8 remains Windows-1252; HTML export places pictures and charts, as inline SVG, at their anchors) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 Every format Excel opens or saves, with GridCraft's read and write support, fidelity and tests. Part of [target-app-parity.md](target-app-parity.md). **File formats: ~50% ready, 80–120 h** @@ -93,6 +93,16 @@ Server, Access, ODBC, …), web pages, SharePoint and online services, through P GridCraft imports CSV/TSV, JSON and HTML tables only, with no query editor, no connections and no refresh. Counted under features (Get & Transform, ~10% ready), not here. +## Python cells + +| Where | GridCraft | Notes | +|---|---|---| +| `.xlsx` | read and write | Written as [MS-XLSX] 2.3.10 specifies: the cell's formula is `_xlfn._xlws.PY(i,n)`, where `i` is the index of the cell's script in the workbook's Python part (`xl/python.xml`, `application/vnd.ms-excel.python+xml`) and `n` the return type, 0 for an Excel value or 1 for a Python object, with the cached value. The code keeps its `xl()` references as text, so the formula has no further arguments. A spilling result is written as a dynamic array with its spilled values cached. Read back with or without the `_xlfn.` and `_xlws.` prefixes: an index names a script of the Python part, whose `%Px%` references become the text of the formula's argument `x`, and the older form with the code in string literals, `PY("…"&"…",n)`, still reads. So workbooks laid out as the format specifies open with Python cells, their cached values and spills. Python errors are cached as `#VALUE!`, as Excel caches `#SPILL!` and `#CALC!` | +| `.xls`, `.xlsb`, `.ods` | read | A `PY("…",0)` or `PY("…",1)` formula read from any format becomes a Python cell; the `.xlsb` and `.ods` imports keep values only, so Python cells there come in as their cached values | +| GridCraft JSON | read and write | The whole Python cell: code, return type, the last result and the exact error | +| Limits on open | — | Code that doesn't fit Excel's 8,192-character formula as `PY("…",n)` keeps only its cached value, with a warning, and so does a formula naming a script the Python part doesn't hold. The Python part is read up to 64 MB and 1,000,000 scripts, and the cells take at most 64 MB of code from it in all (several formulas can name one script). A kept result that is not a well-formed array of at most 1,000,000 cells is dropped, a spill range larger than that is ignored, and results are rebuilt for at most 4,000,000 cells over the workbook. Opening runs nothing | +| Not kept | — | Python object results stored as rich values (Excel's `richData` parts) come in as their cached values. The Python part's environment definition (its id and initialization code) is not read, and a save writes none | + ## Work (76–114 h) | Work | Hours | @@ -110,6 +120,7 @@ refresh. Counted under features (Get & Transform, ~10% ready), not here. | Date | Change | Summary | |---|---|---| +| 2026-10-25 | minor | Python cells: `.xlsx` reads and writes `_xlfn._xlws.PY(i,0 or 1)` with the code in the Python part (and reads the string-literal form) with cached values and spills, other formats adopt PY formulas, GridCraft JSON keeps the whole cell, and opening limits code size, result size and spill ranges | | 2026-10-21 | minor | HTML (Save as Web Page) writes pictures (`data:` URI ``) and charts (inline SVG from the chart crate's drawing primitives, text included) at their anchors; the table uses the sheet's column widths and row heights when the sheet has either, and output for sheets without pictures or charts is unchanged | | 2026-10-20 | trivial | XLSX shared strings: Japanese phonetic runs (ふりがな, `` with ``) are read into the workbook and written back next to the string with the same text, so an open + save no longer deletes them; bad offsets and readings are dropped with a warning, a `` without runs is kept on its own, and when duplicate `` items share a text but not a reading only the first reading is written (reported on open) (`phonetic_tests.rs`) | | 2026-10-20 | minor | A workbook theme's major/minor per-script typefaces (``, also Hebr, Jpan and the rest) are read and written back in file order after ``; they were dropped before. A hostile file is held to 64 entries of 256 characters | diff --git a/docs/function-parity.md b/docs/function-parity.md index bf57fcd3e..5337204b5 100644 --- a/docs/function-parity.md +++ b/docs/function-parity.md @@ -1,6 +1,6 @@ # Function parity: worksheet functions and formulas -> **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-25 · **Change:** minor (the LAMBDA row lists the recursion limit and its `#NUM!`, and the error a defined name holding an error gives when called; earlier: number text with accounting parentheses coerces like Excel 365; INDEX with one index on a 2-D reference is #REF! as in Excel; earlier: SUBTOTAL and AGGREGATE cap the cells they visit, #2175) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 +> **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-25 · **Change:** minor (the Python errors `#PYTHON!`, `#BUSY!`, `#TIMEOUT!` and `#BLOCKED!`, and a Python in cells row; earlier: the LAMBDA row lists the recursion limit and its `#NUM!`, and the error a defined name holding an error gives when called; earlier: number text with accounting parentheses coerces like Excel 365; INDEX with one index on a 2-D reference is #REF! as in Excel; earlier: SUBTOTAL and AGGREGATE cap the cells they visit, #2175) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 Excel's worksheet functions, category by category, against GridCraft's function library. Part of [target-app-parity.md](target-app-parity.md); gaps feed [gaps.md](gaps.md). @@ -93,7 +93,8 @@ This improves series preparation; it does not change the model-fitting or season | External workbook references (`[Book.xlsx]Sheet1!A1`) | parsed; links dropped on open, no update | | Localized function names and `;` separators | **missing** (#45); en-US only | | Long flat operator chains (`=A1+A2+…`, `-`, `*`, `/`, `^`, `&`, comparisons) | done: no operand limit below Excel's 8,192-character formula limit (Excel, observed: `=1+1+…` with 4,096 operands is exactly 8,192 characters with the `=` and calculates; one more character is refused). Evaluated (#484), printed (#628), stored, copied, adjusted, saved and reopened without recursing once per operand, on a 1 MiB stack (#916); written to XLSX as plain formulas, not dynamic arrays; Spanish entry accepts them | -| Error values | the seven classic errors plus `#SPILL!`, `#CALC!`, `#GETTING_DATA` done; `#FIELD!`, `#BLOCKED!`, `#CONNECT!`, `#BUSY!`, `#UNKNOWN!`, `#PYTHON!` missing (minor: they come from data types, cloud and Python features) | +| Error values | the seven classic errors plus `#SPILL!`, `#CALC!`, `#GETTING_DATA`, and the Python errors `#PYTHON!`, `#BUSY!`, `#TIMEOUT!`, `#BLOCKED!` done; `#FIELD!`, `#CONNECT!`, `#UNKNOWN!` missing (minor: they come from data types and cloud features). `ERROR.TYPE` gives 11 for `#BLOCKED!` as Excel does; Microsoft documents no number for `#PYTHON!`, `#BUSY!` or `#TIMEOUT!`, and GridCraft gives them 19, 20 and 21 | +| Python in cells | partial: Python cells are stored, edited (`python.set`), copied, moved and saved, and Excel's `_xlfn._xlws.PY` cells open with their cached results; the code doesn't run, so new or changed code shows `#BLOCKED!`. Typing `=PY(` is refused with a pointer to `python.set`, so `PY` stays counted as missing above | ## Depth (estimated) @@ -123,6 +124,7 @@ workbooks; results checked in as data, not Excel files) and fixing what it finds | Date | Change | Summary | |---|---|---| +| 2026-10-25 | minor | Python in cells (stored, not run): `#PYTHON!`, `#BUSY!`, `#TIMEOUT!` and `#BLOCKED!` done, with their `ERROR.TYPE` numbers; `PY` still counted as missing | | 2026-10-25 | minor | LAMBDA row: a LAMBDA that calls itself more than 140 times, and a formula nested past the depth limit, give `#NUM!` (#1809; nesting gave `#CALC!` before). `=Foo(1)` where the defined name Foo holds `#N/A` gives `#N/A`, not `#NAME?` | | 2026-10-23 | minor | Number text coercion (VALUE, arithmetic, typed entry): a dollar sign before accounting parentheses (`$(1,234)`, `$ (5)`) and a percent sign after them (`(5)%`) read as negatives; `(-5)`, `((5))`, `($-5)`, `$5%`, `$(5)%` and dates or times in parentheses are `#VALUE!`, as observed in Excel 365 | | 2026-10-23 | minor | INDEX with only a row index on a reference of several rows and columns (`INDEX(A1:C3,2)`, `INDEX(A1:C3,0)`) is `#REF!`, as observed in Excel, so it also fails as a range endpoint and inside other functions; an empty column argument (`INDEX(A1:C3,2,)`) still gives the row. Checked that a column array with one index stays a 1×1 array (`TYPE` 64), which is what Excel returns | From 2d56f1b9363814b0d9b25ef6d06ad6393aeb447f Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:09:50 -0500 Subject: [PATCH 05/24] Engine: a macro's check for writes over data keeps a Python cell The footprint dry run and the per-step guard of macro runs (new on main) compared formulas and values only. A step that replaced a Python cell with its own cached value, or with other code that also shows #BLOCKED!, was not counted as a write over data, so the code could be lost without the question. A step that only blocked a Python cell's spill, which leaves its code as it was, was counted as one. Python code is now content as a formula is: a change of code or return type is a write, and a change of the cached value alone is a recalculation. --- crates/engine/src/macros/run.rs | 19 ++++++++++--- crates/engine/src/tests/python_cells.rs | 36 +++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/macros/run.rs b/crates/engine/src/macros/run.rs index fcb6768f3..71c23aabe 100644 --- a/crates/engine/src/macros/run.rs +++ b/crates/engine/src/macros/run.rs @@ -1237,26 +1237,39 @@ pub(crate) fn pair_sheets(before: &Workbook, after: &Workbook) -> (Vec<(usize, O (pairs, spare.collect()) } -/// Only a formula's cached value changed (a recalculation, not a write). +/// Only a formula's (or a Python cell's) cached value changed (a recalculation, not a write): a +/// Python cell whose spill a step blocked shows #SPILL! with its code untouched. fn recalculated_only(b: Option<&Cell>, a: Option<&Cell>) -> bool { match (b, a) { (Some(b), Some(a)) => match (&b.formula, &a.formula) { (Some(x), Some(y)) => x.text == y.text && x.array == y.array && b.style == a.style, + (None, None) => match (&b.py, &a.py) { + (Some(x), Some(y)) => x.code == y.code && x.object == y.object && b.style == a.style, + _ => false, + }, _ => false, }, _ => false, } } +/// Python code is content, as a formula is, whatever the cell shows. fn has_content(c: &Cell) -> bool { - c.formula.is_some() || !(c.value.is_empty() || c.value.as_text() == Some("")) + c.formula.is_some() || c.py.is_some() || !(c.value.is_empty() || c.value.as_text() == Some("")) } +/// Whether a write changed what the person entered: the formula, the Python code and its return +/// type, or the value. A Python cell replaced by its own cached value, or by other code that +/// shows the same `#BLOCKED!`, has changed. fn content_differs(b: Option<&Cell>, a: Option<&Cell>) -> bool { match (b, a) { (Some(b), Some(a)) => match (&b.formula, &a.formula) { (Some(x), Some(y)) => x.text != y.text, - (None, None) => b.value != a.value, + (None, None) => match (&b.py, &a.py) { + (Some(x), Some(y)) => x.code != y.code || x.object != y.object, + (None, None) => b.value != a.value, + _ => true, + }, _ => true, }, (Some(b), None) => has_content(b), diff --git a/crates/engine/src/tests/python_cells.rs b/crates/engine/src/tests/python_cells.rs index 91a1fed77..417ed1e4a 100644 --- a/crates/engine/src/tests/python_cells.rs +++ b/crates/engine/src/tests/python_cells.rs @@ -227,3 +227,39 @@ fn go_to_special_counts_a_python_cell_as_a_formula_not_a_constant() { assert_eq!(r["count"], 1, "{r}"); assert_eq!(r["selection"], "A3", "{r}"); } + +#[test] +fn a_macro_counts_python_code_as_data_and_a_blocked_spill_as_no_write() { + let mut s = s(); + // F2 shows 1, its cached value: a step that enters 1 there still replaces the code. + plant(&mut s, 0, "F2", "1"); + // H2's code shows #BLOCKED!, as any other new code there would. + s.execute("python.set", json!({"cell": "H2", "code": "'a'"})).unwrap(); + // J2's result spills over J2:J3. A step that writes J3 blocks it, which doesn't write J2. + s.execute("python.set", json!({"cell": "J2", "code": "[5, 6]"})).unwrap(); + { + let d = s.doc_mut().unwrap(); + let wb = Arc::make_mut(&mut d.wb); + let cell = wb.sheet_mut(0).unwrap().cells.get_mut(CellRef::parse("J2").unwrap()).unwrap(); + let data = vec![Value::Number(5.0), Value::Number(6.0)]; + let value = Value::Array(Arc::new(gridcraft_core::Array::new(2, 1, data).unwrap())); + Arc::make_mut(cell.py.as_mut().unwrap()).output = + Some(gridcraft_model::PyOutput { value, type_name: "list".into(), preview: String::new(), convertible: true }); + } + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(super::v(&s, "J3"), Value::Number(6.0)); + super::macros::put( + &mut s, + json!({"name": "Py", "steps": [ + {"command": "cell.set", "params": {"cell": "F2", "input": "1"}}, + {"command": "python.set", "params": {"cell": "H2", "code": "'b'"}}, + {"command": "cell.set", "params": {"cell": "J3", "input": "x"}} + ]}), + ); + let c = s.execute("macro.check", json!({"name": "Py"})).unwrap(); + assert_eq!(c["footprint"]["sheets"][0]["cells"], json!(["F2", "H2", "J3"]), "{c}"); + assert_eq!(c["footprint"]["sheets"][0]["conflicts"], json!(["F2", "H2"]), "{c}"); + let e = s.execute("macro.run", json!({"name": "Py", "onConflict": "fail"})).unwrap_err().to_string(); + assert!(e.contains("F2") && e.contains("H2") && !e.contains("J2"), "{e}"); + assert_eq!(code(&s, 0, "F2").as_deref(), Some("1"), "nothing was changed"); +} From 14ab0f8ee3ea80502fd79a01a0ac137883a06c28 Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:10:31 -0500 Subject: [PATCH 06/24] Engine: Replace edits a Python cell's code and keeps it a Python cell Replace matched a Python cell on its code and then rebuilt the cell from the replaced text as if it had been typed, so the cell became a text cell holding the edited code and the Python cell was lost. The replacement now edits the code: the cell stays a Python cell with its return type, its result is dropped (#BLOCKED!, as for any changed code) and a spill it had is cleared. Code that would no longer fit in a PY formula is left unchanged and counted in a new skipped field of the result. --- crates/engine/src/cmd/edit.rs | 2 +- crates/engine/src/cmd/edit/find.rs | 29 ++++++++++++++++++++++ crates/engine/src/tests/python_cells.rs | 33 +++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/cmd/edit.rs b/crates/engine/src/cmd/edit.rs index 3bba84748..f2972be5a 100644 --- a/crates/engine/src/cmd/edit.rs +++ b/crates/engine/src/cmd/edit.rs @@ -129,7 +129,7 @@ pub fn specs() -> Vec { "Replace…", ["Home", "Editing", "Find & Select"], Some("Ctrl+H"), - "{what, with, matchCase?, wholeCell?, all?: true, within?: sheet|workbook}", + "{what, with, matchCase?, wholeCell?, all?: true, within?: sheet|workbook} → {replaced, skipped?} (in a Python cell the code is edited; skipped counts Python cells whose code would no longer fit)", has_doc, replace ), diff --git a/crates/engine/src/cmd/edit/find.rs b/crates/engine/src/cmd/edit/find.rs index bce0ba728..06a908e14 100644 --- a/crates/engine/src/cmd/edit/find.rs +++ b/crates/engine/src/cmd/edit/find.rs @@ -115,6 +115,7 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { let all = bool_param(p, "all").unwrap_or(true); let workbook = str_param(p, "within") == Some("workbook"); let active = s.doc()?.selection.active; + let mut skipped = 0usize; let n = edit(s, |cx| { let sheets: Vec = if workbook { (0..cx.wb.sheets.len()).collect() } else { vec![cx.wb.active_sheet()] }; let mut n = 0; @@ -135,6 +136,31 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { } else { replace_ci(&text, &what, &with) }; + // A Python cell's text is its code: the replacement edits the code and the cell + // stays a Python cell. Code that would no longer fit in a PY formula is left as it was. + let old_py = cx.wb.sheet(si).and_then(|sh| sh.cell(c)).and_then(|x| x.py.clone()); + if let Some(old) = old_py { + let py = gridcraft_model::PyCode { object: old.object, ..gridcraft_model::PyCode::new(&new) }; + if py.code == old.code { + continue; + } + if !py.fits() { + skipped += 1; + continue; + } + let sh = cx.sheet_mut(si)?; + if let Some(r) = sh.spill_ranges.remove(&c) { + sh.clear_spill(r); + } + let style = sh.style_id(c); + sh.set_cell( + c, + Cell { value: Value::Error(gridcraft_core::CellError::Blocked), formula: None, style, py: Some(std::sync::Arc::new(py)) }, + ); + cx.touch(si, c); + n += 1; + continue; + } let cell = input_to_cell(&new, si, c, &mut cx.wb).unwrap_or_else(|_| Some(Cell::value(Value::text(new.as_str())))); let sh = cx.sheet_mut(si)?; match cell { @@ -149,6 +175,9 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { } Ok(n) })?; + if skipped > 0 { + return Ok(json!({"replaced": n, "skipped": skipped})); + } Ok(json!({"replaced": n})) } diff --git a/crates/engine/src/tests/python_cells.rs b/crates/engine/src/tests/python_cells.rs index 417ed1e4a..ef1df605a 100644 --- a/crates/engine/src/tests/python_cells.rs +++ b/crates/engine/src/tests/python_cells.rs @@ -263,3 +263,36 @@ fn a_macro_counts_python_code_as_data_and_a_blocked_spill_as_no_write() { assert!(e.contains("F2") && e.contains("H2") && !e.contains("J2"), "{e}"); assert_eq!(code(&s, 0, "F2").as_deref(), Some("1"), "nothing was changed"); } + +#[test] +fn replace_edits_a_python_cells_code_and_keeps_it_a_python_cell() { + let mut s = s(); + plant(&mut s, 0, "A1", "x = 1\nx"); + { + let d = s.doc_mut().unwrap(); + let mut wb = (*d.wb).clone(); + let cell = wb.sheet_mut(0).unwrap().cells.get_mut(CellRef::parse("A1").unwrap()).unwrap(); + Arc::make_mut(cell.py.as_mut().unwrap()).object = true; + d.wb = Arc::new(wb); + } + let r = s.execute("edit.replace", json!({"what": "x", "with": "y"})).unwrap(); + assert_eq!(r["replaced"], 1); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("y = 1\ny")); + let d = s.doc().unwrap(); + let cell = d.wb.sheet(0).unwrap().cell(CellRef::parse("A1").unwrap()).unwrap(); + assert!(cell.py.as_ref().unwrap().object, "the return type is kept"); + assert_eq!(cell.value, Value::Error(gridcraft_core::CellError::Blocked)); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("x = 1\nx")); +} + +#[test] +fn replace_leaves_python_code_that_would_no_longer_fit() { + let mut s = s(); + let long = "a".repeat(gridcraft_model::python::MAX_FORMULA_CHARS / 2); + plant(&mut s, 0, "A1", &long); + let r = s.execute("edit.replace", json!({"what": "a", "with": "bb"})).unwrap(); + assert_eq!(r["replaced"], 0); + assert_eq!(r["skipped"], 1); + assert_eq!(code(&s, 0, "A1").as_deref(), Some(long.as_str())); +} From 58aa00bae4bbea1fa462aa35325b2b83683ede08 Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:11:23 -0500 Subject: [PATCH 07/24] Docs: xl() references in Python code stay put when a sort moves the cell --- docs/architecture.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index 3bb3f9212..ba6162020 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -144,6 +144,13 @@ characters (the function's syntax, which some files hold), the `_xlfn._xlws.` pr 8,192-character limit measured on that formula (`PyCode::fits`), and the `xl("…")` literals in code, which structural edits rewrite like formula references. +Only those edits rewrite them: inserting or deleting cells, rows and columns, a cut and paste, and +renaming or deleting a sheet. Copying, filling and sorting carry the code unchanged, so an `xl()` +reference acts as an absolute reference. A sort that moves a Python cell to another row leaves its +code reading the cells it named before (a formula moved by the same sort has its relative references +shifted), and references to the sorted cells from other cells, formulas and Python code alike, keep +naming the same positions. + Calc treats a Python cell as a graph node that reads nothing: a pass takes its kept result instead of evaluating it, so an array result spills, is blocked and unblocked, and reaches its dependents through the same code as a formula's. The xlsx reader and writer map Python cells to and from @@ -178,7 +185,7 @@ rpm, tarball), FreeBSD, and web (WASM; WebGPU or WebGL2). Release workflows in | Date | Change | Summary | |---|---|---| -| 2026-11-01 | minor | Python cells as data: `model::PyCode`, the file forms, calc laying out kept results, and the limits on open | +| 2026-11-01 | minor | Python cells as data: `model::PyCode`, the file forms, calc laying out kept results, the limits on open, and what copying, filling and sorting do to `xl()` references | | 2026-10-25 | trivial | The hidden names that hold macros are `_gc_macro__` | | 2026-10-24 | minor | Macro engine on the building blocks (`macros/`, `cmd/macros.rs`) | | 2026-10-24 | minor | Engine building blocks for multi-command actions: undo groups, recording buffer, document trust, chunked hidden-name storage | From 4fbd1dc337551f1d34f6f731ad2e017b1eb333c1 Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:16:05 -0500 Subject: [PATCH 08/24] Engine: a Hidden cell on a protected sheet hides its Python code A cell formatted Hidden on a protected sheet hid its formula but not its Python code: display::formula_hidden only looked for a formula at the cell (a Python cell was covered only while its result spilled), and the formula bar had its own check for formulas alone. Both now treat Python code as they treat a formula, so the formula bar shows nothing for a hidden Python cell, as it does for a hidden formula. Editing the cell is refused as before, because it is locked. --- crates/engine/src/display.rs | 6 +++--- crates/engine/src/tests/python_cells.rs | 20 ++++++++++++++++++++ crates/ui-egui/src/formula_bar.rs | 20 +++++++++++++++++++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/display.rs b/crates/engine/src/display.rs index aa8bdb22d..2189d6c64 100644 --- a/crates/engine/src/display.rs +++ b/crates/engine/src/display.rs @@ -90,14 +90,14 @@ pub fn edit_text(wb: &Workbook, sheet: &Sheet, c: CellRef) -> String { if reads_back(&iso) { iso } else { raw } } -/// Whether `c` has a formula that must not be shown: its cell is formatted Hidden on a -/// protected sheet. A spilled cell follows its anchor. +/// Whether `c` has a formula or Python code that must not be shown: its cell is formatted +/// Hidden on a protected sheet. A spilled cell follows its anchor. pub fn formula_hidden(wb: &Workbook, sheet: &Sheet, c: CellRef) -> bool { if !sheet.is_protected() { return false; } let at = match sheet.cell(c) { - Some(x) if x.formula.is_some() => c, + Some(x) if x.formula.is_some() || x.py.is_some() => c, _ => match sheet.spill_ranges.iter().find(|(_, r)| r.contains(c)) { Some((anchor, _)) => *anchor, None => return false, diff --git a/crates/engine/src/tests/python_cells.rs b/crates/engine/src/tests/python_cells.rs index ef1df605a..7971f7811 100644 --- a/crates/engine/src/tests/python_cells.rs +++ b/crates/engine/src/tests/python_cells.rs @@ -296,3 +296,23 @@ fn replace_leaves_python_code_that_would_no_longer_fit() { assert_eq!(r["skipped"], 1); assert_eq!(code(&s, 0, "A1").as_deref(), Some(long.as_str())); } + +#[test] +fn a_hidden_python_cell_on_a_protected_sheet_hides_its_code() { + let mut s = s(); + s.execute("python.set", json!({"cell": "A1", "code": "secret = 1\nsecret"})).unwrap(); + s.execute("python.set", json!({"cell": "B1", "code": "'shown'"})).unwrap(); + s.execute("home.formatCells", json!({"range": "A1", "style": {"protection": {"locked": true, "hidden": true}}})).unwrap(); + let hidden = |s: &Session, a: &str| { + let d = s.doc().unwrap(); + let sh = d.wb.active().unwrap(); + crate::display::formula_hidden(&d.wb, sh, CellRef::parse(a).unwrap()) + }; + assert!(!hidden(&s, "A1"), "an unprotected sheet hides nothing"); + s.execute("review.protectSheet", json!({})).unwrap(); + assert!(hidden(&s, "A1"), "Hidden on a protected sheet hides the code, as it hides a formula"); + assert!(!hidden(&s, "B1")); + // Editing it is refused where a formula's would be: the cell is locked. + assert!(s.execute("python.set", json!({"cell": "A1", "code": "2"})).is_err()); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("secret = 1\nsecret")); +} diff --git a/crates/ui-egui/src/formula_bar.rs b/crates/ui-egui/src/formula_bar.rs index 3db0c3da4..557e3dac0 100644 --- a/crates/ui-egui/src/formula_bar.rs +++ b/crates/ui-egui/src/formula_bar.rs @@ -173,7 +173,8 @@ fn active_input(app: &SheetApp) -> String { .unwrap_or_default(); } if let Some(c) = sh.cell(a) { - if c.formula.is_some() && d.wb.styles.get(c.style).protection.hidden && sh.is_protected() { + // A Hidden cell on a protected sheet shows neither its formula nor its Python code. + if (c.formula.is_some() || c.py.is_some()) && d.wb.styles.get(c.style).protection.hidden && sh.is_protected() { return String::new(); } return if let Some(f) = &c.formula { @@ -753,6 +754,23 @@ mod spill_tests { app.run("sheet.activate", json!({"sheet": "Sheet1"})).unwrap(); check(&mut app, "A2", "=FILTER(D1:D3,D1:D3>0)", true); } + + #[test] + fn a_hidden_python_cell_on_a_protected_sheet_shows_no_code() { + let mut app = app(); + app.run("python.set", json!({"cell": "A1", "code": "secret = 1\nsecret"})).unwrap(); + app.run("python.set", json!({"cell": "B1", "code": "'shown'"})).unwrap(); + set(&mut app, "C1", "=1+1"); + app.run("home.formatCells", json!({"range": "A1:C1", "style": {"protection": {"locked": true, "hidden": true}}})).unwrap(); + app.run("home.formatCells", json!({"range": "B1", "style": {"protection": {"locked": true, "hidden": false}}})).unwrap(); + select(&mut app, "A1"); + assert_eq!(active_input(&app), "secret = 1\nsecret", "an unprotected sheet hides nothing"); + app.run("review.protectSheet", json!({})).unwrap(); + for (cell, text) in [("A1", ""), ("B1", "'shown'"), ("C1", "")] { + select(&mut app, cell); + assert_eq!(active_input(&app), text, "{cell}"); + } + } #[test] /// A disabled spill formula paints disabled and does not edit; clicking an active formula starts formula-bar editing in both light and dark themes, with accessibility checks. fn formula_bar_click_and_theme_presentation() { From 44e9ddad45dae080d3eefaf878233b659d641ff2 Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:33:22 -0500 Subject: [PATCH 09/24] Engine: a Python result cut by the adoption budget takes its cached spill with it When a file's cached Python spills were larger than the adoption budget, the cells past it kept only their own value, but their spill ranges and spilled values stayed on the sheet, which went on showing values no result held. The spill of such a cell is now cleared, as for a dropped result; the warning is unchanged. --- crates/engine/src/io.rs | 5 ++++ crates/engine/src/tests/python_files.rs | 34 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 2622f0ae8..7e8aa1da7 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -168,6 +168,11 @@ pub fn adopt_python(wb: &mut Workbook) -> Vec { for (anchor, range) in spills { let Some(left) = budget.checked_sub(range.count()) else { skipped += 1; + // The cell keeps only its own value, so its cached spill goes too: the sheet would + // otherwise show spilled values that no result holds. + if let Some(r) = sh.spill_ranges.remove(&anchor) { + sh.clear_spill(r); + } continue; }; budget = left; diff --git a/crates/engine/src/tests/python_files.rs b/crates/engine/src/tests/python_files.rs index 00b597445..6870664f2 100644 --- a/crates/engine/src/tests/python_files.rs +++ b/crates/engine/src/tests/python_files.rs @@ -260,6 +260,40 @@ fn many_cells_naming_one_long_script_take_from_one_budget() { assert_eq!(sh.value(CellRef::parse("A20000").unwrap()), Value::Number(1.0), "the others keep their values"); } +#[test] +fn a_result_cut_by_the_budget_takes_its_cached_spill_with_it() { + // Five Python cells each with a cached 1,000 x 1,000 spill: the budget rebuilds four results; + // the fifth cell keeps only its own value, and its spilled values no longer show. + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + for i in 0..5u32 { + let anchor = CellRef::new(0, i * 1_000); + sh.set_cell(anchor, Cell { py: Some(Arc::new(PyCode::new("x"))), ..Cell::value(Value::Number(1.0)) }); + sh.spill_ranges.insert(anchor, RangeRef::new(anchor, CellRef::new(999, i * 1_000 + 999))); + sh.spill.insert(CellRef::new(1, i * 1_000), Value::Number(7.0)); + sh.spill.insert(CellRef::new(999, i * 1_000 + 999), Value::Number(8.0)); + } + let warnings = crate::io::adopt_python(&mut wb); + assert!(warnings.iter().any(|w| w.contains("Python")), "{warnings:?}"); + let sh = wb.sheet(0).unwrap(); + let mut cut = 0; + for i in 0..5u32 { + let anchor = CellRef::new(0, i * 1_000); + let rebuilt = sh.cell(anchor).and_then(|c| c.py.as_ref()).is_some_and(|p| p.output.is_some()); + let (below, corner) = (sh.value(CellRef::new(1, i * 1_000)), sh.value(CellRef::new(999, i * 1_000 + 999))); + if rebuilt { + assert_eq!((below, corner), (Value::Number(7.0), Value::Number(8.0)), "a result within the budget keeps its spill"); + assert!(sh.spill_ranges.contains_key(&anchor)); + } else { + cut += 1; + assert_eq!((below, corner), (Value::Empty, Value::Empty), "no stale spilled values beyond the anchor"); + assert!(!sh.spill_ranges.contains_key(&anchor)); + assert_eq!(sh.value(anchor), Value::Number(1.0), "the cell keeps its own cached value"); + } + } + assert_eq!(cut, 1); +} + #[test] fn native_json_keeps_python_cells_and_their_results() { let (mut s, long) = build(); From 113aeff07cb281fb473b68c3fbd0773caee7ad26 Mon Sep 17 00:00:00 2001 From: prxnvt <53838599+prxnvt@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:36:04 -0500 Subject: [PATCH 10/24] XLSX: a damaged Python part is a warning, and the workbook opens with its values A Python part whose zip entry failed to read (a bad checksum or compressed stream) made the whole workbook fail to open: the error was passed up, and the failed read had also ended the package's read budget, so even a warning would have left the sheets unread. The part is now read with Package::read_optional_limited, which charges a failed read only that part's allowance, and every error other than TooLarge is a warning. The Python cells then keep only their cached values, as when the part is missing. Malformed XML in the part was already a warning; the test covers it too. --- crates/engine/src/tests/python_files.rs | 49 +++++++++++++++++++++++++ crates/xlsx/src/package.rs | 36 ++++++++++++++++++ crates/xlsx/src/python_part.rs | 9 ++++- 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/tests/python_files.rs b/crates/engine/src/tests/python_files.rs index 6870664f2..5f6d3343c 100644 --- a/crates/engine/src/tests/python_files.rs +++ b/crates/engine/src/tests/python_files.rs @@ -243,6 +243,55 @@ fn a_py_formula_naming_a_missing_script_keeps_only_its_value() { assert_eq!(wb.sheet(0).unwrap().cell(CellRef::parse("A1").unwrap()).unwrap().py.as_ref().map(|p| p.code.as_str()), Some("5")); } +/// `bytes` with the part `name` stored uncompressed and one byte of its data changed, so the +/// archive is intact but reading the part fails its checksum. +fn with_damaged_part(bytes: &[u8], name: &str, marker: &str) -> Vec { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut out = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for i in 0..z.len() { + let mut f = z.by_index(i).unwrap(); + let entry = f.name().to_string(); + let mut data = Vec::new(); + f.read_to_end(&mut data).unwrap(); + let options = zip::write::SimpleFileOptions::default(); + let options = if entry == name { options.compression_method(zip::CompressionMethod::Stored) } else { options }; + out.start_file(entry, options).unwrap(); + out.write_all(&data).unwrap(); + } + let mut zipped = out.finish().unwrap().into_inner(); + let at = zipped.windows(marker.len()).position(|w| w == marker.as_bytes()).expect("the marker is stored as it is"); + zipped[at + 1] ^= 0x20; + zipped +} + +#[test] +fn a_damaged_python_part_is_a_warning_and_the_workbook_opens_with_its_values() { + let data = "_xlfn._xlws.PY(0,0)59"; + let good = excel_shaped(data, Some(&["5"])); + // The part's data fails its checksum (a damaged archive entry). + let damaged = with_damaged_part(&good, "xl/python/python1.xml", ""); + // The part is not well-formed XML, before its first script. + let broken = |at: &'static str, with: &'static str| { + with_parts(&good, move |name, text| if name == "xl/python/python1.xml" { text.replace(at, with) } else { text.to_string() }, &[]) + }; + let malformed = broken("", "