From da0cb382ae94824047ccf310722d3a600c329509 Mon Sep 17 00:00:00 2001 From: aiongg Date: Mon, 26 Oct 2026 20:27:03 +0000 Subject: [PATCH] refactor(gridcraft-cli): move check.rs items to check/ --- apps/gridcraft-cli/src/check.rs | 1024 +--------------------- apps/gridcraft-cli/src/check/calc.rs | 237 +++++ apps/gridcraft-cli/src/check/fidelity.rs | 185 ++++ apps/gridcraft-cli/src/check/package.rs | 329 +++++++ apps/gridcraft-cli/src/check/report.rs | 264 ++++++ 5 files changed, 1034 insertions(+), 1005 deletions(-) create mode 100644 apps/gridcraft-cli/src/check/calc.rs create mode 100644 apps/gridcraft-cli/src/check/fidelity.rs create mode 100644 apps/gridcraft-cli/src/check/package.rs create mode 100644 apps/gridcraft-cli/src/check/report.rs diff --git a/apps/gridcraft-cli/src/check.rs b/apps/gridcraft-cli/src/check.rs index 318b43aad..5fc37b14c 100644 --- a/apps/gridcraft-cli/src/check.rs +++ b/apps/gridcraft-cli/src/check.rs @@ -26,22 +26,31 @@ use serde_json::{Value as Json, json}; use super::{Args, USAGE, number_text, pretty}; +mod calc; +mod fidelity; +mod package; +mod report; + +pub use calc::oracle; +#[cfg(test)] +use calc::volatile_names; +#[cfg(test)] +use calc::{bool_cached_as_number, values_match}; +#[cfg(test)] +use fidelity::diff_workbooks; +pub use fidelity::round_trip; +use package::full_calc_on_load; +#[cfg(test)] +use package::validate_package; +use report::print_human; +pub use report::report_json; + /// Version tag of the `--json` output; bumped when a field changes meaning or disappears. pub const SCHEMA: &str = "gridcraft-check/1"; - -/// Functions whose result depends on the clock, a random generator or the environment. -const VOLATILE: [&str; 6] = ["RAND", "RANDARRAY", "RANDBETWEEN", "NOW", "TODAY", "INFO"]; /// Directory recursion limit (symlink loops end here). const MAX_DEPTH: usize = 32; /// Most files collected from the inputs. const MAX_FILES: usize = 100_000; -/// Largest package part read while validating a written package. -const MAX_XML_PART: u64 = 64 * 1024 * 1024; -/// Most zip entries looked at while validating a written package. -const MAX_ENTRIES: usize = 200_000; -/// Relationship kinds (lower case) whose absence after a save isn't a loss: the calculation chain is -/// a cache Excel rebuilds, and GridCraft never writes it. -const REBUILT_KINDS: [&str; 1] = ["calcchain"]; #[derive(Clone, Copy, Debug)] pub struct Options { @@ -357,1000 +366,5 @@ fn check_inner(path: &str, bytes: &[u8], opts: &Options) -> FileReport { report } -// ---------------------------------------------------------------- calc oracle - -fn functions_of(e: &Expr) -> Vec { - let mut set = BTreeSet::new(); - e.walk(&mut |x| { - if let Expr::Call(n, _) = x { - set.insert(n.clone()); - } - }); - set.into_iter().collect() -} - -fn is_volatile(e: &Expr, volatile_names: &HashSet) -> bool { - let mut v = false; - e.walk(&mut |x| match x { - Expr::Call(n, args) => { - let filename = n == "CELL" && matches!(args.first(), Some(Expr::Text(t)) if t.eq_ignore_ascii_case("filename")); - if filename || VOLATILE.contains(&n.as_str()) { - v = true; - } - } - Expr::Name(n) if volatile_names.contains(&n.to_ascii_uppercase()) => v = true, - _ => {} - }); - v -} - -/// Upper-cased defined names whose formula is volatile, directly or through other names. -fn volatile_names(wb: &Workbook) -> HashSet { - let parsed: Vec<(String, Option)> = - wb.names.iter().map(|n| (n.name.to_ascii_uppercase(), gridcraft_engine::formula::parse(&n.formula).ok())).collect(); - let mut set = HashSet::new(); - // Each round adds at least one name or stops, so `len + 1` rounds reach the fixed point. - for _ in 0..=parsed.len() { - let before = set.len(); - for (name, e) in &parsed { - if !set.contains(name) && e.as_ref().is_some_and(|e| is_volatile(e, &set)) { - set.insert(name.clone()); - } - } - if set.len() == before { - break; - } - } - set -} - -/// The formula as the formula bar shows it. -fn formula_text(cell: &Cell) -> String { - cell.bar_text() -} - -fn cell_name(wb: &Workbook, (sheet, c): Key) -> String { - let name = wb.sheet(sheet).map(|s| s.name.as_str()).unwrap_or("?"); - format!("{}!{}", quote_sheet(name), c.a1()) -} - -/// Whether the producer cached GridCraft's logical result as the number 1 or 0. Excel types a -/// logical result `t="b"`; a writer that leaves the type out (`TRUE()1`, as -/// Business Central's "Open in Excel" exports do) stores the same value, read back as a number. -pub fn bool_cached_as_number(expected: &Value, actual: &Value) -> bool { - matches!((expected, actual), (Value::Number(n), Value::Bool(b)) if *n == if *b { 1.0 } else { 0.0 }) -} - -/// Whether GridCraft's `actual` agrees with the producer's `expected`. -pub fn values_match(expected: &Value, actual: &Value, tolerance: f64) -> bool { - match (expected, actual) { - (Value::Number(a), Value::Number(b)) => (a - b).abs() <= tolerance * a.abs().max(b.abs()).max(1.0), - (Value::Text(a), Value::Text(b)) => a == b, - (Value::Bool(a), Value::Bool(b)) => a == b, - (Value::Error(a), Value::Error(b)) => a == b, - _ => false, - } -} - -fn type_name(v: &Value) -> &'static str { - match v { - Value::Empty => "blank", - Value::Number(_) => "number", - Value::Text(_) => "text", - Value::Bool(_) => "bool", - Value::Error(e) => e.as_str(), - Value::Array(_) => "array", - } -} - -fn mismatch_kind(expected: &Value, actual: &Value) -> &'static str { - match (expected, actual) { - (_, Value::Empty) => "blank", - (Value::Number(_), Value::Number(_)) => "numeric", - (Value::Text(_), Value::Text(_)) => "text", - (Value::Bool(_), Value::Bool(_)) => "bool", - (Value::Error(_), _) | (_, Value::Error(_)) => "error", - _ => "type", - } -} - -/// Mismatched cells indexed by (sheet, column) → rows, for precedent-area queries. -struct MismatchIndex { - cols: HashMap<(usize, u32), BTreeSet>, - /// Columns with mismatches per sheet (for very wide areas). - sheet_cols: HashMap>, -} - -impl MismatchIndex { - fn new(keys: &[Key]) -> MismatchIndex { - let mut cols: HashMap<(usize, u32), BTreeSet> = HashMap::new(); - let mut sheet_cols: HashMap> = HashMap::new(); - for &(s, c) in keys { - cols.entry((s, c.col)).or_default().insert(c.row); - sheet_cols.entry(s).or_default().insert(c.col); - } - MismatchIndex { cols, sheet_cols } - } - /// Whether a mismatched cell other than `own` lies in the area. - fn any_in(&self, sheet: usize, r0: u32, r1: u32, c0: u32, c1: u32, own: Key) -> bool { - let Some(cols) = self.sheet_cols.get(&sheet) else { return false }; - cols.range(c0..=c1) - .any(|&col| self.cols.get(&(sheet, col)).is_some_and(|rows| rows.range(r0..=r1).any(|&row| (sheet, CellRef::new(row, col)) != own))) - } -} - -/// Compares every formula cell of `cached` (as the producer saved it) with `fresh` (recalculated -/// by GridCraft). Without `trust_cache` nothing is compared (see [`CalcReport::stale_cache`]). -pub fn oracle(cached: &Workbook, fresh: &Workbook, calc: &Calc, trust_cache: bool, opts: &Options) -> CalcReport { - let mut r = CalcReport { stale_cache: !trust_cache, ..Default::default() }; - let vnames = volatile_names(cached); - let mut volatile: HashSet = HashSet::new(); - for (si, sh) in cached.sheets.iter().enumerate() { - for (c, cell) in sh.cells.iter() { - if let Some(f) = &cell.formula { - r.formula_cells += 1; - if f.expr_arc().is_some_and(|e| is_volatile(&e, &vnames)) { - volatile.insert((si, c)); - } - } - } - } - // Everything downstream of a volatile formula is volatile too; each key is pushed once. - let mut stack: Vec = volatile.iter().copied().collect(); - let mut deps = Vec::new(); - while let Some((s, c)) = stack.pop() { - deps.clear(); - calc.graph.dependents(s, c, &mut deps); - if let Some(range) = fresh.sheet(s).and_then(|sh| sh.spill_ranges.get(&c)) { - calc.graph.dependents_of_range(s, *range, &mut deps); - } - for k in deps.drain(..) { - if volatile.insert(k) { - stack.push(k); - } - } - } - - let mut bad: Vec<(Key, Vec)> = Vec::new(); - for (si, sh) in cached.sheets.iter().enumerate() { - for (c, cell) in sh.cells.iter() { - let Some(f) = &cell.formula else { continue }; - if volatile.contains(&(si, c)) { - r.volatile += 1; - continue; - } - if cell.value.is_empty() || !trust_cache { - r.no_oracle += 1; - continue; - } - r.checked += 1; - let actual = fresh.sheet(si).and_then(|s| s.cell(c)).map(|x| x.value.scalar()).unwrap_or_default(); - let funcs = match f.expr_arc() { - Some(e) => { - let names = functions_of(&e); - if names.is_empty() { vec!["(no function)".to_string()] } else { names } - } - None => vec!["(unparsed)".to_string()], - }; - let bool_as_number = bool_cached_as_number(&cell.value, &actual); - let ok = bool_as_number || values_match(&cell.value, &actual, opts.tolerance); - if bool_as_number { - r.bool_as_number += 1; - } - for n in &funcs { - let st = r.functions.entry(n.clone()).or_default(); - st.checked += 1; - if !ok { - st.mismatched += 1; - } - } - if ok { - r.matched += 1; - } else { - r.mismatched += 1; - *r.signatures.entry(format!("{} → {}", type_name(&cell.value), type_name(&actual))).or_default() += 1; - bad.push(((si, c), funcs)); - } - } - } - - // A mismatch is primary when none of its precedent cells mismatch. - let keys: Vec = bad.iter().map(|(k, _)| *k).collect(); - let index = MismatchIndex::new(&keys); - let mut listed: Vec<(bool, Key, Vec)> = Vec::with_capacity(bad.len()); - for (k, funcs) in bad { - let upstream = calc - .graph - .precedents_of(k) - .iter() - .any(|a| index.any_in(a.sheet, a.range.start.row, a.range.end.row, a.range.start.col, a.range.end.col, k)); - if !upstream { - for n in &funcs { - r.functions.entry(n.clone()).or_default().primary += 1; - } - } - listed.push((!upstream, k, funcs)); - } - // Primary mismatches first, each group in sheet then row-major order (already sorted). - listed.sort_by_key(|(primary, _, _)| !*primary); - for (primary, k, functions) in listed.into_iter().take(opts.max_report) { - let cell = cached.sheet(k.0).and_then(|s| s.cell(k.1)); - let expected = cell.map(|c| c.value.clone()).unwrap_or_default(); - let actual = fresh.sheet(k.0).and_then(|s| s.cell(k.1)).map(|x| x.value.scalar()).unwrap_or_default(); - r.mismatches.push(Mismatch { - cell: cell_name(cached, k), - formula: cell.map(formula_text).unwrap_or_default(), - kind: mismatch_kind(&expected, &actual), - expected, - actual, - functions, - primary, - }); - } - r -} - -// ---------------------------------------------------------------- round trip - -/// Writes `wb` with GridCraft's XLSX writer, validates the package against itself and against -/// `source` (the file `wb` was read from), reads it back and compares. -pub fn round_trip(wb: &Workbook, source: &[u8], opts: &Options) -> RoundTrip { - let bytes = match gridcraft_engine::io::save_bytes(wb, "roundtrip.xlsx") { - Ok(b) => b, - Err(e) => return RoundTrip { error: Some(format!("write failed: {e}")), categories: vec![] }, - }; - let mut package = Diffs::new("package", opts.max_report); - match validate_package(&bytes) { - Ok(issues) => issues.into_iter().for_each(|i| package.push(|| i)), - Err(e) => package.push(|| e), - } - if let (Ok(before), Ok(after)) = (relationship_kinds(source), relationship_kinds(&bytes)) { - for (key, kind) in &before { - if !after.contains_key(key) && !REBUILT_KINDS.contains(&key.as_str()) { - package.push(|| format!("lost relationship {kind}")); - } - } - } - let back = match gridcraft_engine::io::open_bytes("roundtrip.xlsx", &bytes) { - Ok((b, _)) => b, - Err(e) => return RoundTrip { error: Some(format!("reading the written file failed: {e}")), categories: vec![package.done()] }, - }; - let mut categories = diff_workbooks(wb, &back, opts); - categories.push(package.done()); - RoundTrip { error: None, categories } -} - -fn value_show(v: &Value) -> String { - match v { - Value::Empty => "(blank)".to_string(), - Value::Text(t) => serde_json::to_string(t.as_ref()).unwrap_or_default(), - Value::Number(n) => number_text(*n), - Value::Bool(b) => if *b { "TRUE" } else { "FALSE" }.to_string(), - Value::Error(e) => e.as_str().to_string(), - Value::Array(_) => v.scalar().display(), - } -} - -fn opt_formula(c: Option<&Cell>) -> String { - c.filter(|c| c.formula.is_some()).map(formula_text).unwrap_or_default() -} - -/// Per-sheet comparison of two lists, reporting what one side has and the other lacks. -fn diff_sets(d: &mut Diffs, sheet: &str, a: &BTreeSet, b: &BTreeSet) { - for x in a.difference(b) { - d.push(|| format!("{sheet}: lost {x}")); - } - for x in b.difference(a) { - d.push(|| format!("{sheet}: gained {x}")); - } -} - -/// A sheet's conditional-format rules in precedence order, as comparable text. The priority number -/// itself is left out, so renumbering without changing the order isn't a difference. -fn cond_format_list(s: &Sheet) -> Vec { - let mut cfs: Vec<_> = s.cond_formats.iter().collect(); - cfs.sort_by_key(|c| c.priority); - cfs.iter().map(|c| format!("{:?} {:?} stopIfTrue={}", c.ranges, c.rule, c.stop_if_true)).collect() -} - -fn diff_count(d: &mut Diffs, sheet: &str, a: usize, b: usize) { - if a != b { - d.push(|| format!("{sheet}: {a} → {b}")); - } -} - -/// A value as the file format can hold it: `#SPILL!`/`#CALC!` are cached as `#VALUE!` and an -/// unresolved circular reference as 0 (what Excel writes), so those aren't round-trip losses. -fn as_cached(v: Value) -> Value { - match v { - Value::Error(CellError::Spill | CellError::Calc) => Value::Error(CellError::Value), - Value::Error(CellError::Circ) => Value::Number(0.0), - v => v, - } -} - -fn num_fmt<'a>(wb: &'a Workbook, c: Option<&Cell>) -> &'a str { - wb.styles.get(c.map(|c| c.style).unwrap_or_default()).num_fmt.as_str() -} - -/// Feature-by-feature comparison of a workbook (`a`) with what reading it back gave (`b`). Sheets -/// are paired by position. -pub fn diff_workbooks(a: &Workbook, b: &Workbook, opts: &Options) -> Vec { - let m = opts.max_report; - let mut sheets = Diffs::new("sheets", m); - let (an, bn): (Vec<&str>, Vec<&str>) = (a.sheets.iter().map(|s| s.name.as_str()).collect(), b.sheets.iter().map(|s| s.name.as_str()).collect()); - if an != bn { - sheets.push(|| format!("{an:?} → {bn:?}")); - } - let mut values = Diffs::new("values", m); - let mut formulas = Diffs::new("formulas", m); - let mut num_fmts = Diffs::new("numFmts", m); - let mut merges = Diffs::new("merges", m); - let mut tables = Diffs::new("tables", m); - let mut cond_formats = Diffs::new("condFormats", m); - let mut validations = Diffs::new("validations", m); - let mut charts = Diffs::new("charts", m); - let mut comments = Diffs::new("comments", m); - let mut hyperlinks = Diffs::new("hyperlinks", m); - let mut pivots = Diffs::new("pivots", m); - for (sa, sb) in a.sheets.iter().zip(b.sheets.iter()) { - let name = quote_sheet(&sa.name); - let mut cell = |c: CellRef, ca: Option<&Cell>, cb: Option<&Cell>| { - let cached = |x: Option<&Cell>| as_cached(x.map(|x| x.value.scalar()).unwrap_or_default()); - let (va, vb) = (cached(ca), cached(cb)); - if va != vb { - values.push(|| format!("{name}!{}: {} → {}", c.a1(), value_show(&va), value_show(&vb))); - } - let (fa, fb) = (opt_formula(ca), opt_formula(cb)); - if fa != fb { - formulas.push(|| format!("{name}!{}: {fa} → {fb}", c.a1())); - } - let (na, nb) = (num_fmt(a, ca), num_fmt(b, cb)); - if na != nb { - num_fmts.push(|| format!("{name}!{}: {na} → {nb}", c.a1())); - } - }; - for (c, ca) in sa.cells.iter() { - cell(c, Some(ca), sb.cell(c)); - } - for (c, cb) in sb.cells.iter() { - if sa.cell(c).is_none() { - cell(c, None, Some(cb)); - } - } - let set = |f: &dyn Fn(&Sheet) -> Vec, s: &Sheet| f(s).into_iter().collect::>(); - let merge_list = |s: &Sheet| -> Vec { s.merges.iter().map(|r| r.a1()).collect() }; - diff_sets(&mut merges, &name, &set(&merge_list, sa), &set(&merge_list, sb)); - let table_list = |s: &Sheet| -> Vec { - s.tables - .iter() - .map(|t| format!("{} {} [{}]", t.name, t.range.a1(), t.columns.iter().map(|c| c.name.as_str()).collect::>().join(", "))) - .collect() - }; - diff_sets(&mut tables, &name, &set(&table_list, sa), &set(&table_list, sb)); - let (cfa, cfb) = (cond_format_list(sa), cond_format_list(sb)); - if cfa.len() != cfb.len() { - cond_formats.push(|| format!("{name}: {} → {}", cfa.len(), cfb.len())); - } else if let Some((x, y)) = cfa.iter().zip(&cfb).find(|(x, y)| x != y) { - cond_formats.push(|| format!("{name}: {x} → {y}")); - } - diff_count(&mut validations, &name, sa.validations.len(), sb.validations.len()); - let chart_list = |s: &Sheet| s.charts.iter().map(|c| format!("{:?} ({} series)", c.kind, c.series.len())).collect::>(); - let (ca, cb) = (chart_list(sa), chart_list(sb)); - if ca != cb { - charts.push(|| format!("{name}: {ca:?} → {cb:?}")); - } - diff_count(&mut comments, &name, sa.comments.len(), sb.comments.len()); - diff_count(&mut hyperlinks, &name, sa.hyperlinks.len(), sb.hyperlinks.len()); - diff_count(&mut pivots, &name, sa.pivots.len(), sb.pivots.len()); - } - let mut names = Diffs::new("names", m); - let name_list = |wb: &Workbook| -> BTreeSet { - wb.names - .iter() - .map(|n| { - let scope = n.scope.map(|i| format!("{}!", quote_sheet(wb.sheet(i).map(|s| s.name.as_str()).unwrap_or("?")))).unwrap_or_default(); - format!("{scope}{}={}", n.name, n.formula) - }) - .collect() - }; - diff_sets(&mut names, "workbook", &name_list(a), &name_list(b)); - vec![ - sheets.done(), - values.done(), - formulas.done(), - num_fmts.done(), - merges.done(), - names.done(), - tables.done(), - cond_formats.done(), - validations.done(), - charts.done(), - comments.done(), - hyperlinks.done(), - pivots.done(), - ] -} - -// ---------------------------------------------------------------- package validation - -fn local_name(name: &[u8]) -> &[u8] { - match name.iter().rposition(|&b| b == b':') { - Some(i) => name.get(i + 1..).unwrap_or(name), - None => name, - } -} - -/// `(element local name, attributes)` for every start/empty element of an XML part. -fn xml_elements(xml: &[u8]) -> Result)>, String> { - use quick_xml::events::Event; - let mut reader = quick_xml::Reader::from_reader(xml); - let mut buf = Vec::new(); - let mut out = Vec::new(); - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(e)) | Ok(Event::Empty(e)) => { - let attrs = e - .attributes() - .with_checks(false) - .flatten() - .map(|a| { - let v = match a.normalized_value(quick_xml::XmlVersion::Explicit1_0) { - Ok(v) => v.into_owned(), - Err(_) => String::from_utf8_lossy(&a.value).into_owned(), - }; - (String::from_utf8_lossy(local_name(a.key.as_ref())).into_owned(), v) - }) - .collect(); - out.push((String::from_utf8_lossy(local_name(e.name().as_ref())).into_owned(), attrs)); - } - Ok(Event::Eof) => break, - Ok(_) => {} - Err(e) => return Err(format!("invalid XML: {e}")), - } - buf.clear(); - } - Ok(out) -} - -fn attr<'a>(attrs: &'a [(String, String)], k: &str) -> Option<&'a str> { - attrs.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str()) -} - -/// Lower-cased part name without a leading slash: OPC part names compare case-insensitively. -fn part_key(name: &str) -> String { - name.trim_start_matches('/').to_ascii_lowercase() -} - -/// `%XX` escapes in a relationship target. -fn percent_decode(s: &str) -> String { - let b = s.as_bytes(); - let mut out = Vec::with_capacity(b.len()); - let mut i = 0; - while let Some(&c) = b.get(i) { - let hex = |j: usize| b.get(j).and_then(|h| (*h as char).to_digit(16)); - match (c, hex(i + 1), hex(i + 2)) { - (b'%', Some(h), Some(l)) => { - out.push((h * 16 + l) as u8); - i += 3; - } - _ => { - out.push(c); - i += 1; - } - } - } - String::from_utf8_lossy(&out).into_owned() -} - -/// Resolves a relationship target against the source part's directory. -fn resolve_target(source_dir: &str, target: &str) -> String { - let target = percent_decode(target.split('#').next().unwrap_or(target)); - let joined = match target.strip_prefix('/') { - Some(abs) => abs.to_string(), - None if source_dir.is_empty() => target, - None => format!("{source_dir}/{target}"), - }; - let mut parts: Vec<&str> = Vec::new(); - for seg in joined.split('/') { - match seg { - "" | "." => {} - ".." => { - parts.pop(); - } - s => parts.push(s), - } - } - parts.join("/") -} - -type Zip<'a> = zip::ZipArchive>; - -fn zip_names(zip: &mut Zip<'_>) -> Vec { - (0..zip.len().min(MAX_ENTRIES)).filter_map(|i| zip.by_index_raw(i).ok().filter(|f| !f.is_dir()).map(|f| f.name().to_string())).collect() -} - -fn read_zip_part(zip: &mut Zip<'_>, name: &str) -> Result, String> { - let f = zip.by_name(name).map_err(|e| format!("{name}: {e}"))?; - let mut buf = Vec::new(); - f.take(MAX_XML_PART).read_to_end(&mut buf).map_err(|e| format!("{name}: {e}"))?; - Ok(buf) -} - -/// Whether the producer flagged its cached results as stale with ``: -/// libraries that don't calculate write it, and store 0 or nothing as every formula's value. -fn full_calc_on_load(bytes: &[u8]) -> bool { - let Ok(mut zip) = zip::ZipArchive::new(Cursor::new(bytes)) else { return false }; - let names = zip_names(&mut zip); - let find = |key: &str| names.iter().find(|n| part_key(n) == key).cloned(); - let mut workbook = "xl/workbook.xml".to_string(); - if let Some(rels) = find("_rels/.rels") - && let Ok(xml) = read_zip_part(&mut zip, &rels) - && let Ok(els) = xml_elements(&xml) - && let Some(target) = els - .iter() - .find(|(e, a)| e == "Relationship" && attr(a, "Type").is_some_and(|t| t.ends_with("/officeDocument"))) - .and_then(|(_, a)| attr(a, "Target")) - { - workbook = resolve_target("", target); - } - let Some(name) = find(&part_key(&workbook)) else { return false }; - let Ok(xml) = read_zip_part(&mut zip, &name) else { return false }; - xml_elements(&xml) - .ok() - .into_iter() - .flatten() - .any(|(e, a)| e == "calcPr" && attr(&a, "fullCalcOnLoad").is_some_and(|v| v == "1" || v.eq_ignore_ascii_case("true"))) -} - -/// Problems Excel would repair in a written package: parts without a content type, content-type -/// overrides for missing parts, relationship files for missing parts, duplicate relationship ids -/// and internal relationships pointing at missing parts. `Err` when the package can't be read. -pub fn validate_package(bytes: &[u8]) -> Result, String> { - let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| format!("not a zip package: {e}"))?; - let names = zip_names(&mut zip); - let parts: HashSet = names.iter().map(|n| part_key(n)).collect(); - let mut read_part = |name: &str| read_zip_part(&mut zip, name); - let mut issues = Vec::new(); - let ct_name = names.iter().find(|n| part_key(n) == "[content_types].xml").cloned(); - let (mut defaults, mut overrides) = (HashSet::new(), HashSet::new()); - match ct_name { - None => issues.push("no [Content_Types].xml".to_string()), - Some(n) => { - for (el, attrs) in xml_elements(&read_part(&n)?).map_err(|e| format!("[Content_Types].xml: {e}"))? { - match el.as_str() { - "Default" => { - if let Some(x) = attr(&attrs, "Extension") { - defaults.insert(x.to_ascii_lowercase()); - } - } - "Override" => { - if let Some(p) = attr(&attrs, "PartName") { - overrides.insert(part_key(p)); - } - } - _ => {} - } - } - } - } - let mut sorted: Vec<&String> = names.iter().collect(); - sorted.sort(); - for n in &sorted { - let key = part_key(n); - if key == "[content_types].xml" { - continue; - } - let ext = key.rsplit('/').next().and_then(|f| f.rsplit_once('.')).map(|(_, e)| e.to_string()); - if !overrides.contains(&key) && !ext.is_some_and(|e| defaults.contains(&e)) { - issues.push(format!("part {n} has no content type")); - } - } - let mut missing_overrides: Vec<&String> = overrides.iter().filter(|p| !parts.contains(*p)).collect(); - missing_overrides.sort(); - for p in missing_overrides { - issues.push(format!("content type override for missing part /{p}")); - } - for n in sorted { - let key = part_key(n); - let Some(rels_file) = key.strip_suffix(".rels") else { continue }; - // `dir/_rels/name.rels` describes `dir/name`; `_rels/.rels` the package itself. - let (dir, file) = match rels_file.rsplit_once('/') { - Some((d, f)) => (d, f), - None => continue, - }; - let Some(src_dir) = dir.strip_suffix("_rels").map(|d| d.trim_end_matches('/')) else { continue }; - let source = if src_dir.is_empty() { file.to_string() } else { format!("{src_dir}/{file}") }; - if !source.is_empty() && !parts.contains(&source) { - issues.push(format!("{n} describes missing part {source}")); - } - let mut ids = HashSet::new(); - for (el, attrs) in xml_elements(&read_part(n)?).map_err(|e| format!("{n}: {e}"))? { - if el != "Relationship" { - continue; - } - let id = attr(&attrs, "Id").unwrap_or(""); - if !ids.insert(id.to_string()) { - issues.push(format!("{n}: duplicate relationship id {id}")); - } - if attr(&attrs, "TargetMode").is_some_and(|m| m.eq_ignore_ascii_case("External")) { - continue; - } - let target = attr(&attrs, "Target").unwrap_or(""); - let resolved = resolve_target(src_dir, target); - if !parts.contains(&resolved.to_ascii_lowercase()) { - let kind = attr(&attrs, "Type").and_then(|t| t.rsplit('/').next()).unwrap_or("?"); - issues.push(format!("{n}: {id} ({kind}) targets missing part {resolved}")); - } - } - } - Ok(issues) -} - -/// Every relationship kind (last segment of the type, the same in the transitional and strict -/// namespaces) in the package's relationship parts, keyed in lower case. Relationships that carry -/// nothing are left out, as a save may rightly drop them: a link to a place in the workbook -/// (`#Sheet1!A1`, written as a cell's `location`) and a target part with nothing in it. -fn relationship_kinds(bytes: &[u8]) -> Result, String> { - let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| format!("not a zip package: {e}"))?; - let names = zip_names(&mut zip); - let by_key: HashMap = names.iter().map(|n| (part_key(n), n)).collect(); - // Several relationships name the same part; each fresh check decompressed it again, so - // repeated references to one highly compressible part multiplied the work. The verdict is - // remembered per (part, named content) and the decompression the package can cause is - // budgeted as a whole (finding: repeated decompression). - let mut verdicts: HashMap<(String, &'static str), bool> = HashMap::new(); - let mut budget: u64 = 512 * 1024 * 1024; - let mut kinds = BTreeMap::new(); - for n in names.iter().filter(|n| part_key(n).ends_with(".rels")) { - let Some(src_dir) = rels_source_dir(&part_key(n)) else { continue }; - let Ok(els) = read_zip_part(&mut zip, n).and_then(|xml| xml_elements(&xml)) else { continue }; - for (_, attrs) in els.iter().filter(|(el, _)| el == "Relationship") { - let Some(kind) = attr(attrs, "Type").and_then(|t| t.rsplit('/').next()) else { continue }; - let target = attr(attrs, "Target").unwrap_or(""); - if kind.eq_ignore_ascii_case("hyperlink") && target.starts_with('#') { - continue; - } - let external = attr(attrs, "TargetMode").is_some_and(|m| m.eq_ignore_ascii_case("External")); - if !external - && let Some(part) = by_key.get(&resolve_target(&src_dir, target).to_ascii_lowercase()) - && !holds_something(&mut zip, part, kind, &mut verdicts, &mut budget) - { - continue; - } - kinds.entry(kind.to_ascii_lowercase()).or_insert_with(|| kind.to_string()); - } - } - Ok(kinds) -} - -/// The directory of the part a rels part describes: `xl` for `xl/_rels/workbook.xml.rels`, the -/// empty string for the package's `_rels/.rels`. -fn rels_source_dir(key: &str) -> Option { - let (dir, _) = key.strip_suffix(".rels")?.rsplit_once('/')?; - Some(dir.strip_suffix("_rels")?.trim_end_matches('/').to_string()) -} - -/// Whether a part has content: an element under its root, or for comments (whose authors list -/// comes first) a comment, for VML a shape. Read only as far as needed; an unreadable part counts. -/// `verdicts` remembers the answer per (part, named content); `budget` is the decompression this -/// package's remaining checks may cause, and once it is spent the part counts as having content. -fn holds_something(zip: &mut Zip<'_>, name: &str, kind: &str, verdicts: &mut HashMap<(String, &'static str), bool>, budget: &mut u64) -> bool { - let wanted: Option<&'static [u8]> = match kind.to_ascii_lowercase().as_str() { - "comments" => Some(b"comment"), - "vmldrawing" => Some(b"shape"), - _ => None, - }; - let marker = match wanted { - Some(b"comment") => "comment", - Some(_) => "shape", - None => "any", - }; - if let Some(v) = verdicts.get(&(name.to_ascii_lowercase(), marker)) { - return *v; - } - let v = holds_something_uncached(zip, name, marker, budget); - verdicts.insert((name.to_ascii_lowercase(), marker), v); - v -} - -fn holds_something_uncached(zip: &mut Zip<'_>, name: &str, marker: &'static str, budget: &mut u64) -> bool { - use quick_xml::events::Event; - let Ok(f) = zip.by_name(name) else { return true }; - if *budget == 0 { - return true; - } - let take = f.size().min(MAX_XML_PART).min(*budget); - *budget -= take; - let mut reader = quick_xml::Reader::from_reader(std::io::BufReader::new(f.take(take))); - let wanted: Option<&'static [u8]> = match marker { - "comment" => Some(b"comment"), - "shape" => Some(b"shape"), - _ => None, - }; - let mut buf = Vec::new(); - let mut in_root = false; - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(e)) | Ok(Event::Empty(e)) => { - let found = match wanted { - Some(w) => local_name(e.name().as_ref()) == w, - None => in_root, - }; - if found { - return true; - } - in_root = true; - } - Ok(Event::Eof) => return false, - Ok(_) => {} - Err(_) => return true, - } - buf.clear(); - } -} - -// ---------------------------------------------------------------- output - -#[derive(Default)] -struct Totals { - files: usize, - opened: usize, - failed: usize, - formula_cells: usize, - checked: usize, - matched: usize, - mismatched: usize, - no_oracle: usize, - volatile: usize, - bool_as_number: usize, - round_trip_failed: usize, - round_trip_differing: usize, - categories: BTreeMap<&'static str, usize>, - functions: BTreeMap, - signatures: BTreeMap, -} - -fn totals(reports: &[FileReport]) -> Totals { - let mut t = Totals { files: reports.len(), ..Default::default() }; - for r in reports { - if r.error.is_some() { - t.failed += 1; - continue; - } - t.opened += 1; - t.formula_cells += r.calc.formula_cells; - t.checked += r.calc.checked; - t.matched += r.calc.matched; - t.mismatched += r.calc.mismatched; - t.no_oracle += r.calc.no_oracle; - t.volatile += r.calc.volatile; - t.bool_as_number += r.calc.bool_as_number; - if r.round_trip.error.is_some() { - t.round_trip_failed += 1; - } - if !r.round_trip.is_clean() { - t.round_trip_differing += 1; - } - for c in r.round_trip.differing() { - *t.categories.entry(c).or_default() += 1; - } - for (n, s) in &r.calc.functions { - t.functions.entry(n.clone()).or_default().add(s); - } - for (sig, n) in &r.calc.signatures { - *t.signatures.entry(sig.clone()).or_default() += n; - } - } - t -} - -/// Functions with mismatches, worst first: most primary mismatches, then most mismatches. -fn ranked(functions: &BTreeMap) -> Vec<(&String, &FnStat)> { - let mut v: Vec<(&String, &FnStat)> = functions.iter().filter(|(_, s)| s.mismatched > 0).collect(); - v.sort_by(|a, b| b.1.primary.cmp(&a.1.primary).then(b.1.mismatched.cmp(&a.1.mismatched)).then(a.0.cmp(b.0))); - v -} - -fn functions_json(functions: &BTreeMap) -> Json { - let mut v: Vec<(&String, &FnStat)> = functions.iter().collect(); - v.sort_by(|a, b| b.1.primary.cmp(&a.1.primary).then(b.1.mismatched.cmp(&a.1.mismatched)).then(a.0.cmp(b.0))); - Json::Array(v.into_iter().map(|(n, s)| json!({"name": n, "checked": s.checked, "mismatched": s.mismatched, "primary": s.primary})).collect()) -} - -pub fn report_json(reports: &[FileReport], opts: &Options) -> Json { - let files: Vec = reports - .iter() - .map(|r| { - let c = &r.calc; - let mismatches: Vec = c - .mismatches - .iter() - .map(|m| { - json!({ - "cell": m.cell, "formula": m.formula, "expected": value_json(&m.expected), "actual": value_json(&m.actual), - "kind": m.kind, "functions": m.functions, "primary": m.primary, - }) - }) - .collect(); - let categories: serde_json::Map = r - .round_trip - .categories - .iter() - .map(|cat| (cat.name.to_string(), json!({"equal": cat.diffs == 0, "diffs": cat.diffs, "examples": cat.examples}))) - .collect(); - json!({ - "path": r.path, - "opened": r.error.is_none(), - "error": r.error, - "warnings": r.warnings, - "calc": { - "formulaCells": c.formula_cells, "checked": c.checked, "matched": c.matched, "mismatched": c.mismatched, - "noOracle": c.no_oracle, "staleCache": c.stale_cache, "volatile": c.volatile, "boolAsNumber": c.bool_as_number, - "mismatches": mismatches, "functions": functions_json(&c.functions), - "signatures": c.signatures, - }, - "roundTrip": {"error": r.round_trip.error, "differing": r.round_trip.differing(), "categories": categories}, - }) - }) - .collect(); - let t = totals(reports); - json!({ - "schema": SCHEMA, - "tolerance": opts.tolerance, - "files": files, - "totals": { - "files": t.files, "opened": t.opened, "failedToOpen": t.failed, "formulaCells": t.formula_cells, "checked": t.checked, - "matched": t.matched, "mismatched": t.mismatched, "noOracle": t.no_oracle, "volatile": t.volatile, "boolAsNumber": t.bool_as_number, - "roundTripFailed": t.round_trip_failed, "roundTripDiffering": t.round_trip_differing, "categoriesDiffering": t.categories, - }, - "functions": functions_json(&t.functions), - "signatures": t.signatures, - }) -} - -/// One report line: line breaks and tabs shown escaped so that they stay visible as differences. -fn one_line(s: &str) -> String { - s.replace('\n', "\\n").replace('\r', "\\r").replace('\t', "\\t") -} - -/// `…tail` of a long label. -fn fit(s: &str, w: usize) -> String { - let n = s.chars().count(); - if n <= w { s.to_string() } else { format!("…{}", s.chars().skip(n + 1 - w).collect::()) } -} - -fn print_human(reports: &[FileReport], opts: &Options) { - for r in reports { - outln!("== {}", r.path); - if let Some(e) = &r.error { - outln!(" can't open: {e}"); - continue; - } - let c = &r.calc; - outln!( - " formulas {}: {} checked, {} matched, {} mismatched, {} no oracle, {} volatile", - c.formula_cells, - c.checked, - c.matched, - c.mismatched, - c.no_oracle, - c.volatile - ); - if c.stale_cache { - outln!(" (the producer asked for a full recalculation on load: its cached values aren't results)"); - } - if c.bool_as_number > 0 { - outln!(" ({} matched TRUE/FALSE results were saved by the producer as 1/0, without t=\"b\")", c.bool_as_number); - } - for m in &c.mismatches { - let tag = if m.primary { "mismatch" } else { " (downstream)" }; - outln!( - " {tag} {} {} expected {} got {} [{}]", - m.cell, - one_line(&m.formula), - value_show(&m.expected), - value_show(&m.actual), - m.functions.join(" ") - ); - } - if c.mismatched > c.mismatches.len() { - outln!(" … {} more mismatches", c.mismatched - c.mismatches.len()); - } - let rt = &r.round_trip; - if let Some(e) = &rt.error { - outln!(" round trip: {e}"); - } - let differing: Vec<&Category> = rt.categories.iter().filter(|c| c.diffs > 0).collect(); - if differing.is_empty() && rt.error.is_none() { - outln!(" round trip: all equal"); - } - for cat in differing { - outln!(" round trip {}: {} difference(s)", cat.name, cat.diffs); - for e in &cat.examples { - outln!(" {}", one_line(e)); - } - if cat.diffs > cat.examples.len() { - outln!(" … {} more", cat.diffs - cat.examples.len()); - } - } - } - outln!(""); - let w = reports.iter().map(|r| r.path.chars().count()).max().unwrap_or(4).clamp(4, 50); - outln!( - "{:8} {:>8} {:>8} {:>8} {:>9} {:>8} round trip", - "file", - "formulas", - "checked", - "matched", - "mismatch", - "no-oracle", - "volatile" - ); - for r in reports { - let label = fit(&r.path, w); - if let Some(e) = &r.error { - outln!("{label: "FAILED".to_string(), - (None, d) if d.is_empty() => "ok".to_string(), - (None, d) => d.join(","), - }; - outln!( - "{label:8} {:>8} {:>8} {:>8} {:>9} {:>8} {rt}", - c.formula_cells, - c.checked, - c.matched, - c.mismatched, - c.no_oracle, - c.volatile - ); - } - let t = totals(reports); - let top = ranked(&t.functions); - if !top.is_empty() { - outln!(""); - outln!("top mismatching functions (primary = no mismatching precedent):"); - outln!(" {:<20} {:>8} {:>10} {:>8}", "function", "primary", "mismatched", "checked"); - for (n, s) in top.iter().take(opts.max_report.max(1)) { - outln!(" {:<20} {:>8} {:>10} {:>8}", n, s.primary, s.mismatched, s.checked); - } - } - let mut sigs: Vec<(&String, &usize)> = t.signatures.iter().collect(); - sigs.sort_by(|a, b| b.1.cmp(a.1).then(a.0.cmp(b.0))); - if !sigs.is_empty() { - outln!(""); - outln!("mismatch signatures (expected → GridCraft):"); - for (sig, n) in sigs.iter().take(opts.max_report.max(1)) { - outln!(" {n:>8} {sig}"); - } - } - outln!(""); - outln!("files: {} ({} opened, {} failed to open)", t.files, t.opened, t.failed); - outln!( - "formula cells: {} ({} checked: {} matched, {} mismatched; {} no oracle; {} volatile)", - t.formula_cells, - t.checked, - t.matched, - t.mismatched, - t.no_oracle, - t.volatile - ); - if t.bool_as_number > 0 { - outln!("booleans saved as 1/0 by the producer: {} (counted as matched)", t.bool_as_number); - } - let cats: Vec = t.categories.iter().map(|(c, n)| format!("{c} {n}")).collect(); - outln!( - "round trip: {} of {} files differ{}{}", - t.round_trip_differing, - t.opened, - if t.round_trip_failed > 0 { format!(" ({} failed)", t.round_trip_failed) } else { String::new() }, - if cats.is_empty() { String::new() } else { format!("; categories (files): {}", cats.join(", ")) } - ); -} - #[cfg(test)] mod tests; diff --git a/apps/gridcraft-cli/src/check/calc.rs b/apps/gridcraft-cli/src/check/calc.rs new file mode 100644 index 000000000..a3f33d296 --- /dev/null +++ b/apps/gridcraft-cli/src/check/calc.rs @@ -0,0 +1,237 @@ +use super::*; + +/// Functions whose result depends on the clock, a random generator or the environment. +const VOLATILE: [&str; 6] = ["RAND", "RANDARRAY", "RANDBETWEEN", "NOW", "TODAY", "INFO"]; + +// ---------------------------------------------------------------- calc oracle + +fn functions_of(e: &Expr) -> Vec { + let mut set = BTreeSet::new(); + e.walk(&mut |x| { + if let Expr::Call(n, _) = x { + set.insert(n.clone()); + } + }); + set.into_iter().collect() +} + +fn is_volatile(e: &Expr, volatile_names: &HashSet) -> bool { + let mut v = false; + e.walk(&mut |x| match x { + Expr::Call(n, args) => { + let filename = n == "CELL" && matches!(args.first(), Some(Expr::Text(t)) if t.eq_ignore_ascii_case("filename")); + if filename || VOLATILE.contains(&n.as_str()) { + v = true; + } + } + Expr::Name(n) if volatile_names.contains(&n.to_ascii_uppercase()) => v = true, + _ => {} + }); + v +} + +/// Upper-cased defined names whose formula is volatile, directly or through other names. +pub(super) fn volatile_names(wb: &Workbook) -> HashSet { + let parsed: Vec<(String, Option)> = + wb.names.iter().map(|n| (n.name.to_ascii_uppercase(), gridcraft_engine::formula::parse(&n.formula).ok())).collect(); + let mut set = HashSet::new(); + // Each round adds at least one name or stops, so `len + 1` rounds reach the fixed point. + for _ in 0..=parsed.len() { + let before = set.len(); + for (name, e) in &parsed { + if !set.contains(name) && e.as_ref().is_some_and(|e| is_volatile(e, &set)) { + set.insert(name.clone()); + } + } + if set.len() == before { + break; + } + } + set +} + +/// The formula as the formula bar shows it. +pub(super) fn formula_text(cell: &Cell) -> String { + cell.bar_text() +} + +fn cell_name(wb: &Workbook, (sheet, c): Key) -> String { + let name = wb.sheet(sheet).map(|s| s.name.as_str()).unwrap_or("?"); + format!("{}!{}", quote_sheet(name), c.a1()) +} + +/// Whether the producer cached GridCraft's logical result as the number 1 or 0. Excel types a +/// logical result `t="b"`; a writer that leaves the type out (`TRUE()1`, as +/// Business Central's "Open in Excel" exports do) stores the same value, read back as a number. +pub fn bool_cached_as_number(expected: &Value, actual: &Value) -> bool { + matches!((expected, actual), (Value::Number(n), Value::Bool(b)) if *n == if *b { 1.0 } else { 0.0 }) +} + +/// Whether GridCraft's `actual` agrees with the producer's `expected`. +pub fn values_match(expected: &Value, actual: &Value, tolerance: f64) -> bool { + match (expected, actual) { + (Value::Number(a), Value::Number(b)) => (a - b).abs() <= tolerance * a.abs().max(b.abs()).max(1.0), + (Value::Text(a), Value::Text(b)) => a == b, + (Value::Bool(a), Value::Bool(b)) => a == b, + (Value::Error(a), Value::Error(b)) => a == b, + _ => false, + } +} + +fn type_name(v: &Value) -> &'static str { + match v { + Value::Empty => "blank", + Value::Number(_) => "number", + Value::Text(_) => "text", + Value::Bool(_) => "bool", + Value::Error(e) => e.as_str(), + Value::Array(_) => "array", + } +} + +fn mismatch_kind(expected: &Value, actual: &Value) -> &'static str { + match (expected, actual) { + (_, Value::Empty) => "blank", + (Value::Number(_), Value::Number(_)) => "numeric", + (Value::Text(_), Value::Text(_)) => "text", + (Value::Bool(_), Value::Bool(_)) => "bool", + (Value::Error(_), _) | (_, Value::Error(_)) => "error", + _ => "type", + } +} + +/// Mismatched cells indexed by (sheet, column) → rows, for precedent-area queries. +struct MismatchIndex { + cols: HashMap<(usize, u32), BTreeSet>, + /// Columns with mismatches per sheet (for very wide areas). + sheet_cols: HashMap>, +} + +impl MismatchIndex { + fn new(keys: &[Key]) -> MismatchIndex { + let mut cols: HashMap<(usize, u32), BTreeSet> = HashMap::new(); + let mut sheet_cols: HashMap> = HashMap::new(); + for &(s, c) in keys { + cols.entry((s, c.col)).or_default().insert(c.row); + sheet_cols.entry(s).or_default().insert(c.col); + } + MismatchIndex { cols, sheet_cols } + } + /// Whether a mismatched cell other than `own` lies in the area. + fn any_in(&self, sheet: usize, r0: u32, r1: u32, c0: u32, c1: u32, own: Key) -> bool { + let Some(cols) = self.sheet_cols.get(&sheet) else { return false }; + cols.range(c0..=c1) + .any(|&col| self.cols.get(&(sheet, col)).is_some_and(|rows| rows.range(r0..=r1).any(|&row| (sheet, CellRef::new(row, col)) != own))) + } +} + +/// Compares every formula cell of `cached` (as the producer saved it) with `fresh` (recalculated +/// by GridCraft). Without `trust_cache` nothing is compared (see [`CalcReport::stale_cache`]). +pub fn oracle(cached: &Workbook, fresh: &Workbook, calc: &Calc, trust_cache: bool, opts: &Options) -> CalcReport { + let mut r = CalcReport { stale_cache: !trust_cache, ..Default::default() }; + let vnames = volatile_names(cached); + let mut volatile: HashSet = HashSet::new(); + for (si, sh) in cached.sheets.iter().enumerate() { + for (c, cell) in sh.cells.iter() { + if let Some(f) = &cell.formula { + r.formula_cells += 1; + if f.expr_arc().is_some_and(|e| is_volatile(&e, &vnames)) { + volatile.insert((si, c)); + } + } + } + } + // Everything downstream of a volatile formula is volatile too; each key is pushed once. + let mut stack: Vec = volatile.iter().copied().collect(); + let mut deps = Vec::new(); + while let Some((s, c)) = stack.pop() { + deps.clear(); + calc.graph.dependents(s, c, &mut deps); + if let Some(range) = fresh.sheet(s).and_then(|sh| sh.spill_ranges.get(&c)) { + calc.graph.dependents_of_range(s, *range, &mut deps); + } + for k in deps.drain(..) { + if volatile.insert(k) { + stack.push(k); + } + } + } + + let mut bad: Vec<(Key, Vec)> = Vec::new(); + for (si, sh) in cached.sheets.iter().enumerate() { + for (c, cell) in sh.cells.iter() { + let Some(f) = &cell.formula else { continue }; + if volatile.contains(&(si, c)) { + r.volatile += 1; + continue; + } + if cell.value.is_empty() || !trust_cache { + r.no_oracle += 1; + continue; + } + r.checked += 1; + let actual = fresh.sheet(si).and_then(|s| s.cell(c)).map(|x| x.value.scalar()).unwrap_or_default(); + let funcs = match f.expr_arc() { + Some(e) => { + let names = functions_of(&e); + if names.is_empty() { vec!["(no function)".to_string()] } else { names } + } + None => vec!["(unparsed)".to_string()], + }; + let bool_as_number = bool_cached_as_number(&cell.value, &actual); + let ok = bool_as_number || values_match(&cell.value, &actual, opts.tolerance); + if bool_as_number { + r.bool_as_number += 1; + } + for n in &funcs { + let st = r.functions.entry(n.clone()).or_default(); + st.checked += 1; + if !ok { + st.mismatched += 1; + } + } + if ok { + r.matched += 1; + } else { + r.mismatched += 1; + *r.signatures.entry(format!("{} → {}", type_name(&cell.value), type_name(&actual))).or_default() += 1; + bad.push(((si, c), funcs)); + } + } + } + + // A mismatch is primary when none of its precedent cells mismatch. + let keys: Vec = bad.iter().map(|(k, _)| *k).collect(); + let index = MismatchIndex::new(&keys); + let mut listed: Vec<(bool, Key, Vec)> = Vec::with_capacity(bad.len()); + for (k, funcs) in bad { + let upstream = calc + .graph + .precedents_of(k) + .iter() + .any(|a| index.any_in(a.sheet, a.range.start.row, a.range.end.row, a.range.start.col, a.range.end.col, k)); + if !upstream { + for n in &funcs { + r.functions.entry(n.clone()).or_default().primary += 1; + } + } + listed.push((!upstream, k, funcs)); + } + // Primary mismatches first, each group in sheet then row-major order (already sorted). + listed.sort_by_key(|(primary, _, _)| !*primary); + for (primary, k, functions) in listed.into_iter().take(opts.max_report) { + let cell = cached.sheet(k.0).and_then(|s| s.cell(k.1)); + let expected = cell.map(|c| c.value.clone()).unwrap_or_default(); + let actual = fresh.sheet(k.0).and_then(|s| s.cell(k.1)).map(|x| x.value.scalar()).unwrap_or_default(); + r.mismatches.push(Mismatch { + cell: cell_name(cached, k), + formula: cell.map(formula_text).unwrap_or_default(), + kind: mismatch_kind(&expected, &actual), + expected, + actual, + functions, + primary, + }); + } + r +} diff --git a/apps/gridcraft-cli/src/check/fidelity.rs b/apps/gridcraft-cli/src/check/fidelity.rs new file mode 100644 index 000000000..446a8949a --- /dev/null +++ b/apps/gridcraft-cli/src/check/fidelity.rs @@ -0,0 +1,185 @@ +use super::calc::formula_text; +use super::package::{REBUILT_KINDS, relationship_kinds, validate_package}; +use super::*; + +// ---------------------------------------------------------------- round trip + +/// Writes `wb` with GridCraft's XLSX writer, validates the package against itself and against +/// `source` (the file `wb` was read from), reads it back and compares. +pub fn round_trip(wb: &Workbook, source: &[u8], opts: &Options) -> RoundTrip { + let bytes = match gridcraft_engine::io::save_bytes(wb, "roundtrip.xlsx") { + Ok(b) => b, + Err(e) => return RoundTrip { error: Some(format!("write failed: {e}")), categories: vec![] }, + }; + let mut package = Diffs::new("package", opts.max_report); + match validate_package(&bytes) { + Ok(issues) => issues.into_iter().for_each(|i| package.push(|| i)), + Err(e) => package.push(|| e), + } + if let (Ok(before), Ok(after)) = (relationship_kinds(source), relationship_kinds(&bytes)) { + for (key, kind) in &before { + if !after.contains_key(key) && !REBUILT_KINDS.contains(&key.as_str()) { + package.push(|| format!("lost relationship {kind}")); + } + } + } + let back = match gridcraft_engine::io::open_bytes("roundtrip.xlsx", &bytes) { + Ok((b, _)) => b, + Err(e) => return RoundTrip { error: Some(format!("reading the written file failed: {e}")), categories: vec![package.done()] }, + }; + let mut categories = diff_workbooks(wb, &back, opts); + categories.push(package.done()); + RoundTrip { error: None, categories } +} + +pub(super) fn value_show(v: &Value) -> String { + match v { + Value::Empty => "(blank)".to_string(), + Value::Text(t) => serde_json::to_string(t.as_ref()).unwrap_or_default(), + Value::Number(n) => number_text(*n), + Value::Bool(b) => if *b { "TRUE" } else { "FALSE" }.to_string(), + Value::Error(e) => e.as_str().to_string(), + Value::Array(_) => v.scalar().display(), + } +} + +fn opt_formula(c: Option<&Cell>) -> String { + c.filter(|c| c.formula.is_some()).map(formula_text).unwrap_or_default() +} + +/// Per-sheet comparison of two lists, reporting what one side has and the other lacks. +fn diff_sets(d: &mut Diffs, sheet: &str, a: &BTreeSet, b: &BTreeSet) { + for x in a.difference(b) { + d.push(|| format!("{sheet}: lost {x}")); + } + for x in b.difference(a) { + d.push(|| format!("{sheet}: gained {x}")); + } +} + +/// A sheet's conditional-format rules in precedence order, as comparable text. The priority number +/// itself is left out, so renumbering without changing the order isn't a difference. +fn cond_format_list(s: &Sheet) -> Vec { + let mut cfs: Vec<_> = s.cond_formats.iter().collect(); + cfs.sort_by_key(|c| c.priority); + cfs.iter().map(|c| format!("{:?} {:?} stopIfTrue={}", c.ranges, c.rule, c.stop_if_true)).collect() +} + +fn diff_count(d: &mut Diffs, sheet: &str, a: usize, b: usize) { + if a != b { + d.push(|| format!("{sheet}: {a} → {b}")); + } +} + +/// A value as the file format can hold it: `#SPILL!`/`#CALC!` are cached as `#VALUE!` and an +/// unresolved circular reference as 0 (what Excel writes), so those aren't round-trip losses. +fn as_cached(v: Value) -> Value { + match v { + Value::Error(CellError::Spill | CellError::Calc) => Value::Error(CellError::Value), + Value::Error(CellError::Circ) => Value::Number(0.0), + v => v, + } +} + +fn num_fmt<'a>(wb: &'a Workbook, c: Option<&Cell>) -> &'a str { + wb.styles.get(c.map(|c| c.style).unwrap_or_default()).num_fmt.as_str() +} + +/// Feature-by-feature comparison of a workbook (`a`) with what reading it back gave (`b`). Sheets +/// are paired by position. +pub fn diff_workbooks(a: &Workbook, b: &Workbook, opts: &Options) -> Vec { + let m = opts.max_report; + let mut sheets = Diffs::new("sheets", m); + let (an, bn): (Vec<&str>, Vec<&str>) = (a.sheets.iter().map(|s| s.name.as_str()).collect(), b.sheets.iter().map(|s| s.name.as_str()).collect()); + if an != bn { + sheets.push(|| format!("{an:?} → {bn:?}")); + } + let mut values = Diffs::new("values", m); + let mut formulas = Diffs::new("formulas", m); + let mut num_fmts = Diffs::new("numFmts", m); + let mut merges = Diffs::new("merges", m); + let mut tables = Diffs::new("tables", m); + let mut cond_formats = Diffs::new("condFormats", m); + let mut validations = Diffs::new("validations", m); + let mut charts = Diffs::new("charts", m); + let mut comments = Diffs::new("comments", m); + let mut hyperlinks = Diffs::new("hyperlinks", m); + let mut pivots = Diffs::new("pivots", m); + for (sa, sb) in a.sheets.iter().zip(b.sheets.iter()) { + let name = quote_sheet(&sa.name); + let mut cell = |c: CellRef, ca: Option<&Cell>, cb: Option<&Cell>| { + let cached = |x: Option<&Cell>| as_cached(x.map(|x| x.value.scalar()).unwrap_or_default()); + let (va, vb) = (cached(ca), cached(cb)); + if va != vb { + values.push(|| format!("{name}!{}: {} → {}", c.a1(), value_show(&va), value_show(&vb))); + } + let (fa, fb) = (opt_formula(ca), opt_formula(cb)); + if fa != fb { + formulas.push(|| format!("{name}!{}: {fa} → {fb}", c.a1())); + } + let (na, nb) = (num_fmt(a, ca), num_fmt(b, cb)); + if na != nb { + num_fmts.push(|| format!("{name}!{}: {na} → {nb}", c.a1())); + } + }; + for (c, ca) in sa.cells.iter() { + cell(c, Some(ca), sb.cell(c)); + } + for (c, cb) in sb.cells.iter() { + if sa.cell(c).is_none() { + cell(c, None, Some(cb)); + } + } + let set = |f: &dyn Fn(&Sheet) -> Vec, s: &Sheet| f(s).into_iter().collect::>(); + let merge_list = |s: &Sheet| -> Vec { s.merges.iter().map(|r| r.a1()).collect() }; + diff_sets(&mut merges, &name, &set(&merge_list, sa), &set(&merge_list, sb)); + let table_list = |s: &Sheet| -> Vec { + s.tables + .iter() + .map(|t| format!("{} {} [{}]", t.name, t.range.a1(), t.columns.iter().map(|c| c.name.as_str()).collect::>().join(", "))) + .collect() + }; + diff_sets(&mut tables, &name, &set(&table_list, sa), &set(&table_list, sb)); + let (cfa, cfb) = (cond_format_list(sa), cond_format_list(sb)); + if cfa.len() != cfb.len() { + cond_formats.push(|| format!("{name}: {} → {}", cfa.len(), cfb.len())); + } else if let Some((x, y)) = cfa.iter().zip(&cfb).find(|(x, y)| x != y) { + cond_formats.push(|| format!("{name}: {x} → {y}")); + } + diff_count(&mut validations, &name, sa.validations.len(), sb.validations.len()); + let chart_list = |s: &Sheet| s.charts.iter().map(|c| format!("{:?} ({} series)", c.kind, c.series.len())).collect::>(); + let (ca, cb) = (chart_list(sa), chart_list(sb)); + if ca != cb { + charts.push(|| format!("{name}: {ca:?} → {cb:?}")); + } + diff_count(&mut comments, &name, sa.comments.len(), sb.comments.len()); + diff_count(&mut hyperlinks, &name, sa.hyperlinks.len(), sb.hyperlinks.len()); + diff_count(&mut pivots, &name, sa.pivots.len(), sb.pivots.len()); + } + let mut names = Diffs::new("names", m); + let name_list = |wb: &Workbook| -> BTreeSet { + wb.names + .iter() + .map(|n| { + let scope = n.scope.map(|i| format!("{}!", quote_sheet(wb.sheet(i).map(|s| s.name.as_str()).unwrap_or("?")))).unwrap_or_default(); + format!("{scope}{}={}", n.name, n.formula) + }) + .collect() + }; + diff_sets(&mut names, "workbook", &name_list(a), &name_list(b)); + vec![ + sheets.done(), + values.done(), + formulas.done(), + num_fmts.done(), + merges.done(), + names.done(), + tables.done(), + cond_formats.done(), + validations.done(), + charts.done(), + comments.done(), + hyperlinks.done(), + pivots.done(), + ] +} diff --git a/apps/gridcraft-cli/src/check/package.rs b/apps/gridcraft-cli/src/check/package.rs new file mode 100644 index 000000000..bcbbef10f --- /dev/null +++ b/apps/gridcraft-cli/src/check/package.rs @@ -0,0 +1,329 @@ +use super::*; + +/// Largest package part read while validating a written package. +const MAX_XML_PART: u64 = 64 * 1024 * 1024; + +/// Most zip entries looked at while validating a written package. +const MAX_ENTRIES: usize = 200_000; + +/// Relationship kinds (lower case) whose absence after a save isn't a loss: the calculation chain is +/// a cache Excel rebuilds, and GridCraft never writes it. +pub(super) const REBUILT_KINDS: [&str; 1] = ["calcchain"]; + +// ---------------------------------------------------------------- package validation + +fn local_name(name: &[u8]) -> &[u8] { + match name.iter().rposition(|&b| b == b':') { + Some(i) => name.get(i + 1..).unwrap_or(name), + None => name, + } +} + +/// `(element local name, attributes)` for every start/empty element of an XML part. +fn xml_elements(xml: &[u8]) -> Result)>, String> { + use quick_xml::events::Event; + let mut reader = quick_xml::Reader::from_reader(xml); + let mut buf = Vec::new(); + let mut out = Vec::new(); + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(e)) | Ok(Event::Empty(e)) => { + let attrs = e + .attributes() + .with_checks(false) + .flatten() + .map(|a| { + let v = match a.normalized_value(quick_xml::XmlVersion::Explicit1_0) { + Ok(v) => v.into_owned(), + Err(_) => String::from_utf8_lossy(&a.value).into_owned(), + }; + (String::from_utf8_lossy(local_name(a.key.as_ref())).into_owned(), v) + }) + .collect(); + out.push((String::from_utf8_lossy(local_name(e.name().as_ref())).into_owned(), attrs)); + } + Ok(Event::Eof) => break, + Ok(_) => {} + Err(e) => return Err(format!("invalid XML: {e}")), + } + buf.clear(); + } + Ok(out) +} + +fn attr<'a>(attrs: &'a [(String, String)], k: &str) -> Option<&'a str> { + attrs.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str()) +} + +/// Lower-cased part name without a leading slash: OPC part names compare case-insensitively. +fn part_key(name: &str) -> String { + name.trim_start_matches('/').to_ascii_lowercase() +} + +/// `%XX` escapes in a relationship target. +fn percent_decode(s: &str) -> String { + let b = s.as_bytes(); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while let Some(&c) = b.get(i) { + let hex = |j: usize| b.get(j).and_then(|h| (*h as char).to_digit(16)); + match (c, hex(i + 1), hex(i + 2)) { + (b'%', Some(h), Some(l)) => { + out.push((h * 16 + l) as u8); + i += 3; + } + _ => { + out.push(c); + i += 1; + } + } + } + String::from_utf8_lossy(&out).into_owned() +} + +/// Resolves a relationship target against the source part's directory. +fn resolve_target(source_dir: &str, target: &str) -> String { + let target = percent_decode(target.split('#').next().unwrap_or(target)); + let joined = match target.strip_prefix('/') { + Some(abs) => abs.to_string(), + None if source_dir.is_empty() => target, + None => format!("{source_dir}/{target}"), + }; + let mut parts: Vec<&str> = Vec::new(); + for seg in joined.split('/') { + match seg { + "" | "." => {} + ".." => { + parts.pop(); + } + s => parts.push(s), + } + } + parts.join("/") +} + +type Zip<'a> = zip::ZipArchive>; + +fn zip_names(zip: &mut Zip<'_>) -> Vec { + (0..zip.len().min(MAX_ENTRIES)).filter_map(|i| zip.by_index_raw(i).ok().filter(|f| !f.is_dir()).map(|f| f.name().to_string())).collect() +} + +fn read_zip_part(zip: &mut Zip<'_>, name: &str) -> Result, String> { + let f = zip.by_name(name).map_err(|e| format!("{name}: {e}"))?; + let mut buf = Vec::new(); + f.take(MAX_XML_PART).read_to_end(&mut buf).map_err(|e| format!("{name}: {e}"))?; + Ok(buf) +} + +/// Whether the producer flagged its cached results as stale with ``: +/// libraries that don't calculate write it, and store 0 or nothing as every formula's value. +pub(super) fn full_calc_on_load(bytes: &[u8]) -> bool { + let Ok(mut zip) = zip::ZipArchive::new(Cursor::new(bytes)) else { return false }; + let names = zip_names(&mut zip); + let find = |key: &str| names.iter().find(|n| part_key(n) == key).cloned(); + let mut workbook = "xl/workbook.xml".to_string(); + if let Some(rels) = find("_rels/.rels") + && let Ok(xml) = read_zip_part(&mut zip, &rels) + && let Ok(els) = xml_elements(&xml) + && let Some(target) = els + .iter() + .find(|(e, a)| e == "Relationship" && attr(a, "Type").is_some_and(|t| t.ends_with("/officeDocument"))) + .and_then(|(_, a)| attr(a, "Target")) + { + workbook = resolve_target("", target); + } + let Some(name) = find(&part_key(&workbook)) else { return false }; + let Ok(xml) = read_zip_part(&mut zip, &name) else { return false }; + xml_elements(&xml) + .ok() + .into_iter() + .flatten() + .any(|(e, a)| e == "calcPr" && attr(&a, "fullCalcOnLoad").is_some_and(|v| v == "1" || v.eq_ignore_ascii_case("true"))) +} + +/// Problems Excel would repair in a written package: parts without a content type, content-type +/// overrides for missing parts, relationship files for missing parts, duplicate relationship ids +/// and internal relationships pointing at missing parts. `Err` when the package can't be read. +pub fn validate_package(bytes: &[u8]) -> Result, String> { + let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| format!("not a zip package: {e}"))?; + let names = zip_names(&mut zip); + let parts: HashSet = names.iter().map(|n| part_key(n)).collect(); + let mut read_part = |name: &str| read_zip_part(&mut zip, name); + let mut issues = Vec::new(); + let ct_name = names.iter().find(|n| part_key(n) == "[content_types].xml").cloned(); + let (mut defaults, mut overrides) = (HashSet::new(), HashSet::new()); + match ct_name { + None => issues.push("no [Content_Types].xml".to_string()), + Some(n) => { + for (el, attrs) in xml_elements(&read_part(&n)?).map_err(|e| format!("[Content_Types].xml: {e}"))? { + match el.as_str() { + "Default" => { + if let Some(x) = attr(&attrs, "Extension") { + defaults.insert(x.to_ascii_lowercase()); + } + } + "Override" => { + if let Some(p) = attr(&attrs, "PartName") { + overrides.insert(part_key(p)); + } + } + _ => {} + } + } + } + } + let mut sorted: Vec<&String> = names.iter().collect(); + sorted.sort(); + for n in &sorted { + let key = part_key(n); + if key == "[content_types].xml" { + continue; + } + let ext = key.rsplit('/').next().and_then(|f| f.rsplit_once('.')).map(|(_, e)| e.to_string()); + if !overrides.contains(&key) && !ext.is_some_and(|e| defaults.contains(&e)) { + issues.push(format!("part {n} has no content type")); + } + } + let mut missing_overrides: Vec<&String> = overrides.iter().filter(|p| !parts.contains(*p)).collect(); + missing_overrides.sort(); + for p in missing_overrides { + issues.push(format!("content type override for missing part /{p}")); + } + for n in sorted { + let key = part_key(n); + let Some(rels_file) = key.strip_suffix(".rels") else { continue }; + // `dir/_rels/name.rels` describes `dir/name`; `_rels/.rels` the package itself. + let (dir, file) = match rels_file.rsplit_once('/') { + Some((d, f)) => (d, f), + None => continue, + }; + let Some(src_dir) = dir.strip_suffix("_rels").map(|d| d.trim_end_matches('/')) else { continue }; + let source = if src_dir.is_empty() { file.to_string() } else { format!("{src_dir}/{file}") }; + if !source.is_empty() && !parts.contains(&source) { + issues.push(format!("{n} describes missing part {source}")); + } + let mut ids = HashSet::new(); + for (el, attrs) in xml_elements(&read_part(n)?).map_err(|e| format!("{n}: {e}"))? { + if el != "Relationship" { + continue; + } + let id = attr(&attrs, "Id").unwrap_or(""); + if !ids.insert(id.to_string()) { + issues.push(format!("{n}: duplicate relationship id {id}")); + } + if attr(&attrs, "TargetMode").is_some_and(|m| m.eq_ignore_ascii_case("External")) { + continue; + } + let target = attr(&attrs, "Target").unwrap_or(""); + let resolved = resolve_target(src_dir, target); + if !parts.contains(&resolved.to_ascii_lowercase()) { + let kind = attr(&attrs, "Type").and_then(|t| t.rsplit('/').next()).unwrap_or("?"); + issues.push(format!("{n}: {id} ({kind}) targets missing part {resolved}")); + } + } + } + Ok(issues) +} + +/// Every relationship kind (last segment of the type, the same in the transitional and strict +/// namespaces) in the package's relationship parts, keyed in lower case. Relationships that carry +/// nothing are left out, as a save may rightly drop them: a link to a place in the workbook +/// (`#Sheet1!A1`, written as a cell's `location`) and a target part with nothing in it. +pub(super) fn relationship_kinds(bytes: &[u8]) -> Result, String> { + let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| format!("not a zip package: {e}"))?; + let names = zip_names(&mut zip); + let by_key: HashMap = names.iter().map(|n| (part_key(n), n)).collect(); + // Several relationships name the same part; each fresh check decompressed it again, so + // repeated references to one highly compressible part multiplied the work. The verdict is + // remembered per (part, named content) and the decompression the package can cause is + // budgeted as a whole (finding: repeated decompression). + let mut verdicts: HashMap<(String, &'static str), bool> = HashMap::new(); + let mut budget: u64 = 512 * 1024 * 1024; + let mut kinds = BTreeMap::new(); + for n in names.iter().filter(|n| part_key(n).ends_with(".rels")) { + let Some(src_dir) = rels_source_dir(&part_key(n)) else { continue }; + let Ok(els) = read_zip_part(&mut zip, n).and_then(|xml| xml_elements(&xml)) else { continue }; + for (_, attrs) in els.iter().filter(|(el, _)| el == "Relationship") { + let Some(kind) = attr(attrs, "Type").and_then(|t| t.rsplit('/').next()) else { continue }; + let target = attr(attrs, "Target").unwrap_or(""); + if kind.eq_ignore_ascii_case("hyperlink") && target.starts_with('#') { + continue; + } + let external = attr(attrs, "TargetMode").is_some_and(|m| m.eq_ignore_ascii_case("External")); + if !external + && let Some(part) = by_key.get(&resolve_target(&src_dir, target).to_ascii_lowercase()) + && !holds_something(&mut zip, part, kind, &mut verdicts, &mut budget) + { + continue; + } + kinds.entry(kind.to_ascii_lowercase()).or_insert_with(|| kind.to_string()); + } + } + Ok(kinds) +} + +/// The directory of the part a rels part describes: `xl` for `xl/_rels/workbook.xml.rels`, the +/// empty string for the package's `_rels/.rels`. +fn rels_source_dir(key: &str) -> Option { + let (dir, _) = key.strip_suffix(".rels")?.rsplit_once('/')?; + Some(dir.strip_suffix("_rels")?.trim_end_matches('/').to_string()) +} + +/// Whether a part has content: an element under its root, or for comments (whose authors list +/// comes first) a comment, for VML a shape. Read only as far as needed; an unreadable part counts. +/// `verdicts` remembers the answer per (part, named content); `budget` is the decompression this +/// package's remaining checks may cause, and once it is spent the part counts as having content. +fn holds_something(zip: &mut Zip<'_>, name: &str, kind: &str, verdicts: &mut HashMap<(String, &'static str), bool>, budget: &mut u64) -> bool { + let wanted: Option<&'static [u8]> = match kind.to_ascii_lowercase().as_str() { + "comments" => Some(b"comment"), + "vmldrawing" => Some(b"shape"), + _ => None, + }; + let marker = match wanted { + Some(b"comment") => "comment", + Some(_) => "shape", + None => "any", + }; + if let Some(v) = verdicts.get(&(name.to_ascii_lowercase(), marker)) { + return *v; + } + let v = holds_something_uncached(zip, name, marker, budget); + verdicts.insert((name.to_ascii_lowercase(), marker), v); + v +} + +fn holds_something_uncached(zip: &mut Zip<'_>, name: &str, marker: &'static str, budget: &mut u64) -> bool { + use quick_xml::events::Event; + let Ok(f) = zip.by_name(name) else { return true }; + if *budget == 0 { + return true; + } + let take = f.size().min(MAX_XML_PART).min(*budget); + *budget -= take; + let mut reader = quick_xml::Reader::from_reader(std::io::BufReader::new(f.take(take))); + let wanted: Option<&'static [u8]> = match marker { + "comment" => Some(b"comment"), + "shape" => Some(b"shape"), + _ => None, + }; + let mut buf = Vec::new(); + let mut in_root = false; + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(e)) | Ok(Event::Empty(e)) => { + let found = match wanted { + Some(w) => local_name(e.name().as_ref()) == w, + None => in_root, + }; + if found { + return true; + } + in_root = true; + } + Ok(Event::Eof) => return false, + Ok(_) => {} + Err(_) => return true, + } + buf.clear(); + } +} diff --git a/apps/gridcraft-cli/src/check/report.rs b/apps/gridcraft-cli/src/check/report.rs new file mode 100644 index 000000000..54227e797 --- /dev/null +++ b/apps/gridcraft-cli/src/check/report.rs @@ -0,0 +1,264 @@ +use super::fidelity::value_show; +use super::*; + +// ---------------------------------------------------------------- output + +#[derive(Default)] +struct Totals { + files: usize, + opened: usize, + failed: usize, + formula_cells: usize, + checked: usize, + matched: usize, + mismatched: usize, + no_oracle: usize, + volatile: usize, + bool_as_number: usize, + round_trip_failed: usize, + round_trip_differing: usize, + categories: BTreeMap<&'static str, usize>, + functions: BTreeMap, + signatures: BTreeMap, +} + +fn totals(reports: &[FileReport]) -> Totals { + let mut t = Totals { files: reports.len(), ..Default::default() }; + for r in reports { + if r.error.is_some() { + t.failed += 1; + continue; + } + t.opened += 1; + t.formula_cells += r.calc.formula_cells; + t.checked += r.calc.checked; + t.matched += r.calc.matched; + t.mismatched += r.calc.mismatched; + t.no_oracle += r.calc.no_oracle; + t.volatile += r.calc.volatile; + t.bool_as_number += r.calc.bool_as_number; + if r.round_trip.error.is_some() { + t.round_trip_failed += 1; + } + if !r.round_trip.is_clean() { + t.round_trip_differing += 1; + } + for c in r.round_trip.differing() { + *t.categories.entry(c).or_default() += 1; + } + for (n, s) in &r.calc.functions { + t.functions.entry(n.clone()).or_default().add(s); + } + for (sig, n) in &r.calc.signatures { + *t.signatures.entry(sig.clone()).or_default() += n; + } + } + t +} + +/// Functions with mismatches, worst first: most primary mismatches, then most mismatches. +fn ranked(functions: &BTreeMap) -> Vec<(&String, &FnStat)> { + let mut v: Vec<(&String, &FnStat)> = functions.iter().filter(|(_, s)| s.mismatched > 0).collect(); + v.sort_by(|a, b| b.1.primary.cmp(&a.1.primary).then(b.1.mismatched.cmp(&a.1.mismatched)).then(a.0.cmp(b.0))); + v +} + +fn functions_json(functions: &BTreeMap) -> Json { + let mut v: Vec<(&String, &FnStat)> = functions.iter().collect(); + v.sort_by(|a, b| b.1.primary.cmp(&a.1.primary).then(b.1.mismatched.cmp(&a.1.mismatched)).then(a.0.cmp(b.0))); + Json::Array(v.into_iter().map(|(n, s)| json!({"name": n, "checked": s.checked, "mismatched": s.mismatched, "primary": s.primary})).collect()) +} + +pub fn report_json(reports: &[FileReport], opts: &Options) -> Json { + let files: Vec = reports + .iter() + .map(|r| { + let c = &r.calc; + let mismatches: Vec = c + .mismatches + .iter() + .map(|m| { + json!({ + "cell": m.cell, "formula": m.formula, "expected": value_json(&m.expected), "actual": value_json(&m.actual), + "kind": m.kind, "functions": m.functions, "primary": m.primary, + }) + }) + .collect(); + let categories: serde_json::Map = r + .round_trip + .categories + .iter() + .map(|cat| (cat.name.to_string(), json!({"equal": cat.diffs == 0, "diffs": cat.diffs, "examples": cat.examples}))) + .collect(); + json!({ + "path": r.path, + "opened": r.error.is_none(), + "error": r.error, + "warnings": r.warnings, + "calc": { + "formulaCells": c.formula_cells, "checked": c.checked, "matched": c.matched, "mismatched": c.mismatched, + "noOracle": c.no_oracle, "staleCache": c.stale_cache, "volatile": c.volatile, "boolAsNumber": c.bool_as_number, + "mismatches": mismatches, "functions": functions_json(&c.functions), + "signatures": c.signatures, + }, + "roundTrip": {"error": r.round_trip.error, "differing": r.round_trip.differing(), "categories": categories}, + }) + }) + .collect(); + let t = totals(reports); + json!({ + "schema": SCHEMA, + "tolerance": opts.tolerance, + "files": files, + "totals": { + "files": t.files, "opened": t.opened, "failedToOpen": t.failed, "formulaCells": t.formula_cells, "checked": t.checked, + "matched": t.matched, "mismatched": t.mismatched, "noOracle": t.no_oracle, "volatile": t.volatile, "boolAsNumber": t.bool_as_number, + "roundTripFailed": t.round_trip_failed, "roundTripDiffering": t.round_trip_differing, "categoriesDiffering": t.categories, + }, + "functions": functions_json(&t.functions), + "signatures": t.signatures, + }) +} + +/// One report line: line breaks and tabs shown escaped so that they stay visible as differences. +fn one_line(s: &str) -> String { + s.replace('\n', "\\n").replace('\r', "\\r").replace('\t', "\\t") +} + +/// `…tail` of a long label. +fn fit(s: &str, w: usize) -> String { + let n = s.chars().count(); + if n <= w { s.to_string() } else { format!("…{}", s.chars().skip(n + 1 - w).collect::()) } +} + +pub(super) fn print_human(reports: &[FileReport], opts: &Options) { + for r in reports { + outln!("== {}", r.path); + if let Some(e) = &r.error { + outln!(" can't open: {e}"); + continue; + } + let c = &r.calc; + outln!( + " formulas {}: {} checked, {} matched, {} mismatched, {} no oracle, {} volatile", + c.formula_cells, + c.checked, + c.matched, + c.mismatched, + c.no_oracle, + c.volatile + ); + if c.stale_cache { + outln!(" (the producer asked for a full recalculation on load: its cached values aren't results)"); + } + if c.bool_as_number > 0 { + outln!(" ({} matched TRUE/FALSE results were saved by the producer as 1/0, without t=\"b\")", c.bool_as_number); + } + for m in &c.mismatches { + let tag = if m.primary { "mismatch" } else { " (downstream)" }; + outln!( + " {tag} {} {} expected {} got {} [{}]", + m.cell, + one_line(&m.formula), + value_show(&m.expected), + value_show(&m.actual), + m.functions.join(" ") + ); + } + if c.mismatched > c.mismatches.len() { + outln!(" … {} more mismatches", c.mismatched - c.mismatches.len()); + } + let rt = &r.round_trip; + if let Some(e) = &rt.error { + outln!(" round trip: {e}"); + } + let differing: Vec<&Category> = rt.categories.iter().filter(|c| c.diffs > 0).collect(); + if differing.is_empty() && rt.error.is_none() { + outln!(" round trip: all equal"); + } + for cat in differing { + outln!(" round trip {}: {} difference(s)", cat.name, cat.diffs); + for e in &cat.examples { + outln!(" {}", one_line(e)); + } + if cat.diffs > cat.examples.len() { + outln!(" … {} more", cat.diffs - cat.examples.len()); + } + } + } + outln!(""); + let w = reports.iter().map(|r| r.path.chars().count()).max().unwrap_or(4).clamp(4, 50); + outln!( + "{:8} {:>8} {:>8} {:>8} {:>9} {:>8} round trip", + "file", + "formulas", + "checked", + "matched", + "mismatch", + "no-oracle", + "volatile" + ); + for r in reports { + let label = fit(&r.path, w); + if let Some(e) = &r.error { + outln!("{label: "FAILED".to_string(), + (None, d) if d.is_empty() => "ok".to_string(), + (None, d) => d.join(","), + }; + outln!( + "{label:8} {:>8} {:>8} {:>8} {:>9} {:>8} {rt}", + c.formula_cells, + c.checked, + c.matched, + c.mismatched, + c.no_oracle, + c.volatile + ); + } + let t = totals(reports); + let top = ranked(&t.functions); + if !top.is_empty() { + outln!(""); + outln!("top mismatching functions (primary = no mismatching precedent):"); + outln!(" {:<20} {:>8} {:>10} {:>8}", "function", "primary", "mismatched", "checked"); + for (n, s) in top.iter().take(opts.max_report.max(1)) { + outln!(" {:<20} {:>8} {:>10} {:>8}", n, s.primary, s.mismatched, s.checked); + } + } + let mut sigs: Vec<(&String, &usize)> = t.signatures.iter().collect(); + sigs.sort_by(|a, b| b.1.cmp(a.1).then(a.0.cmp(b.0))); + if !sigs.is_empty() { + outln!(""); + outln!("mismatch signatures (expected → GridCraft):"); + for (sig, n) in sigs.iter().take(opts.max_report.max(1)) { + outln!(" {n:>8} {sig}"); + } + } + outln!(""); + outln!("files: {} ({} opened, {} failed to open)", t.files, t.opened, t.failed); + outln!( + "formula cells: {} ({} checked: {} matched, {} mismatched; {} no oracle; {} volatile)", + t.formula_cells, + t.checked, + t.matched, + t.mismatched, + t.no_oracle, + t.volatile + ); + if t.bool_as_number > 0 { + outln!("booleans saved as 1/0 by the producer: {} (counted as matched)", t.bool_as_number); + } + let cats: Vec = t.categories.iter().map(|(c, n)| format!("{c} {n}")).collect(); + outln!( + "round trip: {} of {} files differ{}{}", + t.round_trip_differing, + t.opened, + if t.round_trip_failed > 0 { format!(" ({} failed)", t.round_trip_failed) } else { String::new() }, + if cats.is_empty() { String::new() } else { format!("; categories (files): {}", cats.join(", ")) } + ); +}