diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index b1a06db35..9dde6a60c 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -368,6 +368,24 @@ fn iterative_calculation_of_text_and_errors() { assert_eq!(t.get("B1"), Value::Error(CellError::Div0)); } +/// `&`, CONCAT and TEXTJOIN turn a number into at most 15 significant digits, decimal unless that +/// would need 16 integer digits or more than 21 characters below 1. +#[test] +fn text_joins_convert_numbers_to_15_significant_digits() { + let mut t = T::new(); + for (formula, text) in [ + ("=(1E15-0.5)&\"\"", "1E+15"), + ("=(999999999999999)&\"\"", "999999999999999"), + ("=1234567890123456&\"\"", "1.23456789012346E+15"), + ("=(9.99999999999999E-9)&\"\"", "9.99999999999999E-09"), + ("=CONCAT(1/3*1E-5)", "3.33333333333333E-06"), + ("=TEXTJOIN(\",\",TRUE,1/3*1E-4,0.1+0.2)", "0.0000333333333333333,0.3"), + ] { + t.set("A1", formula); + assert_eq!(t.get("A1"), Value::from(text), "{formula}"); + } +} + #[test] fn circular_references_without_iterative_calculation() { let mut t = T::new(); diff --git a/crates/core/src/value.rs b/crates/core/src/value.rs index 4d76ef70e..3ff852a1d 100644 --- a/crates/core/src/value.rs +++ b/crates/core/src/value.rs @@ -444,14 +444,27 @@ pub fn sort_compare(a: &Value, b: &Value) -> Ordering { } } -/// Number → text the way Excel converts in `&` and `TEXT(x,"General")`-free contexts: up to 15 -/// significant digits, scientific for very large or very small magnitudes. +/// Number → text the way Excel converts in `&`, CONCAT and TEXTJOIN: at most 15 significant +/// digits, rounded first; scientific from 1E+15, below 1E-9, and wherever the decimal would be +/// longer than 21 characters, not counting the sign (`0.` plus 19 decimals). Excel-verified only +/// for positive 15-digit mantissas: `0.0000692361330972509` stays decimal and +/// `1.28233995888454E-06` does not, as Excel caches them. The limit for shorter mantissas from +/// 1E-6 to 1E-9 (e.g. `0.0000099999999999999` decimal) and for negatives (sign excluded) are +/// inferred. pub fn number_to_text(n: f64) -> String { if n == 0.0 || !n.is_finite() { return "0".into(); } + // Round to 15 significant digits first, then pick decimal or scientific from the rounded + // magnitude, so 999999999999999.5 reads `1E+15`, never a 16-digit integer. let a = n.abs(); - if !(1e-9..1e15).contains(&a) && a.round() != a || a >= 1e15 { + let sci = format!("{:.14e}", n); + let (mant, exp) = sci.split_once('e').unwrap_or((&sci, "0")); + let mag: i32 = exp.parse().unwrap_or(0); + let sig = mant.bytes().filter(u8::is_ascii_digit).count() - mant.bytes().rev().take_while(|&b| b == b'0').count(); + // Characters after `0.` for |n| < 1: the leading zeros, then the significant digits. + let decimals_below_one = -mag - 1 + sig as i32; + if !(-9..15).contains(&mag) || decimals_below_one > 19 { return sci_text(n, 15); } // A value just below 1e15 can round up to 1e15 at 15 significant digits. Only that narrow @@ -460,7 +473,6 @@ pub fn number_to_text(n: f64) -> String { return sci_text(n, 15); } // 15 significant digits, trailing zeros trimmed. - let mag = a.log10().floor() as i32; let decimals = (14 - mag).clamp(0, 30) as usize; let s = format!("{:.*}", decimals, n); trim_decimal(&s) @@ -499,9 +511,44 @@ mod tests { assert_eq!(number_to_text(1234567890123456.0), "1.23456789012346E+15"); assert_eq!(number_to_text(0.0001), "0.0001"); assert_eq!(number_to_text(1e-10), "1E-10"); + assert_eq!(number_to_text(1e15 - 0.5), "1E+15"); + assert_eq!(number_to_text(1e15 - 0.1), "1E+15"); + assert_eq!(number_to_text(-(1e15 - 0.5)), "-1E+15"); + assert_eq!(number_to_text(999999999999999.0), "999999999999999"); + assert_eq!(number_to_text(9.999999999999999e-10), "0.000000001"); + assert_eq!(number_to_text(-9.999999999999999e-10), "-0.000000001"); assert_eq!(number_to_text(1.0 / 3.0), "0.333333333333333"); } + /// Excel switches to scientific when the 15-digit decimal would be longer than 21 characters + /// (`0.` plus 19 decimals). Excel-verified: IMTAN, IMSECH, IMCSCH and IMEXP results cached by + /// Excel for Mac in the `formulas` project's test.xlsx (ENGINEERING!S53, R47, S37, V37, V39). + #[test] + fn number_text_small_magnitudes() { + assert_eq!(number_to_text(6.92361330972509e-5), "0.0000692361330972509"); + assert_eq!(number_to_text(2.41377759972715e-4), "0.000241377759972715"); + assert_eq!(number_to_text(1.28233995888454e-6), "1.28233995888454E-06"); + assert_eq!(number_to_text(-4.48088972789593e-6), "-4.48088972789593E-06"); + assert_eq!(number_to_text(5.98317105631764e-7), "5.98317105631764E-07"); + assert_eq!(number_to_text(-2.99158552811404e-7), "-2.99158552811404E-07"); + // Just below powers of ten: the 21-character limit, and the rounded magnitude. + assert_eq!(number_to_text(9.99999999999999e-5), "0.0000999999999999999"); + assert_eq!(number_to_text(9.99999999999999e-6), "9.99999999999999E-06"); + assert_eq!(number_to_text(9.9999999999999e-6), "0.0000099999999999999"); + assert_eq!(number_to_text(9.99999999999999e-9), "9.99999999999999E-09"); + assert_eq!(number_to_text(-9.99999999999999e-9), "-9.99999999999999E-09"); + assert_eq!(number_to_text(9.99999999999999e-10), "9.99999999999999E-10"); + assert_eq!(number_to_text(9.999999999999999e-6), "0.00001"); + assert_eq!(number_to_text(0.00001), "0.00001"); + assert_eq!(number_to_text(0.000001), "0.000001"); + assert_eq!(number_to_text(1.5e-9), "0.0000000015"); + // Large side: 15 significant digits, scientific from 1E+15. + assert_eq!(number_to_text(99999999999999.9), "99999999999999.9"); + assert_eq!(number_to_text(999999999999999.4), "999999999999999"); + assert_eq!(number_to_text(9999999999999998.0), "1E+16"); + assert_eq!(number_to_text(9.99999999999999e15), "9.99999999999999E+15"); + } + #[test] fn number_text_rounds_up_to_scientific_boundary() { assert_eq!(number_to_text(999999999999999.9), "1E+15"); diff --git a/crates/engine/src/display.rs b/crates/engine/src/display.rs index 4dbdf46e8..e6766ff59 100644 --- a/crates/engine/src/display.rs +++ b/crates/engine/src/display.rs @@ -5,7 +5,7 @@ use std::sync::Mutex; use gridcraft_core::{CellRef, RangeRef, Value}; use gridcraft_model::{Sheet, Workbook}; -use gridcraft_numfmt::{Formatted, NumberFormat, format_value}; +use gridcraft_numfmt::{Formatted, NumberFormat, format_value, format_value_full}; /// Parsed number formats are cached by code. pub(crate) fn parsed(code: &str) -> NumberFormat { @@ -116,7 +116,8 @@ pub fn formula_hidden(wb: &Workbook, sheet: &Sheet, c: CellRef) -> bool { wb.styles.get(sheet.style_id(at)).protection.hidden } -/// The text a cell shows (full precision General; the grid narrows General to fit the column). +/// The text a cell shows (General up to the 11-character cell width; the grid narrows General +/// further to fit the column). pub fn cell_text(wb: &Workbook, sheet: &Sheet, c: CellRef) -> String { if sheet.show_formulas && let Some(f) = sheet.cell(c).and_then(|x| x.formula.as_ref()) @@ -124,20 +125,35 @@ pub fn cell_text(wb: &Workbook, sheet: &Sheet, c: CellRef) -> String { { return format!("={}", f.text); } - value_text(wb, sheet, c) + shown_text(wb, sheet, c) } /// A cell's value with its number format applied, independent of column width: no `#####` -/// and no General narrowing (Save As CSV/Text writes this). Formulas give their result. +/// and no General narrowing (Save As CSV/Text writes this). Formulas give their result. A number +/// shown as General (also through a coloured or sectioned General, or a Text format) is written +/// as [`format_value_full`] writes it, never with an exponent: `&`'s 15 significant digits below +/// 1E+15 (`124412140710`, not `1.24412E+11`), the shortest round-trip digits from 1E+15 (every +/// integer below 1e16 exactly). Excel writes the cell display. pub fn value_text(wb: &Workbook, sheet: &Sheet, c: CellRef) -> String { + let style = wb.styles.get(sheet.style_id(c)); + with_fill_space(format_value_full(&sheet.value(c), &parsed(style.num_fmt.as_str()), wb.date_system)) +} + +/// A cell's value through its number format, General at the 11-character cell width. +fn shown_text(wb: &Workbook, sheet: &Sheet, c: CellRef) -> String { let style = wb.styles.get(sheet.style_id(c)); value_text_as(wb, &sheet.value(c), style.num_fmt.as_str()) } -/// [`value_text`] for a value and a format code given apart from a cell (print takes the code -/// from the cell's effective look, which a conditional format can override). +/// A value through a format code given apart from a cell, as a cell shows it (General at the +/// 11-character cell width). Print takes the code from the cell's effective look, which a +/// conditional format can override. pub fn value_text_as(wb: &Workbook, v: &Value, code: &str) -> String { - let f = format(v, code, wb); + with_fill_space(format(v, code, wb)) +} + +/// The formatted text with a single space where a `*x` repeat fill would expand. +fn with_fill_space(f: Formatted) -> String { match f.fill { Some((_, pos)) => { let mut t = f.text; diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index b1fc08845..a4c4f63d8 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -558,20 +558,13 @@ pub fn source_path(path: &str) -> Option { /// Save As CSV/Text writes "the text and values as they are displayed in cells" with "all /// characters in each cell" (Microsoft Support, *Excel formatting and features that are not /// transferred to other file formats*): number formats applied, never column-width `#####`. +/// One intentional difference: a number shown as General is not narrowed to the cell width and +/// never gets an exponent (`gridcraft_numfmt::format_value_full`). Below 1E+15 it is written as +/// `&` converts it (15 significant digits); from 1E+15 as the shortest digits that read back as +/// the same value (every integer below 1e16 exactly, so long identifiers survive); small numbers +/// that `&` writes in scientific notation as their 15 digits in full. fn csv(wb: &Workbook, sh: &Sheet, delimiter: u8) -> Result> { - gridcraft_xlsx::write_csv(sh, delimiter, |c| { - // General's grid-width formatting rounds large identifiers to scientific notation. - // Delimited files have no column width, so preserve the whole numeric value instead. - if wb.styles.get(sh.style_id(c)).num_fmt.as_str().eq_ignore_ascii_case("General") - && let gridcraft_core::Value::Number(number) = sh.value(c) - && number.is_finite() - { - number.to_string() - } else { - crate::display::value_text(wb, sh, c) - } - }) - .map_err(|e| EngineError::Other(e.to_string())) + gridcraft_xlsx::write_csv(sh, delimiter, |c| crate::display::value_text(wb, sh, c)).map_err(|e| EngineError::Other(e.to_string())) } /// The active sheet as CSV of raw cell values, for automation: numbers at full precision, dates and diff --git a/crates/engine/src/io/tests.rs b/crates/engine/src/io/tests.rs index 626d0f952..d2a4c0888 100644 --- a/crates/engine/src/io/tests.rs +++ b/crates/engine/src/io/tests.rs @@ -586,9 +586,80 @@ fn delimited_exports_preserve_general_precision_and_explicit_number_formats() { sheet.set_cell(CellRef::new(0, 0), Cell { value: Value::Number(124412140710.0), ..Default::default() }); sheet.set_cell(CellRef::new(0, 1), Cell { value: Value::Number(1.23456789012345), ..Default::default() }); sheet.set_cell(CellRef::new(0, 2), Cell { value: Value::Number(124412140710.0), style: scientific, ..Default::default() }); + // Every integer below 1e16 that f64 holds (also 2^53 and the even integers above it) keeps + // every digit; larger numbers keep the shortest digits that read back as the same value, + // and a number `&` would write with an exponent is written out in full, so reopening the + // file can't give it a scientific format that then rounds it on the next save. + let cells = [ + (1234567890123456.0, "1234567890123456"), + (-9007199254740991.0, "-9007199254740991"), + (9007199254740992.0, "9007199254740992"), + (9007199254740994.0, "9007199254740994"), + (9123456789012346.0, "9123456789012346"), + (-9999999999999998.0, "-9999999999999998"), + (12345678901234567.0, "12345678901234568"), + (1e20, "100000000000000000000"), + (1e15 - 0.5, "999999999999999.5"), + (1.0 / 3.0 * 1e-6, "0.000000333333333333333"), + (1.23456789012346e-10, "0.000000000123456789012346"), + ]; + for (col, (n, _)) in cells.iter().enumerate() { + sheet.set_cell(CellRef::new(0, col as u32 + 3), Cell { value: Value::Number(*n), ..Default::default() }); + } for (path, separator) in [("data.csv", ","), ("data.tsv", "\t")] { let output = String::from_utf8(save_bytes(&workbook, path).unwrap()).unwrap(); - assert_eq!(output, format!("124412140710{separator}1.23456789012345{separator}1.24E+11\r\n")); + let tail: Vec<&str> = cells.iter().map(|(_, text)| *text).collect(); + assert_eq!(output, format!("124412140710{s}1.23456789012345{s}1.24E+11{s}{}\r\n", tail.join(separator), s = separator)); + // Reopening the file and saving it again changes nothing. + let (reopened, _) = open_bytes(path, output.as_bytes()).unwrap(); + assert_eq!(String::from_utf8(save_bytes(&reopened, path).unwrap()).unwrap(), output); + } +} + +/// A number shown as General is written as `&` converts it (15 significant digits), whatever +/// colour, sections or literals the format adds, and in a Text cell. Excel writes the cell +/// display (`1.24412E+11`, `0.333333333`); GridCraft keeps the digits so part codes survive +/// (integers below 1e16 keep all their digits, and nothing gets an exponent; see the test above). +#[test] +fn delimited_exports_write_general_numbers_as_text_converts_them() { + use gridcraft_core::Value; + use gridcraft_model::{Cell, NumFmt, StyleId}; + let mut wb = Workbook::new(); + let mut styled = |code: &str| wb.styles.derive(StyleId::DEFAULT, |s| s.num_fmt = NumFmt::new(code)); + let (blue, signed, red, text, unit) = + (styled("[Blue]General"), styled("General;-General"), styled("General;[Red]-General"), styled("@"), styled("General\" mm\"")); + let code = 124412140710.0; + let sh = wb.sheet_mut(0).unwrap(); + let cells = [ + (Value::Number(1.0 / 3.0), StyleId::DEFAULT), + (Value::Number(0.1 + 0.2), StyleId::DEFAULT), + (Value::Number(1e20), StyleId::DEFAULT), + (Value::Number(code), blue), + (Value::Number(9007199254740994.0), blue), + (Value::Number(-1.0 / 3.0), blue), + (Value::Number(-code), signed), + (Value::Number(-code), red), + (Value::Number(code), text), + (Value::Number(code), unit), + ]; + for (col, (value, style)) in cells.into_iter().enumerate() { + sh.set_cell(CellRef::new(0, col as u32), Cell { value, formula: None, py: None, style }); + } + for (path, sep) in [("out.csv", ","), ("out.txt", "\t")] { + let output = String::from_utf8(save_bytes(&wb, path).unwrap()).unwrap(); + let expected = [ + "0.333333333333333", + "0.3", + "100000000000000000000", + "124412140710", + "9007199254740994", + "-0.333333333333333", + "-124412140710", + "-124412140710", + "124412140710", + "124412140710 mm", + ]; + assert_eq!(output, format!("{}\r\n", expected.join(sep))); } } diff --git a/crates/functions/src/engineering.rs b/crates/functions/src/engineering.rs index a163d8fda..0979784b5 100644 --- a/crates/functions/src/engineering.rs +++ b/crates/functions/src/engineering.rs @@ -617,6 +617,11 @@ mod tests { cx(ev("IMCSC", vec![t("4+3i")]), -0.075_489_832_915_863_7, 0.064_877_471_370_635_5); is_text(ev("IMSECH", vec![t("0")]), "1"); is_err(ev("IMCSCH", vec![t("0")]), CellError::Num); + // Excel-cached results (Excel for Mac, `formulas` project test.xlsx, ENGINEERING sheet): a + // part needing more than 21 characters as a decimal is written in scientific notation. + is_text(ev("IMCSCH", vec![t("13+5j")]), "1.28233995888454E-06+4.33496947416312E-06j"); + is_text(ev("IMEXP", vec![t("-13+25j")]), "2.24044486393732E-06-2.99158552811404E-07j"); + is_text(ev("IMSECH", vec![t("7+25j")]), "0.00180771853884451+0.000241377759972715j"); // sech z · cosh z = 1 let Value::Text(c) = ev("IMCOSH", vec![t("4+3i")]) else { panic!() }; let Value::Text(sc) = ev("IMSECH", vec![t("4+3i")]) else { panic!() }; diff --git a/crates/numfmt/src/lib.rs b/crates/numfmt/src/lib.rs index 524ff5d47..97b426172 100644 --- a/crates/numfmt/src/lib.rs +++ b/crates/numfmt/src/lib.rs @@ -293,23 +293,22 @@ impl NumberFormat { } /// Formats a number. The flag is true when the value cannot be shown (date out of range). - fn format_number(&self, n: f64, sys: DateSystem) -> (Formatted, bool) { - let general = |n: f64| Formatted { - text: format_general_fit(n, GENERAL_WIDTH).unwrap_or_else(|| OVERFLOW.into()), - color: None, - fill: None, - numeric: true, - }; + /// With `full`, General is not narrowed to the cell width but written by [`general_full`]. + fn format_number(&self, n: f64, sys: DateSystem, full: bool) -> (Formatted, bool) { + let general_text = |n: f64, width: usize| if full { Some(general_full(n)) } else { format_general_fit(n, width) }; + let general = + |n: f64| Formatted { text: general_text(n, GENERAL_WIDTH).unwrap_or_else(|| OVERFLOW.into()), color: None, fill: None, numeric: true }; if self.is_general() { return (general(n), false); } let Some((sec, drop_sign)) = self.select(n) else { return (general(n), false) }; let minus = n < 0.0 && !drop_sign; let out: Option = match sec.kind { - SecKind::Text => Some(render::render_text(sec, &format_general_fit(n, GENERAL_WIDTH).unwrap_or_default())), + SecKind::Text => Some(render::render_text(sec, &general_text(n, GENERAL_WIDTH).unwrap_or_default())), SecKind::General => { let width = if minus { GENERAL_WIDTH - 1 } else { GENERAL_WIDTH }; - Some(render::render_general(sec, n.abs(), width)) + let text = general_text(n.abs(), width).unwrap_or_else(|| "#".repeat(width.clamp(1, 11))); + Some(render::render_general(sec, &text)) } SecKind::Date => { if minus { @@ -405,16 +404,45 @@ impl NumberFormat { /// Formats a cell value for display. pub fn format_value(v: &Value, fmt: &NumberFormat, sys: DateSystem) -> Formatted { + format_value_with(v, fmt, sys, false) +} + +/// Formats a cell value as Save As CSV/Text writes it: like [`format_value`], except that a number +/// shown as General (a General or Text section, with or without colour and literals) is not +/// narrowed to the cell width but written by `general_full`: `124412140710`, not `1.24412E+11`. +/// Excel writes the cell display here; GridCraft keeps the digits. +pub fn format_value_full(v: &Value, fmt: &NumberFormat, sys: DateSystem) -> Formatted { + format_value_with(v, fmt, sys, true) +} + +/// General for Save As CSV/Text: the text `&` gives (15 significant digits), but never with an +/// exponent, because reopening the file would give the cell a scientific format that rounds it +/// on the next save. Where `&` writes an exponent, a number of 1 or more is written as the +/// shortest digits that read back as the same value (every integer f64 holds below 1e16 exactly, +/// so 16-digit card numbers survive), and a smaller one as its 15 significant digits in full. +fn general_full(n: f64) -> String { + let text = gridcraft_core::number_to_text(n); + if !text.contains('E') { + text + } else if n.abs() >= 1.0 { + format!("{n}") + } else { + let mag: i32 = format!("{n:.14e}").split_once('e').and_then(|(_, e)| e.parse().ok()).unwrap_or(0); + gridcraft_core::value::trim_decimal(&format!("{:.*}", (14 - mag).max(0) as usize, n)) + } +} + +fn format_value_with(v: &Value, fmt: &NumberFormat, sys: DateSystem, full: bool) -> Formatted { match v { Value::Empty => Formatted::default(), Value::Number(n) if !n.is_finite() => Formatted { text: CellError::Num.as_str().into(), ..Formatted::default() }, - Value::Number(n) => fmt.format_number(*n, sys).0, + Value::Number(n) => fmt.format_number(*n, sys, full).0, Value::Text(t) => fmt.format_text(t), Value::Bool(b) => Formatted { text: if *b { "TRUE" } else { "FALSE" }.into(), ..Formatted::default() }, Value::Error(e) => Formatted { text: e.as_str().into(), ..Formatted::default() }, Value::Array(a) => match a.data.first() { Some(Value::Array(_)) | None => Formatted::default(), - Some(first) => format_value(first, fmt, sys), + Some(first) => format_value_with(first, fmt, sys, full), }, } } @@ -525,7 +553,7 @@ pub fn text_function(v: &Value, code: &str, sys: DateSystem) -> Result Some(out) } -/// Renders a number through a General section (literals around the General keyword). -pub(crate) fn render_general(sec: &Section, v: f64, width: usize) -> Out { +/// Renders a number through a General section: `general` (the number already written as General) +/// with the section's literals around it. +pub(crate) fn render_general(sec: &Section, general: &str) -> Out { let mut out = Out::default(); for t in &sec.toks { match t { - Tok::General => out.push(&format_general_fit(v, width).unwrap_or_else(|| "#".repeat(width.clamp(1, 11)))), + Tok::General => out.push(general), Tok::Fill(c) => out.fill(*c), other => out.push_tok(other), } diff --git a/crates/numfmt/src/tests.rs b/crates/numfmt/src/tests.rs index 7bbfaf66f..2c1cf04ec 100644 --- a/crates/numfmt/src/tests.rs +++ b/crates/numfmt/src/tests.rs @@ -668,7 +668,7 @@ fn text_uncached(v: &Value, code: &str, sys: DateSystem) -> Result