diff --git a/.github/workflows/packaging-lint.yml b/.github/workflows/packaging-lint.yml index 2d1213a4e..758ecac06 100644 --- a/.github/workflows/packaging-lint.yml +++ b/.github/workflows/packaging-lint.yml @@ -27,6 +27,10 @@ jobs: name: Windows installation policy (no app build) runs-on: windows-latest steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eba6ece89..e09499812 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,10 @@ name: Release on: + # A nightly dry run on main: every release build except macOS, nothing signed or released + # (craft-common-actions release.yml `dry-run`). Finds release-only breakage before release day. + schedule: + - cron: "56 1 * * *" push: branches: [release] workflow_dispatch: @@ -23,6 +27,7 @@ jobs: # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. uses: storytold/craft-common-actions/.github/workflows/release.yml@main with: + dry-run: ${{ github.event_name == 'schedule' }} app-name: gridcraft app-title: GridCraft craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index e78fe1ea3..f03ab8355 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -18,4 +18,7 @@ jobs: sonarcloud: # Tracked at @main so org-wide CI improvements propagate without PR churn. uses: storytold/craft-common-actions/.github/workflows/sonarcloud.yml@main + with: + # Match the main CI gate so coverage exercises every glyph-sensitive UI case. + fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 secrets: inherit diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 7dd388ccd..d63b32f16 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -47,5 +47,7 @@ Generated-in-code art is original and has no file to list: the UI icon set (`cra | `docs/images/pivot.png` | GridCraft contributors | screenshot of GridCraft rendered offscreen (`crates/ui-egui/examples/snapshot.rs`) showing an original sample workbook | MIT OR Apache-2.0 | README screenshot; no third-party content | | `docs/images/format-chart.png` | GridCraft contributors | screenshot of GridCraft rendered offscreen (`crates/ui-egui/examples/snapshot.rs`) showing an original sample workbook | MIT OR Apache-2.0 | README screenshot; no third-party content | | `docs/images/page-break.png` | GridCraft contributors | screenshot of GridCraft rendered offscreen (`crates/ui-egui/examples/snapshot.rs`) showing an original sample workbook | MIT OR Apache-2.0 | README screenshot; no third-party content | -| Core 14 AFM metric numbers in `crates/pdf/src/symbol.rs` (Symbol: character codes and advance widths) and `crates/pdf/src/metrics.rs` (Helvetica, Times and Courier advance widths); no font file | Adobe Systems Incorporated | Adobe's Core 14 AFM font metrics files (`Symbol.afm` 001.008 and the Helvetica, Times and Courier AFMs) | Adobe's Core 14 metrics notice: the files may be used, copied and distributed for any purpose and without charge, provided the notice is kept (quoted in `crates/pdf/src/metrics.rs`) | Numbers only, typed in from the public metrics. PDF export embeds no font: Symbol and the Times faces are base-14 fonts every PDF reader supplies, and text outside WinAnsiEncoding is set in a predefined, non-embedded Adobe-GB1 CID font | +| Core 14 AFM metric numbers in `crates/pdf/src/symbol.rs` (Symbol: character codes and advance widths) and `crates/pdf/src/metrics.rs` (Helvetica, Times and Courier advance widths); no font file | Adobe Systems Incorporated | Adobe's Core 14 AFM font metrics files (`Symbol.afm` 001.008 and the Helvetica, Times and Courier AFMs) | Adobe's Core 14 metrics notice: the files may be used, copied and distributed for any purpose and without charge, provided the notice is kept (quoted in `crates/pdf/src/metrics.rs`) | Numbers only, typed in from the public metrics. PDF export embeds none of these fonts: Symbol and the Times faces are base-14 fonts every PDF reader supplies, and non-Arabic text outside WinAnsiEncoding is set in a predefined, non-embedded Adobe-GB1 CID font | | Unicode values of the Symbol encoding in `crates/pdf/src/symbol.rs` (the code-to-character map; numbers only, no file copied) | Unicode, Inc. | `symbol.txt`, "Adobe Symbol Encoding to Unicode", table version 1.0 (2011-07-12), at unicode.org/Public/MAPPINGS/VENDORS/ADOBE/ | Unicode, Inc.'s notice in the file: "Unicode, Inc. hereby grants the right to freely use the information supplied in this file in the creation of products supporting the Unicode Standard, and to make copies of this file in any form for internal or external distribution as long as this notice remains attached." The file is provided as-is, without warranties | Separate from Adobe's Core 14 notice (previous row), which covers the character codes and widths; the Unicode values come from this table | + +Arabic PDF text uses Noto Sans Arabic from the optional [craft-fonts build input](https://github.com/storytold/craft-fonts), pinned by the release workflow. Author: The Noto Project Authors. Licence: SIL Open Font License 1.1. Native unbundled export uses an installed Noto Sans Arabic face. No new asset file is committed. diff --git a/Cargo.lock b/Cargo.lock index 543cddfe1..53fead994 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -739,6 +739,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cmpv2" version = "0.2.0" @@ -757,7 +763,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "spki", "x509-cert", @@ -827,6 +833,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" + [[package]] name = "core-foundation" version = "0.9.4" @@ -907,15 +919,28 @@ dependencies = [ "libc", ] +[[package]] +name = "craft-cfb" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" + +[[package]] +name = "craft-csv" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "craft-fonts-build" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +source = "git+https://github.com/storytold/craft-libs?rev=aa84415bae3b765fef6b7ef42404f66ad1cff9d9#aa84415bae3b765fef6b7ef42404f66ad1cff9d9" [[package]] name = "craft-i18n" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +source = "git+https://github.com/storytold/craft-libs?rev=aa84415bae3b765fef6b7ef42404f66ad1cff9d9#aa84415bae3b765fef6b7ef42404f66ad1cff9d9" dependencies = [ "fixed_decimal", "icu_decimal", @@ -935,10 +960,38 @@ dependencies = [ "writeable", ] +[[package]] +name = "craft-offcrypto" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" +dependencies = [ + "aes", + "base64 0.23.1", + "craft-cfb", + "craft-xml", + "getrandom 0.3.4", + "hmac", + "sha1 0.11.0", + "sha2 0.11.1", + "thiserror 2.0.21", + "zeroize", +] + +[[package]] +name = "craft-opc" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" +dependencies = [ + "craft-xml", + "craft-zip", + "log", + "thiserror 2.0.21", +] + [[package]] name = "craft-print" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +source = "git+https://github.com/storytold/craft-libs?rev=aa84415bae3b765fef6b7ef42404f66ad1cff9d9#aa84415bae3b765fef6b7ef42404f66ad1cff9d9" dependencies = [ "craft-winprint-gdi", "log", @@ -949,10 +1002,15 @@ dependencies = [ "winsafe", ] +[[package]] +name = "craft-sheet" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" + [[package]] name = "craft-ui" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +source = "git+https://github.com/storytold/craft-libs?rev=aa84415bae3b765fef6b7ef42404f66ad1cff9d9#aa84415bae3b765fef6b7ef42404f66ad1cff9d9" dependencies = [ "craft-fonts-build", "craft-i18n", @@ -972,7 +1030,7 @@ dependencies = [ [[package]] name = "craft-update" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" dependencies = [ "chrono", "dunce", @@ -982,6 +1040,7 @@ dependencies = [ "image", "log", "plist", + "rustix 1.1.5", "semver", "serde", "serde_json", @@ -998,7 +1057,7 @@ dependencies = [ [[package]] name = "craft-web-api" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" +source = "git+https://github.com/storytold/craft-libs?rev=aa84415bae3b765fef6b7ef42404f66ad1cff9d9#aa84415bae3b765fef6b7ef42404f66ad1cff9d9" dependencies = [ "js-sys", "serde_json", @@ -1009,7 +1068,7 @@ dependencies = [ [[package]] name = "craft-web-host" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" +source = "git+https://github.com/storytold/craft-libs?rev=aa84415bae3b765fef6b7ef42404f66ad1cff9d9#aa84415bae3b765fef6b7ef42404f66ad1cff9d9" dependencies = [ "wasm-bindgen", "web-sys", @@ -1018,7 +1077,7 @@ dependencies = [ [[package]] name = "craft-window" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" dependencies = [ "egui", "objc2 0.6.5", @@ -1034,6 +1093,38 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "craft-xls" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" +dependencies = [ + "craft-cfb", + "craft-sheet", + "md-5", + "miniz_oxide 0.9.1", + "sha1 0.10.7", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-xml" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" +dependencies = [ + "quick-xml 0.41.0", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-zip" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=bb57d422d0bc842cbde430955825553ab7b976f4#bb57d422d0bc842cbde430955825553ab7b976f4" +dependencies = [ + "flate2", + "thiserror 2.0.21", + "zip 9.1.0", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -1114,6 +1205,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctutils" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f" +dependencies = [ + "cmov", +] + [[package]] name = "cursor-icon" version = "1.2.0" @@ -1126,7 +1226,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "flagset", "pem-rfc7468", @@ -1191,7 +1291,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", + "const-oid 0.10.3", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -1882,9 +1984,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -2022,7 +2126,7 @@ dependencies = [ [[package]] name = "gridcraft" -version = "0.16.0" +version = "0.19.1" dependencies = [ "arboard", "chrono", @@ -2034,6 +2138,7 @@ dependencies = [ "getrandom 0.4.3", "gridcraft-engine", "gridcraft-mcp", + "gridcraft-sandbox", "gridcraft-ui-egui", "image", "log", @@ -2049,7 +2154,7 @@ dependencies = [ [[package]] name = "gridcraft-calc" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", "gridcraft-formula", @@ -2061,7 +2166,7 @@ dependencies = [ [[package]] name = "gridcraft-chart" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-calc", "gridcraft-core", @@ -2072,10 +2177,11 @@ dependencies = [ [[package]] name = "gridcraft-cli" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-engine", "gridcraft-mcp", + "gridcraft-sandbox", "log", "quick-xml 0.41.0", "serde_json", @@ -2084,7 +2190,7 @@ dependencies = [ [[package]] name = "gridcraft-core" -version = "0.16.0" +version = "0.19.1" dependencies = [ "chrono", "js-sys", @@ -2095,8 +2201,9 @@ dependencies = [ [[package]] name = "gridcraft-engine" -version = "0.16.0" +version = "0.19.1" dependencies = [ + "craft-csv", "gridcraft-calc", "gridcraft-chart", "gridcraft-core", @@ -2112,13 +2219,14 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.21", + "unicode-segmentation", "web-time", "zip 3.0.0", ] [[package]] name = "gridcraft-formula" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", "serde", @@ -2127,7 +2235,7 @@ dependencies = [ [[package]] name = "gridcraft-functions" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", "regex", @@ -2135,7 +2243,7 @@ dependencies = [ [[package]] name = "gridcraft-math" -version = "0.16.0" +version = "0.19.1" dependencies = [ "ratex-font", "ratex-layout", @@ -2145,7 +2253,7 @@ dependencies = [ [[package]] name = "gridcraft-mcp" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-engine", "log", @@ -2154,7 +2262,7 @@ dependencies = [ [[package]] name = "gridcraft-model" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", "gridcraft-formula", @@ -2167,18 +2275,35 @@ dependencies = [ [[package]] name = "gridcraft-numfmt" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", ] [[package]] name = "gridcraft-pdf" -version = "0.16.0" +version = "0.19.1" +dependencies = [ + "harfrust", + "sha2 0.11.1", + "skrifa", + "unicode-bidi", +] + +[[package]] +name = "gridcraft-sandbox" +version = "0.19.1" +dependencies = [ + "landlock", + "libc", + "nix", + "seccompiler", + "thiserror 2.0.21", +] [[package]] name = "gridcraft-ui-egui" -version = "0.16.0" +version = "0.19.1" dependencies = [ "craft-ui", "craft-update", @@ -2205,7 +2330,7 @@ dependencies = [ [[package]] name = "gridcraft-web" -version = "0.16.0" +version = "0.19.1" dependencies = [ "craft-web-api", "craft-web-host", @@ -2224,16 +2349,19 @@ dependencies = [ [[package]] name = "gridcraft-xlsx" -version = "0.16.0" +version = "0.19.1" dependencies = [ - "aes", + "craft-cfb", + "craft-csv", + "craft-offcrypto", + "craft-opc", + "craft-xls", "gridcraft-core", "gridcraft-formula", "gridcraft-functions", "gridcraft-math", "gridcraft-model", "quick-xml 0.41.0", - "sha1 0.11.0", "sha2 0.11.1", "thiserror 2.0.21", "zip 3.0.0", @@ -2338,6 +2466,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.5.0" @@ -2873,6 +3010,17 @@ dependencies = [ "smallvec", ] +[[package]] +name = "landlock" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cca98e95f35b29d469dade6724c6f96cec9236640f745a0e99b0334ec320ab1" +dependencies = [ + "enumflags2", + "libc", + "thiserror 2.0.21", +] + [[package]] name = "lazy_static" version = "1.5.1" @@ -2967,6 +3115,16 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "memchr" version = "2.8.3" @@ -4544,6 +4702,15 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "seccompiler" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae55de56877481d112a559bbc12667635fdaf5e005712fd4e2b2fa50ffc884" +dependencies = [ + "libc", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -4770,7 +4937,7 @@ checksum = "da90c3d9af638898a5fdc347479b18f950bb0dde11ecf2244f94cd28135da53b" dependencies = [ "aws-lc-rs", "base64 0.22.1", - "const-oid", + "const-oid 0.9.6", "der", "digest 0.10.7", "pem", @@ -4844,7 +5011,7 @@ dependencies = [ "chrono", "cmpv2", "cms", - "const-oid", + "const-oid 0.9.6", "der", "hex", "rand 0.10.3", @@ -4883,7 +5050,7 @@ dependencies = [ "base64 0.22.1", "chrono", "cms", - "const-oid", + "const-oid 0.9.6", "hex", "pem", "rustls-pki-types", @@ -6486,7 +6653,7 @@ dependencies = [ [[package]] name = "winprint" version = "0.2.1" -source = "git+https://github.com/storytold/winprint.rs?rev=51934db382f225454cf152a8657eb03789be63c4#51934db382f225454cf152a8657eb03789be63c4" +source = "git+https://github.com/storytold/winprint.rs?rev=bc06a7c48b768eb27a828c7986f6d2868b245ee0#bc06a7c48b768eb27a828c7986f6d2868b245ee0" dependencies = [ "flate2", "fmt-derive", @@ -6501,6 +6668,7 @@ dependencies = [ "windows", "windows-core", "windows-implement", + "windows-numerics", "xml", ] @@ -6607,7 +6775,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "sha1 0.10.7", "signature", @@ -6675,7 +6843,7 @@ checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" [[package]] name = "xtask" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-engine", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 1941c9d1e..debe80119 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*", "apps/*", "xtask"] default-members = ["crates/*", "apps/gridcraft", "apps/gridcraft-cli", "xtask"] [workspace.package] -version = "0.16.0" +version = "0.19.1" edition = "2024" license = "MIT OR Apache-2.0" rust-version = "1.95" @@ -14,6 +14,7 @@ repository = "https://github.com/storytold/gridcraft" # internal gridcraft-core = { path = "crates/core" } gridcraft-numfmt = { path = "crates/numfmt" } +gridcraft-sandbox = { path = "crates/sandbox" } gridcraft-formula = { path = "crates/formula" } gridcraft-functions = { path = "crates/functions" } gridcraft-model = { path = "crates/model" } @@ -25,10 +26,21 @@ gridcraft-ui-egui = { path = "crates/ui-egui" } gridcraft-mcp = { path = "crates/mcp" } gridcraft-pdf = { path = "crates/pdf" } gridcraft-math = { path = "crates/math" } -# Every craft-libs crate (craft-update and craft-window here, craft-ui in ui-egui, craft-print in -# the app) comes from one craft-libs revision. +# Every craft-libs crate (craft-update, craft-window and the file-format crates here, craft-ui in +# ui-egui, craft-print in the app, craft-web-api and craft-web-host in the web app) comes from one +# craft-libs revision. # craft-update: app-specific pretend versions apply to startup and manual checks. -craft-update = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d", version = "=0.1.0", features = ["egui"] } +craft-update = { git = "https://github.com/storytold/craft-libs", rev = "bb57d422d0bc842cbde430955825553ab7b976f4", version = "=0.1.0", features = ["egui"] } +# Compound files ([MS-CFB]) and password-encrypted OOXML packages ([MS-OFFCRYPTO]). +craft-cfb = { git = "https://github.com/storytold/craft-libs", rev = "bb57d422d0bc842cbde430955825553ab7b976f4", version = "0.1.0" } +craft-offcrypto = { git = "https://github.com/storytold/craft-libs", rev = "bb57d422d0bc842cbde430955825553ab7b976f4", version = "0.1.0" } +# OPC zip packages (parts, relationships, the zip writer) under `crates/xlsx/src/package.rs`. +craft-opc = { git = "https://github.com/storytold/craft-libs", rev = "bb57d422d0bc842cbde430955825553ab7b976f4", version = "0.1.0" } +# Delimited text (CSV, TSV) and SYLK; `crates/xlsx/src/csv.rs` and `crates/engine/src/io/sylk.rs` +# turn them into workbooks. +craft-csv = { git = "https://github.com/storytold/craft-libs", rev = "bb57d422d0bc842cbde430955825553ab7b976f4", version = "0.1.0" } +# Excel 97-2003 (.xls, BIFF8) workbooks; `crates/xlsx/src/xls.rs` maps them onto the model. +craft-xls = { git = "https://github.com/storytold/craft-libs", rev = "bb57d422d0bc842cbde430955825553ab7b976f4", version = "0.1.0" } # foundation thiserror = "2" @@ -43,9 +55,6 @@ regex = { version = "1", default-features = false, features = ["std", "unicode", quick-xml = "0.41" zip = { version = "3", default-features = false, features = ["deflate"] } sha2 = { version = "0.11", default-features = false } -# Decrypting password-protected workbooks ([MS-OFFCRYPTO]): AES and SHA-1, pure Rust (wasm-safe). -aes = { version = "0.9", default-features = false } -sha1 = { version = "0.11", default-features = false } image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } # equations: RaTeX (MIT) lays LaTeX out as positioned glyphs, rules and paths. Exact pins: it is @@ -71,7 +80,7 @@ skrifa = { version = "0.44", default-features = false, features = ["std"] } epaint_default_fonts = "0.37" # One native window per workbook: the window list, Window menu titles and the immediate-viewport # driver. Same craft-libs revision as craft-update above. -craft-window = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d", default-features = false, features = ["egui"] } +craft-window = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "bb57d422d0bc842cbde430955825553ab7b976f4", default-features = false, features = ["egui"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/ROADMAP.md b/ROADMAP.md index 51f7e51d3..dc43b3a55 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -149,6 +149,8 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-31 | Recently Used lists the functions most recently entered in a formula from any source (ribbon, control channel, MCP, scripts), kept with the preferences and returned by `formulas.recentlyUsed`; the Insert Function dialog takes Excel's title and a working category list (Most Recently Used, All, each category) | | 2026-10-31 | Formulas › Function Library as a big Insert Function button, the AutoSum split button and three columns of labelled small buttons, each category book in its own colour; category menus end with Insert Function… on that category; More Functions cascades into the remaining categories; Home › Editing shares the AutoSum menu; the MR key tip opens Recently Used | | 2026-10-31 | System and embedded CJK fallback text shares the Latin baseline in cells, editors and window titles (#1265, #2621). Symbol and emoji alignment, manual row heights and clipping are unchanged | +| 2026-10-26 | PDF preserves Context/LTR/RTL reading direction through wrapping and drawing; RTL sheet cells, borders, headings, objects and sparklines mirror consistently | +| 2026-10-26 | Arabic PDF export shapes licensed glyph outlines and preserves searchable logical text; characters the font lacks, or a missing Arabic font, fall back to the standard fonts instead of failing the export | | 2026-10-26 | Arabic bidi display/editing preserve logical source text; per-cell reading direction is editable and survives undo, XLSX and conditional-format priority | | 2026-10-26 | Arabic text shows in cells and the interface through a system, fontconfig or craft-fonts Arabic face; workbook fonts keep their script fallbacks | | 2026-10-25 | AutoSum over a selection does what Excel does: a selected row gets one total to its right (in its last cell if that is empty), not a one-cell total under each cell; a block whose last column is empty gets each row's total there, one whose last row is empty each column's, and with both the corner gets the total of the block; otherwise totals go under each column as before | @@ -199,8 +201,13 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-13 | XLSX export writes a picture used on several sheets once, as one `xl/media` part every drawing points to (it wrote a copy per drawing: a 16-sheet workbook with one 112 KB logo grew from 0.4 to 1.9 MB) (#985) | | 2026-10-13 | XLSX: `[Content_Types].xml`, `_rels/.rels`, the workbook rels and `xl/workbook.xml` are the first zip entries (document properties last), so libmagic and PHP `finfo` detect the file type even with large properties (#1090) | | 2026-10-13 | Conditional formatting: a rule gathers range statistics only when it reads them (top/bottom, above average, duplicates, scales, bars, icons) and borrows them instead of copying them per cell, so the per-frame lookup of an expression or cell-value rule no longer visits its whole range (#1098, part 1) | +| 2026-10-13 | Grid: resizing several selected columns or rows previews on the dragged one and applies the size to all on release; Esc cancels (#1082) | | 2026-10-12 | Next/previous sheet navigation stops at the first/last visible worksheet instead of activating a hidden boundary sheet; the current selection is retained | +| 2026-10-12 | PDF writer: objects are written straight to the output, identical pictures are stored once, style operators are emitted only when they change, `q` nesting is limited to 28 and polyline joins no longer leak into later strokes (#816) | +| 2026-10-12 | Insert › Text › WordArt (`insert.wordArt`, `wordArt.set`): floating decorative text with 10 original styles, outline and shadow, in-place editing, XLSX and print (#784) | +| 2026-10-12 | Number formatting: one table of built-in formats (xlsx); numbers, dates and General render with one scratch buffer instead of per-token strings; output unchanged, checked against digests recorded from the old code (#817) | | 2026-10-21 | Macro engine ([macros.md](docs/macros.md)): typed, declarative macros saved in the workbook (a JSON field; hidden names in xlsx); declared inputs checked with messages that name the cells; tokens (`Input.col(2)`, `Row`, `Output`, `Name:Total`, `{{…}}` in formulas); one undo step with full rollback; a footprint dry run with ask, fail, overwrite, shift down or shift right; a policy that keeps macros away from files and the app, with approval for macros in opened files; Repeat, If and Stop steps; a debugger; a relative recorder; `macro.*` commands beside the unchanged Automate scripts | +| 2026-10-13 | Performance: a chart is resolved and rendered once per change of its workbook, zoom or fonts instead of on every frame; idle frames, hovering and scrolling reuse it (#1139, part of #1098) | | 2026-10-13 | Charts: work and memory are bounded by series x points (2,000,000 per chart: every series keeps its first points; 16 trendlines per chart), hairline columns and bars are merged per pixel column, series ranges are clipped before they are evaluated, the moving average is linear and rendering clean data no longer copies it (#1147) | | 2026-10-13 | Charts: one combo plot-kind rule for resolving and drawing; ticks of a scale off the step grid keep their own values (#1146) | | 2026-10-12 | Security: the `--control` server requires an auth token (random, kept in a 0600 file or chosen with `GRIDCRAFT_CONTROL_TOKEN`) on each connection's first line and closes bad connections (#777) | @@ -281,6 +288,8 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-31 | minor | Progress log: Recently Used and Insert Function categories | | 2026-10-31 | minor | Progress log: Function Library | | 2026-10-31 | minor | Progress log: shared CJK/Latin baselines in cells, editors and window titles | +| 2026-10-26 | minor | Progress log: PDF reading direction and RTL sheets | +| 2026-10-26 | minor | Progress log: Arabic PDF text | | 2026-10-26 | minor | Progress log: Arabic bidi display and editing | | 2026-10-26 | minor | Progress log: Arabic font fallback | | 2026-10-25 | minor | Progress log: AutoSum over a selected row, and a selection's empty last row and column | @@ -313,6 +322,10 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-13 | trivial | Progress log: shared pictures written once on XLSX export (#985) | | 2026-10-13 | minor | Progress log: conditional-format lookups no longer scan the range for rules without statistics (#1098) | | 2026-10-12 | minor | Progress log: hidden-sheet navigation at workbook boundaries | +| 2026-10-13 | trivial | Progress log: charts cached between frames (#1139) | +| 2026-10-12 | minor | Progress log: PDF writer cleanup (#816) | +| 2026-10-12 | minor | Catalog 276 → 277 / 292: Insert › WordArt (#784) | +| 2026-10-12 | minor | Numfmt cleanup: one built-in format table (gridcraft-xlsx), fewer allocations rendering numbers, dates and General, equality test against the previous output (#817); progress log row | | 2026-10-12 | minor | The control server requires an auth token and closes bad connections (#777); progress log row | | 2026-10-12 | minor | Rows auto-fit oversized fonts on open and whole-row/column formatting (#833); progress log row | | 2026-10-12 | minor | Progress log: multi-area row/column delete (#758) | diff --git a/apps/gridcraft-cli/Cargo.toml b/apps/gridcraft-cli/Cargo.toml index 21e958005..83c45d0f2 100644 --- a/apps/gridcraft-cli/Cargo.toml +++ b/apps/gridcraft-cli/Cargo.toml @@ -14,6 +14,7 @@ name = "gridcraft-cli" path = "src/main.rs" [dependencies] +gridcraft-sandbox = { workspace = true } gridcraft-engine = { workspace = true } gridcraft-mcp = { workspace = true } serde_json = { workspace = true } diff --git a/apps/gridcraft-cli/src/check.rs b/apps/gridcraft-cli/src/check.rs index 5fc37b14c..cfd583baf 100644 --- a/apps/gridcraft-cli/src/check.rs +++ b/apps/gridcraft-cli/src/check.rs @@ -214,7 +214,7 @@ pub fn main(args: &[String]) -> ExitCode { Err(_) => return usage(format!("--max-report needs a whole number, not `{n}`")), } } - let Discovery { files, failures: mut reports } = collect_inputs(&a.positional); + let Discovery { files, failures: mut reports, .. } = collect_inputs(&a.positional); reports.extend(files.iter().map(|p| { let path = p.to_string_lossy().into_owned(); match gridcraft_engine::io::read_file(&path) { @@ -233,6 +233,9 @@ pub fn main(args: &[String]) -> ExitCode { #[derive(Default)] struct Discovery { files: Vec, + /// Identity (dev+inode) of each collected file, so a rescan reached by a shallower path doesn't add it twice. + #[cfg(unix)] + file_ids: std::collections::HashSet<(u64, u64)>, failures: Vec, } @@ -257,11 +260,11 @@ impl Discovery { /// (`~$…`) are skipped. fn collect_inputs(inputs: &[String]) -> Discovery { let mut out = Discovery::default(); - let mut seen = std::collections::HashSet::new(); + let mut seen = std::collections::HashMap::new(); for i in inputs { let p = PathBuf::from(i); if p.is_dir() { - if !walk(&p, 0, &mut out, &mut seen) { + if !walk(&p, 0, false, &mut out, &mut seen) { break; } } else if !out.add_file(p) { @@ -277,19 +280,29 @@ fn collect_inputs(inputs: &[String]) -> Discovery { out } -/// Returns false only when the global file budget stops traversal. -fn walk(dir: &Path, depth: usize, out: &mut Discovery, seen: &mut std::collections::HashSet) -> bool { - // A directory is visited once by identity (dev+inode on unix): several links to the same - // directory are aliases, not exponential branches of the scan. Other platforms keep the - // depth cap only. +/// Returns false only when the global file budget stops traversal. `seen` holds the shallowest +/// depth each directory was scanned from; `rescan` is set below a directory reached again from a +/// shallower path (its depth-limited subtree may now fit), where already collected files are skipped. +#[cfg_attr(not(unix), allow(unused_mut))] +fn walk(dir: &Path, depth: usize, mut rescan: bool, out: &mut Discovery, seen: &mut std::collections::HashMap<(u64, u64), usize>) -> bool { + // A directory is visited once by identity (dev+inode on unix) unless reached from a shallower + // path: several links to the same directory are aliases, not exponential branches of the scan + // (each directory is scanned at most MAX_DEPTH + 1 times). Other platforms keep the depth cap only. #[cfg(unix)] { use std::os::unix::fs::MetadataExt as _; - if let Ok(m) = std::fs::metadata(dir) - && let Some(id) = m.dev().checked_mul(1 << 60).and_then(|d| d.checked_add(m.ino())) - && !seen.insert(id) - { - return true; + if let Ok(m) = std::fs::metadata(dir) { + let identity = (m.dev(), m.ino()); + // A directory past the depth limit is not scanned, so it is not recorded: asked for + // by name later, it is walked from depth 0. + match seen.get(&identity) { + Some(&shallowest) if shallowest <= depth => return true, + Some(_) => rescan = true, + None => {} + } + if depth <= MAX_DEPTH { + seen.insert(identity, depth); + } } } if depth > MAX_DEPTH { @@ -324,13 +337,20 @@ fn walk(dir: &Path, depth: usize, out: &mut Discovery, seen: &mut std::collectio } }; if metadata.is_dir() { - if !walk(&p, depth + 1, out, seen) { + if !walk(&p, depth + 1, rescan, out, seen) { + return false; + } + } else if p.extension().and_then(std::ffi::OsStr::to_str).is_some_and(|e| e.eq_ignore_ascii_case("xlsx") || e.eq_ignore_ascii_case("xlsm")) { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + if !out.file_ids.insert((metadata.dev(), metadata.ino())) && rescan { + continue; + } + } + if !out.add_file(p) { return false; } - } else if p.extension().and_then(std::ffi::OsStr::to_str).is_some_and(|e| e.eq_ignore_ascii_case("xlsx") || e.eq_ignore_ascii_case("xlsm")) - && !out.add_file(p) - { - return false; } } true diff --git a/apps/gridcraft-cli/src/check/tests.rs b/apps/gridcraft-cli/src/check/tests.rs index 57baa2198..d85cb3a3a 100644 --- a/apps/gridcraft-cli/src/check/tests.rs +++ b/apps/gridcraft-cli/src/check/tests.rs @@ -11,8 +11,8 @@ fn discovery_reports_a_directory_read_error() { let path = std::env::temp_dir().join(format!("gridcraft-check-not-a-directory-{}", std::process::id())); std::fs::write(&path, "not a directory").unwrap(); let mut discovery = Discovery::default(); - let mut seen = std::collections::HashSet::new(); - assert!(walk(&path, 0, &mut discovery, &mut seen)); + let mut seen = std::collections::HashMap::new(); + assert!(walk(&path, 0, false, &mut discovery, &mut seen)); assert!(discovery.files.is_empty()); assert_eq!(discovery.failures.len(), 1); assert_eq!(discovery.failures[0].path, path.to_string_lossy()); @@ -31,6 +31,47 @@ fn discovery_reports_omissions_only_after_the_file_limit() { assert!(discovery.failures[0].error.as_ref().unwrap().contains("file limit")); } +/// A new directory under the temp dir: `create_dir` fails on a name somebody else made first. +#[cfg(unix)] +fn fresh_dir(tag: &str) -> PathBuf { + let mut n = 0u32; + loop { + let dir = std::env::temp_dir().join(format!("gridcraft-check-{tag}-{}-{n}", std::process::id())); + match std::fs::create_dir(&dir) { + Ok(()) => return dir, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => n += 1, + Err(e) => panic!("{e}"), + } + } +} + +/// A directory named again by a deeper path, or by a link back to a parent, lists each workbook once. +#[cfg(unix)] +#[test] +fn discovery_lists_each_workbook_once_whatever_path_reaches_it() { + let dir = fresh_dir("once"); + let sub = dir.join("sub"); + std::fs::create_dir(&sub).unwrap(); + std::fs::write(dir.join("a.xlsx"), b"x").unwrap(); + std::fs::write(sub.join("b.xlsx"), b"x").unwrap(); + std::os::unix::fs::symlink(&dir, sub.join("loop")).unwrap(); + let names = |d: &Discovery| { + let mut v: Vec = d.files.iter().map(|p| p.file_name().unwrap().to_string_lossy().into_owned()).collect(); + v.sort(); + v + }; + // The link back to the top is one directory, not another branch of the scan. + let found = collect_inputs(&[dir.to_string_lossy().into_owned()]); + assert_eq!(names(&found), ["a.xlsx", "b.xlsx"]); + // `sub` named after its parent is reached from a shallower depth: scanned again, nothing listed twice. + let found = collect_inputs(&[dir.to_string_lossy().into_owned(), sub.to_string_lossy().into_owned()]); + assert_eq!(names(&found), ["a.xlsx", "b.xlsx"]); + // Named first, the same `sub` is the shallowest path and the parent's scan skips it. + let found = collect_inputs(&[sub.to_string_lossy().into_owned(), dir.to_string_lossy().into_owned()]); + assert_eq!(names(&found), ["a.xlsx", "b.xlsx"]); + let _ = std::fs::remove_dir_all(dir); +} + fn at(a1: &str) -> CellRef { CellRef::parse(a1).unwrap() } diff --git a/apps/gridcraft-cli/src/main.rs b/apps/gridcraft-cli/src/main.rs index 5170ba0c5..2c49c6033 100644 --- a/apps/gridcraft-cli/src/main.rs +++ b/apps/gridcraft-cli/src/main.rs @@ -11,6 +11,7 @@ //! gridcraft-cli functions [--json] [--search X] [--category C] //! gridcraft-cli mcp [--connect PORT] [--in FILE | --sample NAME] //! gridcraft-cli send [json] +//! gridcraft-cli sandbox-check [--read PATH]... [--exec PATH]... [-- PROGRAM ARGS...] //! gridcraft-cli version | --version //! ``` #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] @@ -30,6 +31,7 @@ macro_rules! outln { } mod check; +mod sandbox_check; use gridcraft_engine::Session; use serde_json::{Value, json}; @@ -49,6 +51,8 @@ const USAGE: &str = "usage: gridcraft-cli functions [--json] [--search X] [--category C] gridcraft-cli mcp [--connect PORT] [--in FILE | --sample NAME] gridcraft-cli send [json] + gridcraft-cli sandbox-check [--read PATH]... [--exec PATH]... [-- PROGRAM ARGS...] + whether this system confines programs (no program: a confined self-test) gridcraft-cli version"; /// stdout went away. A reader that stopped early (a closed pipe) ends the program quietly; any @@ -62,6 +66,11 @@ fn stdout_failed(e: std::io::Error) -> ! { } fn main() -> ExitCode { + // A confined launcher run (and its probes) comes first, before anything else starts. + let raw: Vec = std::env::args_os().skip(1).collect(); + if let Some(code) = gridcraft_sandbox::handle_args(&raw) { + return ExitCode::from(u8::try_from(code).unwrap_or(1)); + } let args: Vec = std::env::args().skip(1).collect(); let rest = args.get(1..).unwrap_or_default(); let r = match args.first().map(String::as_str) { @@ -79,6 +88,7 @@ fn main() -> ExitCode { Some("cat") => cat(rest), Some("run") => run(rest), Some("check") => return check::main(rest), + Some("sandbox-check") => return sandbox_check::main(rest), Some("commands") => commands(rest), Some("functions") => functions(rest), Some("mcp") => mcp(rest), diff --git a/apps/gridcraft-cli/src/sandbox_check.rs b/apps/gridcraft-cli/src/sandbox_check.rs new file mode 100644 index 000000000..e40bb9ac5 --- /dev/null +++ b/apps/gridcraft-cli/src/sandbox_check.rs @@ -0,0 +1,98 @@ +//! `gridcraft-cli sandbox-check [--read PATH]... [--exec PATH]... [-- PROGRAM ARGS...]`: whether +//! this system confines programs, for a person or a reviewer checking a machine. +//! +//! Without a program it starts a confined copy of itself that tries what the sandbox must deny +//! (reading outside the policy, writing, the network, starting a process) and what it must still +//! allow (a thread, reading an allowed file), and prints one line per attempt. With a program it +//! runs that program confined, with the given read and executable paths and the system's shared +//! libraries, and passes its output and exit status through; its own lines then go to standard +//! error. Every path is absolute. Exit status: 0 when every attempt is +//! as required (or the program succeeded), 1 otherwise (a refused launch included), 2 when GridCraft +//! has no sandbox for this kind of system. + +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::ExitCode; + +use gridcraft_sandbox::{Canary, Policy, REFUSED_EXIT_CODE, REPORT_FLAG, Support}; + +pub fn main(args: &[String]) -> ExitCode { + match run(args) { + Ok(code) => code, + Err(e) => { + eprintln!("gridcraft-cli: sandbox-check: {e}"); + ExitCode::FAILURE + } + } +} + +fn run(args: &[String]) -> Result { + let mut policy = Policy::new().with_system_libraries(); + let mut it = args.iter(); + let mut program: Option<(PathBuf, Vec)> = None; + while let Some(a) = it.next() { + match a.as_str() { + "--read" => policy = policy.read(it.next().ok_or("--read needs a path")?), + "--exec" => policy = policy.exec(it.next().ok_or("--exec needs a path")?), + "--" => { + let p = it.next().ok_or("a program follows `--`")?; + program = Some((PathBuf::from(p), it.by_ref().map(OsString::from).collect())); + } + other => return Err(format!("unknown option `{other}`")), + } + } + // With a program, its standard output is passed through untouched, so this command's own + // lines go to standard error; the self-test's report is the output, on standard output. + let report = program.is_none(); + let say = |line: String| { + if report { + outln!("{line}"); + } else { + eprintln!("{line}"); + } + }; + match gridcraft_sandbox::support() { + Support::Available(how) => say(format!("sandbox: available ({how})")), + Support::Unsupported(why) => { + say(format!("sandbox: unsupported ({why})")); + return Ok(ExitCode::from(2)); + } + } + let canary = Canary::create().map_err(|e| e.to_string())?; + let exe = std::env::current_exe().map_err(|e| e.to_string())?; + let (program, program_args) = match program { + Some((p, a)) => (p, a), + // A confined copy of this program, reporting each attempt; it reads its own executable as + // the allowed file. + None => { + let home = std::env::var_os("HOME").unwrap_or_default(); + (exe.clone(), vec![REPORT_FLAG.into(), canary.file().into(), exe.clone().into(), home]) + } + }; + let policy = policy.exec(program.clone()); + let mut cmd = gridcraft_sandbox::launch_command(&policy, &canary, &program, &program_args).map_err(|e| e.to_string())?; + let status = cmd.status().map_err(|e| format!("couldn't start the launcher: {e}"))?; + drop(canary); + match status.code() { + Some(0) => { + say(if report { "result: every attempt is as the sandbox requires" } else { "result: the program ran confined and succeeded" }.into()); + Ok(ExitCode::SUCCESS) + } + Some(REFUSED_EXIT_CODE) => { + // The launcher's refusal and a program's own exit share the code; the launcher's reason, + // when it refused, is the `gridcraft-sandbox: refused:` line above. + say(format!( + "result: exit code {REFUSED_EXIT_CODE}: the launcher refused to run the program (its reason is above), or the program itself exited with {REFUSED_EXIT_CODE}" + )); + Ok(ExitCode::FAILURE) + } + other => { + say(format!( + "result: {} ({})", + if report { "some attempts are not as the sandbox requires" } else { "the program failed" }, + other.map_or("no exit code".into(), |c| format!("exit code {c}")) + )); + Ok(ExitCode::FAILURE) + } + } +} diff --git a/apps/gridcraft-cli/tests/it/cli.rs b/apps/gridcraft-cli/tests/it/cli.rs index dde018ca8..e59295bc4 100644 --- a/apps/gridcraft-cli/tests/it/cli.rs +++ b/apps/gridcraft-cli/tests/it/cli.rs @@ -205,6 +205,44 @@ fn check_reports_a_depth_limited_scan_while_checking_readable_files() { let _ = std::fs::remove_dir_all(dir); } +#[cfg(unix)] +#[test] +fn a_directory_past_the_depth_limit_is_still_scanned_when_named_as_an_input() { + let dir = tmpdir("check-depth-named"); + // The first folder past the limit (33 levels down): the scan from the top reports it unreached. + let mut deep = dir.clone(); + for _ in 0..33 { + deep.push("d"); + } + std::fs::create_dir_all(&deep).unwrap(); + ok(&["run", "--out", deep.join("deep.xlsx").to_str().unwrap(), "--quiet"]); + let out = run(&["check", dir.to_str().unwrap(), deep.to_str().unwrap(), "--json"]); + let report: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap(); + // The first scan reports the subtree it could not reach; the named directory is checked. + assert_eq!(report["totals"]["opened"], 1, "{report}"); + let _ = std::fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn a_directory_first_reached_at_the_depth_limit_is_completed_when_named_as_an_input() { + let dir = tmpdir("check-depth-shallower"); + // The 32nd folder is scanned from the top, but its child (and the workbook in it) is past the limit. + let mut limit = dir.clone(); + for _ in 0..32 { + limit.push("d"); + } + let child = limit.join("c"); + std::fs::create_dir_all(&child).unwrap(); + ok(&["run", "--out", limit.join("at_limit.xlsx").to_str().unwrap(), "--quiet"]); + ok(&["run", "--out", child.join("below.xlsx").to_str().unwrap(), "--quiet"]); + let out = run(&["check", dir.to_str().unwrap(), limit.to_str().unwrap(), "--json"]); + let report: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap(); + // Named from depth 0 the subtree is read: the workbook below the limit is checked, the one at it only once. + assert_eq!(report["totals"]["opened"], 2, "{report}"); + let _ = std::fs::remove_dir_all(dir); +} + /// A script line whose parameters are under a key `run` does not read used to run the command /// with no parameters at all, and the run still exited 0: three mistaken lines made Sheet2, /// Sheet3 and Sheet4 instead of the named sheets, and nothing said so (#2256). diff --git a/apps/gridcraft-cli/tests/it/main.rs b/apps/gridcraft-cli/tests/it/main.rs index 5906b3967..2078aedfd 100644 --- a/apps/gridcraft-cli/tests/it/main.rs +++ b/apps/gridcraft-cli/tests/it/main.rs @@ -1,3 +1,4 @@ //! The crate's integration tests, one module per file, built as one test binary. mod cli; +mod sandbox; mod text_to_columns_quotes; diff --git a/apps/gridcraft-cli/tests/it/sandbox.rs b/apps/gridcraft-cli/tests/it/sandbox.rs new file mode 100644 index 000000000..9c6bd5cb7 --- /dev/null +++ b/apps/gridcraft-cli/tests/it/sandbox.rs @@ -0,0 +1,490 @@ +//! The confined launcher, through the real binary: what a confined program can't do, what it +//! still can, and that a launch whose check fails runs nothing. Linux and macOS confine; other +//! systems report that they can't. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +/// Whether this machine confines programs: a supported system whose self-test passes. On a Linux +/// or macOS machine that reports no sandbox (another architecture, no `sandbox-exec`) or whose +/// kernel doesn't enforce Landlock, the confinement tests have nothing to check and pass with a +/// note. On CI (`CI` set) the machine must confine, so a failure there is never skipped. +fn confines() -> bool { + static CONFINES: std::sync::OnceLock = std::sync::OnceLock::new(); + *CONFINES.get_or_init(|| { + if !cfg!(any(target_os = "linux", target_os = "macos")) { + return false; + } + let o = cli(&["sandbox-check"]); + let out = text(&o); + let missing = out.contains("sandbox: unsupported") || out.contains("Landlock is not enforced"); + if missing && std::env::var_os("CI").is_none() { + eprintln!("this machine doesn't confine programs; skipping:\n{out}"); + return false; + } + true + }) +} + +fn cli(args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")).args(args).output().unwrap() +} + +fn text(o: &Output) -> String { + format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)) +} + +/// A directory of the test's own in the temporary directory, with one file. +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().canonicalize().unwrap().join(format!("gridcraft-sandbox-it-{name}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("allowed.txt"), "allowed text\n").unwrap(); + dir +} + +/// `cat` as this system names it. +fn cat() -> &'static str { + if Path::new("/bin/cat").exists() { "/bin/cat" } else { "/usr/bin/cat" } +} + +#[test] +fn the_self_test_finds_every_denial() { + // A home directory of the test's own, so the home-directory attempt always runs. + let home = scratch("home"); + let o = Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")).arg("sandbox-check").env("HOME", &home).output().unwrap(); + let out = text(&o); + if !cfg!(any(target_os = "linux", target_os = "macos")) { + assert_eq!(o.status.code(), Some(2), "{out}"); + assert!(out.contains("sandbox: unsupported"), "{out}"); + return; + } + if !confines() { + return; + } + assert!(o.status.success(), "{out}"); + for line in [ + "denied: read the canary", + "denied: change the canary's permissions", + "denied: truncate the canary", + "denied: write next to the canary", + "denied: write in the temporary directory", + "denied: TCP connection", + "denied: start a process", + "denied: signal another process", + "denied: UDP socket", + "denied: list the home directory", + "works: start a thread", + "works: read an allowed file", + ] { + assert!(out.contains(line), "missing `{line}` in:\n{out}"); + } + #[cfg(target_os = "linux")] + assert!(out.contains("denied: switch a descriptor's notifications on"), "missing the notification switch in:\n{out}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_confined_program_reads_only_what_it_is_given() { + if !confines() { + return; + } + let dir = scratch("read"); + let outside = scratch("outside"); + let d = dir.to_string_lossy().into_owned(); + let allowed = dir.join("allowed.txt").to_string_lossy().into_owned(); + let o = cli(&["sandbox-check", "--read", &d, "--", cat(), &allowed]); + assert!(o.status.success(), "{}", text(&o)); + // The program's output is passed through untouched; this command's own lines go to stderr. + assert_eq!(String::from_utf8_lossy(&o.stdout), "allowed text\n", "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stderr).contains("result: the program ran confined"), "{}", text(&o)); + // The same program, a file outside the policy: the read is refused and nothing is printed. + let other = outside.join("allowed.txt").to_string_lossy().into_owned(); + let o = cli(&["sandbox-check", "--read", &d, "--", cat(), &other]); + assert!(!o.status.success(), "{}", text(&o)); + assert!(!String::from_utf8_lossy(&o.stdout).contains("allowed text"), "{}", text(&o)); + std::fs::remove_dir_all(dir).unwrap(); + std::fs::remove_dir_all(outside).unwrap(); +} + +#[test] +fn a_launch_whose_check_fails_runs_nothing() { + // A run that says it is already confined and isn't finds the canary readable and must refuse. + // On macOS that is the second stage started without `sandbox-exec`; on Linux the launcher never + // uses that flag, and the same refusal must hold. + let canary = scratch("canary"); + let echo = if Path::new("/bin/echo").exists() { "/bin/echo" } else { "/usr/bin/echo" }; + let probe = canary.join("allowed.txt").to_string_lossy().into_owned(); + // A dynamically linked `echo` needs the system's shared libraries and its loader. + // The second stage takes the process the first looked at: here, the parent of what `cli` starts. + let mut launch = vec![ + "--sandbox-confined".to_string(), + std::process::id().to_string(), + "--exec".into(), + echo.into(), + "--exec".into(), + env!("CARGO_BIN_EXE_gridcraft-cli").into(), + ]; + for lib in ["/lib", "/lib64", "/usr/lib", "/usr/lib64"].into_iter().filter(|d| Path::new(d).exists()) { + launch.extend(["--exec".to_string(), lib.into()]); + } + launch.extend(["--probe".to_string(), probe.clone(), "--".into(), echo.into(), "the program ran".into()]); + let o = cli(&launch.iter().map(String::as_str).collect::>()); + let out = text(&o); + assert_eq!(o.status.code(), Some(97), "{out}"); + assert!(!out.contains("the program ran"), "{out}"); + assert!(out.contains("gridcraft-sandbox: refused"), "{out}"); + assert!(out.contains("read the canary"), "the check refused, not the arguments: {out}"); + // Without that process, the second stage has nothing to try a signal on. + let without: Vec<&str> = std::iter::once("--sandbox-confined").chain(launch[2..].iter().map(String::as_str)).collect(); + let o = cli(&without); + assert_eq!(o.status.code(), Some(97), "{}", text(&o)); + assert!(text(&o).contains("needs the process"), "{}", text(&o)); + // The normal path: the launcher confines itself first, and the program runs. + if confines() { + launch.splice(0..2, ["--sandbox-launch".to_string()]); + // The launcher may be under a folder in the list, on either system. + let mut by_folder = launch.clone(); + by_folder[4] = Path::new(env!("CARGO_BIN_EXE_gridcraft-cli")).parent().unwrap().to_string_lossy().into_owned(); + let o = cli(&by_folder.iter().map(String::as_str).collect::>()); + assert!(o.status.success(), "{}", text(&o)); + assert!(text(&o).contains("the program ran"), "{}", text(&o)); + let o = cli(&launch.iter().map(String::as_str).collect::>()); + let out = text(&o); + assert!(o.status.success() && out.contains("the program ran"), "{out}"); + } + // A canary inside an allowed path is refused before anything runs, everywhere. + let dir = canary.to_string_lossy().into_owned(); + let o = cli(&["--sandbox-launch", "--read", &dir, "--exec", echo, "--probe", &probe, "--", echo, "the program ran"]); + assert_eq!(o.status.code(), Some(97), "{}", text(&o)); + assert!(!text(&o).contains("the program ran")); + std::fs::remove_dir_all(canary).unwrap(); +} + +/// A Python found on this system: the interpreter to run confined, and its installation, the path it +/// needs to read. `None` when there is no `python3`. +fn system_python() -> Option<(String, String)> { + // On macOS `/usr/bin/python3` is a stub that starts Xcode's Python, outside any read path given + // here; an installed Python comes first. + let candidates = if cfg!(target_os = "macos") { + ["/opt/homebrew/bin/python3", "/usr/local/bin/python3"] + } else { + ["/usr/bin/python3", "/usr/local/bin/python3"] + }; + let python = candidates.into_iter().find(|p| Path::new(p).exists())?; + let prefix = Command::new(python).args(["-c", "import sys; print(sys.base_prefix)"]).output().unwrap(); + let prefix = String::from_utf8_lossy(&prefix.stdout).trim().to_string(); + // A framework build's `bin/python3` only starts `Python.app`'s interpreter as a new process, + // which the sandbox refuses; the interpreter itself is run instead. + let framework = Path::new(&prefix).join("Resources/Python.app/Contents/MacOS/Python"); + let real = if framework.exists() { framework } else { std::fs::canonicalize(python).unwrap() }; + Some((real.to_string_lossy().into_owned(), prefix)) +} + +/// Confined, a program can't signal another process (this test, signal 0 delivering nothing), but can +/// still signal itself and its own threads. On Linux, where the filter names them, it also can't +/// change another process's limits or priority, or type into a terminal: no call here reaches a real +/// terminal, since its standard input isn't one. Linux also refuses the `fcntl` commands that would +/// have the kernel signal a process for it: naming the owner, choosing the signal, or switching the +/// notification on. +#[test] +fn a_confined_program_leaves_other_processes_alone() { + if !confines() { + return; + } + let Some((python, prefix)) = system_python() else { + return; + }; + let code = r#" +import os, signal, sys, threading +other = int(sys.argv[1]) +def refused(call, *args): + try: + call(*args) + except PermissionError: + return True + except OSError as e: + print("other error", call, args, e) + return False +assert refused(os.kill, other, 0), "a signal to another process" +os.kill(os.getpid(), 0) +seen = [] +signal.signal(signal.SIGUSR1, lambda *_: seen.append(1)) +signal.raise_signal(signal.SIGUSR1) +worker = threading.Thread(target=lambda: signal.pthread_kill(threading.main_thread().ident, signal.SIGUSR1)) +worker.start() +worker.join() +assert len(seen) >= 1, "a signal to itself" +if sys.platform == "linux": + import ctypes, errno, fcntl, resource, termios + libc = ctypes.CDLL(None, use_errno=True) + assert libc.shmget(0, 4096, 0o600) == -1 and ctypes.get_errno() == errno.EPERM, "System V shared memory" + assert libc.msgget(0, 0o600) == -1 and ctypes.get_errno() == errno.EPERM, "a System V message queue" + assert libc.semget(0, 1, 0o600) == -1 and ctypes.get_errno() == errno.EPERM, "a System V semaphore" + if hasattr(libc, "mq_open"): + assert libc.mq_open(b"/gridcraft-check", os.O_RDONLY) == -1 and ctypes.get_errno() == errno.EPERM, "a POSIX message queue" + # macOS answers these for the processes the profile lets it signal, which are only itself. + assert refused(os.kill, -1, 0), "a signal to every process" + assert refused(os.kill, 0, 0), "a signal to the process group" + assert refused(resource.prlimit, other, resource.RLIMIT_NOFILE), "another process's limits" + assert refused(os.setpriority, os.PRIO_PROCESS, other, 0), "another process's priority" + assert refused(os.setpriority, os.PRIO_USER, 0, 0), "every process's priority" + assert refused(os.sched_setaffinity, other, {0}), "another process's processors" + assert refused(os.pidfd_open, other), "a handle on another process" + read, write = os.pipe() + assert refused(fcntl.fcntl, write, fcntl.F_SETOWN, other), "giving a descriptor's notifications to another process" + # 15 is F_SETOWN_EX and 10 is F_SETSIG, which Python's fcntl doesn't name; the filter refuses + # them whatever owner or signal they name, so the argument doesn't matter. + assert refused(fcntl.fcntl, write, 15), "naming a descriptor's owner the extended way" + assert refused(fcntl.fcntl, write, 10), "choosing a descriptor's notification signal" + assert refused(fcntl.fcntl, write, fcntl.F_SETFL, fcntl.fcntl(write, fcntl.F_GETFL) | os.O_ASYNC), "switching a descriptor's notifications on" + resource.prlimit(0, resource.RLIMIT_NOFILE) + resource.prlimit(os.getpid(), resource.RLIMIT_NOFILE) + os.setpriority(os.PRIO_PROCESS, 0, os.getpriority(os.PRIO_PROCESS, 0)) + assert refused(fcntl.ioctl, 0, termios.TIOCSTI, b"x"), "typing into a terminal" + assert refused(fcntl.ioctl, 0, termios.TIOCSTI | (1 << 32), b"x"), "the same with high bits set" + assert refused(fcntl.ioctl, 0, 0x541C, b"x"), "TIOCLINUX" +print("left alone") +"#; + let me = std::process::id().to_string(); + let o = cli(&["sandbox-check", "--read", &prefix, "--", &python, "-I", "-c", code, &me]); + assert!(o.status.success(), "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stdout).contains("left alone"), "{}", text(&o)); +} + +/// A socket the parent made, which the program inherited as a standard stream, can't be pointed at +/// another process through the ioctls that do what `fcntl`'s notification commands do: `FIOSETOWN` +/// (0x8901) and `FIOASYNC` (0x5452), which Python's `fcntl` passes straight through. Linux only: +/// macOS's profile has no rule for them, and its standard streams here are pipes. +#[cfg(target_os = "linux")] +#[test] +fn a_confined_program_cant_point_an_inherited_socket_at_another_process() { + if !confines() { + return; + } + let Some((python, prefix)) = system_python() else { + return; + }; + let code = r#" +import fcntl, sys +other = int(sys.argv[1]) +def refused(call, *args): + try: + call(*args) + except PermissionError: + return True + except OSError as e: + print("other error", e) + return False +assert refused(fcntl.ioctl, 0, 0x8901, other), "giving a socket's notifications to another process" +assert refused(fcntl.ioctl, 0, 0x5452, 1), "switching a socket's notifications on" +print("socket quiet") +"#; + let (mine, _) = std::os::unix::net::UnixStream::pair().unwrap(); + let socket: std::os::fd::OwnedFd = mine.into(); + let me = std::process::id().to_string(); + let o = Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")) + .args(["sandbox-check", "--read", &prefix, "--", &python, "-I", "-c", code, &me]) + .stdin(std::process::Stdio::from(socket)) + .output() + .unwrap(); + let out = text(&o); + assert!(o.status.success(), "{out}"); + assert!(String::from_utf8_lossy(&o.stdout).contains("socket quiet"), "{out}"); +} + +/// The same inherited socket, by both ioctls that name the process to notify, `FIOSETOWN` (0x8901) +/// and `SIOCSPGRP` (0x8902), with the argument a real pointer. Named by value the kernel reads the +/// number as an address and fails with `EFAULT` whether or not the filter refuses it, so that form +/// shows only that a rule matched a request number; by pointer each call succeeds unconfined and is +/// `EPERM` confined, which is the transition the page claims. Kept apart from the test above so a +/// rule dropped for one request number fails for that request, not behind another. Linux only, as +/// those rules are: macOS's profile has none, and its standard streams here are pipes. +#[cfg(target_os = "linux")] +#[test] +fn a_confined_program_cant_point_an_inherited_socket_at_another_process_by_pointer() { + if !confines() { + return; + } + let Some((python, prefix)) = system_python() else { + return; + }; + let code = r#" +import fcntl, socket, struct, sys +other = int(sys.argv[1]) +def refused(call, *args): + try: + call(*args) + except PermissionError: + return True + except OSError as e: + print("other error", e) + return False +# Read first, so a standard stream that is not a socket can't pass for one: a pipe has no socket name. +# Detached, not dropped: a socket object closes the descriptor it wrapped when it is collected, and then +# the refusals below would be read off an invalid descriptor. +wrapped = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, fileno=0) +name = wrapped.getsockname() +wrapped.detach() +assert name == "" +# The process group is a pointer, as a C caller passes it; naming it by value has the kernel read the +# number as an address and fail with EFAULT, which would leave the refusal unexplained by the filter. +group = bytearray(struct.pack("i", other)) +assert refused(fcntl.ioctl, 0, 0x8901, group), "giving a socket's notifications to another process" +assert refused(fcntl.ioctl, 0, 0x8902, group), "giving a socket's notifications to another process's group" +print("socket owner quiet") +"#; + let (mine, _) = std::os::unix::net::UnixStream::pair().unwrap(); + let socket: std::os::fd::OwnedFd = mine.into(); + let me = std::process::id().to_string(); + let o = Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")) + .args(["sandbox-check", "--read", &prefix, "--", &python, "-I", "-c", code, &me]) + .stdin(std::process::Stdio::from(socket)) + .output() + .unwrap(); + let out = text(&o); + assert!(o.status.success(), "{out}"); + assert!(String::from_utf8_lossy(&o.stdout).contains("socket owner quiet"), "{out}"); +} + +/// Confined, a program changes no file, whether the policy lets it read that file or not, by path or +/// by a descriptor: not its contents, permissions, owner, extended attributes, times or flags. +/// Skipped when there is no `python3`. +#[test] +fn a_confined_program_changes_no_file() { + if !confines() { + return; + } + let Some((python, prefix)) = system_python() else { + return; + }; + let code = r#" +import errno, fcntl, os, struct, sys +readable, outside = sys.argv[1:3] +def refused(what, call, *args): + try: + call(*args) + except PermissionError: + return + except OSError as e: + sys.exit(f"{what}: another error, not a refusal: {e}") + sys.exit(f"{what}: it worked") +fd = os.open(readable, os.O_RDONLY) +for target in (readable, outside, fd): + refused(f"permissions of {target}", os.chmod, target, 0) + refused(f"owner of {target}", os.chown, target, os.getuid(), os.getgid()) + refused(f"times of {target}", os.utime, target, (0, 0)) + if sys.platform == "linux": + refused(f"setting an attribute of {target}", os.setxattr, target, "user.gridcraft", b"1") + refused(f"removing an attribute of {target}", os.removexattr, target, "user.gridcraft") +for path in (readable, outside): + refused(f"truncate {path}", os.truncate, path, 0) + refused(f"open {path} with O_TRUNC", os.open, path, os.O_RDONLY | os.O_TRUNC) + refused(f"open {path} to write", os.open, path, os.O_WRONLY) +# A descriptor opened to read can't be truncated either way; the file's contents are checked after. +try: + os.ftruncate(fd, 0) +except OSError: + pass +if sys.platform == "linux": + import ctypes + libc = ctypes.CDLL(None, use_errno=True) + # Newer calls, which the kernel running this may not have: the filter answers before it would. + for name, number in (("fchmodat2", 452), ("setxattrat", 463), ("removexattrat", 466), ("file_setattr", 469)): + ctypes.set_errno(0) + result = libc.syscall(number, -100, readable.encode(), 0, 0) + assert result == -1 and ctypes.get_errno() == errno.EPERM, f"{name}: {result}, {ctypes.get_errno()}" + # On a descriptor opened only to read, these change the file's flags and attributes. + refused("FS_IOC_SETFLAGS", fcntl.ioctl, fd, 0x40086602, struct.pack("l", 0)) + refused("FS_IOC_FSSETXATTR", fcntl.ioctl, fd, 0x401C5820, bytes(28)) + refused("FS_IOC_SET_ENCRYPTION_POLICY", fcntl.ioctl, fd, 0x800C6613, bytes(12)) + # What a program still needs: its own descriptors' flags, and a file of its own in memory. + os.set_inheritable(fd, True) + os.set_inheritable(fd, False) + os.set_blocking(fd, False) + own = os.memfd_create("gridcraft-check") + os.ftruncate(own, 16) + assert os.fstat(own).st_size == 16 +print("unchanged") +"#; + let inside = scratch("unchanged-inside"); + let elsewhere = scratch("unchanged-outside"); + let (readable, outside) = (inside.join("allowed.txt"), elsewhere.join("allowed.txt")); + let state = |file: &Path| { + let m = std::fs::metadata(file).unwrap(); + (std::fs::read(file).unwrap(), m.permissions(), m.modified().unwrap()) + }; + let before = (state(&readable), state(&outside)); + let d = inside.to_string_lossy().into_owned(); + let (r, x) = (readable.to_string_lossy().into_owned(), outside.to_string_lossy().into_owned()); + let o = cli(&["sandbox-check", "--read", &prefix, "--read", &d, "--", &python, "-I", "-c", code, &r, &x]); + assert!(o.status.success(), "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stdout).contains("unchanged"), "{}", text(&o)); + assert_eq!((state(&readable), state(&outside)), before); + std::fs::remove_dir_all(inside).unwrap(); + std::fs::remove_dir_all(elsewhere).unwrap(); +} + +/// A Python found on this system starts and imports its standard library while confined, with +/// read access to its installation only. Skipped when there is no `python3`. +#[test] +fn python_starts_confined() { + if !confines() { + return; + } + let Some((real, prefix)) = system_python() else { + return; + }; + let o = + cli(&["sandbox-check", "--read", &prefix, "--", &real, "-I", "-c", "import json, math, datetime, decimal; print(json.dumps([math.pi > 3]))"]); + assert!(o.status.success(), "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stdout).contains("[true]"), "{}", text(&o)); + // Confined Python can't open the network or a file outside its installation. + let o = cli(&["sandbox-check", "--read", &prefix, "--", &real, "-I", "-c", "import socket; socket.create_connection(('127.0.0.1', 9), 2)"]); + assert!(!o.status.success(), "{}", text(&o)); + assert!(text(&o).contains("PermissionError") || text(&o).contains("Operation not permitted"), "{}", text(&o)); +} + +#[test] +fn a_confined_program_inherits_no_secrets() { + if !confines() { + return; + } + let env = if Path::new("/usr/bin/env").exists() { "/usr/bin/env" } else { "/bin/env" }; + let o = Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")) + .args(["sandbox-check", "--", env]) + .env("GRIDCRAFT_CONTROL_TOKEN", "secret-token") + .env("AWS_SECRET_ACCESS_KEY", "secret-key") + .env("LANG", "en_US.UTF-8") + .output() + .unwrap(); + let out = text(&o); + assert!(o.status.success(), "{out}"); + assert!(!out.contains("secret-"), "the environment reached the program:\n{out}"); + assert!(out.contains("LANG=en_US.UTF-8"), "the locale is kept:\n{out}"); + assert!(out.contains("PATH=/usr/bin:/bin"), "{out}"); +} + +/// A descriptor the parent leaves open (here fd 9, opened by a shell on a file outside any policy) +/// would survive `exec` with access the sandbox doesn't revoke, so the launcher closes it: the +/// program runs, and can't read through it. (A desktop app's system libraries leave such files open, +/// so refusing to run would block Python there.) +#[test] +fn an_inherited_descriptor_never_reaches_the_program() { + if !confines() { + return; + } + let outside = scratch("leak"); + let leaked = outside.join("allowed.txt").to_string_lossy().into_owned(); + let cli = env!("CARGO_BIN_EXE_gridcraft-cli"); + // The shell opens fd 9 (not close-on-exec) and then becomes the command, which inherits it. The + // confined shell then tries to read through it, with a built-in: starting `cat` is refused. Not + // fd 3: on macOS the system's libraries hold a socket of their own there under the sandbox. + let bash = if cfg!(target_os = "macos") { "--exec /bin/bash" } else { "" }; + let program = "read line <&9 && echo \"leaked: $line\"; echo finished"; + let script = format!("exec 9<'{leaked}'; exec '{cli}' sandbox-check {bash} -- /bin/sh -c '{program}'"); + let o = Command::new("/bin/sh").args(["-c", &script]).output().unwrap(); + let out = text(&o); + assert!(out.contains("finished"), "the program ran:\n{out}"); + assert!(!out.contains("leaked:"), "the program read through the descriptor:\n{out}"); + std::fs::remove_dir_all(outside).unwrap(); +} diff --git a/apps/gridcraft-web/Cargo.toml b/apps/gridcraft-web/Cargo.toml index 09c6d7e2a..2eb51df3c 100644 --- a/apps/gridcraft-web/Cargo.toml +++ b/apps/gridcraft-web/Cargo.toml @@ -18,8 +18,8 @@ gridcraft-engine = { workspace = true } gridcraft-ui-egui = { workspace = true } # The host messages (storytold/craft-libs): one contract and one bridge for the web builds. # The page command API (storytold/craft-libs#413): window.gridcraft over the control channel. -craft-web-api = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } -craft-web-host = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } +craft-web-api = { git = "https://github.com/storytold/craft-libs", rev = "aa84415bae3b765fef6b7ef42404f66ad1cff9d9", version = "0.1.0" } +craft-web-host = { git = "https://github.com/storytold/craft-libs", rev = "aa84415bae3b765fef6b7ef42404f66ad1cff9d9", version = "0.1.0" } egui = { workspace = true } log = { workspace = true } serde_json = { workspace = true } diff --git a/apps/gridcraft/Cargo.toml b/apps/gridcraft/Cargo.toml index 75acee3ff..72f5d0855 100644 --- a/apps/gridcraft/Cargo.toml +++ b/apps/gridcraft/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] gridcraft-engine = { workspace = true } gridcraft-ui-egui = { workspace = true } +gridcraft-sandbox = { workspace = true } eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11", "clipboard"] } egui = { workspace = true } serde_json = { workspace = true } @@ -34,7 +35,7 @@ winit = { version = "0.30", default-features = false, features = ["x11", "waylan [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Same craft-libs revision as the workspace's craft-update and craft-window (see the root # Cargo.toml). -craft-print = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d" } +craft-print = { git = "https://github.com/storytold/craft-libs", rev = "aa84415bae3b765fef6b7ef42404f66ad1cff9d9" } craft-update = { workspace = true } rfd = { workspace = true } arboard = { workspace = true, features = ["image-data"] } diff --git a/apps/gridcraft/src/main.rs b/apps/gridcraft/src/main.rs index cc67e241a..f5232b17c 100644 --- a/apps/gridcraft/src/main.rs +++ b/apps/gridcraft/src/main.rs @@ -241,6 +241,12 @@ fn icon() -> Option { } fn main() -> eframe::Result<()> { + // A confined launcher run (`gridcraft_sandbox::launch_command`) comes before everything else: + // it opens no window, writes no log, and becomes the program it confines. + let raw: Vec = std::env::args_os().skip(1).collect(); + if let Some(code) = gridcraft_sandbox::handle_args(&raw) { + std::process::exit(code); + } // First, so the panic hook and every start-up warning are recorded (`logging`). let logger = logging::install(); // Commands catch their panics (`Session::execute`), but the default hook only prints to @@ -334,12 +340,16 @@ fn main() -> eframe::Result<()> { for f in &files { app.open_path(f); } + // Linux/BSD: Excel's layout — no window-manager title bar above ours; our title bar carries the + // window controls. `GRIDCRAFT_SYSTEM_TITLEBAR=1` keeps the system frame. + app.window_controls = cfg!(not(any(target_os = "macos", target_os = "windows"))) && std::env::var_os("GRIDCRAFT_SYSTEM_TITLEBAR").is_none(); let mut viewport = egui::ViewportBuilder::default() .with_title("GridCraft") .with_inner_size([1440.0, 900.0]) .with_min_inner_size([640.0, 420.0]) .with_app_id("ai.storyteller.gridcraft") - .with_drag_and_drop(true); + .with_drag_and_drop(true) + .with_decorations(!app.window_controls); if cfg!(target_os = "macos") { viewport = viewport.with_fullsize_content_view(true).with_titlebar_shown(false).with_title_shown(false); } diff --git a/crates/calc/src/eval.rs b/crates/calc/src/eval.rs index 2fda4442b..525440474 100644 --- a/crates/calc/src/eval.rs +++ b/crates/calc/src/eval.rs @@ -39,6 +39,15 @@ const MAX_DEPTH: usize = 400; /// every legitimate call; hostile work that grows past one dimension (a LAMBDA that recurses /// or nest `MAP` into `MAP`) then stops at the same bound instead of squaring. const LAMBDA_FUEL: u64 = MAX_CELLS; +/// Work one cell's formula may do: one unit per expression evaluated, per cell read from a +/// reference, per array value handed to a function, per pair of areas intersected and per +/// character of a defined name's formula parsed where the name is used. Each limit +/// above bounds one kind of step, but a compact formula could still repeat bounded steps +/// exponentially often (defined names that each use the one before twice, a computed criterion +/// whose database function reruns it, an aggregate over a large array inside every LAMBDA call). +/// Past this the formula is `#CALC!`. It is far above what a formula that reads every cell of a +/// full-size array a few times needs. +const EVAL_WORK: u64 = 16 * MAX_CELLS; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Area { @@ -145,6 +154,16 @@ pub struct Evaluator<'h> { /// runs out of work after the same number of calls that a legitimately wide one could /// make, instead of an amount of work that squares with array nesting (#2382). fuel: u64, + /// Work this cell still has left (see [`EVAL_WORK`]), shared by every branch of the formula + /// and by the formulas it computes on the way (names, computed criteria). + work: u64, + /// A spend found too little work left. A spend that leaves exactly nothing is fine, so this + /// can't be told from `work` alone; a database function checks it after its computed + /// criteria ran, whose failures it can't otherwise see. + exhausted: bool, + /// Computed criteria being evaluated, outermost first: one that is reached again from inside + /// itself counts as not met instead of repeating its work at every level. + active_criteria: Vec<(usize, CellRef)>, } struct FnCtx<'a, 'h> { @@ -174,6 +193,9 @@ impl Ctx for FnCtx<'_, '_> { fn computed_criterion(&mut self, row: usize, col: usize, db: &Array, record: usize) -> Option { let area = self.criteria?; let at = area.range.start.offset(row as i64, col as i64).filter(|c| area.range.contains(*c))?; + if self.ev.active_criteria.contains(&(area.sheet, at)) { + return Some(false); + } let expr = self.ev.host.workbook().sheet(area.sheet)?.cell(at)?.formula.as_ref()?.expr_arc()?; let shifted = gridcraft_formula::adjust::shift_relative((*expr).clone(), record as i64 - 1, 0); let env = (0..db.cols) @@ -193,12 +215,24 @@ impl Ctx for FnCtx<'_, '_> { calls: self.ev.calls, stack_top: self.ev.stack_top, fuel: self.ev.fuel, + work: self.ev.work, + exhausted: false, + active_criteria: std::mem::take(&mut self.ev.active_criteria), }; + ev.active_criteria.push((area.sheet, at)); let v = matches!(ev.value(&shifted).scalar().to_bool(), Ok(true)); // The child's spend counts against this cell's budget. self.ev.fuel = ev.fuel; + self.ev.work = ev.work; + self.ev.exhausted |= ev.exhausted; + let mut active = std::mem::take(&mut ev.active_criteria); + active.pop(); + self.ev.active_criteria = active; Some(v) } + fn exhausted(&self) -> bool { + self.ev.exhausted + } fn has_lambda(&self) -> bool { self.lambda.is_some() } @@ -216,7 +250,42 @@ impl Ctx for FnCtx<'_, '_> { impl<'h> Evaluator<'h> { pub fn new(host: &'h mut dyn Host, sheet: usize, at: CellRef) -> Self { - Evaluator { host, sheet, at, env: Vec::new(), depth: 0, calls: 0, stack_top: 0, fuel: LAMBDA_FUEL } + Evaluator { + host, + sheet, + at, + env: Vec::new(), + depth: 0, + calls: 0, + stack_top: 0, + fuel: LAMBDA_FUEL, + work: EVAL_WORK, + exhausted: false, + active_criteria: Vec::new(), + } + } + + /// Spends `units` of this cell's work; `false` (and nothing left) when there isn't enough. + pub(crate) fn spend(&mut self, units: u64) -> bool { + match self.work.checked_sub(units) { + Some(w) => { + self.work = w; + true + } + None => { + self.work = 0; + self.exhausted = true; + false + } + } + } + + /// Charges the values of an array (one unit per value; one for anything else). + pub(crate) fn spend_on(&mut self, v: &Value) -> bool { + match v { + Value::Array(a) => self.spend(a.data.len() as u64), + _ => self.spend(1), + } } /// Evaluates to a value (references are dereferenced; multi-cell → array). @@ -266,18 +335,27 @@ impl<'h> Evaluator<'h> { } // A reference keeps its size, so the array has its shape and its blanks count // (`=A1:A10`, `SUMPRODUCT(--(A1:A10=""))`). + let range = self.read_range(a); + let (h, w) = (range.height() as usize, range.width() as usize); + if (h as u64) * (w as u64) > MAX_CELLS { + return Value::Error(CellError::Num); + } + if !self.spend((h as u64) * (w as u64)) { + return Value::Error(CellError::Calc); + } + let data = self.host.range_values(a.sheet, range); + Array::new(h, w, data).map(Value::from).unwrap_or(Value::Error(CellError::Value)) + } + + /// The block that reading `a` gives: `a` itself, or when it is large (or runs to the sheet's + /// edge) the part inside the sheet's used range. + pub(crate) fn read_range(&mut self, a: &Area) -> RangeRef { let cells = u64::from(a.range.height()) * u64::from(a.range.width()); // `A:A`, `1:1` and `A2:A1048576` run to the edge of the sheet: they stand for "the rest of // the column", not for a block of a chosen size. let to_edge = a.range.end.row == MAX_ROWS - 1 || a.range.end.col == MAX_COLS - 1; let full_size = cells <= FULL_SIZE_CELLS || (!to_edge && cells <= FULL_SIZE_BOUNDED_CELLS); - let range = if full_size { a.range } else { self.trim(a) }; - let (h, w) = (range.height() as usize, range.width() as usize); - if (h as u64) * (w as u64) > MAX_CELLS { - return Value::Error(CellError::Num); - } - let data = self.host.range_values(a.sheet, range); - Array::new(h, w, data).map(Value::from).unwrap_or(Value::Error(CellError::Value)) + if full_size { a.range } else { self.trim(a) } } /// Trims the bottom/right of an area to the sheet's used range (keeps the top-left so @@ -310,6 +388,9 @@ impl<'h> Evaluator<'h> { self.host.stack_exhausted(); return err(CellError::Num); } + if !self.spend(1) { + return err(CellError::Calc); + } self.depth += 1; if self.depth > MAX_DEPTH { self.depth -= 1; @@ -382,6 +463,11 @@ impl<'h> Evaluator<'h> { if let Some(def) = wb.name(bare, scope_sheet) { let text = def.formula.clone(); let scope = def.scope.unwrap_or(self.sheet); + // Each use parses and evaluates the definition again: names that each use the one + // before twice double the work per name, so it is charged. + if !self.spend(text.len() as u64) { + return err(CellError::Calc); + } return match gridcraft_formula::parse_name(&text) { Ok(expr) => { // Relative references in a name are relative to A1: used here, they move by @@ -579,6 +665,10 @@ impl<'h> Evaluator<'h> { let (x, y) = (self.eval(a), self.eval(b)); match (x, y) { (Ev::R(mut xa), Ev::R(ya)) => { + // A union reused through LET doubles at each step: its areas count. + if !self.spend((xa.len() as u64).saturating_add(ya.len() as u64)) { + return err(CellError::Calc); + } xa.extend(ya); Ev::R(xa) } @@ -590,6 +680,11 @@ impl<'h> Evaluator<'h> { let (x, y) = (self.eval(a), self.eval(b)); match (x, y) { (Ev::R(xa), Ev::R(ya)) => { + // Every pair is looked at, and each may be kept: charged before the loop, + // so unions squared through LET can't build an enormous list. + if !self.spend((xa.len() as u64).saturating_mul(ya.len() as u64)) { + return err(CellError::Calc); + } let mut out = Vec::new(); for p in &xa { for q in &ya { @@ -683,6 +778,113 @@ mod fuel_tests { ev.value(&gridcraft_formula::parser::parse(src).unwrap_or_else(|e| panic!("{e}"))) } + /// Evaluates `src` in `wb` with `work` units of work, and the work left. + fn eval_work(wb: Workbook, work: u64, src: &str) -> (Value, u64) { + let mut host = Empty { wb }; + let mut ev = Evaluator::new(&mut host, 0, CellRef::new(0, 0)); + ev.work = work; + let v = ev.value(&gridcraft_formula::parser::parse(src).unwrap_or_else(|e| panic!("{e}"))); + (v, ev.work) + } + + fn is_calc(v: &Value) -> bool { + matches!(v, Value::Error(CellError::Calc)) + } + + /// Serves the workbook's stored values, for database functions to read. + struct Stored { + wb: Workbook, + } + impl Host for Stored { + fn workbook(&self) -> &Workbook { + &self.wb + } + fn cell_value(&mut self, sheet: usize, c: CellRef) -> Value { + self.wb.sheet(sheet).and_then(|s| s.cell(c)).map_or(Value::Empty, |cell| cell.value.clone()) + } + fn now_serial(&self) -> f64 { + 0.0 + } + fn random(&mut self) -> f64 { + 0.0 + } + } + + /// A computed criterion that runs out of work counts as not met for its record, which the + /// database function can't see: it must answer #CALC!, not a count that leaves records out. + #[test] + fn a_computed_criterion_out_of_work_makes_the_database_function_calc() { + use gridcraft_model::{Cell, Formula}; + let mut wb = Workbook::new(); + let name = |name: String, formula: String| gridcraft_model::DefinedName { name, scope: None, formula, comment: String::new(), hidden: false }; + wb.names.push(name("Step0".into(), "1".into())); + for i in 1..=60 { + wb.names.push(name(format!("Step{i}"), format!("Step{}+Step{}", i - 1, i - 1))); + } + let at = |s: &str| CellRef::parse(s).unwrap(); + let sheet = wb.sheet_mut(0).unwrap(); + for (r, (n, age)) in [("name", Value::text("age")), ("a", Value::number(1.0)), ("b", Value::number(2.0))].into_iter().enumerate() { + sheet.cells.set(at(&format!("A{}", r + 1)), Cell::value(Value::text(n))); + sheet.cells.set(at(&format!("B{}", r + 1)), Cell::value(age)); + } + sheet.cells.set(at("D1"), Cell::value(Value::text("Formula"))); + let host_wb = |criterion: &str| { + let mut wb = wb.clone(); + // A criteria cell is read by its value first (a blank one matches everything): give it + // the value a recalculation would have stored. + let mut cell = Cell::formula(Formula::new(criterion)); + cell.value = Value::Bool(true); + wb.sheet_mut(0).unwrap().cells.set(at("D2"), cell); + wb + }; + let run = |criterion: &str| { + let mut host = Stored { wb: host_wb(criterion) }; + let mut ev = Evaluator::new(&mut host, 0, CellRef::new(9, 9)); + ev.work = 100_000; + ev.value(&gridcraft_formula::parser::parse("=DCOUNT(A1:B3,\"age\",D1:D2)").unwrap()) + }; + assert_eq!(run("=B2>0"), Value::number(2.0), "a cheap criterion counts every record"); + assert!(is_calc(&run("=Step60>0")), "{:?}", run("=Step60>0")); + } + + #[test] + fn names_that_double_at_each_step_run_out_of_work() { + // Step1 = Step0+Step0, Step2 = Step1+Step1, ...: Step60 would evaluate Step0 2^60 times. + let mut wb = Workbook::default(); + let name = |name: String, formula: String| gridcraft_model::DefinedName { name, scope: None, formula, comment: String::new(), hidden: false }; + wb.names.push(name("Step0".into(), "1".into())); + for i in 1..=60 { + wb.names.push(name(format!("Step{i}"), format!("Step{}+Step{}", i - 1, i - 1))); + } + let (v, _) = eval_work(wb.clone(), EVAL_WORK, "=Step5"); + assert_eq!(v, Value::number(32.0)); + let (v, left) = eval_work(wb, 100_000, "=Step60"); + assert!(is_calc(&v), "{v:?}"); + assert_eq!(left, 0); + } + + #[test] + fn intersections_of_unions_are_charged_per_pair() { + let wb = Workbook::default(); + // 4 x 4 pairs, 4 of which meet. + let src = "=AREAS((A1,B1,C1,D1) (A1,B1,C1,D1))"; + let (v, _) = eval_work(wb.clone(), EVAL_WORK, src); + assert_eq!(v, Value::number(4.0)); + let (v, _) = eval_work(wb, 20, src); + assert!(is_calc(&v), "{v:?}"); + } + + #[test] + fn an_aggregate_inside_every_lambda_call_spends_work() { + let wb = Workbook::default(); + // 100 calls, each summing the 10,000 values of a captured array: about a million units. + let src = "=LET(big,SEQUENCE(10000),SUM(MAP(SEQUENCE(100),LAMBDA(x,SUM(big)))))"; + let (v, _) = eval_work(wb.clone(), 2_000_000, src); + assert_eq!(v, Value::number(100.0 * 50_005_000.0)); + let (v, _) = eval_work(wb, 200_000, src); + assert!(is_calc(&v), "{v:?}"); + } + #[test] fn lambda_calls_spend_fuel_and_run_out_quietly() { // Three applications, one charge each: 3 inside the budget. (A nested MAP wouldn't do diff --git a/crates/calc/src/eval/call.rs b/crates/calc/src/eval/call.rs index 8ac58c8d1..eb426398a 100644 --- a/crates/calc/src/eval/call.rs +++ b/crates/calc/src/eval/call.rs @@ -52,13 +52,16 @@ impl<'h> Evaluator<'h> { "ISFORMULA" | "FORMULATEXT" => { let Some(Ev::R(a)) = args.first().map(|a| self.eval(a)) else { return err(CellError::NA) }; let Some(a) = a.first() else { return err(CellError::NA) }; - let f = self.host.workbook().sheet(a.sheet).and_then(|s| s.cell(a.range.start)).and_then(|c| c.formula.clone()); + let wb = self.host.workbook(); + let f = wb.sheet(a.sheet).and_then(|s| s.cell(a.range.start)).and_then(|c| c.formula.clone()); + // A formula formatted Hidden on a protected sheet stays hidden from FORMULATEXT too. + let hidden = wb.sheet(a.sheet).is_some_and(|s| s.is_protected() && wb.styles.get(s.style_id(a.range.start)).protection.hidden); if name == "ISFORMULA" { Ev::V(Value::Bool(f.is_some())) } else { match f { - Some(f) => Ev::V(Value::text(format!("={}", f.text))), - None => err(CellError::NA), + Some(f) if !hidden => Ev::V(Value::text(format!("={}", f.text))), + _ => err(CellError::NA), } } } @@ -203,9 +206,9 @@ impl<'h> Evaluator<'h> { let index = self.value(&args[0]); if let Value::Array(_) = index { let Some(spec) = gridcraft_functions::lookup("CHOOSE") else { return err(CellError::Name) }; - let mut evaluated = vec![Arg { value: index, from_ref: false }]; + let mut evaluated = vec![Arg::val(index)]; for argument in &args[1..] { - evaluated.push(Arg { value: self.value(argument), from_ref: false }); + evaluated.push(Arg::val(self.value(argument))); } let mut context = FnCtx { ev: self, criteria: None, lambda: None }; return gridcraft_functions::call(spec, &evaluated, &mut context).pipe(Ev::V); @@ -351,10 +354,25 @@ impl<'h> Evaluator<'h> { } ev => self.deref(ev), }; - evaluated.push(Arg { value, from_ref }); + // Only an argument left out (`f(a,,b)`) is omitted; one that computes a blank is given. + let blank_value = !from_ref && value == Value::Empty && lambda_at != Some(i) && !matches!(a, Expr::Missing); + evaluated.push(Arg { value, from_ref, blank_value }); } + // The function may go through every value it is given (a SUM inside each call of a + // LAMBDA over a large array): the values count against the cell's work. + let values = evaluated.iter().fold(0u64, |n, a| n.saturating_add(if let Value::Array(x) = &a.value { x.data.len() as u64 } else { 1 })); + if !self.spend(values) { + return err(CellError::Calc); + } + let database = criteria.is_some(); let mut ctx = FnCtx { ev: self, criteria, lambda }; - gridcraft_functions::call(spec, &evaluated, &mut ctx).pipe(Ev::V) + let v = gridcraft_functions::call(spec, &evaluated, &mut ctx); + // A computed criterion that ran out of work counted as not met: the result leaves + // records out. + if database && self.exhausted { + return err(CellError::Calc); + } + Ev::V(v) } /// Performs aggregation operations without copying its ranges: blocks are folded over the stored @@ -392,6 +410,12 @@ impl<'h> Evaluator<'h> { acc.count = acc.count.saturating_add(count); return Ok(()); } + // Folded without building an array, but still walked: the part inside the used + // range counts against the cell's work. + let r = self.read_range(a); + if !self.spend(u64::from(r.height()) * u64::from(r.width())) { + return Err(err(CellError::Calc)); + } let mut error = None; let walked = self.host.visit_range(a.sheet, a.range, &mut |v| match acc.add_cell(kind, v) { Ok(()) => true, @@ -411,10 +435,13 @@ impl<'h> Evaluator<'h> { /// Adds one argument's value by calling `SUM` and `COUNT` on it, so coercion and error rules /// stay in one place (`AVERAGE` of an argument is its `SUM` over its `COUNT`). fn aggregate_value(&mut self, kind: Agg, value: Value, from_ref: bool, acc: &mut Acc) -> Result<(), Ev> { + if !from_ref && !self.spend_on(&value) { + return Err(err(CellError::Calc)); + } let (Some(sum), Some(count)) = (gridcraft_functions::lookup("SUM"), gridcraft_functions::lookup("COUNT")) else { return Err(err(CellError::Name)); }; - let args = [Arg { value, from_ref }]; + let args = [Arg { value, from_ref, blank_value: false }]; if kind != Agg::Count { match gridcraft_functions::call(sum, &args, &mut FnCtx { ev: self, criteria: None, lambda: None }) { Value::Number(n) => acc.sum += n, diff --git a/crates/calc/src/eval/reference_fns.rs b/crates/calc/src/eval/reference_fns.rs index 4f5154c2d..6cec46deb 100644 --- a/crates/calc/src/eval/reference_fns.rs +++ b/crates/calc/src/eval/reference_fns.rs @@ -118,11 +118,29 @@ impl<'h> Evaluator<'h> { for c in 0..w { let pos = (num(&Some(ra.get_broadcast(r, c))), num(&Some(ca.get_broadcast(r, c)))); let v = match pos { + // A position that picks several cells is #CALC!. That is decided from the + // shape: reading the block first, for every position, let a short formula + // read one large range over and over only to discard it. + (Ok(row), Ok(col)) if index_picks_several_values(&src, row, col, two) => Value::Error(CellError::Calc), (Ok(row), Ok(col)) => { let one = self.index_one(src.clone(), row, col, area_n, two); - match self.deref(one) { - Value::Array(a) if a.rows * a.cols > 1 => Value::Error(CellError::Calc), - v => v.scalar(), + let cells = match &one { + Ev::R(areas) => match areas.as_slice() { + [a] => self.area_cells(a), + _ => 0, + }, + _ => 0, + }; + if cells > MAX_CELLS { + // As reading it would answer. + Value::Error(CellError::Num) + } else if cells > 1 { + Value::Error(CellError::Calc) + } else { + match self.deref(one) { + Value::Array(a) if a.rows * a.cols > 1 => Value::Error(CellError::Calc), + v => v.scalar(), + } } } (Err(e), _) | (_, Err(e)) => Value::Error(e), @@ -139,6 +157,16 @@ impl<'h> Evaluator<'h> { self.index_one(src, row, col, area_n, two) } + /// Cells that reading `a` gives: its own size, or for a large block the part inside the + /// sheet's used range (what [`Evaluator::area_value`] reads). + fn area_cells(&mut self, a: &Area) -> u64 { + if a.range.is_single() { + return 1; + } + let r = self.read_range(a); + u64::from(r.height()) * u64::from(r.width()) + } + /// INDEX for one row and column position (0 = the whole column or row). fn index_one(&mut self, src: Ev, row: i64, col: i64, area_n: i64, two_args: bool) -> Ev { if row < 0 || col < 0 { @@ -203,7 +231,7 @@ impl<'h> Evaluator<'h> { let horizontal = l.range.height() == 1 && l.range.width() > 1; let same_len = if vertical { r.range.height() == l.range.height() } else { r.range.width() == l.range.width() }; if !matches!(key, Value::Array(_)) && (vertical || horizontal) && same_len { - let mut xargs = vec![Arg { value: key, from_ref: false }, Arg { value: self.area_value(&l), from_ref: true }]; + let mut xargs = vec![Arg::val(key), Arg::reference(self.area_value(&l))]; // An empty match_mode before a given search_mode is the default (0), not the end. let given = |i: usize| args.get(i).is_some_and(|e| !matches!(e, Expr::Missing)); let last = if given(5) { @@ -215,10 +243,11 @@ impl<'h> Evaluator<'h> { }; for i in 4..=last { match args.get(i) { - Some(Expr::Missing) | None => xargs.push(Arg { value: Value::Number(0.0), from_ref: false }), + Some(Expr::Missing) | None => xargs.push(Arg::val(Value::Number(0.0))), Some(e) => { let value = self.value(e); - xargs.push(Arg { value, from_ref: false }); + let blank_value = value == Value::Empty; + xargs.push(Arg { value, from_ref: false, blank_value }); } } } @@ -407,3 +436,71 @@ impl<'h> Evaluator<'h> { Ev::V(gridcraft_functions::aggregate_values(func, &values)) } } + +/// Whether INDEX on an array picks a whole row or column of several values at position +/// (`row`, `col`), which the array-of-positions form answers with #CALC!. +fn index_picks_several_values(src: &Ev, row: i64, col: i64, two_args: bool) -> bool { + let Ev::V(Value::Array(arr)) = src else { return false }; + let (row, col) = if two_args && arr.rows == 1 { (1, row) } else { (row, col) }; + if row < 0 || col < 0 || row as u64 > arr.rows as u64 || col as u64 > arr.cols as u64 { + return false; + } + (row == 0 && arr.rows > 1) || (col == 0 && arr.cols > 1) +} + +#[cfg(test)] +mod tests { + use super::super::Host; + use super::*; + use gridcraft_model::Workbook; + + /// A sheet of blanks that counts the blocks it is asked for. + struct Counting { + wb: Workbook, + cells_read: usize, + } + impl Host for Counting { + fn workbook(&self) -> &Workbook { + &self.wb + } + fn cell_value(&mut self, _sheet: usize, _c: CellRef) -> Value { + Value::Empty + } + fn now_serial(&self) -> f64 { + 0.0 + } + fn random(&mut self) -> f64 { + 0.0 + } + fn range_values(&mut self, _sheet: usize, range: RangeRef) -> Vec { + self.cells_read += range.count() as usize; + vec![Value::Empty; range.count() as usize] + } + } + + fn eval(src: &str) -> (Value, usize) { + let mut host = Counting { wb: Workbook::default(), cells_read: 0 }; + let mut ev = Evaluator::new(&mut host, 0, CellRef::new(0, 0)); + let v = ev.value(&gridcraft_formula::parser::parse(src).unwrap_or_else(|e| panic!("{e}"))); + (v, host.cells_read) + } + + fn all_calc(v: &Value, n: usize) -> bool { + matches!(v, Value::Array(a) if a.data.len() == n && a.data.iter().all(|x| *x == Value::Error(CellError::Calc))) + } + + #[test] + fn index_positions_that_pick_several_cells_are_calc_without_reading_them() { + // Each zero row position picks the whole of A1:A50000: #CALC!, decided from the shape. It + // used to read the 50,000 cells once per position before discarding them. + let (v, read) = eval("=INDEX(A1:A50000,{0,0,0,0},1)"); + assert!(all_calc(&v, 4), "{v:?}"); + assert_eq!(read, 0); + // The same for an array source: a whole column of a 3x3 array. + let (v, _) = eval("=INDEX({1,2,3;4,5,6;7,8,9},{0,0},2)"); + assert!(all_calc(&v, 2), "{v:?}"); + // Single cells are still read. + let (v, _) = eval("=INDEX({1,2,3;4,5,6;7,8,9},{1,3},2)"); + assert!(matches!(&v, Value::Array(a) if a.data == vec![Value::number(2.0), Value::number(8.0)]), "{v:?}"); + } +} diff --git a/crates/calc/src/eval/special.rs b/crates/calc/src/eval/special.rs index 0151ea2f9..2e7083457 100644 --- a/crates/calc/src/eval/special.rs +++ b/crates/calc/src/eval/special.rs @@ -27,62 +27,153 @@ pub fn resized_sum_range(sum: RangeRef, height: u32, width: u32) -> RangeRef { RangeRef::new(sum.start, CellRef::new(row, col)) } +/// Most expression nodes [`precedents`] visits before giving up and marking the formula dynamic. +const MAX_PRECEDENT_NODES: usize = 1_000_000; + /// Collects the static references of an expression with their sheets resolved; `dynamic` is /// set when the formula uses INDIRECT/OFFSET/names/tables whose targets can change. +/// +/// Every node is visited once (#2183: range-operator chains used to re-walk shared operands, +/// costing 2^k for k operands), with an explicit stack, so a flat chain as long as Excel allows +/// (`1+1+…`, about 4,000 levels deep) never recurses. Past [`MAX_PRECEDENT_NODES`] the result +/// is conservative: `dynamic` is set so the formula recalculates every time. pub fn precedents(wb: &Workbook, sheet: usize, e: &Expr) -> (Vec, bool) { - let mut out = Vec::new(); - let mut dynamic = false; - let push = |out: &mut Vec, r: &Reference, range: RangeRef| match &r.sheet { - SheetSel::Current => out.push(Area { sheet, range }), - SheetSel::Named(n) => { - if let Some(i) = wb.sheet_index(n) { - out.push(Area { sheet: i, range }); - } + let mut c = Precedents { wb, sheet, out: Vec::new(), dynamic: false, boxes: Vec::new() }; + c.collect(e); + (c.out, c.dynamic) +} + +/// Grows the per-sheet bounding box in `boxes` to cover `area`. +fn union_box(boxes: &mut Vec, area: Area) { + match boxes.iter_mut().find(|x| x.sheet == area.sheet) { + Some(x) => x.range = x.range.union(&area.range), + None => boxes.push(area), + } +} + +/// One entry of [`Precedents::collect`]'s explicit stack. `boxes` indexes +/// [`Precedents::boxes`]: the bounding boxes of the innermost enclosing range operator, if any. +enum PrecedentStep<'e> { + /// Visit an expression and queue its children. + Visit(&'e Expr, Option), + /// Both operands of a range operator have been visited: record the bounding boxes they + /// collected in `inner`, growing the `outer` operator's boxes too. + Close { inner: usize, outer: Option }, +} + +struct Precedents<'w> { + wb: &'w Workbook, + sheet: usize, + out: Vec, + dynamic: bool, + /// Per-sheet bounding boxes, one list per range operator (`A1:INDEX(…)`) met so far. + boxes: Vec>, +} + +impl Precedents<'_> { + /// Records `area`, also growing the bounding boxes of the enclosing range operator, if any. + fn add(&mut self, area: Area, boxes: Option) { + if let Some(b) = boxes.and_then(|i| self.boxes.get_mut(i)) { + union_box(b, area); } - SheetSel::Span(a, b) => { - if let (Some(i), Some(j)) = (wb.sheet_index(a), wb.sheet_index(b)) { - for s in i.min(j)..=i.max(j) { - out.push(Area { sheet: s, range }); + self.out.push(area); + } + + fn push(&mut self, r: &Reference, range: RangeRef, boxes: Option) { + match &r.sheet { + SheetSel::Current => self.add(Area { sheet: self.sheet, range }, boxes), + SheetSel::Named(n) => { + if let Some(i) = self.wb.sheet_index(n) { + self.add(Area { sheet: i, range }, boxes); + } + } + SheetSel::Span(a, b) => { + if let (Some(i), Some(j)) = (self.wb.sheet_index(a), self.wb.sheet_index(b)) { + for s in i.min(j)..=i.max(j) { + self.add(Area { sheet: s, range }, boxes); + } } } } - }; - e.walk(&mut |x| match x { - Expr::Ref(r) => push(&mut out, r, r.range()), - // The cells SUMIF/AVERAGEIF really read from the sum range (see `resized_sum_range`); when - // the criteria range's size isn't known statically, recalculate the formula every time. - Expr::Call(n, args) if resizes_sum_range(n, args.len()) => match (args.first(), args.get(2)) { - (Some(Expr::Ref(c)), Some(Expr::Ref(s))) => { - let (crit, sum) = (c.range(), s.range()); - push(&mut out, s, resized_sum_range(sum, crit.height(), crit.width())); - } - (_, Some(Expr::Number(_) | Expr::Text(_) | Expr::Bool(_) | Expr::Error(_) | Expr::Array(_) | Expr::Missing)) => {} - _ => dynamic = true, - }, - Expr::Call(n, _) - if matches!(n.as_str(), "INDIRECT" | "OFFSET" | "NOW" | "TODAY" | "RAND" | "RANDBETWEEN" | "RANDARRAY" | "CELL" | "INFO") => - { - dynamic = true - } - Expr::Name(_) | Expr::Struct(_) => dynamic = true, - Expr::Call(n, _) if gridcraft_functions::lookup(n).is_none() && !is_special(n) => dynamic = true, - // A range ending in a function's result (`A1:INDEX(B1:B9,n)`) covers the cells between - // the references inside it, not only those references. - Expr::Binary(BinOp::Range, a, b) if !matches!((&**a, &**b), (Expr::Ref(_), Expr::Ref(_))) => { - let mut inner = precedents(wb, sheet, a).0; - inner.extend(precedents(wb, sheet, b).0); - let mut boxes: Vec = Vec::new(); - for area in inner { - match boxes.iter_mut().find(|x| x.sheet == area.sheet) { - Some(x) => x.range = x.range.union(&area.range), - None => boxes.push(area), + } + + /// Visits every node of `root` once, parents before children, left to right. Areas found + /// inside a range operator (`A1:INDEX(…)`) also grow that operator's per-sheet bounding boxes. + fn collect(&mut self, root: &Expr) { + let mut pending = vec![PrecedentStep::Visit(root, None)]; + let mut nodes = 0usize; + while let Some(step) = pending.pop() { + let (x, boxes) = match step { + PrecedentStep::Visit(x, boxes) => (x, boxes), + PrecedentStep::Close { inner, outer } => { + let areas = self.boxes.get_mut(inner).map(std::mem::take).unwrap_or_default(); + for area in areas { + self.add(area, outer); + } + continue; } + }; + nodes = nodes.saturating_add(1); + if nodes > MAX_PRECEDENT_NODES { + self.dynamic = true; + return; } - out.extend(boxes); + self.visit(x, boxes, &mut pending); } - _ => {} - }); - (out, dynamic) + } + + /// Handles one node for [`Precedents::collect`], queueing the children still to visit. + fn visit<'e>(&mut self, x: &'e Expr, boxes: Option, pending: &mut Vec>) { + match x { + Expr::Ref(r) => self.push(r, r.range(), boxes), + // The cells SUMIF/AVERAGEIF really read from the sum range (see `resized_sum_range`); when + // the criteria range's size isn't known statically, recalculate the formula every time. + Expr::Call(n, args) if resizes_sum_range(n, args.len()) => match (args.first(), args.get(2)) { + (Some(Expr::Ref(c)), Some(Expr::Ref(s))) => { + let (crit, sum) = (c.range(), s.range()); + self.push(s, resized_sum_range(sum, crit.height(), crit.width()), boxes); + } + (_, Some(Expr::Number(_) | Expr::Text(_) | Expr::Bool(_) | Expr::Error(_) | Expr::Array(_) | Expr::Missing)) => {} + _ => self.dynamic = true, + }, + Expr::Call(n, _) + if matches!(n.as_str(), "INDIRECT" | "OFFSET" | "NOW" | "TODAY" | "RAND" | "RANDBETWEEN" | "RANDARRAY" | "CELL" | "INFO") => + { + self.dynamic = true + } + Expr::Name(_) | Expr::Struct(_) => self.dynamic = true, + Expr::Call(n, _) if gridcraft_functions::lookup(n).is_none() && !is_special(n) => self.dynamic = true, + // A range ending in a function's result (`A1:INDEX(B1:B9,n)`), or a chain of plain + // references the parser can't fold into one (`A1:B2:C3`), covers the cells between the + // references inside it, not only those references. Both operands are visited once, + // collecting their bounding boxes on the way, which `Close` then records. + Expr::Binary(BinOp::Range, a, b) => { + let inner = self.boxes.len(); + self.boxes.push(Vec::new()); + pending.push(PrecedentStep::Close { inner, outer: boxes }); + pending.push(PrecedentStep::Visit(b, Some(inner))); + pending.push(PrecedentStep::Visit(a, Some(inner))); + return; + } + _ => {} + } + // Children are pushed in reverse so they pop left to right. + let visit = |e: &'e Expr| PrecedentStep::Visit(e, boxes); + match x { + Expr::Array(rows) => pending.extend(rows.iter().flatten().rev().map(visit)), + Expr::Unary(_, e) | Expr::Paren(e) => pending.push(visit(e)), + Expr::Binary(_, a, b) => { + pending.push(visit(b)); + pending.push(visit(a)); + } + Expr::Call(_, args) => pending.extend(args.iter().rev().map(visit)), + Expr::Invoke(c, args) => { + pending.extend(args.iter().rev().map(visit)); + pending.push(visit(c)); + } + _ => {} + } + } } fn is_special(n: &str) -> bool { diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 2b2ae7019..ab1b5bd03 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -48,6 +48,8 @@ pub struct Calc { /// Cells whose formulas have errors from cycles. pub circular: Vec, pub last_recalc_cells: usize, + /// Which anchors held a spill when the current calculation started (see [`Standing`]). + standing: Standing, /// The cells whose value is `#SPILL!`, kept so that finding the formulas an emptied cell or a /// freed spill area may unblock doesn't walk every formula or window of cells. Filled in /// `rebuild`, kept up to date by `cells_changed` and by the write-back of a run; an entry is @@ -55,6 +57,45 @@ pub struct Calc { spill_errors: SpillErrors, } +/// Who gets the cells when two arrays want the same ones. Excel (measured by a reviewer on +/// PR #2483) lets the spill that already exists keep its cells, whichever anchor comes first; +/// the other formula shows `#SPILL!`, and clearing the winner lets it spill. The rule here: +/// +/// 1. an anchor that held a spill before this calculation outranks one that did not; +/// 2. otherwise the anchor first in reading order (row, then column) wins. +/// +/// Rule 2 alone decides when nothing is remembered, so the result never depends on hash order +/// or evaluation order. A full recalculation remembers the layout the sheet held before it +/// started (rule 1), so it agrees with the incremental one; a workbook that is calculated for +/// the first time has no such layout (rule 2 only). +/// +/// Known limits: (a) the xlsx reader does not restore spill layouts, so a workbook opened from a +/// file is laid out by rule 2 alone, although Excel keeps the layout the file records; (b) when +/// both anchors held a spill and one grows into the other, rule 2 decides, where Excel keeps +/// the cells the other already owns. +#[derive(Clone, Debug, Default)] +struct Standing { + /// `true`: `keys` are the anchors that held a spill (full recalculation, which lays every + /// spill out afresh). `false`: `keys` are the formulas that held none (incremental). + held: bool, + keys: HashSet, +} + +impl Standing { + fn had_spill(&self, k: Key) -> bool { + self.keys.contains(&k) == self.held + } + + /// Whether `a` takes cells from `b` when both want them. + fn outranks(&self, a: Key, b: Key) -> bool { + match (self.had_spill(a), self.had_spill(b)) { + (true, false) => true, + (false, true) => false, + _ => (a.1.row, a.1.col) < (b.1.row, b.1.col), + } + } +} + /// A set of cells by row, each row's columns in their own set: a window of rows and columns is /// found by one column range per row that has an entry, so the cost follows the cells in the /// window, not those to the side of it in the same rows. @@ -101,6 +142,7 @@ impl Default for Calc { fixed_now: None, circular: vec![], last_recalc_cells: 0, + standing: Standing::default(), spill_errors: SpillErrors::default(), } } @@ -269,6 +311,14 @@ impl Calc { /// Recalculates every formula (F9 / Ctrl+Alt+F9, and after loading). pub fn recalc_all(&mut self, wb: &mut Workbook) { self.rebuild(wb); + // The layout the sheets hold now decides who keeps cells two arrays both want. + // (A Python cell that holds a spill counts too, as it does in the incremental pass.) + let mut held = HashSet::default(); + for i in 0..wb.sheets.len() { + if let Some(sh) = wb.sheet(i) { + held.extend(sh.spill_ranges.keys().filter(|a| sh.cell(**a).is_some_and(|c| c.formula.is_some() || c.py.is_some())).map(|a| (i, *a))); + } + } // Spills are laid out again from the formulas where they are now (a structural edit moves // or deletes the formulas, not the values they spilled). for i in 0..wb.sheets.len() { @@ -280,7 +330,9 @@ impl Calc { } } let all: Vec = self.graph.nodes.keys().copied().collect(); + self.standing = Standing { held: true, keys: held }; self.run(wb, all, true); + self.standing = Standing::default(); } /// Updates the graph for cells whose content changed and recalculates what depends on them. @@ -391,6 +443,13 @@ impl Calc { } } } + // So can the area a removed formula spilled over. + for (si, r) in &cleared { + match emptied.iter_mut().find(|x| x.0 == *si) { + Some(x) => x.1 = CellRef::new(x.1.row.max(r.end.row), x.1.col.max(r.end.col)), + None => emptied.push((*si, r.end)), + } + } // An emptied cell can unblock an anchor any distance above or to the left of it. for (si, end) in emptied { seeds.extend(self.unblock_candidates(wb, si, end)); @@ -472,10 +531,22 @@ impl Calc { fn run(&mut self, wb: &mut Workbook, mut dirty: Vec, full: bool) { let mut dynamic_again = !self.graph.dynamic.is_empty(); self.last_recalc_cells = 0; + let mut seen: HashSet = HashSet::default(); + if !full { + self.standing = Standing::default(); + } for _round in 0..8 { if dirty.is_empty() { break; } + if !full { + // What each formula held before this calculation touched it. + for &k in &dirty { + if seen.insert(k) && wb.sheet(k.0).is_some_and(|s| !s.spill_ranges.contains_key(&k.1)) { + self.standing.keys.insert(k); + } + } + } // Row-major order is a good topological guess; on-demand evaluation fixes the rest. dirty.sort_by_key(|(s, c)| (*s, c.row, c.col)); let mut exprs: HashMap> = HashMap::with_capacity_and_hasher(dirty.len(), Default::default()); @@ -493,6 +564,7 @@ impl Calc { let tp = prof_now(); let (results, spills, cycles) = { let mut host = PassHost::new(wb, &exprs, &mut rng, now).with_python(python); + host.standing = Some(&self.standing); host.iterate = wb.calc.iterative; for k in &dirty { host.calculate(*k); @@ -591,7 +663,40 @@ impl Calc { if let Some(array) = new { let end = CellRef::new(key.1.row.saturating_add(array.rows as u32 - 1), key.1.col.saturating_add(array.cols as u32 - 1)); let range = RangeRef::new(key.1, end); - let blocked = spill_is_blocked(sheet, key.1, range, end); + // Two arrays that want the same cells (see [`Standing`]): a spill already laid out + // that this anchor outranks is taken over, and recalculated. + let standing = &self.standing; + // A spill can only be in the way where it has cells besides its anchor, and an + // anchor itself blocks the array. When no spilled cell lies in the range, the owners + // are not scanned: a full recalculation of many disjoint spills stays linear. + let outranked: Vec<(CellRef, RangeRef)> = if range.iter().any(|cell| cell != key.1 && sheet.spill.contains_key(&cell)) { + #[cfg(test)] + OWNER_SCANS.with(|n| n.set(n.get() + 1)); + sheet + .spill_ranges + .iter() + .filter(|(o, r)| **o != key.1 && r.intersects(&range) && standing.outranks(*key, (key.0, **o))) + .map(|(o, r)| (*o, *r)) + .collect() + } else { + Vec::new() + }; + let blocked = spill_is_blocked(sheet, *key, range, end, standing); + if !blocked { + for (o, r) in outranked { + for c in r.iter() { + sheet.spill.remove(&c); + } + sheet.spill_ranges.remove(&o); + if let Some(cell) = sheet.cells.get_mut(o) { + cell.value = Value::Error(CellError::Spill); + self.spill_errors.insert((key.0, o)); + } + spill_changes.push((key.0, o)); + self.graph.dependents_of_range(key.0, r, &mut spill_changes); + freed.push((key.0, r)); + } + } if blocked { if let Some(cell) = sheet.cells.get_mut(key.1) { cell.value = Value::Error(CellError::Spill); @@ -628,12 +733,22 @@ impl Calc { } } -fn spill_is_blocked(sheet: &gridcraft_model::Sheet, anchor: CellRef, range: RangeRef, end: CellRef) -> bool { +// Owner scans done by `write_back`, to check that disjoint spills never walk every owner. +#[cfg(test)] +thread_local! { + pub(crate) static OWNER_SCANS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Whether the array at `anchor` can't be laid out over `range`. A spill already laid out that +/// `anchor` outranks does not block it: it is taken over when written back. +fn spill_is_blocked(sheet: &gridcraft_model::Sheet, anchor: Key, range: RangeRef, end: CellRef, standing: &Standing) -> bool { + let outranked = + |cell: CellRef| sheet.spill_ranges.iter().any(|(o, r)| *o != anchor.1 && r.contains(cell) && standing.outranks(anchor, (anchor.0, *o))); end.row >= gridcraft_core::MAX_ROWS || end.col >= gridcraft_core::MAX_COLS - || range.iter().any(|cell| cell != anchor && (sheet.cells.has(cell) || sheet.spill.contains_key(&cell))) + || range.iter().any(|cell| cell != anchor.1 && (sheet.cells.has(cell) || (sheet.spill.contains_key(&cell) && !outranked(cell)))) || sheet.merges.iter().any(|merge| merge.intersects(&range)) - || sheet.tables.iter().any(|table| table.range.contains(anchor)) + || sheet.tables.iter().any(|table| table.range.contains(anchor.1)) } fn fill_spill(sheet: &mut gridcraft_model::Sheet, anchor: CellRef, range: RangeRef, array: &Array) { diff --git a/crates/calc/src/recalc/pass_host.rs b/crates/calc/src/recalc/pass_host.rs index 1d7d1c38c..78bd5d9ef 100644 --- a/crates/calc/src/recalc/pass_host.rs +++ b/crates/calc/src/recalc/pass_host.rs @@ -31,6 +31,8 @@ pub(super) struct PassHost<'a> { /// every row, so once per sheet rather than once per range read. used: HashMap>, rng: &'a mut u64, + /// Who held a spill before the calculation; unset: every formula did (reading order decides). + pub(super) standing: Option<&'a super::Standing>, now: f64, cycle: bool, pub(super) cycles: Vec, @@ -99,6 +101,7 @@ impl<'a> PassHost<'a> { spill_index: SpillIndex::default(), used: HashMap::default(), rng, + standing: None, now, cycle: false, cycles: vec![], @@ -160,19 +163,26 @@ fn spill_hits_overlap(index: &SpillIndex, sheet: usize, hits: &[CellRef], range: } /// Whether an array result at `k` can't spill: it is in a table, or its area runs off the sheet, -/// overlaps merged cells, holds other cells, or another formula's spill (unless that formula is -/// recalculated in this pass, and so may spill elsewhere now). +/// overlaps merged cells, holds other cells, or the spill of another formula that outranks it +/// (see `Standing`; unless that formula is recalculated in this pass, and so may spill elsewhere +/// now). A spill it outranks gives way to it when written back. fn spill_blocked(host: &PassHost<'_>, (sheet, anchor): Key, a: &Array) -> bool { let Some(sh) = host.wb.sheet(sheet) else { return false }; let end = CellRef::new(anchor.row.saturating_add((a.rows as u32).saturating_sub(1)), anchor.col.saturating_add((a.cols as u32).saturating_sub(1))); let range = RangeRef::new(anchor, end); let own = sh.spill_ranges.get(&anchor); + let default = super::Standing::default(); + let standing = host.standing.unwrap_or(&default); let spilled_over = |c: CellRef| { !own.is_some_and(|o| o.contains(c)) && sh.spill_ranges.iter().any(|(o, r)| { // A formula or Python cell recalculated in this pass lays its array out again. - *o != anchor && r.contains(c) && !host.exprs.contains_key(&(sheet, *o)) && !host.python.contains_key(&(sheet, *o)) + *o != anchor + && r.contains(c) + && !host.exprs.contains_key(&(sheet, *o)) + && !host.python.contains_key(&(sheet, *o)) + && standing.outranks((sheet, *o), (sheet, anchor)) }) }; end.row >= gridcraft_core::MAX_ROWS diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 8784a8d69..695b33f72 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -1826,6 +1826,54 @@ fn trim_references() { assert_eq!(t.get("K20"), Value::Error(CellError::Calc)); } +#[test] +fn lookup_of_an_empty_cell_stays_blank() { + // #374: B1 (the value for key "k") is empty. + let mut t = T::new(); + t.set("A1", "k"); + t.set("A2", "j"); + t.set("B2", "v"); + for (at, f, want) in [ + ("D2", "=\"x\"&VLOOKUP(\"k\",A1:B2,2,0)&\"y\"", Value::from("xy")), + ("D3", "=CONCATENATE(\"x\",VLOOKUP(\"k\",A1:B2,2,0),\"y\")", Value::from("xy")), + ("D4", "=ISBLANK(XLOOKUP(\"k\",A1:A2,B1:B2))", Value::Bool(true)), + ("D5", "=\"x\"&XLOOKUP(\"k\",A1:A2,B1:B2)&\"y\"", Value::from("xy")), + // A bare lookup still shows 0, like a reference to an empty cell. + ("D8", "=VLOOKUP(\"k\",A1:B2,2,0)", Value::Number(0.0)), + // Arithmetic still reads the blank as 0. + ("D9", "=VLOOKUP(\"k\",A1:B2,2,0)+1", Value::Number(1.0)), + // A blank lookup given as an argument is still given, not omitted. + ("D10", "=FILTER(1,FALSE,VLOOKUP(\"k\",A1:B2,2,FALSE))", Value::Number(0.0)), + ("D11", "=XLOOKUP(\"z\",A1:A2,B1:B2,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), + // The same holds per element when the call is lifted over an array of blank lookups. + ("D12", "=CEILING.MATH(6.7,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), + ("D13", "=SUM(CEILING.MATH(6.7,VLOOKUP({\"k\",\"k\"},A1:B2,2,0)))", Value::Number(0.0)), + ("D14", "=FLOOR.MATH(6.7,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), + ("D15", "=SUM(FLOOR.MATH(6.7,VLOOKUP({\"k\",\"k\"},A1:B2,2,0)))", Value::Number(0.0)), + ] { + t.set(at, f); + assert_eq!(t.get(at), want, "{f}"); + } +} + +#[test] +fn blank_lookup_values_match_excel_windows_observation() { + // Excel 365 Windows 2609, build 16.0.20430.20032, black-box check 2026-10-13: + // A1="k", B1 and A2 genuinely empty. + // All three value lookups preserve blankness inside ISBLANK/concatenation. + let mut t = T::new(); + t.set("A1", "k"); + for (at, f, want) in [ + ("D1", "=ISBLANK(VLOOKUP(\"k\",A1:B1,2,FALSE))", Value::Bool(true)), + ("D2", "=ISBLANK(HLOOKUP(\"k\",A1:A2,2,FALSE))", Value::Bool(true)), + ("D3", "=ISBLANK(LOOKUP(\"k\",A1:A1,B1:B1))", Value::Bool(true)), + ("D4", "=\"x\"&HLOOKUP(\"k\",A1:A2,2,FALSE)&\"y\"", Value::from("xy")), + ] { + t.set(at, f); + assert_eq!(t.get(at), want, "{f}"); + } +} + #[test] fn cell_width_is_the_column_width_dialog_value_rounded() { // #735: Excel 365 values. Widths in px as `home.columnWidth {"chars"}` sets them @@ -2283,6 +2331,85 @@ fn median_large_values_recalculate_dependents() { } } +/// #2183: `INDEX(…):INDEX(…):…` used to re-walk each operand of the left-associative range chain, +/// costing 2^k for k operands (28 took 40 s; 60 would take millennia), on open and on every edit. +#[test] +fn range_operator_chain_precedents_are_linear() { + let operands: Vec = (1..=60).map(|i| format!("INDEX(A{i},1)")).collect(); + let src = format!("=SUM({})", operands.join(":")); + let e = gridcraft_formula::parse(&src).unwrap(); + let wb = Workbook::new(); + // With the old 2^k walk this call would never return; #2183 asks for under 100 ms. + #[allow(clippy::disallowed_methods)] // a native test: the clock is there + let t0 = std::time::Instant::now(); + let (areas, dynamic) = crate::eval::precedents(&wb, 0, &e); + let took = t0.elapsed(); + assert!(took < std::time::Duration::from_millis(100), "precedents took {took:?}"); + assert!(!dynamic); + // The chain covers A1:A60, not only the 60 cells named in it. + let whole = gridcraft_core::RangeRef::new(c("A1"), c("A60")); + assert!(areas.iter().any(|a| a.sheet == 0 && a.range == whole), "{areas:?}"); + // Each node is visited once: 60 references plus one box per range operator. + assert!(areas.len() < 200, "{} areas", areas.len()); + + // Through the recalculation engine: setting it finishes and editing a cell in the box recalculates. + let mut t = T::new(); + for i in 1..=60 { + t.set(&format!("A{i}"), "1"); + } + t.set("B1", &src); + assert_eq!(t.num("B1"), 60.0); + t.set("A30", "5"); + assert_eq!(t.num("B1"), 64.0); +} + +/// A chained range of plain references (`A1:B2:C3`, which the parser can't fold into one +/// reference) covers their bounding box, as the evaluator expands it: an edit inside the box but +/// outside each operand still recalculates. +#[test] +fn chained_plain_range_precedents_cover_the_bounding_box() { + let e = gridcraft_formula::parse("=SUM(A1:B2:C3)").unwrap(); + let (areas, dynamic) = crate::eval::precedents(&Workbook::new(), 0, &e); + assert!(!dynamic); + let whole = gridcraft_core::RangeRef::new(c("A1"), c("C3")); + assert!(areas.iter().any(|a| a.sheet == 0 && a.range == whole), "{areas:?}"); + + let mut t = T::new(); + t.set("A1", "1"); + t.set("E1", "=SUM(A1:B2:C3)"); + assert_eq!(t.num("E1"), 1.0); + t.set("C1", "4"); + assert_eq!(t.num("E1"), 5.0); +} + +/// `precedents` walks with an explicit stack: an expression nested deeper than any formula +/// within the length limit can build still finds its references, on a small stack. +#[test] +fn precedents_of_deep_nesting_do_not_recurse() { + let (areas, dynamic) = std::thread::Builder::new() + .stack_size(256 * 1024) + .spawn(|| { + let mut e = gridcraft_formula::parse("=A1").unwrap(); + for _ in 0..10_000 { + e = gridcraft_formula::Expr::Paren(Box::new(e)); + } + let found = crate::eval::precedents(&Workbook::new(), 0, &e); + // Dropping the deep tree recurses; leak it so only `precedents` is under test. + std::mem::forget(e); + found + }) + .unwrap() + .join() + .unwrap(); + assert!(!dynamic); + assert_eq!(areas.len(), 1); + // Shallow nesting stays static. + let e = gridcraft_formula::parse("=((((A1))))").unwrap(); + let (areas, dynamic) = crate::eval::precedents(&Workbook::new(), 0, &e); + assert!(!dynamic); + assert_eq!(areas.len(), 1); +} + #[test] fn cell_filename_names_the_workbook_file() { // Excel returns `/[]`, and the classic sheet-name formula splits it on @@ -2832,3 +2959,118 @@ fn formulas_read_a_dirty_python_cell_after_it_is_laid_out() { assert_eq!((t.num("B1"), t.num("B2")), (5.0, 6.0)); assert_eq!((t.num("A1"), t.num("A2"), t.num("A3")), (5.0, 100.0, 111.0)); } + +#[test] +fn disjoint_spills_do_not_scan_every_spill_owner() { + // Many independent two-cell arrays: write-back must not walk all earlier owners for each one. + let mut t = T::new(); + for i in 1..=2000 { + t.wb.sheet_mut(0).unwrap().cells.set(c(&format!("A{}", i * 2 - 1)), Cell::formula(Formula::new("=SEQUENCE(2)"))); + } + crate::recalc::OWNER_SCANS.with(|n| n.set(0)); + t.calc.recalc_all(&mut t.wb); + assert_eq!((t.num("A2"), t.num("A4000")), (2.0, 2.0)); + let scans = crate::recalc::OWNER_SCANS.with(|n| n.get()); + assert!(scans < 10, "{scans} owner scans for 2000 disjoint spills"); +} + +#[test] +fn overlapping_spill_edit_sequences_match_a_full_recalc() { + // Random short edit sequences on a small grid of arrays that run into each other (sizes from + // shared cells, so one edit resizes several at once), with constants and clears in their way. + // After every edit the incremental result must equal a full recalc of the same workbook. + let seeds: u64 = std::env::var("GRIDCRAFT_SPILL_SEEDS").ok().and_then(|v| v.parse().ok()).unwrap_or(200); + let steps: usize = std::env::var("GRIDCRAFT_SPILL_STEPS").ok().and_then(|v| v.parse().ok()).unwrap_or(20); + let only: Option = std::env::var("GRIDCRAFT_SPILL_SEED").ok().and_then(|v| v.parse().ok()); + let next = |state: &mut u64| { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + }; + let seeds: Vec = match only { + Some(s) => vec![s], + None => (0..seeds).collect(), + }; + for seed in seeds { + let mut state = seed; + let mut t = T::new(); + // Sizes in J1:J4. + for (i, n) in [2, 3, 1, 4].iter().enumerate() { + t.set(&format!("J{}", i + 1), &n.to_string()); + } + let mut log: Vec<(String, String)> = Vec::new(); + for step in 0..steps { + let r = next(&mut state); + let at = format!("{}{}", (b'A' + (r % 6) as u8) as char, 1 + (r >> 8) % 6); + let kind = (r >> 16) % 14; + let (at, input) = match kind { + 0..=3 => { + let a = 1 + (r >> 24) % 4; + let b = 1 + (r >> 28) % 4; + (at, format!("=SEQUENCE($J${a},$J${b})")) + } + 4 => (at, format!("=SEQUENCE({},{})", 1 + (r >> 24) % 3, 1 + (r >> 28) % 3)), + 5 => (format!("J{}", 1 + (r >> 24) % 4), (1 + (r >> 28) % 4).to_string()), + 6 => (at, "x".to_string()), + 7 | 8 => (at, String::new()), + 9 => (format!("K{}", 1 + (r >> 24) % 6), "=SUM(A1:F6)".to_string()), + 10 => { + // An earlier edit again (often putting back what a cell held before, as undo does). + match log.len() { + 0 => (at, String::new()), + n => log[(r >> 24) as usize % n].clone(), + } + } + 11 => (format!("{}{}", (b'A' + ((r >> 24) % 6) as u8) as char, 1 + (r >> 32) % 6), (1 + (r >> 40) % 3).to_string()), + // Several cells in one edit (a fill, or clearing a range): a 2x2 block from `at`. + 12 => (format!("{at}+"), String::new()), + _ => (format!("{at}+"), format!("=SEQUENCE($J${},$J${})", 1 + (r >> 24) % 4, 1 + (r >> 28) % 4)), + }; + log.push((at.clone(), input.clone())); + if let Some(from) = at.strip_suffix('+') { + let from = c(from); + let keys: Vec<(usize, CellRef)> = + (0..2).flat_map(|dr| (0..2).map(move |dc| (0, CellRef { row: from.row + dr, col: from.col + dc }))).collect(); + for k in &keys { + let cell = if input.is_empty() { Cell::value(Value::Empty) } else { Cell::formula(Formula::new(&input)) }; + t.wb.sheet_mut(0).unwrap().cells.set(k.1, cell); + } + t.calc.cells_changed(&mut t.wb, &keys); + } else { + t.set(&at, &input); + } + let mut fresh = t.wb.clone(); + Calc::new().recalc_all(&mut fresh); + let (a, b) = (t.wb.sheet(0).unwrap(), fresh.sheet(0).unwrap()); + for row in 0..12 { + for col in 0..12 { + let cell = CellRef { row, col }; + assert_eq!(a.value(cell), b.value(cell), "seed {seed} step {step}: {cell} incremental vs full recalc after {log:?}"); + } + } + let mut ka: Vec<_> = a.spill_ranges.iter().map(|(k, v)| (*k, *v)).collect(); + let mut kb: Vec<_> = b.spill_ranges.iter().map(|(k, v)| (*k, *v)).collect(); + ka.sort_by_key(|(k, _)| (k.row, k.col)); + kb.sort_by_key(|(k, _)| (k.row, k.col)); + assert_eq!(ka, kb, "seed {seed} step {step}: spill layout after {log:?}"); + } + } +} + +/// A computed criterion whose formula runs a database function over the same criteria counts as +/// not met inside itself: before, each record's check ran the function again for every record, at +/// every level, so the work grew exponentially with the nesting allowed. +#[test] +fn computed_criterion_that_reenters_itself_is_not_met() { + let mut t = T::new(); + for (r, (name, age)) in [("name", "age"), ("a", "1"), ("b", "2"), ("c", "3"), ("d", "4"), ("e", "5")].iter().enumerate() { + t.set(&format!("F{}", r + 2), name); + t.set(&format!("G{}", r + 2), age); + } + t.set("N46", "Formula"); + t.set("N47", "=DCOUNT($F$2:$G$7,\"age\",$N$46:$N$47)>0"); + t.set("B60", "=DCOUNT($F$2:$G$7,\"age\",N46:N47)"); + assert_eq!(t.num("B60"), 0.0); +} diff --git a/crates/chart/src/lib.rs b/crates/chart/src/lib.rs index 4b68b8d17..f4e2895fe 100644 --- a/crates/chart/src/lib.rs +++ b/crates/chart/src/lib.rs @@ -26,7 +26,7 @@ use serde::{Deserialize, Serialize}; pub use axis::{Scale, format_number, nice_scale}; pub use raster::rasterize; pub use render::{Swatch, legend_entries, render, supports_trendline}; -pub use resolve::{default_palette, resolve, series_color, series_plot_kind}; +pub use resolve::{default_palette, has_volatile_series, resolve, series_color, series_plot_kind}; pub use sparkline::render_sparkline; /// Straight (non-premultiplied) RGBA colour. diff --git a/crates/chart/src/raster.rs b/crates/chart/src/raster.rs index 0de2450b5..80042b970 100644 --- a/crates/chart/src/raster.rs +++ b/crates/chart/src/raster.rs @@ -7,6 +7,20 @@ use crate::{Prim, Rgba}; /// Largest image we allocate (pixels). const MAX_PIXELS: u64 = 64 * 1024 * 1024; const SUB: usize = 4; +/// Scan-conversion work one [`rasterize`] call may do: edge tests (an active edge at one +/// sub-sample) plus covered pixels blended. Edges are only tested on the rows they cross, but a +/// caller can still pass enough geometry to keep the rasteriser busy for minutes on a small +/// image; once the allowance is spent the remaining fills are skipped. 2^28 is about a second, +/// far beyond any chart. +const MAX_SCAN_WORK: u64 = 1 << 28; +/// Most points one filled path may have: more is skipped before any edge is built. Chart +/// renderers cap and decimate their paths far below this; it only bounds direct callers. +const MAX_FILL_POINTS: usize = 1 << 22; + +/// Most points one stroked polyline may have: more is skipped before any segment path is built +/// (each segment becomes its own quad). Charts hold at most 100,000 points per series and draw far +/// fewer; it only bounds direct callers. +const MAX_STROKE_POINTS: usize = 1 << 17; type Path = Vec>; @@ -15,6 +29,8 @@ struct Canvas { h: usize, buf: Vec, acc: Vec, + /// Scan-conversion work left (see [`MAX_SCAN_WORK`]). + work: u64, } impl Canvas { @@ -41,7 +57,12 @@ impl Canvas { /// Fills a multi-contour path with the even-odd rule. fn fill(&mut self, path: &Path, color: Rgba) { - let mut edges: Vec<[f32; 4]> = Vec::new(); + // Out of work, or more geometry than the allowance could ever scan: no edge list is built. + if self.work == 0 || path.iter().map(Vec::len).fold(0usize, usize::saturating_add) > MAX_FILL_POINTS { + return; + } + // Each edge as [top y, bottom y, x0, y0, x1, y1]. + let mut edges: Vec<[f32; 6]> = Vec::new(); let (mut minx, mut miny, mut maxx, mut maxy) = (f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY); for c in path { if c.len() < 3 || !c.iter().all(|p| p[0].is_finite() && p[1].is_finite()) { @@ -54,7 +75,7 @@ impl Canvas { miny = miny.min(p[1]); maxy = maxy.max(p[1]); if p[1] != q[1] { - edges.push([p[0], p[1], q[0], q[1]]); + edges.push([p[1].min(q[1]), p[1].max(q[1]), p[0], p[1], q[0], q[1]]); } } } @@ -72,8 +93,28 @@ impl Canvas { if self.acc.len() < bw + 1 { self.acc.resize(bw + 1, 0.0); } + // Active-edge list: edges join when the scan reaches their top and leave past their + // bottom, so each row tests only the edges crossing it instead of every edge. + edges.sort_by(|a, b| a[0].total_cmp(&b[0])); + let mut pending = edges.iter().peekable(); + let mut active: Vec<[f32; 6]> = Vec::new(); let mut xs: Vec = Vec::new(); for py in y0..y1 { + while let Some(e) = pending.next_if(|e| e[0] < py as f32 + 1.0) { + active.push(*e); + } + active.retain(|e| e[1] > py as f32); + if active.is_empty() { + continue; + } + // The row's edge tests and its `bw` accumulator clears and blends, charged before any of + // them, so rows with no coverage still count. + let cost = (active.len() * SUB) as u64 + bw as u64; + if cost > self.work { + self.work = 0; + return; + } + self.work -= cost; for v in self.acc.iter_mut().take(bw) { *v = 0.0; } @@ -81,10 +122,9 @@ impl Canvas { for s in 0..SUB { let sy = py as f32 + (s as f32 + 0.5) / SUB as f32; xs.clear(); - for e in &edges { - let (ya, yb) = if e[1] < e[3] { (e[1], e[3]) } else { (e[3], e[1]) }; - if sy >= ya && sy < yb { - xs.push(e[0] + (sy - e[1]) * (e[2] - e[0]) / (e[3] - e[1])); + for e in &active { + if sy >= e[0] && sy < e[1] { + xs.push(e[2] + (sy - e[3]) * (e[4] - e[2]) / (e[5] - e[3])); } } xs.sort_by(f32::total_cmp); @@ -154,6 +194,9 @@ fn rounded_rect(x: f32, y: f32, w: f32, h: f32, r: f32) -> Vec<[f32; 2]> { /// Thick polyline → one quad per segment (each a separate path so overlaps don't cancel). fn stroke_paths(pts: &[[f32; 2]], width: f32, dash: bool, closed: bool) -> Vec { + if pts.len() > MAX_STROKE_POINTS { + return Vec::new(); + } let width = width.clamp(1.0, 50.0); let mut segs: Vec<([f32; 2], [f32; 2])> = pts.windows(2).map(|w| (w[0], w[1])).collect(); if closed && let (Some(a), Some(b)) = (pts.last(), pts.first()) { @@ -212,7 +255,7 @@ pub fn rasterize(prims: &[Prim], w: u32, h: u32, background: Rgba) -> Vec { for _ in 0..w * h { buf.extend_from_slice(&background); } - let mut cv = Canvas { w, h, buf, acc: Vec::new() }; + let mut cv = Canvas { w, h, buf, acc: Vec::new(), work: MAX_SCAN_WORK }; for p in prims { match p { Prim::Rect { x, y, w, h, fill, stroke, radius } => { @@ -274,6 +317,40 @@ pub fn rasterize(prims: &[Prim], w: u32, h: u32, background: Rgba) -> Vec { mod tests { use super::*; + /// Security: each row tests only the edges crossing it. A polygon of 100,000 tiny edges over + /// 4,000 rows tested every edge at every sub-sample of every row (1.6 billion tests). + #[test] + fn many_short_edges_scan_only_their_rows() { + let mut pts: Vec<[f32; 2]> = (0..100_000).map(|i| [if i % 2 == 0 { 0.0 } else { 0.5 }, i as f32 * 0.04]).collect(); + pts.push([2.0, 4_000.0]); + pts.push([2.0, 0.0]); + let img = rasterize(&[Prim::Polygon { pts, fill: [255, 0, 0, 255], stroke: None }], 2, 4_000, [0, 0, 0, 0]); + // The right-hand pixel of every row is inside the polygon. + assert_eq!(img.get(4..8), Some(&[255, 0, 0, 255][..])); + assert_eq!(img.get(4 * (2 * 3_999 + 1)..4 * (2 * 3_999 + 2)), Some(&[255, 0, 0, 255][..])); + } + + /// Security: scan conversion stops once the call's work allowance is spent. + #[test] + fn fills_stop_when_the_work_allowance_is_spent() { + let square = vec![vec![[0.0, 0.0], [4.0, 0.0], [4.0, 4.0], [0.0, 4.0]]]; + let mut cv = Canvas { w: 4, h: 4, buf: vec![0; 64], acc: Vec::new(), work: 2 * SUB as u64 * 4 + 16 }; + cv.fill(&square, [255; 4]); + assert!(cv.buf.iter().all(|&b| b == 255)); + cv.buf = vec![0; 64]; + cv.fill(&square, [255; 4]); + assert!(cv.buf.iter().all(|&b| b == 0)); + } + + /// Security: a polyline over the point cap builds no segment paths at all, solid or dashed. + #[test] + fn oversized_polylines_build_no_segment_paths() { + let line = |n: usize| -> Vec<[f32; 2]> { (0..n).map(|i| [i as f32, (i % 2) as f32]).collect() }; + assert!(stroke_paths(&line(MAX_STROKE_POINTS + 1), 1.0, false, false).is_empty()); + assert!(stroke_paths(&line(MAX_STROKE_POINTS + 1), 1.0, true, false).is_empty()); + assert_eq!(stroke_paths(&line(MAX_STROKE_POINTS), 1.0, false, false).len(), MAX_STROKE_POINTS - 1); + } + #[test] fn dashed_geometry_stays_within_the_dash_budget() { // Forty very long dashed lines, each allowed 10_000 dashes per segment: before the diff --git a/crates/chart/src/render/lines.rs b/crates/chart/src/render/lines.rs index 770aabe02..678181326 100644 --- a/crates/chart/src/render/lines.rs +++ b/crates/chart/src/render/lines.rs @@ -136,7 +136,13 @@ pub(super) fn decimate_area(top: Vec<[f32; 2]>, base: Vec<[f32; 2]>, max: usize) /// Draws a polyline split at gaps (`None`), decimated when long. With `smooth` (the plot box), /// each run is interpolated with a Catmull-Rom spline so the line curves through the data points. +/// +/// The points of the smoothed runs (data points included) are budgeted across all the runs of the line, not per run: +/// short runs between gaps escape decimation, and 100 sub-segments each would turn a +/// 100,000-point series of three-point runs into millions of points. Past the budget the +/// remaining runs are drawn through their data points only. pub(super) fn draw_runs(ctx: &mut Ctx, pts: &[Option<[f32; 2]>], color: Rgba, width: f32, smooth: Option) { + let mut budget = MAX_LINE_PTS * 2; let mut run: Vec<[f32; 2]> = Vec::new(); for p in pts.iter().chain(std::iter::once(&None)) { match p { @@ -145,7 +151,11 @@ pub(super) fn draw_runs(ctx: &mut Ctx, pts: &[Option<[f32; 2]>], color: Rgba, wi if run.len() >= 2 { let r = decimate(std::mem::take(&mut run), MAX_LINE_PTS); let r = match smooth { - Some(plot) => smooth_run(r, plot), + Some(plot) => { + let r = smooth_run(r, plot, budget); + budget = budget.saturating_sub(r.len()); + r + } None => r, }; ctx.line(r, color, width); @@ -158,15 +168,15 @@ pub(super) fn draw_runs(ctx: &mut Ctx, pts: &[Option<[f32; 2]>], color: Rgba, wi } /// Catmull-Rom spline through `pts`: every segment gets the same number of sub-segments (up to -/// 100), chosen so the whole curve stays within `MAX_LINE_PTS * 2` points. Endpoints are +/// 100), chosen so the whole curve stays within `cap` points (one per data point when `cap` is +/// smaller than that). Endpoints are /// duplicated so the curve passes through the first and last points, and the curve is clamped /// to the `plot` box (a spline overshoots its data near sharp turns). -fn smooth_run(pts: Vec<[f32; 2]>, plot: Bx) -> Vec<[f32; 2]> { +fn smooth_run(pts: Vec<[f32; 2]>, plot: Bx, cap: usize) -> Vec<[f32; 2]> { let n = pts.len(); if n < 3 { return pts; } - let cap = MAX_LINE_PTS * 2; let steps = (cap / n).clamp(1, 100); let mut out: Vec<[f32; 2]> = Vec::with_capacity(n.saturating_mul(steps).min(cap) + 1); let at = |i: isize| -> [f32; 2] { @@ -219,7 +229,7 @@ mod smooth_tests { fn long_runs_are_smoothed_uniformly_within_the_cap() { let plot = Bx { x: 0.0, y: 0.0, w: 3000.0, h: 100.0 }; let pts: Vec<[f32; 2]> = (0..1500).map(|i| [i as f32 * 2.0, if i % 2 == 0 { 20.0 } else { 80.0 }]).collect(); - let out = smooth_run(pts, plot); + let out = smooth_run(pts, plot, MAX_LINE_PTS * 2); assert!(out.len() <= MAX_LINE_PTS * 2 + 1, "{}", out.len()); // Two sub-segments per segment all the way to the end, not 100 each and then a straight tail. assert_eq!(out.len(), 1499 * 2 + 1); @@ -231,11 +241,29 @@ mod smooth_tests { let plot = Bx { x: 0.0, y: 10.0, w: 300.0, h: 100.0 }; // A step right at the plot edges: Catmull-Rom overshoots past both. let pts = vec![[0.0, 110.0], [100.0, 110.0], [110.0, 10.0], [200.0, 10.0], [300.0, 10.0]]; - let out = smooth_run(pts, plot); + let out = smooth_run(pts, plot, MAX_LINE_PTS * 2); assert!(out.len() > 5); assert!(out.iter().all(|p| (0.0..=300.0).contains(&p[0]) && (10.0..=110.0).contains(&p[1]))); } + /// Security: smoothing budgets its added points across every run of a line. A series of + /// three-point runs between gaps used to get 100 sub-segments per run, 67 times its data. + #[test] + fn gap_separated_runs_share_one_smoothing_budget() { + let plot = Bx { x: 0.0, y: 0.0, w: 1e6, h: 100.0 }; + let pts: Vec> = + (0..100_000).map(|i| if i % 4 == 3 { None } else { Some([i as f32, if i % 2 == 0 { 20.0 } else { 80.0 }]) }).collect(); + let m = crate::ApproxMeasure; + let mut ctx = Ctx { out: Vec::new(), m: &m }; + draw_runs(&mut ctx, &pts, WHITE, 1.0, Some(plot)); + let total: usize = ctx.out.iter().map(|p| if let Prim::Line { pts, .. } = p { pts.len() } else { 0 }).sum(); + // 75,000 data points, plus at most the budget added by interpolation. + assert!(total <= 75_000 + MAX_LINE_PTS * 2, "{total}"); + assert_eq!(ctx.out.len(), 25_000); + // The first runs are still smoothed. + assert!(matches!(ctx.out.first(), Some(Prim::Line { pts, .. }) if pts.len() > 3)); + } + #[test] fn decimate_area_preserves_paired_vertical_strips() { let top: Vec<[f32; 2]> = (0..1000).map(|i| [i as f32 * 0.25, ((i * 37) % 100) as f32]).collect(); diff --git a/crates/chart/src/resolve.rs b/crates/chart/src/resolve.rs index 11213a5e7..e7663e0d9 100644 --- a/crates/chart/src/resolve.rs +++ b/crates/chart/src/resolve.rs @@ -155,6 +155,131 @@ fn eval_list_in(wb: &Workbook, sheet: usize, formula: &str, limit: usize, depth: } } +/// Whether a series formula of `ch` (chart of `sheet`) calls a function whose result changes +/// between evaluations (`NOW`, `TODAY`, `RAND`, `RANDBETWEEN`, `RANDARRAY`), so a result resolved +/// earlier is stale. Defined names the formulas use (`=FreshClock`, `Sheet1!Rates`, a LAMBDA name) +/// are followed into their own formulas, the way the evaluator looks them up; each name once per +/// chart, so a cycle ends. Names nested deeper than [`MAX_NAME_DEPTH`], or more than +/// [`MAX_NAMES_FOLLOWED`] distinct names, count as volatile: a chart whose stability isn't +/// established is not cached. +pub fn has_volatile_series(wb: &Workbook, sheet: usize, ch: &Chart) -> bool { + let mut seen = Seen::new(); + ch.series.iter().any(|s| { + [s.name.as_deref(), s.categories.as_deref(), Some(s.values.as_str()), s.bubble_sizes.as_deref()] + .into_iter() + .flatten() + .any(|f| formula_volatile(wb, sheet, f, 0, &mut seen)) + }) +} + +/// Names followed inside names before a formula counts as volatile. +const MAX_NAME_DEPTH: u32 = 8; +/// Distinct defined names followed per chart before the chart counts as volatile. +const MAX_NAMES_FOLLOWED: usize = 64; + +/// Defined names already followed for one chart, each with the sheet its formula is read on. +type Seen<'a> = Vec<(&'a gridcraft_model::DefinedName, usize)>; + +fn formula_volatile<'a>(wb: &'a Workbook, sheet: usize, f: &str, depth: u32, seen: &mut Seen<'a>) -> bool { + if calls_volatile(f) { + return true; + } + name_tokens(f).into_iter().any(|t| { + let Some((n, at)) = defined_name(wb, sheet, t) else { return false }; + // Followed already, or being followed (a cycle): a volatile call in it is found there. + if seen.iter().any(|&(m, s)| std::ptr::eq(m, n) && s == at) { + return false; + } + if depth >= MAX_NAME_DEPTH || seen.len() >= MAX_NAMES_FOLLOWED { + return true; + } + seen.push((n, at)); + formula_volatile(wb, at, &n.formula, depth + 1, seen) + }) +} + +/// Whether formula text `f` calls one of the volatile functions by name. +fn calls_volatile(f: &str) -> bool { + const VOLATILE: [&str; 5] = ["NOW", "TODAY", "RAND", "RANDBETWEEN", "RANDARRAY"]; + // Formulas read from a file may carry the `_xlfn.` prefix of newer functions. + let up = f.to_ascii_uppercase().replace("_XLFN.", ""); + VOLATILE.iter().any(|name| { + up.match_indices(name).any(|(i, _)| { + let before = up[..i].chars().next_back(); + let after = up[i + name.len()..].trim_start(); + before.is_none_or(|c| !(c.is_alphanumeric() || c == '_' || c == '.')) && after.starts_with('(') + }) + }) +} + +fn is_ident_char(c: char) -> bool { + c.is_alphanumeric() || c == '_' || c == '.' || c == '\\' +} + +/// End of the identifier starting at byte `from` of `f`. +fn ident_end(f: &str, from: usize) -> usize { + f[from..].char_indices().find(|&(_, c)| !is_ident_char(c)).map_or(f.len(), |(j, _)| from + j) +} + +/// End (past the closing quote) of the `q`-quoted text opening at byte `from`; a doubled quote +/// is an escaped one. `None` when it is not closed. +fn quoted_end(f: &str, from: usize, q: char) -> Option { + let mut i = from + q.len_utf8(); + loop { + let j = i + f[i..].find(q)?; + if f[j + q.len_utf8()..].starts_with(q) { + i = j + 2 * q.len_utf8(); + } else { + return Some(j + q.len_utf8()); + } + } +} + +/// The words of formula text `f` that may be defined names: `Name`, `Sheet1!Name`, +/// `'Bob''s'!Name`. String literals and numbers are skipped; function names, column letters and +/// the like come along too and simply aren't found as names. +fn name_tokens(f: &str) -> Vec<&str> { + let mut out = Vec::new(); + let mut i = 0; + while let Some(c) = f[i..].chars().next() { + let start = i; + let end = match c { + '"' => { + i = quoted_end(f, i, '"').unwrap_or(f.len()); + continue; + } + '\'' => quoted_end(f, i, '\'').unwrap_or(f.len()), + c if is_ident_char(c) => ident_end(f, i), + c => { + i += c.len_utf8(); + continue; + } + }; + i = end; + if f[end..].starts_with('!') { + let e = ident_end(f, end + 1); + if e > end + 1 { + out.push(&f[start..e]); + } + i = e; + } else if c != '\'' && !c.is_ascii_digit() { + out.push(&f[start..end]); + } + } + out +} + +/// The defined name `t` (`Name` or `Sheet1!Name`) seen from `sheet`, and the sheet its own +/// formula is read on. +fn defined_name<'a>(wb: &'a Workbook, sheet: usize, t: &str) -> Option<(&'a gridcraft_model::DefinedName, usize)> { + // `'Bob''s'!Name`: the sheet is unquoted as everywhere else (#2012). + let (at, bare) = match gridcraft_core::split_sheet(t) { + (Some(s), bare) => (wb.sheet_index(&s)?, bare), + (None, bare) => (sheet, bare), + }; + wb.name(bare, at).map(|n| (n, n.scope.unwrap_or(at))) +} + fn to_num(v: &Value) -> Option { match v { Value::Number(n) if n.is_finite() => Some(*n), @@ -184,6 +309,53 @@ fn value_text(v: &Value) -> String { } } +/// Characters kept of one category label: an axis label never shows more (labels are fitted to +/// the axis), and copying more only costs memory. +const MAX_LABEL_CHARS: usize = 255; +/// Bytes of category label text kept per series. One long text broadcast by a short formula +/// (`IF(SEQUENCE(100000), long_text)`) is a single shared value, but turning every element into +/// its own label copied it 100,000 times (gigabytes). Labels past the budget are empty. +const MAX_CATEGORY_BYTES: usize = 8 << 20; + +/// At most [`MAX_LABEL_CHARS`] characters and `room` bytes of `s`, cut at a character boundary. +fn clip_label(s: &str, room: usize) -> String { + let mut end = 0; + for (n, (i, c)) in s.char_indices().enumerate() { + if n >= MAX_LABEL_CHARS || i + c.len_utf8() > room { + break; + } + end = i + c.len_utf8(); + } + s.get(..end).unwrap_or_default().to_string() +} + +/// The label of category `i` (value `v`) in at most `room` bytes: formatted with its source +/// cell's number format when it is a number. +fn category_label(wb: &Workbook, src: Option<(&gridcraft_model::Sheet, RangeRef)>, i: usize, v: &Value, room: usize) -> String { + if room == 0 { + return String::new(); // nothing fits: skip the number formatting + } + if let (Some((s, r)), Value::Number(_)) = (src, v) { + let w = r.width().max(1) as usize; + let at = r.start.offset((i / w) as i64, (i % w) as i64); + if let Some(c) = at { + let code = &wb.styles.get(s.style_id(c)).num_fmt.0; + if !code.eq_ignore_ascii_case("general") { + return clip_label(&format_value(v, &NumberFormat::parse_cached(code), wb.date_system).text, room); + } + } + } + // Text is clipped before it is copied: the source may be one long shared string. + let mut v = v; + while let Value::Array(a) = v { + v = a.get(0, 0).unwrap_or(&Value::Empty); + } + match v { + Value::Text(t) => clip_label(t, room), + _ => clip_label(&value_text(v), room), + } +} + /// Category labels, formatted with each source cell's number format (dates, currency…), and /// the outer levels of a multi-level axis: a category range wider than a column of values (or /// taller than a row of values) groups its innermost labels, year beside month. Each outer level @@ -221,29 +393,29 @@ fn category_table(wb: &Workbook, sheet: usize, formula: &str, values: &str, limi None => eval_list_in(wb, sheet, formula, limit, 0), }; let src = cells.and_then(|(sh, r)| wb.sheet(sh).map(|s| (s, r))); + // The retained levels are known before any label is charged: a cell on a discarded level + // gets no label and spends none of the budget. `None`: a flat list (every cell is a label). + let layout = src.and_then(|(_, r)| { + let levels = levels?; + let (w, h) = (r.width().max(1) as usize, r.height().max(1) as usize); + (vals.len() == w.saturating_mul(h)).then_some((levels, w, h)) // else truncated or not a plain block + }); + let kept = + |i: usize| layout.is_none_or(|(l, w, h)| if l.by_columns { i % w + 1 + l.outer as usize >= w } else { i / w + 1 + l.outer as usize >= h }); + let mut room = MAX_CATEGORY_BYTES; let texts: Vec = vals .iter() .enumerate() .map(|(i, v)| { - if let (Some((s, r)), Value::Number(_)) = (src, v) { - let w = r.width().max(1) as usize; - let at = r.start.offset((i / w) as i64, (i % w) as i64); - if let Some(c) = at { - let code = &wb.styles.get(s.style_id(c)).num_fmt.0; - if !code.eq_ignore_ascii_case("general") { - return format_value(v, &NumberFormat::parse_cached(code), wb.date_system).text; - } - } + if !kept(i) { + return String::new(); } - value_text(v) + let label = category_label(wb, src, i, v, room); + room = room.saturating_sub(label.len()); + label }) .collect(); - let Some((_, r)) = src else { return (texts, vec![]) }; - let Some(levels) = levels else { return (texts, vec![]) }; - let (w, h) = (r.width().max(1) as usize, r.height().max(1) as usize); - if texts.len() != w.saturating_mul(h) { - return (texts, vec![]); // truncated or not a plain block: a flat list - } + let Some((levels, w, h)) = layout else { return (texts, vec![]) }; let at = |row: usize, col: usize| texts.get(row * w + col).cloned().unwrap_or_default(); let column = |c: usize| (0..h).map(|row| at(row, c)).collect::>(); let row = |rr: usize| (0..w).map(|c| at(rr, c)).collect::>(); @@ -258,11 +430,7 @@ fn category_table(wb: &Workbook, sheet: usize, formula: &str, values: &str, limi /// Whether `t` names a defined name seen from `sheet` (`Name`, or `Sheet1!Name` looked up on /// that sheet), the way the evaluator looks names up. fn is_defined_name(wb: &Workbook, sheet: usize, t: &str) -> bool { - // `'Bob''s'!Name`: the sheet is unquoted as everywhere else (#2012). - match gridcraft_core::split_sheet(t) { - (Some(s), bare) => wb.sheet_index(&s).is_some_and(|i| wb.name(bare, i).is_some()), - (None, _) => wb.name(t, sheet).is_some(), - } + defined_name(wb, sheet, t).is_some() } /// A series name: a reference/formula or a defined name (`[0]!Title` in a file) is evaluated, @@ -369,6 +537,46 @@ pub fn resolve(wb: &Workbook, sheet: usize, chart: &Chart) -> ChartData { ChartData { categories, category_levels, series, palette } } +#[cfg(test)] +mod label_budget_tests { + use gridcraft_model::Workbook; + + /// Security: category labels are clipped before they are copied, with one byte budget per + /// series. A formula broadcasting one long text to 100,000 categories used to copy it + /// 100,000 times (3.3 GB). + #[test] + fn category_labels_share_one_byte_budget() { + let wb = Workbook::new(); + let (texts, _) = super::category_table(&wb, 0, r#"=IF(SEQUENCE(100000),REPT("x",32767))"#, "", super::MAX_POINTS); + assert_eq!(texts.len(), 100_000); + assert_eq!(texts.first().map(String::len), Some(super::MAX_LABEL_CHARS)); + assert!(texts.iter().map(String::len).sum::() <= super::MAX_CATEGORY_BYTES); + // Labels are cut at character boundaries. + assert_eq!(super::clip_label("aé", 2), "a"); + assert_eq!(super::clip_label("aé", 3), "aé"); + } + + /// Cells on discarded outer levels (only the innermost labels and three outer levels are + /// kept) spend none of the byte budget. + #[test] + fn discarded_levels_do_not_spend_the_label_budget() { + let mut wb = Workbook::new(); + let rows = 8000u32; + if let Some(sh) = wb.sheet_mut(0) { + for r in 0..rows { + for c in 0..5 { + sh.set_value(gridcraft_core::CellRef::new(r, c), gridcraft_core::Value::text("x".repeat(255))); + } + } + } + // Five columns: column A is a fourth outer level and is discarded. + let (inner, outer) = super::category_table(&wb, 0, "=A1:E8000", "", super::MAX_POINTS); + assert_eq!(outer.len(), 3); + assert_eq!(inner.last().map(String::len), Some(super::MAX_LABEL_CHARS)); + assert_eq!(outer.first().and_then(|l| l.last()).map(String::len), Some(super::MAX_LABEL_CHARS)); + } +} + #[cfg(test)] mod tests { use gridcraft_model::Workbook; diff --git a/crates/chart/src/tests.rs b/crates/chart/src/tests.rs index 8a9b8640e..a6d845387 100644 --- a/crates/chart/src/tests.rs +++ b/crates/chart/src/tests.rs @@ -1680,3 +1680,96 @@ fn a_series_named_by_a_name_on_a_sheet_with_a_bang_resolves() { let d = resolve(&wb, 0, &c); assert_eq!(d.series.first().map(|s| s.name.as_str()), Some("Takings")); } + +#[test] +fn volatile_series_formulas_are_detected() { + let wb = Workbook::new(); + let with = |values: &str, name: Option<&str>| { + let mut c = chart(ChartKind::Line); + c.series.push(Series { + name: name.map(Into::into), + categories: None, + values: values.into(), + bubble_sizes: None, + color: None, + secondary: false, + kind: None, + smooth: false, + quartiles: Default::default(), + trendlines: vec![], + }); + c + }; + assert!(has_volatile_series(&wb, 0, &with("=NOW()", None))); + assert!(has_volatile_series(&wb, 0, &with("=SUM(A1:A3)*rand ()", None))); + assert!(has_volatile_series(&wb, 0, &with("Sheet1!$B$2:$B$9", Some("=TODAY()")))); + assert!(has_volatile_series(&wb, 0, &with("=RANDARRAY(5)", None))); + assert!(has_volatile_series(&wb, 0, &with("=_xlfn.RANDARRAY(5)", None))); + assert!(!has_volatile_series(&wb, 0, &with("Sheet1!$B$2:$B$9", Some("Sheet1!$B$1")))); + assert!(!has_volatile_series(&wb, 0, &with("=MYRAND(A1)", None))); + assert!(!has_volatile_series(&wb, 0, &with("=Sheet1!$NOW", None))); + assert!(!has_volatile_series(&wb, 0, &chart(ChartKind::Line))); +} + +#[test] +fn volatile_defined_names_are_followed() { + let with = |values: &str| { + let mut c = chart(ChartKind::Line); + c.series.push(Series { + name: None, + categories: None, + values: values.into(), + bubble_sizes: None, + color: None, + secondary: false, + kind: None, + smooth: false, + quartiles: Default::default(), + trendlines: vec![], + }); + c + }; + let name = |name: &str, scope: Option, formula: &str| gridcraft_model::DefinedName { + name: name.into(), + scope, + formula: formula.into(), + comment: String::new(), + hidden: false, + }; + let mut wb = Workbook::new(); + wb.sheet_mut(0).unwrap().name = "Bob's".into(); + wb.names.push(name("FreshClock", None, "NOW()")); + wb.names.push(name("Later", None, "FreshClock+1")); + wb.names.push(name("Dice", Some(0), "RANDBETWEEN(1,6)")); + wb.names.push(name("Rates", None, "'Bob''s'!$A$1:$A$3")); + wb.names.push(name("Loop", None, "Loop+1")); + wb.names.push(name("Spin", None, "Spin+NOW()")); + wb.names.push(name("Roll", None, "LAMBDA(n,n*RAND())")); + // A workbook-scoped name, a name used inside another, and a sheet-scoped one, bare or qualified. + assert!(has_volatile_series(&wb, 0, &with("=FreshClock"))); + assert!(has_volatile_series(&wb, 0, &with("=SUM(Later, 1)"))); + assert!(has_volatile_series(&wb, 0, &with("=Dice*2"))); + assert!(has_volatile_series(&wb, 0, &with("='Bob''s'!Dice"))); + assert!(has_volatile_series(&wb, 0, &with("=Roll(3)"))); + // A self-referencing name ends (it evaluates to an error, not a changing value); one that + // also calls NOW is volatile. + assert!(!has_volatile_series(&wb, 0, &with("=Loop"))); + assert!(has_volatile_series(&wb, 0, &with("=Spin"))); + // Many series over the same stable name stay cacheable: each name is followed once. + let mut many = with("=Rates"); + let first = many.series[0].clone(); + many.series.extend(std::iter::repeat_n(first, 100)); + assert!(!has_volatile_series(&wb, 0, &many)); + // Names nested deeper than the limit are not known to be stable. + for i in 0..10 { + wb.names.push(name(&format!("Deep{i}"), None, &format!("Deep{}", i + 1))); + } + assert!(has_volatile_series(&wb, 0, &with("=Deep0"))); + // A name that refers to cells, a name inside a string and a name of another scope are stable. + assert!(!has_volatile_series(&wb, 0, &with("=Rates"))); + assert!(!has_volatile_series(&wb, 0, &with("=\"FreshClock\""))); + // Dice belongs to sheet 0: seen from another sheet it isn't defined. + wb.sheets.push(Arc::new(gridcraft_model::Sheet::new("Other"))); + assert!(!has_volatile_series(&wb, 1, &with("=Dice"))); + assert!(has_volatile_series(&wb, 1, &with("=FreshClock"))); +} diff --git a/crates/core/src/addr.rs b/crates/core/src/addr.rs index 498c3f068..dfe32498a 100644 --- a/crates/core/src/addr.rs +++ b/crates/core/src/addr.rs @@ -147,10 +147,10 @@ impl RangeRef { RangeRef::new(CellRef::new(0, 0), CellRef::new(MAX_ROWS - 1, MAX_COLS - 1)) } pub fn height(&self) -> u32 { - self.end.row - self.start.row + 1 + self.end.row.saturating_sub(self.start.row).saturating_add(1) } pub fn width(&self) -> u32 { - self.end.col - self.start.col + 1 + self.end.col.saturating_sub(self.start.col).saturating_add(1) } pub fn count(&self) -> u64 { self.height() as u64 * self.width() as u64 @@ -321,6 +321,13 @@ pub fn split_areas(reference: &str) -> Vec<&str> { mod tests { use super::*; + #[test] + fn hostile_range_dimensions_saturate_instead_of_wrapping() { + let r = RangeRef { start: CellRef::new(0, 0), end: CellRef::new(u32::MAX, u32::MAX) }; + assert_eq!((r.height(), r.width()), (u32::MAX, u32::MAX)); + assert_eq!(r.count(), u64::from(u32::MAX) * u64::from(u32::MAX)); + } + #[test] fn areas_split_only_at_commas_outside_quoted_sheet_names() { assert_eq!(split_areas("B2:B4,D1"), ["B2:B4", "D1"]); diff --git a/crates/core/src/value.rs b/crates/core/src/value.rs index a0b4fcad6..4d76ef70e 100644 --- a/crates/core/src/value.rs +++ b/crates/core/src/value.rs @@ -454,8 +454,9 @@ pub fn number_to_text(n: f64) -> String { if !(1e-9..1e15).contains(&a) && a.round() != a || a >= 1e15 { return sci_text(n, 15); } - // A value just below 1e15 can round up to 1e15 at 15 significant digits. - if format!("{:.14e}", a).parse::().is_ok_and(|r| r >= 1e15) { + // A value just below 1e15 can round up to 1e15 at 15 significant digits. Only that narrow + // band pays for the exact formatting: it runs for every number literal a formula prints. + if a >= 9.999_999_999_999_9e14 && format!("{:.14e}", a).parse::().is_ok_and(|r| r >= 1e15) { return sci_text(n, 15); } // 15 significant digits, trailing zeros trimmed. @@ -506,6 +507,9 @@ mod tests { assert_eq!(number_to_text(999999999999999.9), "1E+15"); assert_eq!(number_to_text(-999999999999999.9), "-1E+15"); assert_eq!(number_to_text(999999999999999.0), "999999999999999"); + assert_eq!(number_to_text(999999999999999.5), "1E+15"); + assert_eq!(number_to_text(999999999999999.4), "999999999999999"); + assert_eq!(number_to_text(999999999999989.9), "999999999999990"); } #[test] diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 341d1b35c..c68fb117a 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -21,10 +21,12 @@ web-time = { workspace = true } thiserror = { workspace = true } log = { workspace = true } gridcraft-xlsx = { workspace = true } +craft-csv = { workspace = true } gridcraft-chart = { workspace = true } gridcraft-pdf = { workspace = true } gridcraft-math = { workspace = true } image = { workspace = true } +unicode-segmentation = "=1.13.3" [dev-dependencies] gridcraft-xlsx = { workspace = true, features = ["test-support"] } diff --git a/crates/engine/src/cf.rs b/crates/engine/src/cf.rs index 0af7a9df5..e5fe4dedc 100644 --- a/crates/engine/src/cf.rs +++ b/crates/engine/src/cf.rs @@ -1,7 +1,7 @@ //! Conditional formatting evaluation: what extra style, data bar, colour scale or icon a cell //! gets. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use gridcraft_core::{CellRef, RangeRef, Value, compare}; use gridcraft_model::{CfOperator, CfRule, CfValueKind, Color, CondFormat, PatternType, Sheet, Style, Underline, Workbook, default_icon_thresholds}; @@ -47,6 +47,8 @@ enum StatsNeed { fn stats_need(rule: &CfRule) -> Option { match rule { CfRule::Duplicate { .. } | CfRule::Unique { .. } => Some(StatsNeed::Counts), + // only two- and three-colour scales paint: no statistics for the others + CfRule::ColorScale { stops } if !matches!(stops.len(), 2 | 3) => None, CfRule::Top10 { .. } | CfRule::AboveAverage { .. } | CfRule::ColorScale { .. } | CfRule::DataBar { .. } | CfRule::IconSet { .. } => { Some(StatsNeed::Numbers) } @@ -70,15 +72,58 @@ thread_local! { static SCANNED: std::cell::Cell = const { std::cell::Cell::new(0) }; } +/// Most cells one rule's statistics visit, across all of its ranges together: a rule repeating +/// the same range many times must not multiply the scan. +const STATS_SCAN_LIMIT: usize = 2_000_000; + +/// Steps through the spilled cells inside `r`, row-major: `Some(cell)` for each, and `None` for a +/// jump past the spills left or right of the columns on a row (one step per row, however many +/// there are), so unrelated spills between a narrow range's corners cost little. +fn spill_steps(sh: &Sheet, r: RangeRef) -> impl Iterator> + '_ { + let mut cur = Some(r.start); + std::iter::from_fn(move || { + let at = cur.filter(|at| *at <= r.end)?; + let (&c, _) = sh.spill.range(at..=r.end).next()?; + if c.col < r.start.col { + cur = Some(CellRef::new(c.row, r.start.col)); + Some(None) + } else if c.col > r.end.col { + cur = c.row.checked_add(1).map(|row| CellRef::new(row, r.start.col)); + Some(None) + } else { + cur = c.col.checked_add(1).map(|col| CellRef::new(c.row, col)); + Some(Some(c)) + } + }) +} + fn numbers_in(sh: &Sheet, ranges: &[RangeRef], need: StatsNeed) -> (Vec, HashMap) { let mut nums = Vec::new(); let mut counts = HashMap::new(); + let mut budget = STATS_SCAN_LIMIT; + let mut seen: HashSet = HashSet::new(); + let used = sh.used_range(); for r in ranges { - let r = match sh.used_range().and_then(|u| r.intersection(&u)) { + if budget == 0 { + break; + } + let r = match used.and_then(|u| r.intersection(&u)) { Some(x) => x, None => continue, }; - for c in r.iter().take(2_000_000) { + // The same range listed twice counts its cells once. + if !seen.insert(r) { + continue; + } + // Only stored and spilled cells hold values: blank cells of a sparse range cost nothing, + // so a value far down a huge range is still counted. Every cell looked at is charged. + let stored = sh.cells.iter_range(r).map(|(c, _)| (Some(c), true)); + let spilled = spill_steps(sh, r).map(|c| (c, false)); + for (c, is_stored) in stored.chain(spilled).take(budget) { + budget -= 1; + // a skip over spills outside the columns (charged, not read), or a spilled value + // under a stored cell, which was read with it + let Some(c) = c.filter(|c| is_stored || sh.cells.get(*c).is_none()) else { continue }; #[cfg(test)] SCANNED.with(|n| n.set(n.get() + 1)); let v = sh.value(c); @@ -278,7 +323,8 @@ impl CfCache { _ => None, }, CfRule::ColorScale { stops } => { - if let Some(n) = v.as_f64() { + // Only two- and three-colour scales paint: evaluate no thresholds for others. + if let Some(n) = v.as_f64().filter(|_| matches!(stops.len(), 2 | 3)) { let pts: Vec<(f64, [u8; 3])> = stops .iter() .map(|(k, col)| (threshold(wb, sheet, c, st, k), col.resolve(&wb.theme).unwrap_or([255, 255, 255]))) @@ -585,6 +631,89 @@ mod tests { assert_eq!(SCANNED.with(|n| n.get()), 10_000); } + /// A rule listing the same range many times scans it once, and never more than the shared + /// limit however many ranges it holds. + #[test] + fn repeated_ranges_share_one_scan_budget() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1], [2], [3]]})).unwrap(); + s.execute("range.setValues", json!({"range": "XFD1048576", "values": [[4]]})).unwrap(); + let d = s.doc().unwrap(); + let sh = d.wb.sheet(0).unwrap(); + let dense = RangeRef::parse("A1:XFD1048576").unwrap(); + let small = RangeRef::parse("A1:A3").unwrap(); + SCANNED.with(|n| n.set(0)); + let st = RuleStats::build(sh, &vec![small; 1000], StatsNeed::Numbers); + assert_eq!(st.sorted, vec![1.0, 2.0, 3.0]); + assert_eq!(SCANNED.with(|n| n.get()), 3); + let shifted: Vec = (0..50).map(|i| RangeRef::new(CellRef::new(i, 0), dense.end)).collect(); + SCANNED.with(|n| n.set(0)); + RuleStats::build(sh, &shifted, StatsNeed::Numbers); + assert!(SCANNED.with(|n| n.get()) <= STATS_SCAN_LIMIT); + } + + /// Blank cells of a sparse range are not charged to the scan budget: a value at the end of two + /// million-row columns still counts, so duplicates are found. + #[test] + fn sparse_ranges_keep_their_values_under_the_scan_budget() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [[7]]})).unwrap(); + s.execute("range.setValues", json!({"range": "B1048576", "values": [[7]]})).unwrap(); + let d = s.doc().unwrap(); + let sh = d.wb.sheet(0).unwrap(); + let ranges = [RangeRef::parse("A1:A1048576").unwrap(), RangeRef::parse("B1:B1048576").unwrap()]; + SCANNED.with(|n| n.set(0)); + let st = RuleStats::build(sh, &ranges, StatsNeed::Counts); + assert_eq!(st.counts.get("7"), Some(&2)); + assert_eq!(SCANNED.with(|n| n.get()), 2); + } + + /// Spills left and right of a narrow range, between its corners, are skipped a row at a time: + /// they do not use the budget that the in-range spills need. + #[test] + fn spills_outside_a_narrow_range_do_not_use_its_scan_budget() { + let mut s = crate::Session::new(); + s.new_workbook(); + let d = s.doc().unwrap(); + let mut sh = d.wb.sheet(0).unwrap().clone(); + for row in 0..1000 { + for col in [0u32, 5, 6, 7] { + sh.spill.insert(CellRef::new(row, col), Value::Number(1.0)); + } + } + let r = RangeRef::parse("F1:G1000").unwrap(); + let steps: Vec<_> = spill_steps(&sh, r).collect(); + assert_eq!(steps.iter().flatten().count(), 2000); + // column A is skipped one step a row, never one per cell + assert!(steps.len() <= 2000 + 1000, "{}", steps.len()); + let narrow = RangeRef::parse("C1:C1000").unwrap(); + assert_eq!(spill_steps(&sh, narrow).flatten().count(), 0); + } + + /// A colour scale with an unsupported number of stops paints nothing and evaluates none of them. + #[test] + fn colour_scales_with_other_stop_counts_paint_nothing() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1], [5]]})).unwrap(); + let mut wb: Workbook = (*s.doc().unwrap().wb).clone(); + let stop = (CfValueKind::Formula("=1".into()), Color::default()); + let rule = |n: usize| CondFormat { + ranges: vec![RangeRef::parse("A1:A2").unwrap()], + rule: CfRule::ColorScale { stops: vec![stop.clone(); n] }, + priority: 1, + stop_if_true: false, + }; + wb.sheet_mut(0).unwrap().cond_formats.push(rule(4)); + let mut cache = CfCache::new(); + assert_eq!(cache.look(&wb, 0, CellRef::parse("A1").unwrap()).and_then(|l| l.scale_fill), None); + wb.sheet_mut(0).unwrap().cond_formats = vec![rule(2)]; + let mut cache = CfCache::new(); + assert!(cache.look(&wb, 0, CellRef::parse("A1").unwrap()).and_then(|l| l.scale_fill).is_some()); + } + #[test] fn cell_value_operator_names() { let mut s = crate::Session::new(); diff --git a/crates/engine/src/chart_svg.rs b/crates/engine/src/chart_svg.rs index be9e6d227..f5f43041e 100644 --- a/crates/engine/src/chart_svg.rs +++ b/crates/engine/src/chart_svg.rs @@ -10,6 +10,10 @@ use crate::io::esc; /// Widest or tallest chart written; larger anchors are clamped. const MAX_SIDE: f32 = 16_384.0; +/// Most bytes of SVG written for the charts of one web page together. A chart past what is +/// left makes the export fail rather than grow without bound or be dropped without a word. +pub(crate) const MAX_PAGE_SVG_BYTES: usize = 64 * 1024 * 1024; + struct HelveticaMeasure; impl gridcraft_chart::Measure for HelveticaMeasure { @@ -18,24 +22,40 @@ impl gridcraft_chart::Measure for HelveticaMeasure { } } -/// `chart` of sheet `sheet` as an `` element placed by `style`, or `None` for a degenerate size. -pub(crate) fn chart_svg(wb: &Workbook, sheet: usize, chart: &Chart, style: &str) -> Option { +/// `chart` of sheet `sheet` as an `` element placed by `style`, `None` for a degenerate size, +/// or an error for an element longer than `budget` bytes. +pub(crate) fn chart_svg(wb: &Workbook, sheet: usize, chart: &Chart, style: &str, budget: usize) -> crate::Result> { let (w, h) = (chart.anchor.width.min(MAX_SIDE), chart.anchor.height.min(MAX_SIDE)); if !(w >= 1.0 && h >= 1.0) { - return None; + return Ok(None); } let data = gridcraft_chart::resolve(wb, sheet, chart); let prims = gridcraft_chart::render(chart, &data, w, h, &HelveticaMeasure); - Some(prims_svg(&prims, w, h, style)) + match prims_svg_within(&prims, w, h, style, budget) { + Some(svg) => Ok(Some(svg)), + None => Err(crate::EngineError::Other(format!( + "the charts would make the web page over {} MiB. Save as .xlsx or PDF instead.", + MAX_PAGE_SVG_BYTES / (1024 * 1024) + ))), + } } +#[cfg(test)] fn prims_svg(prims: &[Prim], w: f32, h: f32, style: &str) -> String { + prims_svg_within(prims, w, h, style, usize::MAX).unwrap_or_default() +} + +/// The `` element, or `None` as soon as it grows past `budget` bytes. +fn prims_svg_within(prims: &[Prim], w: f32, h: f32, style: &str, budget: usize) -> Option { let mut out = String::new(); let _ = write!( out, "" ); for p in prims { + if out.len() > budget { + return None; + } match p { Prim::Rect { x, y, w, h, fill, stroke, radius } => { let _ = write!(out, " String { } } out.push_str(""); - out + (out.len() <= budget).then_some(out) } /// Fill and stroke attributes of an open shape element; the caller closes it. @@ -149,6 +169,17 @@ fn num(v: f32) -> String { mod tests { use super::*; + #[test] + fn an_svg_past_its_byte_budget_is_left_out() { + let rect = Prim::Rect { x: 1.0, y: 2.0, w: 3.0, h: 4.0, fill: Some([255, 0, 0, 255]), stroke: None, radius: 0.0 }; + let prims = vec![rect; 10_000]; + let full = prims_svg(&prims, 100.0, 50.0, ""); + assert!(full.len() > 100_000); + assert_eq!(prims_svg_within(&prims, 100.0, 50.0, "", full.len()), Some(full.clone())); + assert_eq!(prims_svg_within(&prims, 100.0, 50.0, "", full.len() - 1), None); + assert_eq!(prims_svg_within(&prims, 100.0, 50.0, "", 1_000), None); + } + #[test] fn every_primitive_becomes_an_svg_element() { let red = [255, 0, 0, 255]; diff --git a/crates/engine/src/cmd.rs b/crates/engine/src/cmd.rs index da130fe2a..37e9cd082 100644 --- a/crates/engine/src/cmd.rs +++ b/crates/engine/src/cmd.rs @@ -26,6 +26,7 @@ pub mod spelling; pub mod stash; pub mod view; pub mod whatif; +pub mod wordart; use gridcraft_calc::Key; use gridcraft_core::{CellRef, RangeRef}; @@ -176,6 +177,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(extra::specs()); v.extend(macros::specs()); v.extend(draw::specs()); + v.extend(wordart::specs()); v.extend(spelling::specs()); v.extend(solver::specs()); v.extend(pivot::specs()); @@ -366,6 +368,10 @@ pub struct Ctx<'a> { pub wb: Workbook, pub changed: Vec, pub structural: bool, + /// Recalculate every formula without the other effects of `structural` (table headers are not + /// resynchronized, array formulas are still checked): for edits that change only what a + /// function reads, like sheet protection. + pub recalc_all: bool, pub sel: &'a mut Selection, /// Rows whose automatic height should be recomputed (sheet, row). pub fit_rows: Vec<(usize, u32)>, @@ -400,13 +406,14 @@ pub(crate) fn commit(d: &mut DocState, f: impl FnOnce(&mut Ctx) -> Result) wb: (*d.wb).clone(), changed: Vec::new(), structural: false, + recalc_all: false, sel: &mut sel, fit_rows: Vec::new(), protection_checked: false, edits: Vec::new(), }; let r = f(&mut ctx)?; - let Ctx { mut wb, mut changed, structural, mut fit_rows, protection_checked, edits, .. } = ctx; + let Ctx { mut wb, mut changed, structural, recalc_all, mut fit_rows, protection_checked, edits, .. } = ctx; // Equations whose source changed (or that arrived without a layout, as from a paste) are // laid out here, so painting only ever reads the cache. wb.refresh_equations(); @@ -435,7 +442,7 @@ pub(crate) fn commit(d: &mut DocState, f: impl FnOnce(&mut Ctx) -> Result) if let Some(t0) = t0 { eprintln!("commit: edit {:?}, {} changed", since(t0), changed.len()); } - if structural { + if structural || recalc_all { d.calc.recalc_all(&mut wb); } else if !changed.is_empty() { d.calc.cells_changed(&mut wb, &changed); @@ -446,8 +453,18 @@ pub(crate) fn commit(d: &mut DocState, f: impl FnOnce(&mut Ctx) -> Result) } fit_rows.sort_unstable(); fit_rows.dedup(); + let mut sweep: Option<(usize, MergeSweep)> = None; + let mut left = MERGE_FIT_WORK; for (si, row) in fit_rows { - auto_row_height(&mut wb, si, row); + if sweep.as_ref().is_none_or(|(at, _)| *at != si) { + if let Some((_, old)) = &sweep { + left = old.left(); + } + sweep = Some((si, MergeSweep::new(left))); + } + if let Some((_, sw)) = sweep.as_mut() { + fit_row_height(&mut wb, si, row, sw); + } } if let Some(t0) = t0 { eprintln!("commit: + rows {:?}", since(t0)); @@ -721,6 +738,94 @@ fn wrapped_line_count(line: &str, width: f32, size: f32, bold: bool) -> f32 { /// Line breaks in a cell's text count only when the cell wraps, as in Excel: with Wrap Text off /// a multi-line value shows on one line and AutoFit gives a one-line row. pub fn auto_row_height(wb: &mut Workbook, si: usize, row: u32) { + let mut sweep = MergeSweep::new(MERGE_FIT_WORK); + fit_row_height(wb, si, row, &mut sweep); +} + +/// Most merge comparisons one auto-fit pass (a commit, or opening a workbook) makes. Past it the +/// remaining rows keep the heights they have rather than stall on a sheet whose merges overlap +/// the fitted rows many times over. +pub(crate) const MERGE_FIT_WORK: usize = 50_000_000; + +/// The merged areas of one sheet that touch a row, for rows visited in ascending order: the +/// merges are sorted by first row once and swept, so fitting many rows never rescans every merge +/// per row. All comparisons count against [`MERGE_FIT_WORK`]. +pub(crate) struct MergeSweep { + /// Built on the first row that needs merge checks, so rows that return early never pay for it. + by_start: Option>, + next: usize, + active: Vec, + budget: usize, + /// The first row was answered by a plain scan: a pass that fits one row never pays for a sort. + scanned: bool, +} + +impl MergeSweep { + /// A sweep for one sheet's merges with `budget` comparisons left for the pass (carry + /// [`MergeSweep::left`] of the previous sheet's sweep into the next). Every call to + /// [`MergeSweep::at`] must pass that same sheet. + pub(crate) fn new(budget: usize) -> Self { + Self { by_start: None, next: 0, active: Vec::new(), budget, scanned: false } + } + + /// Comparisons left for the rest of the pass. + pub(crate) fn left(&self) -> usize { + self.budget + } + + /// Takes `n` comparisons from the budget; `false` once it is spent. + fn charge(&mut self, n: usize) -> bool { + match self.budget.checked_sub(n) { + Some(rest) => { + self.budget = rest; + true + } + None => { + self.budget = 0; + false + } + } + } + + /// The merges covering `row`, or `None` when the budget is spent. Rows must not decrease + /// between calls (skipping rows is fine). + fn at(&mut self, sh: &gridcraft_model::Sheet, row: u32) -> Option> { + if self.by_start.is_none() && !self.scanned { + self.scanned = true; + if !self.charge(sh.merges.len()) { + return None; + } + return Some(sh.merges.iter().copied().filter(|m| !m.is_single() && m.start.row <= row && row <= m.end.row).collect()); + } + if self.by_start.is_none() { + // Sorting is about n log2 n comparisons, charged before it starts. + let n = sh.merges.len(); + if !self.charge(n.saturating_mul((usize::BITS - n.leading_zeros()) as usize)) { + return None; + } + let mut v: Vec = sh.merges.iter().copied().filter(|m| !m.is_single()).collect(); + v.sort_unstable_by_key(|m| m.start.row); + self.by_start = Some(v); + } + let by_start = self.by_start.as_ref()?; + while let Some(m) = by_start.get(self.next).filter(|m| m.start.row <= row) { + self.active.push(*m); + self.next += 1; + } + if !self.charge(self.active.len()) { + return None; + } + self.active.retain(|m| m.end.row >= row); + Some(self.active.clone()) + } +} + +/// [`auto_row_height`] with the sheet's merges taken from `sweep`. +pub(crate) fn fit_row_height(wb: &mut Workbook, si: usize, row: u32, sweep: &mut MergeSweep) { + // Out of budget: the row keeps the height it has. + if sweep.left() == 0 { + return; + } let Some(sh) = wb.sheet(si) else { return }; if sh.rows.get(&row).is_some_and(|i| i.custom) { return; @@ -750,9 +855,14 @@ pub fn auto_row_height(wb: &mut Workbook, si: usize, row: u32) { // picked out once, and per cell the candidates are re-checked (normally none or a couple; // a sheet with many merges and many saved columns would otherwise scan all of them per // cell, rows × merges × columns). - let row_merges: Vec = - sh.merges.iter().copied().filter(|m| !m.is_single() && m.start.row <= row && row <= m.end.row).collect(); + // Rows are fitted in ascending order, so the merges touching this one come from a sweep over + // the sheet's merges sorted once, not from a scan of all of them. Out of budget: the row keeps + // the height it has. + let Some(row_merges) = sweep.at(sh, row) else { return }; for (c, cell) in sh.cells.row(row, 0, gridcraft_core::MAX_COLS - 1) { + if !sweep.charge(row_merges.len()) { + return; + } // Merged cells never auto-fit their rows, like Excel (column autofit skips them too). if row_merges.iter().any(|m| m.contains(CellRef::new(row, c))) { continue; @@ -796,3 +906,40 @@ impl Ctx<'_> { self.changed.push((sheet, c)); } } + +#[cfg(test)] +mod merge_sweep_tests { + use super::*; + + #[test] + fn the_budget_carries_from_one_sheets_sweep_to_the_next() { + let mut sheet = gridcraft_model::Sheet::new("S"); + for r in ["A1:B2", "C1:D2", "E1:F2"] { + sheet.merges.push(gridcraft_core::RangeRef::parse(r).unwrap()); + } + let mut first = MergeSweep::new(5); + assert!(first.at(&sheet, 0).is_some(), "the first row is a plain scan: three comparisons"); + assert!(first.at(&sheet, 1).is_none(), "the second row needs the sort: six comparisons, two are left"); + assert_eq!(first.left(), 0); + // The next sheet starts with what is left: nothing, so it fits nothing. + let mut second = MergeSweep::new(first.left()); + assert!(second.at(&sheet, 0).is_none()); + assert!(MergeSweep::new(MERGE_FIT_WORK).at(&sheet, 0).is_some()); + } + + #[test] + fn sorting_the_merges_is_charged() { + let mut sheet = gridcraft_model::Sheet::new("S"); + for r in ["A1:B2", "C1:D2", "E1:F2", "G1:H2"] { + sheet.merges.push(gridcraft_core::RangeRef::parse(r).unwrap()); + } + // The first row costs a plain scan (4); from the second on the merges are sorted, about + // 4 * 3 comparisons: 7 left is not enough, 12 plus the row's 4 is. + let mut sweep = MergeSweep::new(11); + assert!(sweep.at(&sheet, 0).is_some()); + assert!(sweep.at(&sheet, 1).is_none()); + assert_eq!(sweep.left(), 0); + let mut sweep = MergeSweep::new(20); + assert!(sweep.at(&sheet, 0).is_some() && sweep.at(&sheet, 1).is_some()); + } +} diff --git a/crates/engine/src/cmd/analysis.rs b/crates/engine/src/cmd/analysis.rs index 9cd10e50f..92699b75f 100644 --- a/crates/engine/src/cmd/analysis.rs +++ b/crates/engine/src/cmd/analysis.rs @@ -152,26 +152,32 @@ fn input_range(s: &Session, p: &Json, keys: &[&str]) -> Result<(usize, RangeRef) /// Splits a range into variables (columns, or rows when `by_rows`); with `labels` the first /// cell of each names it, else `default(i)` does. fn variables(s: &Session, si: usize, r: RangeRef, by_rows: bool, labels: bool, default: &dyn Fn(usize) -> String) -> Result> { - let d = s.doc()?; - let sh = d.wb.sheet(si).ok_or(EngineError::NoDocument)?; - let (outer, inner) = if by_rows { (r.height(), r.width()) } else { (r.width(), r.height()) }; + let outer = if by_rows { r.height() } else { r.width() }; let mut out = Vec::with_capacity(outer as usize); for i in 0..outer { - let cell = |j: u32| if by_rows { CellRef::new(r.start.row + i, r.start.col + j) } else { CellRef::new(r.start.row + j, r.start.col + i) }; - let mut vals: Vec = (0..inner).map(|j| sh.value(cell(j))).collect(); - let mut name = default(i as usize); - if labels && !vals.is_empty() { - let first = vals.remove(0); - let t = first.display(); - if !t.trim().is_empty() { - name = t; - } - } - out.push(Var { name, vals }); + out.push(variable(s, si, r, by_rows, labels, default, i)?); } Ok(out) } +/// Variable `i` of [`variables`], read alone so a caller that may stop early needn't build them all. +fn variable(s: &Session, si: usize, r: RangeRef, by_rows: bool, labels: bool, default: &dyn Fn(usize) -> String, i: u32) -> Result { + let d = s.doc()?; + let sh = d.wb.sheet(si).ok_or(EngineError::NoDocument)?; + let inner = if by_rows { r.width() } else { r.height() }; + let cell = |j: u32| if by_rows { CellRef::new(r.start.row + i, r.start.col + j) } else { CellRef::new(r.start.row + j, r.start.col + i) }; + let mut vals: Vec = (0..inner).map(|j| sh.value(cell(j))).collect(); + let mut name = default(i as usize); + if labels && !vals.is_empty() { + let first = vals.remove(0); + let t = first.display(); + if !t.trim().is_empty() { + name = t; + } + } + Ok(Var { name, vals }) +} + /// The numbers of a variable; blanks are skipped when `skip_blank`, anything else that isn't a /// number is an error. fn numbers(v: &[Value], skip_blank: bool) -> Result> { @@ -272,11 +278,25 @@ struct Grid { cells: BTreeMap<(u32, u32), Value>, /// Cells shown as percentages. pct: Vec<(u32, u32)>, + /// Rows and columns spanned by `cells`. + rows: u32, + cols: u32, + /// A cell was dropped because the report outgrew [`MAX_OUTPUT_CELLS`]: the report is refused. + too_large: bool, } impl Grid { + /// Stores a value. Past [`MAX_OUTPUT_CELLS`] (the most a report writes) nothing more is kept, + /// so a builder that would produce millions of entries stops allocating them, and + /// [`write_report`] refuses the report. fn put(&mut self, r: u32, c: u32, v: Value) { if !v.is_empty() { + let (rows, cols) = (self.rows.max(r.saturating_add(1)), self.cols.max(c.saturating_add(1))); + if self.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS { + self.too_large = true; + return; + } + (self.rows, self.cols) = (rows, cols); self.cells.insert((r, c), v); } } @@ -296,9 +316,7 @@ impl Grid { } /// (rows, cols) spanned. fn extent(&self) -> (u32, u32) { - let rows = self.cells.keys().map(|(r, _)| r + 1).max().unwrap_or(0); - let cols = self.cells.keys().map(|(_, c)| c + 1).max().unwrap_or(0); - (rows, cols) + (self.rows, self.cols) } } @@ -310,7 +328,7 @@ fn idx(i: usize) -> u32 { fn descriptive(s: &Session, p: &Json) -> Result { let (si, r) = input_range(s, p, &["range", "input"])?; - let vars = variables(s, si, r, by_rows(p), flag(p, "labels"), &|i| format!("Column{}", i + 1))?; + let (by_rows, labels) = (by_rows(p), flag(p, "labels")); let summary = p.get("summary").map(|_| flag(p, "summary")).unwrap_or(true); let conf = confidence(p, "confidence")?; let kl = num(p, "kthLargest").filter(|k| *k >= 1.0).map(|k| k as usize); @@ -319,7 +337,14 @@ fn descriptive(s: &Session, p: &Json) -> Result { return Err(bad(CMD, "choose at least one of summary, confidence, kthLargest or kthSmallest")); } let mut g = Grid::default(); - for (i, v) in vars.iter().enumerate() { + // Variables are read one at a time: a million-row range grouped by rows is a million variables. + for i in 0..(if by_rows { r.height() } else { r.width() }) { + // The report is refused once the grid is too large: the rest would only build discarded labels. + if g.too_large { + break; + } + let v = variable(s, si, r, by_rows, labels, &|i| format!("Column{}", i + 1), i)?; + let i = i as usize; let x = numbers(&v.vals, true)?; let c = idx(i).saturating_mul(2); g.txt(0, c, &v.name); @@ -1749,7 +1774,7 @@ fn sampling(s: &Session, p: &Json, seed: Option) -> Result { fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result { let (rows, cols) = g.extent(); let (rows, cols) = (rows.max(1), cols.max(1)); - if u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS { + if g.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS { return Err(bad(CMD, "the report is too large")); } let d = s.doc()?; @@ -1914,6 +1939,28 @@ mod tests { x } + #[test] + fn a_report_stops_growing_past_the_output_limit() { + // Entries past the limit are not kept, and the report is refused. + let mut g = super::Grid::default(); + g.num(1999, 1999, 1.0); + assert!(!g.too_large); + g.num(0, 2000, 1.0); + assert!(g.too_large); + assert_eq!(g.cells.len(), 1); + assert_eq!(g.extent(), (2000, 2000)); + // Row-grouped statistics of a tall, almost empty column: one variable per row would make + // millions of report entries; the report is refused instead of built. + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1]]})).unwrap(); + s.execute("range.setValues", json!({"range": "A1000000", "values": [[2]]})).unwrap(); + let err = s + .execute("data.analysisToolPak", json!({"tool": "descriptive", "range": "A1:A1000000", "groupedBy": "rows", "output": "C1"})) + .expect_err("report too large"); + assert!(err.to_string().contains("too large"), "{err}"); + assert_eq!(v(&s, "C1"), Value::Empty); + } + #[test] fn forecast_standard_errors_match_the_full_window_rescan() { // #1043: the running sum must give what re-adding every window gave, gaps included. diff --git a/crates/engine/src/cmd/data.rs b/crates/engine/src/cmd/data.rs index 0906f4893..c87906dee 100644 --- a/crates/engine/src/cmd/data.rs +++ b/crates/engine/src/cmd/data.rs @@ -98,7 +98,7 @@ pub fn specs() -> Vec { validation ), cmd!(query "data.validate", "Check Value Against Validation", [], None, "{cell?, input} → {ok, message?}", has_doc, validate_cmd), - cmd!(noundo "data.circleInvalid", "Circle Invalid Data", ["Data", "Data Tools", "Data Validation"], None, "{} → invalid cells", has_doc, circle_invalid), + cmd!(noundo "data.circleInvalid", "Circle Invalid Data", ["Data", "Data Tools", "Data Validation"], None, "{} → {invalid: [cell], truncated: bool (true when the 100,000-cell scan allowance ended before every range was checked)}", has_doc, circle_invalid), CommandSpec { shortcut_mac: Some("Cmd+Shift+K"), ..cmd!( diff --git a/crates/engine/src/cmd/data/filter.rs b/crates/engine/src/cmd/data/filter.rs index cda2160bf..274c0879b 100644 --- a/crates/engine/src/cmd/data/filter.rs +++ b/crates/engine/src/cmd/data/filter.rs @@ -166,14 +166,15 @@ fn filter_blank(v: &Value) -> bool { /// A table's own filter is left to the table, and a row typed after a blank row does not count. /// A formula counts before it is calculated: this runs inside the edit, ahead of the recalc. pub(crate) fn grow_autofilter(sh: &mut Sheet, at: CellRef) { - if !sh.cells.get(at).is_some_and(|c| !filter_blank(&c.value) || c.formula.is_some()) { - return; - } + // The AutoFilter checks come first: they are cheap, and most sheets have none. let Some(af) = sh.autofilter.as_ref() else { return }; let r = af.range; if r.end.row.checked_add(1) != Some(at.row) || at.col < r.start.col || at.col > r.end.col || sh.tables.iter().any(|t| t.range == r) { return; } + if !sh.cells.get(at).is_some_and(|c| !filter_blank(&c.value) || c.formula.is_some()) { + return; + } let grown = RangeRef::new(r.start, CellRef::new(at.row, r.end.col)); if let Some(af) = sh.autofilter.as_mut() { af.range = grown; @@ -184,7 +185,8 @@ pub(crate) fn grow_autofilter(sh: &mut Sheet, at: CellRef) { pub(crate) fn apply_filter(wb: &mut Workbook, sheet: usize) { let Some(sh) = wb.sheet(sheet) else { return }; let Some(af) = sh.autofilter.clone() else { return }; - let r = af.range; + // An imported range can name rows past the grid; none of those exist to hide. + let r = RangeRef::new(af.range.start, CellRef::new(af.range.end.row.min(gridcraft_core::MAX_ROWS - 1), af.range.end.col)); let mut hide = Vec::new(); // Precompute per-criterion top10 threshold and average once, not per row. let mut col_stats: std::collections::HashMap, f64)> = std::collections::HashMap::new(); diff --git a/crates/engine/src/cmd/data/tools.rs b/crates/engine/src/cmd/data/tools.rs index c4ae43ef4..2d0ffc3f9 100644 --- a/crates/engine/src/cmd/data/tools.rs +++ b/crates/engine/src/cmd/data/tools.rs @@ -5,6 +5,9 @@ use super::{ // ---------------------------------------------------------------- tools +/// Most cells (rows times compared columns) one Remove Duplicates call reads. +const MAX_COMPARED_CELLS: u64 = 50_000_000; + pub(super) fn remove_duplicates(s: &mut Session, p: &Json) -> Result { let (sheet, r) = data_target(s, p, "data.removeDuplicates")?; let d = s.doc()?; @@ -24,6 +27,10 @@ pub(super) fn remove_duplicates(s: &mut Session, p: &Json) -> Result { if cols.is_empty() || cols.iter().any(|c| !(r.start.col..=r.end.col).contains(c)) { return Err(bad("data.removeDuplicates", "`columns` must name at least one column of the range")); } + // Rows are compared one by one, so a sparse sheet's enclosing rectangle is bounded like a list. + if r.count() > MAX_COMPARED_CELLS { + return Err(bad("data.removeDuplicates", "the range is too large")); + } let start = r.start.row + header as u32; let mut removed = 0; edit(s, |cx| { @@ -176,8 +183,11 @@ pub(super) fn split_delimited(text: &str, delims: &[char], consecutive: bool, qu let mut in_q = false; let mut last_delim = false; let mut chars = text.chars().peekable(); + let mut at_start = true; while let Some(c) = chars.next() { - if c == quote { + if c == quote && (in_q || at_start) { + at_start = false; + last_delim = false; if in_q && chars.peek() == Some("e) { cur.push(quote); chars.next(); @@ -192,9 +202,11 @@ pub(super) fn split_delimited(text: &str, delims: &[char], consecutive: bool, qu } out.push(std::mem::take(&mut cur)); last_delim = true; + at_start = true; continue; } last_delim = false; + at_start = false; cur.push(c); } out.push(cur); diff --git a/crates/engine/src/cmd/data/validation.rs b/crates/engine/src/cmd/data/validation.rs index e02ef06b3..0d86fbc36 100644 --- a/crates/engine/src/cmd/data/validation.rs +++ b/crates/engine/src/cmd/data/validation.rs @@ -337,11 +337,17 @@ pub(super) fn circle_invalid(s: &mut Session, _: &Json) -> Result { let sheet = d.wb.active_sheet(); let Some(sh) = d.wb.sheet(sheet) else { return Err(EngineError::NoDocument) }; let mut bad_cells = Vec::new(); + // One scan allowance for the whole command: duplicate or overlapping ranges don't multiply it. + let mut budget = 100_000usize; + // Cells were left unchecked because the allowance ran out: said in the result. + let mut truncated = false; for dv in &sh.validations { for r in &dv.ranges { let r = sh.used_range().and_then(|u| r.intersection(&u)); let Some(r) = r else { continue }; - for c in r.iter().take(100_000) { + truncated |= r.count() > budget as u64; + for c in r.iter().take(budget) { + budget -= 1; let text = sh.input_text(c); if !text.is_empty() && check_validation(&d.wb, sheet, c, &text).is_some() { bad_cells.push(c.a1()); @@ -349,5 +355,5 @@ pub(super) fn circle_invalid(s: &mut Session, _: &Json) -> Result { } } } - Ok(json!({"invalid": bad_cells})) + Ok(json!({"invalid": bad_cells, "truncated": truncated})) } diff --git a/crates/engine/src/cmd/edit/find.rs b/crates/engine/src/cmd/edit/find.rs index ddb9ecaf7..94ab15041 100644 --- a/crates/engine/src/cmd/edit/find.rs +++ b/crates/engine/src/cmd/edit/find.rs @@ -14,8 +14,13 @@ fn matches(hay: &str, what: &str, case: bool, whole: bool) -> bool { } /// `*`/`?` wildcard match with `~` escapes. +/// +/// The pattern runs as a bit-parallel automaton over the text: one pass, each character costing +/// one step per 64 pattern positions. Backtracking to the last `*` after a mismatch rescans the +/// text and costs text × pattern comparisons on a long, repetitive cell, for every cell a +/// filter or Find visits. pub(crate) fn wildcard(text: &str, pat: &str) -> bool { - let t: Vec = text.chars().collect(); + // The pattern as (character, escaped) items, runs of `*` folded into one. let mut p: Vec<(char, bool)> = Vec::new(); let mut it = pat.chars(); while let Some(c) = it.next() { @@ -23,32 +28,85 @@ pub(crate) fn wildcard(text: &str, pat: &str) -> bool { if let Some(n) = it.next() { p.push((n, true)); } - } else { + } else if c != '*' || p.last() != Some(&('*', false)) { p.push((c, false)); } } - let (mut ti, mut pi) = (0usize, 0usize); - let (mut star, mut mark) = (None, 0usize); - while ti < t.len() { - if pi < p.len() && (p[pi] == ('?', false) || (p[pi].0 == t[ti] && p[pi] != ('*', false))) { - ti += 1; - pi += 1; - } else if pi < p.len() && p[pi] == ('*', false) { - star = Some(pi); - mark = ti; - pi += 1; - } else if let Some(sp) = star { - pi = sp + 1; - mark += 1; - ti = mark; - } else { - return false; + let words = (p.len() + 1).div_ceil(64); + let bit = |set: &mut [u64], i: usize| set[i / 64] |= 1 << (i % 64); + // Positions holding a `*` and a `?`, and where each literal character sits in the pattern. + let mut stars = vec![0u64; words]; + let mut any = vec![0u64; words]; + let mut at: std::collections::HashMap> = std::collections::HashMap::new(); + for (i, item) in p.iter().enumerate() { + match item { + ('*', false) => bit(&mut stars, i), + ('?', false) => bit(&mut any, i), + (c, _) => at.entry(*c).or_default().push(i), } } - while pi < p.len() && p[pi] == ('*', false) { - pi += 1; + // A literal's bit mask is built when the text first reads it, and kept for at most + // `MAX_CACHED_MASKS` literals: a long pattern of mostly distinct characters would otherwise + // allocate a mask of the pattern's width for each one up front. When there are more literals + // than that, the ones kept are those at the most pattern positions (the dearest to rebuild); + // each of the rest has at most pattern length / `MAX_CACHED_MASKS` positions, so rebuilding + // its mask in `scratch` costs less than one step of the scan. + const MAX_CACHED_MASKS: usize = 256; + let cacheable: Option> = (at.len() > MAX_CACHED_MASKS).then(|| { + let mut by_count: Vec<(&char, usize)> = at.iter().map(|(c, v)| (c, v.len())).collect(); + by_count.sort_unstable_by_key(|&(_, n)| std::cmp::Reverse(n)); + by_count.into_iter().take(MAX_CACHED_MASKS).map(|(c, _)| *c).collect() + }); + let mut masks: std::collections::HashMap> = std::collections::HashMap::new(); + let mut scratch = vec![0u64; words]; + // A set bit i: the text read so far matches the first i pattern items. A `*` matches nothing + // as well, so a state at a `*` is also at the item after it. + let close = |set: &mut Vec| { + let mut carry = 0u64; + for w in 0..words { + let s = set[w] & stars[w]; + let next = (s << 1) | carry; + carry = s >> 63; + set[w] |= next; + } + }; + let mut state = vec![0u64; words]; + bit(&mut state, 0); + close(&mut state); + let mut next = vec![0u64; words]; + for ch in text.chars() { + let lit: Option<&[u64]> = match at.get(&ch) { + None => None, + Some(positions) if cacheable.as_ref().is_none_or(|k| k.contains(&ch)) => { + let mask = masks.entry(ch).or_insert_with(|| { + let mut m = vec![0u64; words]; + positions.iter().for_each(|&i| bit(&mut m, i)); + m + }); + Some(mask) + } + Some(positions) => { + scratch.fill(0); + positions.iter().for_each(|&i| bit(&mut scratch, i)); + Some(&scratch) + } + }; + let mut carry = 0u64; + let mut alive = 0u64; + for w in 0..words { + // Items that read `ch` advance; a `*` reads it and stays. + let m = state[w] & (any[w] | lit.map_or(0, |l| l[w])); + next[w] = (m << 1) | carry | (state[w] & stars[w]); + carry = m >> 63; + alive |= next[w]; + } + if alive == 0 { + return false; + } + std::mem::swap(&mut state, &mut next); + close(&mut state); } - pi == p.len() + state[p.len() / 64] & (1 << (p.len() % 64)) != 0 } pub(super) fn find(s: &mut Session, p: &Json) -> Result { @@ -112,9 +170,19 @@ pub(super) fn find(s: &mut Session, p: &Json) -> Result { Ok(json!({"cell": c.a1(), "sheet": si, "value": t, "count": hits.len()})) } +/// Most characters a cell holds (Excel's limit). +const REPLACE_CELL_MAX_CHARS: usize = 32_767; +/// Longest text one replacement may give a cell, in bytes: the character limit, each up to four +/// bytes. Every occurrence copies the replacement text, so a short cell full of matches and a +/// long replacement would otherwise expand to gigabytes. The character limit is checked on what +/// fits in this. +const REPLACE_CELL_MAX_BYTES: usize = REPLACE_CELL_MAX_CHARS * 4; +/// Most bytes of replaced text one Replace All builds across all its cells. +const REPLACE_TOTAL_MAX_BYTES: usize = 256 * 1024 * 1024; + pub(super) fn replace(s: &mut Session, p: &Json) -> Result { let what = str_param(p, "what").unwrap_or("").to_string(); - let with = str_param(p, "with").unwrap_or("").to_string(); + let with = str_param(p, "with").unwrap_or(""); if what.is_empty() { return Err(bad("edit.replace", "missing `what`")); } @@ -128,9 +196,14 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { let n = edit(s, |cx| { let sheets: Vec = if workbook { (0..cx.wb.sheets.len()).collect() } else { vec![cx.wb.active_sheet()] }; let mut n = 0; + let mut total = 0usize; for si in sheets { let cells: Vec<(CellRef, String)> = - cx.wb.sheet(si).map(|sh| sh.cells.iter().map(|(c, _)| (c, sh.input_text(c))).collect()).unwrap_or_default(); + // A hidden formula's source is not searched: a count or error would reveal its text. + cx.wb + .sheet(si) + .map(|sh| sh.cells.iter().filter(|(c, cell)| cell.formula.is_none() && cell.py.is_none() || !crate::display::formula_hidden(&cx.wb, sh, *c)).map(|(c, _)| (c, sh.input_text(c))).collect()) + .unwrap_or_default(); for (c, text) in cells { if (!all && c != active) || selection.as_ref().is_some_and(|sel| !sel.contains(c)) { continue; @@ -139,12 +212,20 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { continue; } let new = if whole { - with.clone() + (with.len() <= REPLACE_CELL_MAX_BYTES).then(|| with.to_string()) } else if case { - text.replace(&what, &with) + replace_cs_within(&text, &what, with, REPLACE_CELL_MAX_BYTES) } else { - replace_ci(&text, &what, &with) + replace_ci_within(&text, &what, with, REPLACE_CELL_MAX_BYTES) }; + let new = new.filter(|n| n.chars().count() <= REPLACE_CELL_MAX_CHARS); + let Some(new) = new else { + return Err(EngineError::Other(format!("The replacement makes the text of {} too long.", c.a1()))); + }; + total = total.saturating_add(new.len()); + if total > REPLACE_TOTAL_MAX_BYTES { + return Err(EngineError::Other("The replacements make too much text. Replace in a smaller range.".into())); + } // A Python cell's text is its code: the replacement edits the code and the cell // stays a Python cell. Code that would no longer fit in a PY formula is left as it was. let old_py = cx.wb.sheet(si).and_then(|sh| sh.cell(c)).and_then(|x| x.py.clone()); @@ -199,7 +280,37 @@ fn folds_in_place(s: &str) -> bool { }) } +#[cfg(test)] pub(super) fn replace_ci(text: &str, what: &str, with: &str) -> String { + replace_ci_within(text, what, with, usize::MAX).unwrap_or_default() +} + +/// `text` with every `what` replaced by `with`, or `None` once the result passes `limit` bytes +/// (checked before each copy, so nothing larger is allocated). +pub(super) fn replace_cs_within(text: &str, what: &str, with: &str, limit: usize) -> Option { + if what.is_empty() { + return (text.len() <= limit).then(|| text.to_string()); + } + let mut out = String::new(); + let mut i = 0; + for (pos, m) in text.match_indices(what) { + if out.len() + (pos - i) + with.len() > limit { + return None; + } + out.push_str(text.get(i..pos).unwrap_or("")); + out.push_str(with); + i = pos + m.len(); + } + let rest = text.get(i..).unwrap_or(""); + if out.len() + rest.len() > limit { + return None; + } + out.push_str(rest); + Some(out) +} + +/// Case-insensitive [`replace_cs_within`]. +pub(super) fn replace_ci_within(text: &str, what: &str, with: &str, limit: usize) -> Option { let lower = text.to_lowercase(); let w = what.to_lowercase(); // Slicing `text` at positions found in `lower` is only sound when folding kept every byte @@ -207,15 +318,22 @@ pub(super) fn replace_ci(text: &str, what: &str, with: &str) -> String { // boundaries (U+212A KELVIN SIGN shrinks 3→1 bytes, U+0130 grows 2→3), where the tail used // to be dropped. Those fall back to the case-sensitive replace, as length changes already do. if lower.len() != text.len() || w.is_empty() || !folds_in_place(text) || !folds_in_place(what) { - return text.replace(what, with); + return replace_cs_within(text, what, with, limit); } let mut out = String::new(); let mut i = 0; while let Some(pos) = lower.get(i..).and_then(|s| s.find(&w)) { + if out.len() + pos + with.len() > limit { + return None; + } out.push_str(text.get(i..i + pos).unwrap_or("")); out.push_str(with); i += pos + w.len(); } - out.push_str(text.get(i..).unwrap_or("")); - out + let rest = text.get(i..).unwrap_or(""); + if out.len() + rest.len() > limit { + return None; + } + out.push_str(rest); + Some(out) } diff --git a/crates/engine/src/cmd/edit/go_to.rs b/crates/engine/src/cmd/edit/go_to.rs index 0cd7a3de9..3c632e3b9 100644 --- a/crates/engine/src/cmd/edit/go_to.rs +++ b/crates/engine/src/cmd/edit/go_to.rs @@ -17,11 +17,13 @@ pub(in super::super) fn go_to(s: &mut Session, p: &Json) -> Result { } let Some(reference) = str_param(p, "reference").or_else(|| str_param(p, "range")) else { return Err(bad("edit.goTo", "missing `reference`")) }; let d = s.doc()?; + let mut names = Resolver::new(&d.wb); let mut target: Option = None; let mut areas: Vec = Vec::new(); let mut named = false; for part in reference_parts(reference) { - let Some((sheet, found, by_name)) = go_to_target(&d.wb, d.wb.active_sheet(), part, 0) else { + let mut found = Vec::new(); + let Some((sheet, _, by_name)) = names.target(d.wb.active_sheet(), part, 0, Some(&mut found)) else { return Err(EngineError::Other("Reference isn't valid.".into())); }; // One selection holds the areas of one sheet. @@ -77,57 +79,114 @@ pub(super) fn reference_parts(text: &str) -> impl Iterator { }) } -/// Where one part of a Go To reference points, seen from sheet `sheet`: the sheet, its areas, -/// and whether a defined name or a table gave them. `Sheet2!Name` is the name scoped to Sheet2. -/// A name counts when what it refers to is a reference; a constant or a formula is not one. -fn go_to_target(wb: &gridcraft_model::Workbook, sheet: usize, part: &str, depth: usize) -> Option<(usize, Vec, bool)> { - if part.is_empty() || depth > GO_TO_NAME_DEPTH { - return None; - } - let (prefix, body) = split_sheet(part); - let on = match &prefix { - Some(n) => wb.sheet_index(n)?, - None => sheet, - }; - if let Some(r) = RangeRef::parse(&body.replace('$', "")) { - return Some((on, vec![r], false)); - } - let name = match prefix { - Some(_) => wb.names.iter().find(|n| n.scope == Some(on) && gridcraft_formula::same_name(&n.name, body)), - None => wb.name(body, sheet), - }; - if let Some(n) = name { - return name_areas(wb, n, sheet, depth).map(|(t, areas)| (t, areas, true)); - } - if prefix.is_none() - && let Some((si, ti)) = wb.table(body) - { - // As in Excel, a table's name stands for its data, without the header and totals rows. - let t = wb.sheet(si)?.tables.get(ti)?; - return Some((si, vec![t.data_range().unwrap_or(t.range)], true)); - } - None +/// Resolves Go To references against one workbook. Defined names are found through an index +/// built once, and whether a name refers to a reference is remembered: listing the names of a +/// workbook full of names that alias other names would otherwise scan every name for every alias +/// of every name. +struct Resolver<'a> { + wb: &'a gridcraft_model::Workbook, + /// Indexes into `wb.names` by scope and folded name, in workbook order. + by_name: std::collections::HashMap<(Option, String), Vec>, + /// [`Resolver::name_areas`] without areas, by name index, sheet and depth: the target sheet + /// and the number of areas. + memo: std::collections::HashMap<(usize, usize, usize), Option<(usize, usize)>>, } -/// The sheet and areas defined name `n` refers to, seen from sheet `sheet`, when that is a -/// reference. -fn name_areas(wb: &gridcraft_model::Workbook, n: &gridcraft_model::DefinedName, sheet: usize, depth: usize) -> Option<(usize, Vec)> { - let from = n.scope.unwrap_or(sheet); - let formula = n.formula.trim().trim_start_matches('='); - let mut target: Option = None; - let mut areas = Vec::new(); - for p in reference_parts(formula) { - let (s, found, _) = go_to_target(wb, from, p, depth + 1)?; - if target.is_some_and(|t| t != s) { - return None; - } - target = Some(s); - areas.extend(found); - if areas.len() > GO_TO_MAX_AREAS { - return None; +impl<'a> Resolver<'a> { + fn new(wb: &'a gridcraft_model::Workbook) -> Self { + let mut by_name: std::collections::HashMap<(Option, String), Vec> = std::collections::HashMap::new(); + for (i, n) in wb.names.iter().enumerate() { + by_name.entry((n.scope, gridcraft_formula::fold_name(&n.name))).or_default().push(i); } + Self { wb, by_name, memo: std::collections::HashMap::new() } + } + + /// The first name of scope `scope` that `same` accepts, as a scan of `wb.names` finds it. + /// Names `same` accepts fold alike, so only that bucket is checked. + fn find(&self, scope: Option, body: &str, same: impl Fn(&str, &str) -> bool) -> Option { + self.by_name + .get(&(scope, gridcraft_formula::fold_name(body)))? + .iter() + .copied() + .find(|&i| self.wb.names.get(i).is_some_and(|n| same(&n.name, body))) + } + + /// Where one part of a Go To reference points, seen from sheet `sheet`: the sheet, how many + /// areas (appended to `areas` when given), and whether a defined name or a table gave them. + /// `Sheet2!Name` is the name scoped to Sheet2. A name counts when what it refers to is a + /// reference; a constant or a formula is not one. + fn target(&mut self, sheet: usize, part: &str, depth: usize, areas: Option<&mut Vec>) -> Option<(usize, usize, bool)> { + let wb = self.wb; + if part.is_empty() || depth > GO_TO_NAME_DEPTH { + return None; + } + let (prefix, body) = split_sheet(part); + let on = match &prefix { + Some(n) => wb.sheet_index(n)?, + None => sheet, + }; + if let Some(r) = RangeRef::parse(&body.replace('$', "")) { + if let Some(a) = areas { + a.push(r); + } + return Some((on, 1, false)); + } + // As `Workbook::name`: the sheet's own name, else the workbook's. + let name = match prefix { + Some(_) => self.find(Some(on), body, gridcraft_formula::same_name), + None => self.find(Some(sheet), body, str::eq_ignore_ascii_case).or_else(|| self.find(None, body, str::eq_ignore_ascii_case)), + }; + if let Some(i) = name { + return self.name_areas(i, sheet, depth, areas).map(|(t, n)| (t, n, true)); + } + if prefix.is_none() + && let Some((si, ti)) = wb.table(body) + { + // As in Excel, a table's name stands for its data, without the header and totals rows. + let t = wb.sheet(si)?.tables.get(ti)?; + if let Some(a) = areas { + a.push(t.data_range().unwrap_or(t.range)); + } + return Some((si, 1, true)); + } + None + } + + /// The sheet and number of areas defined name `wb.names[i]` refers to, seen from sheet + /// `sheet`, when that is a reference; the areas are appended to `areas` when given. + fn name_areas(&mut self, i: usize, sheet: usize, depth: usize, mut areas: Option<&mut Vec>) -> Option<(usize, usize)> { + let key = (i, sheet, depth); + if areas.is_none() + && let Some(known) = self.memo.get(&key) + { + return *known; + } + let out = self.name_areas_uncached(i, sheet, depth, areas.as_deref_mut()); + if areas.is_none() { + self.memo.insert(key, out); + } + out + } + + fn name_areas_uncached(&mut self, i: usize, sheet: usize, depth: usize, mut areas: Option<&mut Vec>) -> Option<(usize, usize)> { + let n = self.wb.names.get(i)?; + let from = n.scope.unwrap_or(sheet); + let formula = n.formula.trim().trim_start_matches('='); + let mut target: Option = None; + let mut count = 0usize; + for p in reference_parts(formula) { + let (s, found, _) = self.target(from, p, depth + 1, areas.as_deref_mut())?; + if target.is_some_and(|t| t != s) { + return None; + } + target = Some(s); + count += found; + if count > GO_TO_MAX_AREAS { + return None; + } + } + target.map(|t| (t, count)) } - target.map(|t| (t, areas)) } /// The names the Go To dialog lists, seen from sheet `sheet`: the named ranges of the workbook and @@ -137,11 +196,13 @@ fn name_areas(wb: &gridcraft_model::Workbook, n: &gridcraft_model::DefinedName, pub fn go_to_names(wb: &gridcraft_model::Workbook, sheet: usize) -> Vec { let own: std::collections::HashSet = wb.names.iter().filter(|n| n.scope == Some(sheet)).map(|n| gridcraft_formula::fold_name(&n.name)).collect(); + let mut names = Resolver::new(wb); let mut out: Vec = wb .names .iter() - .filter(|n| !n.hidden && name_areas(wb, n, sheet, 0).is_some()) - .filter_map(|n| match n.scope { + .enumerate() + .filter(|(i, n)| !n.hidden && names.name_areas(*i, sheet, 0, None).is_some()) + .filter_map(|(_, n)| match n.scope { None if own.contains(&gridcraft_formula::fold_name(&n.name)) => None, None => Some(n.name.clone()), Some(s) if s == sheet => Some(n.name.clone()), diff --git a/crates/engine/src/cmd/edit/selection.rs b/crates/engine/src/cmd/edit/selection.rs index f171c65a8..e6037c09d 100644 --- a/crates/engine/src/cmd/edit/selection.rs +++ b/crates/engine/src/cmd/edit/selection.rs @@ -63,6 +63,11 @@ pub(super) fn selection_move(s: &mut Session, p: &Json) -> Result { let d = s.doc_mut()?; let Some(sheet) = d.wb.active() else { return Err(EngineError::NoDocument) }; let sel = &mut d.selection; + if extend && !jmp { + // Shift+arrows move the edge the anchor doesn't hold, keeping merged cells whole. + sel.extend_by(sheet, dr, dc); + return Ok(json!({"selection": sel.a1(), "active": sel.active.a1()})); + } // Extension moves the far corner, not the active cell. let from = if extend { sel.moving_end() } else { sel.active }; // Skip hidden rows/columns and step over merged areas. @@ -77,23 +82,32 @@ pub(super) fn selection_move(s: &mut Session, p: &Json) -> Result { jump(sheet, from, dr.signum(), dc.signum()) } else { let mut t = from.offset_clamped(dr, dc); + if !extend && let Some(m) = sheet.merge_at(from) { + // Leaving a merged cell moves from its edge. + if dr > 0 { + t = CellRef::new(m.end.offset_clamped(dr, 0).row, t.col); + } else if dc > 0 { + t = CellRef::new(t.row, m.end.offset_clamped(0, dc).col); + } + } let mut guard = 0; while (sheet.is_row_hidden(t.row) || sheet.is_col_hidden(t.col)) && guard < 100_000 { - let n = t.offset_clamped(dr.signum(), dc.signum()); + // Step each hidden axis along its own direction, so a diagonal exit from a merge + // (Tab-run return column) keeps its column while skipping hidden rows. + let mut n = CellRef::new( + if sheet.is_row_hidden(t.row) { t.offset_clamped(dr.signum(), 0).row } else { t.row }, + if sheet.is_col_hidden(t.col) { t.offset_clamped(0, dc.signum()).col } else { t.col }, + ); + if n == t { + // The hidden axis has no direction (a move along the other axis): walk as before. + n = t.offset_clamped(dr.signum(), dc.signum()); + } if n == t { break; } t = n; guard += 1; } - if !extend && let Some(m) = sheet.merge_at(from) { - // Leaving a merged cell moves from its edge. - if dr > 0 { - t = CellRef::new(m.end.offset_clamped(dr, 0).row, from.col); - } else if dc > 0 { - t = CellRef::new(from.row, m.end.offset_clamped(0, dc).col); - } - } t }; if let Some(m) = sheet.merge_at(to) @@ -140,7 +154,29 @@ pub(super) fn selection_next(s: &mut Session, p: &Json) -> Result { let fwd = bool_param(p, "forward").unwrap_or(true); let by_row = bool_param(p, "byRow").unwrap_or(false); let d = s.doc_mut()?; - d.selection.cycle(fwd, by_row); + let Some(sheet) = d.wb.active() else { return Err(EngineError::NoDocument) }; + let sel = &d.selection; + if sel.is_one_merged_cell(sheet) { + // A selection that is just one merged cell is one cell, as in Excel: Enter and Tab leave + // it for the next cell instead of walking the cells it covers. + let step = if fwd { 1 } else { -1 }; + let (dr, dc) = if by_row { (0, step) } else { (step, 0) }; + selection_move(s, &json!({"dr": dr, "dc": dc}))?; + return Ok(json!({"active": s.doc()?.selection.active.a1()})); + } + // Inside a larger selection the cells a merged cell covers are skipped: it is visited once, + // at its top-left cell. + let area = sel.current(); + let cells = + (u64::from(area.end.row.saturating_sub(area.start.row)) + 1).saturating_mul(u64::from(area.end.col.saturating_sub(area.start.col)) + 1); + let mut next = sel.clone(); + for _ in 0..cells.min(1 << 20) { + next.cycle(fwd, by_row); + if sheet.merge_at(next.active).is_none_or(|m| m.start == next.active) { + break; + } + } + d.selection = next; Ok(json!({"active": d.selection.active.a1()})) } diff --git a/crates/engine/src/cmd/edit/tests.rs b/crates/engine/src/cmd/edit/tests.rs index 49317caa0..160ce9157 100644 --- a/crates/engine/src/cmd/edit/tests.rs +++ b/crates/engine/src/cmd/edit/tests.rs @@ -42,6 +42,28 @@ fn go_to_lists_the_named_ranges_of_every_sheet() { assert_eq!(list["names"], json!(["Areas", "Local", "Price", "Sheet1!Own", "type"]), "{list}"); } +/// Thousands of names that alias other names are listed with one lookup per alias, not a scan of +/// every name for each; aliases resolve through the chain like a single name does. +#[test] +fn go_to_lists_many_aliases_and_follows_them() { + let mut s = go_to_workbook(); + let wb = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().wb); + let alias = |name: String, formula: String| gridcraft_model::DefinedName { name, scope: None, formula, comment: String::new(), hidden: false }; + wb.names.push(alias("Chain0".into(), "Areas".into())); + for i in 1..20_000 { + wb.names.push(alias(format!("Chain{i}"), format!("Chain{},price", i - 1))); + } + let list = s.execute("edit.goTo", json!({"list": true})).unwrap(); + let names: Vec<&str> = list["names"].as_array().unwrap().iter().filter_map(|n| n.as_str()).collect(); + // Chain0..Chain6 are within the name depth Go To follows; longer chains are not ranges. + for i in 0..7 { + assert!(names.contains(&format!("Chain{i}").as_str()), "Chain{i} in {names:?}"); + } + assert!(!names.contains(&"Chain7"), "{names:?}"); + let out = s.execute("edit.goTo", json!({"reference": "chain1"})).unwrap(); + assert_eq!(out["selection"], "A1,C3:C4,B2:B5", "{out}"); +} + #[test] fn go_to_a_name_scoped_to_another_sheet_goes_to_that_sheet() { let mut s = go_to_workbook(); @@ -302,6 +324,103 @@ fn wildcards() { assert!(wildcard("", "*")); } +/// The backtracking matcher `wildcard` replaced, as the reference for its results. +fn wildcard_backtracking(text: &str, pat: &str) -> bool { + let t: Vec = text.chars().collect(); + let mut p: Vec<(char, bool)> = Vec::new(); + let mut it = pat.chars(); + while let Some(c) = it.next() { + if c == '~' { + if let Some(n) = it.next() { + p.push((n, true)); + } + } else { + p.push((c, false)); + } + } + let (mut ti, mut pi) = (0usize, 0usize); + let (mut star, mut mark) = (None, 0usize); + while ti < t.len() { + if pi < p.len() && (p[pi] == ('?', false) || (p[pi].0 == t[ti] && p[pi] != ('*', false))) { + ti += 1; + pi += 1; + } else if pi < p.len() && p[pi] == ('*', false) { + star = Some(pi); + mark = ti; + pi += 1; + } else if let Some(sp) = star { + pi = sp + 1; + mark += 1; + ti = mark; + } else { + return false; + } + } + while pi < p.len() && p[pi] == ('*', false) { + pi += 1; + } + pi == p.len() +} + +#[test] +fn wildcard_agrees_with_backtracking() { + // Every text up to 5 characters over a small alphabet against every pattern up to 4 items. + let alphabet = ['a', 'b', '*', '?', '~']; + let all = |n: usize, chars: &[char]| -> Vec { + let mut out = vec![String::new()]; + let mut level = vec![String::new()]; + for _ in 0..n { + level = level.iter().flat_map(|s| chars.iter().map(move |c| format!("{s}{c}"))).collect(); + out.extend(level.iter().cloned()); + } + out + }; + let texts = all(5, &['a', 'b', '*']); + for pat in all(4, &alphabet) { + for text in &texts { + assert_eq!(wildcard(text, &pat), wildcard_backtracking(text, &pat), "{text:?} ~ {pat:?}"); + } + } + // Patterns longer than one 64-bit word. + let long = format!("*{}*b?", "a".repeat(70)); + for text in [format!("x{}yb1", "a".repeat(70)), format!("{}b", "a".repeat(71)), "a".repeat(200)] { + assert_eq!(wildcard(&text, &long), wildcard_backtracking(&text, &long), "{text}"); + } +} + +#[test] +fn wildcard_with_many_distinct_literals_agrees_with_backtracking() { + // 600 distinct characters in the pattern: more than the masks kept, so the rest are rebuilt. + let chars: Vec = (0x4E00u32..0x4E00 + 600).filter_map(char::from_u32).collect(); + let word: String = chars.iter().collect(); + let pat = format!("*{word}?{}", chars[3]); + for text in [format!("x{word}yq{}", chars[3]), word.clone(), format!("{word}{word}"), chars.iter().rev().collect::()] { + assert_eq!(wildcard(&text, &pat), wildcard_backtracking(&text, &pat), "{} chars", text.chars().count()); + } + assert!(wildcard(&format!("{word}zz{}", chars[3]).replace("zz", "z"), &pat)); +} + +#[test] +fn wildcard_keeps_masks_of_frequent_literals_past_the_cache_limit() { + // 300 distinct characters come before the repeated `a`: past the cached masks, `a` (5,000 positions) + // still has to be read from a kept mask, or it is rebuilt for each of the 20,000 `a` in the text. + let rare: String = (0x4E00u32..0x4E00 + 300).filter_map(char::from_u32).collect(); + let pat = format!("*{rare}{}b", "a".repeat(5_000)); + let text = "a".repeat(20_000); + assert!(!wildcard(&text, &pat)); + assert!(wildcard(&format!("{rare}{}b", "a".repeat(5_000)), &pat)); + assert!(!wildcard(&format!("{rare}{}", "a".repeat(5_000)), &pat)); +} + +#[test] +fn wildcard_on_long_repetitive_text_is_linear() { + // The backtracking matcher compared about text × pattern characters here (10^8). + let text = "a".repeat(20_000); + let pat = format!("*{}b", "a".repeat(5_000)); + assert!(!wildcard(&text, &pat)); + assert!(wildcard(&format!("{text}b"), &pat)); +} + #[test] fn tsv() { assert_eq!(parse_tsv("a\tb\n1\t2\n"), vec![vec!["a", "b"], vec!["1", "2"]]); @@ -424,6 +543,50 @@ fn drag_and_drop_on_another_sheet_stays_on_that_sheet() { assert_eq!(wb.sheet(two).unwrap().value(CellRef::new(0, 1)), Value::Empty); } +#[test] +fn replace_refuses_text_that_would_grow_past_a_cell() { + // 10,000 matches × a 1,000-character replacement would make a 10 MB cell (and a few such + // cells gigabytes); the replacement is refused before that text is built. + let mut s = Session::new(); + s.new_workbook(); + let long = "a".repeat(10_000); + s.execute("range.setValues", json!({"range": "A1", "values": [[long.clone()], ["abc"]]})).unwrap(); + for case in [false, true] { + let err = s.execute("edit.replace", json!({"what": "a", "with": "x".repeat(1_000), "matchCase": case})).expect_err("too long"); + assert!(err.to_string().contains("too long"), "{err}"); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["value"], json!(long)); + assert_eq!(s.execute("cell.get", json!({"cell": "A2"})).unwrap()["value"], json!("abc")); + } + // A replacement that stays within a cell's limit still happens. + s.execute("edit.replace", json!({"what": "b", "with": "x".repeat(100)})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "A2"})).unwrap()["value"], json!(format!("a{}c", "x".repeat(100)))); +} + +#[test] +fn replace_refuses_a_result_past_the_character_limit_though_it_fits_the_byte_limit() { + let mut s = Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [["z"]]})).unwrap(); + for whole in [false, true] { + let err = s.execute("edit.replace", json!({"what": "z", "with": "x".repeat(32_768), "wholeCell": whole})).expect_err("32,768 characters"); + assert!(err.to_string().contains("too long"), "{err}"); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["value"], json!("z")); + } + s.execute("edit.replace", json!({"what": "z", "with": "x".repeat(32_767)})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["value"], json!("x".repeat(32_767))); +} + +#[test] +fn bounded_replace_stops_at_its_limit() { + use super::find::{replace_ci_within, replace_cs_within}; + assert_eq!(replace_cs_within("abab", "a", "xy", 6).as_deref(), Some("xybxyb")); + assert_eq!(replace_cs_within("abab", "a", "xy", 5), None); + assert_eq!(replace_ci_within("AbAb", "a", "xy", 6).as_deref(), Some("xybxyb")); + assert_eq!(replace_ci_within("AbAb", "a", "xy", 5), None); + assert_eq!(replace_cs_within("abc", "z", "", 3).as_deref(), Some("abc")); + assert_eq!(replace_cs_within("abc", "z", "", 2), None); +} + #[test] fn replace_case_insensitive() { assert_eq!(replace_ci("Apple apple APPLE", "apple", "pear"), "pear pear pear"); diff --git a/crates/engine/src/cmd/extra/data.rs b/crates/engine/src/cmd/extra/data.rs index 9a1ae7417..5af7df0a8 100644 --- a/crates/engine/src/cmd/extra/data.rs +++ b/crates/engine/src/cmd/extra/data.rs @@ -42,6 +42,15 @@ pub(super) fn table_resize(s: &mut Session, p: &Json) -> Result { }) } +/// Most cells (rows × columns, blanks included) Get Data imports. The rows are built densely +/// before they are written, so a few sparse records or one far-away CSV cell must not expand +/// into a rectangle of billions of slots. +const MAX_IMPORT_CELLS: u64 = 10_000_000; + +fn too_many_cells() -> EngineError { + EngineError::Other(format!("The data is too large to import ({MAX_IMPORT_CELLS} cells at most).")) +} + pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result { let bytes = if let Some(t) = str_param(p, "text") { t.as_bytes().to_vec() @@ -57,15 +66,27 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result { if text.trim_start().starts_with('[') { let arr: Vec = serde_json::from_str(&text).map_err(|e| bad("data.getData", e.to_string()))?; let mut keys: Vec = Vec::new(); + let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); for o in &arr { if let Some(m) = o.as_object() { for k in m.keys() { - if !keys.contains(k) { + if !seen.contains(k.as_str()) { + // Stop collecting at the column limit. + if keys.len() >= gridcraft_core::MAX_COLS as usize { + return Err(EngineError::Other("The data has more fields than a sheet has columns.".into())); + } + seen.insert(k.as_str()); keys.push(k.clone()); } } } } + if arr.len() as u64 + 1 > u64::from(gridcraft_core::MAX_ROWS) { + return Err(EngineError::Other("The data has more rows than a sheet has.".into())); + } + if (arr.len() as u64 + 1).saturating_mul(keys.len().max(1) as u64) > MAX_IMPORT_CELLS { + return Err(too_many_cells()); + } let mut out = vec![keys.iter().map(|k| json!(k)).collect::>()]; for o in &arr { out.push(keys.iter().map(|k| o.get(k).cloned().unwrap_or(Json::Null)).collect()); @@ -75,10 +96,17 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result { let delim = str_param(p, "delimiter") .and_then(|d| d.chars().next()) .unwrap_or(if text.lines().next().is_some_and(|l| l.contains('\t')) { '\t' } else { ',' }); - let opts = gridcraft_xlsx::CsvOptions { delimiter: delim as u8, ..Default::default() }; - let wb = gridcraft_xlsx::read_csv(&bytes, &opts).map_err(|e| EngineError::Other(e.to_string()))?; + let opts = gridcraft_xlsx::CsvOptions { delimiter: delim as u8, max_used_cells: Some(MAX_IMPORT_CELLS), ..Default::default() }; + // The cell limit is enforced while parsing, before an oversized file is stored. + let wb = gridcraft_xlsx::read_csv(&bytes, &opts).map_err(|e| match e { + gridcraft_xlsx::IoError::TooLarge(_) => too_many_cells(), + e => EngineError::Other(e.to_string()), + })?; let sh = wb.sheet(0).ok_or(EngineError::NoDocument)?; let Some(u) = sh.used_range() else { return Err(EngineError::Other("The file is empty.".into())) }; + if u.count() > MAX_IMPORT_CELLS { + return Err(too_many_cells()); + } (u.start.row..=u.end.row) .map(|r| { (u.start.col..=u.end.col) @@ -129,3 +157,35 @@ pub(super) fn queries(s: &mut Session, _: &Json) -> Result { .collect(), )) } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::Session; + + fn s() -> Session { + let mut s = Session::new(); + s.new_workbook(); + s + } + + #[test] + fn sparse_imports_are_refused_before_their_rows_are_built() { + let mut s = s(); + let sheets = s.doc().unwrap().wb.sheets.len(); + // 5,000 records with one field each: 5,001 × 5,000 slots once every record gets every field. + let records: Vec<_> = (0..5_000).map(|i| json!({ format!("k{i}"): i })).collect(); + let text = serde_json::to_string(&records).unwrap(); + let err = s.execute("data.getData", json!({"text": text})).expect_err("too many cells"); + assert!(err.to_string().contains("too large"), "{err}"); + // A CSV whose last line reaches 16,001 columns, 1,000 lines down. + let text = format!("1{}{}x", "\n".repeat(999), ",".repeat(16_000)); + let err = s.execute("data.fromTextCsv", json!({"text": text})).expect_err("too many cells"); + assert!(err.to_string().contains("too large"), "{err}"); + assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets); + // Small imports still work. + let out = s.execute("data.getData", json!({"text": r#"[{"a": 1}, {"b": 2}]"#})).unwrap(); + assert_eq!((out["rows"].as_u64(), out["cols"].as_u64()), (Some(3), Some(2)), "{out}"); + } +} diff --git a/crates/engine/src/cmd/forecast.rs b/crates/engine/src/cmd/forecast.rs index d0941a9ca..88de4b433 100644 --- a/crates/engine/src/cmd/forecast.rs +++ b/crates/engine/src/cmd/forecast.rs @@ -36,13 +36,28 @@ fn max_periods(history: usize) -> u32 { u32::try_from(MAX_WORK / h).unwrap_or(u32::MAX).clamp(1, MAX_PERIODS) } +/// Most forecast rows when the model is fit to the first `fit` of `history` rows, which span +/// `slots` timeline slots once their gaps are filled. With a Forecast Start inside the history, +/// every row after it is forecast too (a hindcast row holds the same ETS formulas, each refitting +/// the model over the `slots` slots), so those rows (counted from the raw rows) count against +/// [`MAX_WORK`] before any forecast row does. `None` when the hindcast alone exceeds it. +fn max_periods_from(history: usize, fit: usize, slots: usize) -> Option { + if fit >= history { + return Some(max_periods(history)); + } + let f = u64::try_from(slots.max(fit)).unwrap_or(u64::MAX).max(1); + let hindcast = u64::try_from(history - fit).unwrap_or(u64::MAX); + let rest = (MAX_WORK / f).checked_sub(hindcast).filter(|r| *r >= 1)?; + Some(u32::try_from(rest).unwrap_or(u32::MAX).min(MAX_PERIODS)) +} + pub fn specs() -> Vec { vec![cmd!( "data.forecastSheet", "Forecast Sheet", ["Data", "Forecast"], None, - "{timeline?: \"A2:A25\", values?: \"B2:B25\" (default: the first two columns of `range`, the selection or the current region; a text first cell is the header), end?: timeline value or date text | periods?: n (default: a third of the history, at least 1; ≤10,000 and history × periods ≤ 500,000; history ≤ 10,000 rows), forecastStart?: a timeline value or date text on the history, before its last point (hindcast: the model is fit to the points before it, the forecast columns start there and run over the actuals; default and the last point: forecast after the history), confidenceInterval?: bool (default true), confidence?: 0.95 or 95, seasonality?: \"auto\" | 0 (none) | 2–8,760, fillMissing?: interpolate|zeros, aggregate?: average|count|counta|max|median|min|sum, statistics?: bool, chart?: line|column} → {sheet, table, chart, history, periods}. Inserts a sheet before the active one: timeline, values, Forecast (FORECAST.ETS), Lower/Upper Confidence Bound (FORECAST.ETS.CONFINT), optional FORECAST.ETS.STAT table, and a chart.", + "{timeline?: \"A2:A25\", values?: \"B2:B25\" (default: the first two columns of `range`, the selection or the current region; a text first cell is the header), end?: timeline value or date text | periods?: n (default: a third of the history, at least 1; ≤10,000 and history × periods ≤ 500,000, or with a forecastStart inside the history (hindcast rows are forecast too) (the greater of the fitted rows and fitted slots × (periods + rows from forecastStart on, that row included)) ≤ 500,000; history ≤ 10,000 rows), forecastStart?: a timeline value or date text on the history, before its last point (hindcast: the model is fit to the points before it, the forecast columns start there and run over the actuals; default and the last point: forecast after the history), confidenceInterval?: bool (default true), confidence?: 0.95 or 95, seasonality?: \"auto\" | 0 (none) | 2–8,760, fillMissing?: interpolate|zeros, aggregate?: average|count|counta|max|median|min|sum, statistics?: bool, chart?: line|column} → {sheet, table, chart, history, periods}. Inserts a sheet before the active one: timeline, values, Forecast (FORECAST.ETS), Lower/Upper Confidence Bound (FORECAST.ETS.CONFINT), optional FORECAST.ETS.STAT table, and a chart.", has_doc, forecast_sheet )] @@ -257,13 +272,23 @@ fn forecast_sheet(s: &mut Session, p: &Json) -> Result { } } }; - let cap = max_periods(rows.len()); + // The slots the model is fit over: the fit rows' span of the timeline, gaps included. + let slots = match (fit < rows.len(), times.get(fit.wrapping_sub(1)), times.first()) { + (true, Some(last_fit), Some(first)) => match (timeline.steps_after_last(*last_fit), timeline.steps_after_last(*first)) { + (Some(b), Some(a)) => ((b - a).round() + 1.0).clamp(0.0, 1e9) as usize, + _ => fit, + }, + _ => fit, + }; + let cap = max_periods_from(rows.len(), fit, slots).ok_or_else(|| { + EngineError::Other("The forecast start is too early for this much history: choose a later forecast start or less history.".into()) + })?; let periods = match end { Some(end) => steps_to(&timeline, end, cap)?, None => match number(p, "periods")? { Some(n) if n >= 1.0 && n.ceil() <= f64::from(cap) => n.ceil() as u32, Some(_) => return Err(bad(CMD, format!("`periods` is from 1 to {cap} for {} rows of history", rows.len()))), - None => default_periods(rows.len()), + None => default_periods(rows.len()).min(cap), }, }; let future: Vec = @@ -532,6 +557,51 @@ mod tests { assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets - 1); } + #[test] + fn hindcast_rows_count_against_the_work_limit() { + use super::{MAX_WORK, max_periods, max_periods_from}; + assert_eq!(max_periods_from(1_300, 1_300, 1_300), Some(max_periods(1_300))); + // 2,000 rows fit to the first 1,899: 101 hindcast rows, then 162 forecast rows. + assert_eq!(max_periods_from(2_000, 1_899, 1_899), Some(162)); + // Fit to the first 1,000: the 1,000 hindcast rows alone are twice the limit. + assert_eq!(max_periods_from(2_000, 1_000, 1_000), None); + // 800 rows fit to the first 400, which span 571 slots (gaps filled): each formula refits + // 571 slots, so 475 forecast rows after the 400 hindcast rows, not 850. + assert_eq!(max_periods_from(800, 400, 571), Some(475)); + assert!((475 + 400) * 571 <= MAX_WORK as usize); + for (h, f) in [(2_000usize, 1_899usize), (10_000, 9_990), (100, 2), (60, 50)] { + if let Some(p) = max_periods_from(h, f, f) { + assert!((u64::from(p) + (h - f) as u64) * f as u64 <= MAX_WORK, "{h} {f}"); + } + } + let mut s = Session::new(); + s.new_workbook(); + let mut rows = vec![json!(["t", "y"])]; + rows.extend((1..=2_000).map(|t| json!([t, (t % 7) as f64 + t as f64 * 0.5]))); + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + let sheets = s.doc().unwrap().wb.sheets.len(); + let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_001})).expect_err("hindcast too long"); + assert!(err.to_string().contains("forecast start is too early"), "{err}"); + let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_900, "periods": 163})).expect_err("past the cap"); + assert!(err.to_string().contains("from 1 to 162"), "{err}"); + assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets); + } + + #[test] + fn the_hindcast_cap_follows_the_slots_the_fit_spans_not_its_rows() { + // 800 rows, 20% of the slots missing: the first 400 rows span 500 slots, so each ETS + // formula refits 500 slots and the cap is 1,000 - 400 = 600 (it was 850 by rows). + let mut s = Session::new(); + s.new_workbook(); + let times: Vec = (1u32..).filter(|t| !matches!(t % 10, 3 | 6)).take(800).collect(); + let mut rows = vec![json!(["t", "y"])]; + rows.extend(times.iter().map(|t| json!([t, (t % 7) as f64 + f64::from(*t) * 0.5]))); + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + let start = times[400]; + let err = s.execute("data.forecastSheet", json!({"range": "A1:B801", "forecastStart": start, "periods": 601})).expect_err("past the cap"); + assert!(err.to_string().contains("from 1 to 600"), "{err}"); + } + #[test] fn history_and_periods_are_capped() { use super::{MAX_HISTORY, MAX_PERIODS, MAX_WORK, default_periods, max_periods}; diff --git a/crates/engine/src/cmd/format.rs b/crates/engine/src/cmd/format.rs index aa2638801..90213a5e1 100644 --- a/crates/engine/src/cmd/format.rs +++ b/crates/engine/src/cmd/format.rs @@ -891,6 +891,21 @@ mod tests { assert!((0.75..=1.15).contains(&ratio), "Arabic {} vs Latin {}", width(0), width(1)); } + #[test] + fn autofit_of_many_separate_columns_fits_only_the_selected_ones() { + let mut s = crate::Session::new(); + s.new_workbook(); + let long = "a much longer text than the others"; + // Columns A, C, E, G, I, K are selected (six runs), the ones between hold long text too. + let row: Vec<_> = (0..11).map(|c| if c % 2 == 0 { json!("x") } else { json!(long) }).collect(); + s.execute("range.setValues", json!({"range": "A1", "values": [row]})).unwrap(); + s.execute("home.autofitColumnWidth", json!({"cols": "A:A,C:C,E:E,G:G,I:I,K:K"})).unwrap(); + let doc = s.doc().unwrap(); + let size = |c: u32| doc.wb.active().and_then(|sh| sh.cols.get(&c)).and_then(|e| e.size); + assert!([0, 2, 4, 6, 8, 10].iter().all(|&c| size(c).is_some())); + assert!([1, 3, 5, 7, 9].iter().all(|&c| size(c).is_none())); + } + #[test] fn arabic_word_ligatures_are_wider_than_letters() { let em = |t: &str| approx_text_width(t, 72.0, false) / 96.0; diff --git a/crates/engine/src/cmd/format/sizes.rs b/crates/engine/src/cmd/format/sizes.rs index 8b304c889..5a4ba863a 100644 --- a/crates/engine/src/cmd/format/sizes.rs +++ b/crates/engine/src/cmd/format/sizes.rs @@ -194,16 +194,35 @@ pub(super) fn autofit_cols(s: &mut Session, p: &Json) -> Result { let measure = s.text_width.clone(); edit(s, |cx| { let Some(sh) = cx.wb.sheet(si) else { return Ok(Json::Null) }; - let mut widths = Vec::new(); let mut measured: std::collections::HashMap> = Default::default(); let mut distinct = 0usize; - let used = sh.used_range(); - for (a, b) in lines { - for c in a..=b.min(a + 16383) { - let Some(u) = used else { continue }; - let col = RangeRef::new(CellRef::new(u.start.row, c), CellRef::new(u.end.row, c)); - let mut max: f32 = 0.0; - for (cell, _) in sh.cells.iter_range(col) { + // Selected columns, then one pass over the populated cells (the store is row-oriented, so a + // scan per column would revisit every row for each selected column, empty or not). + let cols: std::collections::BTreeSet = lines.iter().flat_map(|&(a, b)| a..=b.min(a + 16383)).collect(); + let mut maxes: std::collections::BTreeMap = Default::default(); + if let Some(u) = sh.used_range() { + // One pass per run of adjacent selected columns, so columns between distant selections + // (A and XFD) are never visited. + let mut runs: Vec<(u32, u32)> = Vec::new(); + for &c in &cols { + match runs.last_mut() { + Some((_, end)) if *end + 1 == c => *end = c, + _ => runs.push((c, c)), + } + } + // Every run is a pass over all the stored rows: many runs (alternating columns) are one + // pass over their whole span instead, the unselected columns skipped. + if runs.len() > 4 + && let (Some(&(first, _)), Some(&(_, last))) = (runs.first(), runs.last()) + { + runs = vec![(first, last)]; + } + for (first, last) in runs { + let span = RangeRef::new(CellRef::new(u.start.row, first), CellRef::new(u.end.row, last)); + for (cell, _) in sh.cells.iter_range(span) { + if !cols.contains(&cell.col) { + continue; + } if sh.merge_at(cell).is_some_and(|m| !m.is_single()) { continue; } @@ -226,14 +245,14 @@ pub(super) fn autofit_cols(s: &mut Session, p: &Json) -> Result { } None => approx_text_width(line, st.font.size, st.font.bold), }; - max = max.max(w + 6.0 + st.align.indent as f32 * 9.0 + sh.filter_button_space(cell)); + let w = w + 6.0 + st.align.indent as f32 * 9.0 + sh.filter_button_space(cell); + let max = maxes.entry(cell.col).or_insert(0.0); + *max = max.max(w); } } - if max > 0.0 { - widths.push((c, max.clamp(8.0, 1785.0))); - } } } + let widths: Vec<(u32, f32)> = maxes.into_iter().filter(|&(_, m)| m > 0.0).map(|(c, m)| (c, m.clamp(8.0, 1785.0))).collect(); let sh = cx.sheet_mut(si)?; for (c, w) in widths { let e = sh.cols.entry(c).or_default(); diff --git a/crates/engine/src/cmd/insert.rs b/crates/engine/src/cmd/insert.rs index d832ecddf..4bd564db3 100644 --- a/crates/engine/src/cmd/insert.rs +++ b/crates/engine/src/cmd/insert.rs @@ -1222,9 +1222,15 @@ fn shape_set_text(s: &mut Session, p: &Json) -> Result { let sheet = d.wb.active_sheet(); let sh = d.wb.active().ok_or_else(|| bad("shape.setText", "no such text box"))?; let shape = sh.shapes.iter().find(|shape| shape.id == id).ok_or_else(|| bad("shape.setText", "no such text box"))?; - if shape.kind != ShapeKind::TextBox { + if !matches!(shape.kind, ShapeKind::TextBox | ShapeKind::WordArt) { return Err(bad("shape.setText", "the shape is not a text box")); } + if shape.kind == ShapeKind::WordArt && text.is_empty() { + return Err(bad("shape.setText", "WordArt text must not be empty")); + } + if shape.kind == ShapeKind::WordArt && text.chars().nth(super::wordart::MAX_CHARS).is_some() { + return Err(bad("shape.setText", format!("`text` is longer than {} characters", super::wordart::MAX_CHARS))); + } // Sheet protection locks drawing objects (there is no "edit objects" allowance yet). if sh.is_protected() { return Err(EngineError::Other(PROTECTED.into())); diff --git a/crates/engine/src/cmd/pivot/tests.rs b/crates/engine/src/cmd/pivot/tests.rs index 3748d0705..c40ddf1be 100644 --- a/crates/engine/src/cmd/pivot/tests.rs +++ b/crates/engine/src/cmd/pivot/tests.rs @@ -209,6 +209,41 @@ fn show_values_as() { assert_eq!(text(&s, "B3"), "Biggest sale"); } +/// Running Total over nested rows adds up each child's value across the top-level items, in the +/// order they appear: the same value as summing the plain values of the earlier items' rows. +#[test] +fn running_total_over_nested_rows_adds_up_across_the_top_level_items() { + let mut s = sales(); + s.execute("insert.pivotTable", json!({"rows": ["Region", "Product"], "values": ["Sales"]})).unwrap(); + let r = pt(&s).last_range.unwrap(); + let rows: Vec = (r.start.row + 1..r.end.row).collect(); + let read = |s: &Session| -> Vec<(String, f64)> { + rows.iter().map(|row| (text(s, &CellRef::new(*row, 0).a1()), val(s, &CellRef::new(*row, 1).a1()).as_f64().unwrap_or(0.0))).collect() + }; + let before = read(&s); + s.execute("pivot.valueSettings", json!({"index": 0, "showAs": "runningTotal"})).unwrap(); + let after = read(&s); + // Region rows carry the report's top-level items (not indented); the others are products. + let indent = |s: &Session, row: u32| { + let d = s.doc().unwrap(); + let sh = d.wb.active().unwrap(); + d.wb.styles.get(sh.style_id(CellRef::new(row, 0))).align.indent + }; + let (mut regions, mut products) = (0.0, std::collections::HashMap::::new()); + for (i, row) in rows.iter().enumerate() { + let name = &before[i].0; + let total = if indent(&s, *row) == 0 { + regions += before[i].1; + regions + } else { + let t = products.entry(name.clone()).or_insert(0.0); + *t += before[i].1; + *t + }; + assert!((after[i].1 - total).abs() < 1e-9, "row {name}: {} vs {total}", after[i].1); + } +} + #[test] fn compact_vs_tabular_shapes() { let mut s = sales(); diff --git a/crates/engine/src/cmd/print.rs b/crates/engine/src/cmd/print.rs index b3ce40d58..0f6ba272c 100644 --- a/crates/engine/src/cmd/print.rs +++ b/crates/engine/src/cmd/print.rs @@ -13,13 +13,15 @@ use gridcraft_core::{CellRef, MAX_COLS, MAX_ROWS, RangeRef, Value}; #[cfg(test)] use gridcraft_model::{Alignment, Image, SheetViewMode}; use gridcraft_model::{ - Anchor, BorderLine, BorderStyle, Borders, Color, HAlign, Orientation, PageOrder, PatternType, PrintSettings, ShapeKind, Sheet, Style, StyleId, - StyleTable, Underline, VAlign, Visibility, Workbook, + Anchor, BorderLine, BorderStyle, Borders, Color, HAlign, Orientation, PageOrder, PatternType, PrintSettings, ReadingOrder, ShapeKind, Sheet, + Style, StyleId, StyleTable, Underline, VAlign, Visibility, Workbook, }; -use gridcraft_pdf::{Font, Page, PdfDoc, Point, Rect, Rgb}; +use gridcraft_pdf::{Baseline, Font, Page, PdfDoc, Point, Rect, Rgb, TextDirection}; use serde_json::{Value as Json, json}; -use super::{CommandSpec, Result, Session, bad, bool_param, cmd, edit, f64_param, has_doc, split_sheet, str_param, target_sheet, u32_param}; +use super::{ + CommandSpec, EngineError, Result, Session, bad, bool_param, cmd, edit, f64_param, has_doc, split_sheet, str_param, target_sheet, u32_param, +}; mod draw; mod header_footer; @@ -32,8 +34,69 @@ use header_footer::draw_header_footer; pub use header_footer::{ HF_FIELDS, HfContext, HfSection, codes_to_fields, expand_header_footer, fields_to_codes, hf_date_time, join_header_footer, split_header_footer, }; -/// Decoded pictures (bytes, width, height) keyed by (sheet, picture); `None` = undecodable. -type PicCache = std::collections::HashMap<(usize, usize), Option<(Vec, u32, u32)>>; +type DecodedPicture = (Vec, u32, u32); + +/// What one PDF render shares across the pages an object spans. +#[derive(Default)] +struct PicCache { + /// Decoded pictures (bytes, width, height) keyed by (sheet, picture); `None` = undecodable. + decoded: std::collections::HashMap<(usize, usize), Option>, + /// Laid-out charts (with their [`chart_cost`]) keyed by (sheet, chart), rendered once however + /// many pages they touch. Only charts that can still be drawn are kept. + charts: std::collections::HashMap<(usize, usize), (Vec, usize)>, + /// Charts too big for what was left of the budget: never drawn, so not rendered again. + rejected_charts: std::collections::HashSet<(usize, usize)>, + /// Bytes of picture pixels embedded so far, and chart cost units emitted so far. Each page an + /// object touches gets its own copy in the PDF, so a workbook-controlled anchor over thousands + /// of pages would multiply them without bound. + image_bytes: usize, + chart_prims: usize, +} + +impl PicCache { + /// The chart's primitives for one more page, or `None` once they no longer fit + /// [`MAX_CHART_PRIMS`]. `render` runs at most once per chart; a chart that doesn't fit is + /// dropped at once and remembered, so its vectors are never retained. + fn chart(&mut self, key: (usize, usize), render: impl FnOnce() -> Vec) -> Option<&[gridcraft_chart::Prim]> { + if self.rejected_charts.contains(&key) { + return None; + } + let (prims, cost) = match self.charts.remove(&key) { + Some(cached) => cached, + None => { + let prims = render(); + let cost = chart_cost(&prims); + (prims, cost) + } + }; + if self.chart_prims.saturating_add(cost) > MAX_CHART_PRIMS { + self.rejected_charts.insert(key); + return None; + } + self.chart_prims += cost; + Some(&self.charts.entry(key).or_insert((prims, cost)).0) + } +} + +/// What a chart's drawing costs in the PDF: one unit per primitive plus one per path point (a +/// smoothed line holds thousands of points, each written out) and per point of a wedge's outline. +fn chart_cost(prims: &[gridcraft_chart::Prim]) -> usize { + use gridcraft_chart::Prim; + prims + .iter() + .map(|p| match p { + Prim::Line { pts, .. } | Prim::Polygon { pts, .. } => pts.len().saturating_add(1), + // At most 400 steps along each of the outer and inner arcs (see `draw_prims`). + Prim::Wedge { .. } => 803, + _ => 1, + }) + .fold(0, usize::saturating_add) +} + +/// Most picture pixel bytes (RGBA, before PDF packing) one render embeds across all pages. +const MAX_EMBEDDED_IMAGE_BYTES: usize = 512 * 1024 * 1024; +/// Most chart cost units (primitives and path points, see [`chart_cost`]) one render draws across all pages. +const MAX_CHART_PRIMS: usize = 4_000_000; #[cfg(test)] use layout::bands; @@ -90,8 +153,17 @@ pub(crate) const PX_TO_PT: f32 = 0.75; /// Most pages one export produces. pub const MAX_PAGES: usize = 2000; +/// A rendered PDF and whether some of its Arabic text was set without an Arabic face. +pub struct RenderedPdf { + pub bytes: Vec, + pub pages: usize, + pub missing_arabic_font: bool, +} + +const MISSING_ARABIC_FONT: &str = "Some Arabic text could not be drawn in this PDF because no Noto Sans Arabic face was found for it."; + /// Renders laid-out sheets to a PDF. -pub fn render_pdf(wb: &Workbook, layouts: &[SheetLayout], file: &str, path: &str) -> (Vec, usize) { +pub fn render_pdf(wb: &Workbook, layouts: &[SheetLayout], file: &str, path: &str) -> Result { let mut doc = PdfDoc::new(); doc.info.title = if wb.props.title.is_empty() { file.to_string() } else { wb.props.title.clone() }; doc.info.author = wb.props.author.clone(); @@ -102,7 +174,7 @@ pub fn render_pdf(wb: &Workbook, layouts: &[SheetLayout], file: &str, path: &str let mut printed_blocks = Vec::new(); // Decoded pictures are reused across the pages their anchor covers: a floating picture // spanning 40 pages was decoded (and pushed into the PDF) 40 times (#2389). - let mut pics = PicCache::new(); + let mut pics = PicCache::default(); for lay in layouts { let Some(sh) = wb.sheet(lay.sheet) else { continue }; let mut cf = if sh.cond_formats.is_empty() { None } else { Some(crate::cf::CfCache::new()) }; @@ -127,7 +199,16 @@ pub fn render_pdf(wb: &Workbook, layouts: &[SheetLayout], file: &str, path: &str let b = make_block(wb, lay.sheet, sh, *rs, *cs, (x, y), lay); draw_block(pg, &b, lay.settings.gridlines, &mut cf); draw_block_links(pg, &b, page_index, &mut pending_links); - printed_blocks.push(PrintedBlock { page: page_index, sheet: lay.sheet, rows: *rs, cols: *cs, x, y, k }); + printed_blocks.push(PrintedBlock { + page: page_index, + sheet: lay.sheet, + rows: *rs, + cols: *cs, + x, + y, + k, + mirror_axis: sh.right_to_left.then_some(b.mirror_axis), + }); if lay.settings.headings { draw_headings(pg, &b, rsi == 0, csi == 0, (ox, oy), (lay.heading_w * k, lay.heading_h * k)); } @@ -143,6 +224,7 @@ pub fn render_pdf(wb: &Workbook, layouts: &[SheetLayout], file: &str, path: &str } if lay.settings.headings { let fill = [0xF2, 0xF2, 0xF2]; + let ox = if sh.right_to_left { lay.page_w - ox - lay.heading_w * k } else { ox }; pg.fill_rect(ox, oy, lay.heading_w * k, lay.heading_h * k, fill); pg.rect_stroke(ox, oy, lay.heading_w * k, lay.heading_h * k, 0.3, [0xA0, 0xA0, 0xA0]); } @@ -166,18 +248,24 @@ pub fn render_pdf(wb: &Workbook, layouts: &[SheetLayout], file: &str, path: &str page.link_page(Rect::new(lx, ly, w, h), target_page, Point::new(x, y)); } } - let n = doc.page_count(); - (doc.finish(), n) + let pages = doc.page_count(); + let missing_arabic_font = doc.missing_arabic_font(); + let bytes = doc.finish().map_err(|error| EngineError::Other(error.to_string()))?; + Ok(RenderedPdf { bytes, pages, missing_arabic_font }) } // ---------------------------------------------------------------- commands -fn build_pdf(s: &Session, p: &Json, cmd: &str) -> Result<(Vec, usize)> { +fn build_pdf(s: &mut Session, p: &Json, cmd: &str) -> Result<(Vec, usize)> { let opts = PrintOptions::from_params(p, cmd)?; let d = s.doc()?; let lays = layouts(&d.wb, &opts); let path = d.path.clone().unwrap_or_default(); - Ok(render_pdf(&d.wb, &lays, &d.display_title(), &path)) + let pdf = render_pdf(&d.wb, &lays, &d.display_title(), &path)?; + if pdf.missing_arabic_font { + s.ui_requests.push(crate::UiRequest::Message(MISSING_ARABIC_FONT.into())); + } + Ok((pdf.bytes, pdf.pages)) } pub(crate) fn export_pdf(s: &mut Session, p: &Json) -> Result { diff --git a/crates/engine/src/cmd/print/draw.rs b/crates/engine/src/cmd/print/draw.rs index 7a49b8e85..35fd41bd4 100644 --- a/crates/engine/src/cmd/print/draw.rs +++ b/crates/engine/src/cmd/print/draw.rs @@ -32,6 +32,9 @@ fn cell_look(wb: &Workbook, si: usize, sh: &Sheet, c: CellRef, cf: &mut Option { pub(super) ys: Vec, pub(super) k: f32, pub(super) scale: f32, + pub(super) mirror_axis: f32, } impl Block<'_> { + fn point_x(&self, x: f32) -> f32 { + if self.sh.right_to_left { self.mirror_axis - x } else { x } + } + pub(super) fn text_x(&self, x: f32, width: f32) -> f32 { + if self.sh.right_to_left { self.mirror_axis - x - width } else { x } + } + pub(super) fn drawn_rect(&self, r: RangeRef) -> (f32, f32, f32, f32) { + let (x, y, w, h) = self.rect(r); + (self.text_x(x, w), y, w, h) + } fn x(&self, col: u32) -> f32 { if col >= self.cols.0 && col <= self.cols.1 + 1 { return self.xs.get((col - self.cols.0) as usize).copied().unwrap_or(0.0); @@ -228,6 +242,82 @@ fn draw_border(pg: &mut Page, b: &BorderLine, wb: &Workbook, start: Point, end: pg.stroke_line_dashed(Point::new(x0, y0), Point::new(x1, y1), lw, color, &border_dash(b.style, k)); } +/// Merged areas active at one row of a downward sweep, kept disjoint: by first column, their last +/// column and last row. Adding, dropping and looking up a merge costs log(active merges). +#[derive(Default)] +struct MergeSweep { + cols: std::collections::BTreeMap, + /// (last row, first column) of the active merges, soonest ending first. + ends: std::collections::BinaryHeap>, +} + +impl MergeSweep { + /// Moves down to `row` (rows never decrease), dropping the merges that end above it. + fn advance(&mut self, row: u32) { + while let Some(std::cmp::Reverse((end, col))) = self.ends.peek().copied() { + if end >= row { + break; + } + self.ends.pop(); + self.cols.remove(&col); + } + } + + /// Does an active merge meet columns `c0..=c1`? The intervals are disjoint, so only the last + /// one starting at or before `c1` can. + fn meets(&self, c0: u32, c1: u32) -> bool { + self.cols.range(..=c1).next_back().is_some_and(|(_, (end, _))| *end >= c0) + } + + /// Adds merge `m`, at row `row`, unless it overlaps an active one. + fn add(&mut self, m: RangeRef, row: u32) -> bool { + self.advance(row); + if self.meets(m.start.col, m.end.col) { + return false; + } + self.cols.insert(m.start.col, (m.end.col, m.end.row)); + self.ends.push(std::cmp::Reverse((m.end.row, m.start.col))); + true + } +} + +/// The merged areas that meet `block`, ordered by the row where they enter it, then by column, +/// leaving out any that overlaps one kept before it (only a damaged file has overlapping merges; +/// Excel drops them when it repairs one). One sweep down the rows instead of a scan of every +/// merge per cell. +fn block_merges(sh: &Sheet, block: RangeRef) -> Vec { + let top = |m: &RangeRef| m.start.row.max(block.start.row); + let mut merges: Vec = sh.merges.iter().filter(|m| m.intersects(&block)).copied().collect(); + merges.sort_by_key(|m| (top(m), m.start.col)); + let mut sweep = MergeSweep::default(); + merges.retain(|m| sweep.add(*m, top(m))); + merges +} + +/// Answers "is this cell inside a merge?" for cells asked in row-major order, from the merges of +/// [`block_merges`]. +struct Covered<'a> { + merges: &'a [RangeRef], + top: u32, + next: usize, + sweep: MergeSweep, +} + +impl<'a> Covered<'a> { + fn new(merges: &'a [RangeRef], block: RangeRef) -> Self { + Self { merges, top: block.start.row, next: 0, sweep: MergeSweep::default() } + } + + fn covers(&mut self, c: CellRef) -> bool { + while let Some(m) = self.merges.get(self.next).filter(|m| m.start.row.max(self.top) <= c.row) { + self.sweep.add(*m, m.start.row.max(self.top)); + self.next += 1; + } + self.sweep.advance(c.row); + self.sweep.meets(c.col, c.col) + } +} + /// Draws the cells of a block: fills, gridlines, data bars, borders and text. pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Option) { let (bx, by) = (b.xs.first().copied().unwrap_or(0.0), b.ys.first().copied().unwrap_or(0.0)); @@ -237,29 +327,30 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt } let block = b.range(); let sh = b.sh; - let merges: Vec = sh.merges.iter().filter(|m| m.intersects(&block)).copied().collect(); - let covered = |c: CellRef| merges.iter().any(|m| m.contains(c)); - // Cells worth visiting. + let merges = block_merges(sh, block); + let by_start: std::collections::HashMap = merges.iter().map(|m| (m.start, *m)).collect(); + // Cells worth visiting, in row-major order (which `Covered` needs). + let mut covered = Covered::new(&merges, block); let cells: Vec = if block.count() <= DENSE_BLOCK { - block.iter().filter(|c| !covered(*c)).collect() + block.iter().filter(|c| !covered.covers(*c)).collect() } else { let mut set: Vec = sh.cells.iter_range(block).map(|(c, _)| c).collect(); set.extend(sh.spill.range(block.start..=block.end).map(|(c, _)| *c).filter(|c| block.contains(*c))); set.sort_unstable(); set.dedup(); - set.retain(|c| !covered(*c)); + set.retain(|c| !covered.covers(*c)); set }; pg.save(); - pg.clip_rect(bx, by, bw, bh); + pg.clip_rect(b.text_x(bx, bw), by, bw, bh); if gridlines { let grey = [0xB0, 0xB0, 0xB0]; let lw = (0.35 * b.scale).clamp(0.1, 0.5); for x in &b.xs { - pg.stroke_line(*x, by, *x, by + bh, lw, grey); + pg.stroke_line(b.point_x(*x), by, b.point_x(*x), by + bh, lw, grey); } for y in &b.ys { - pg.stroke_line(bx, *y, bx + bw, *y, lw, grey); + pg.stroke_line(b.point_x(bx), *y, b.point_x(bx + bw), *y, lw, grey); } } // Cheap pre-filter: blank, unstyled cells outside tables and conditional formats draw nothing. @@ -277,14 +368,14 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt .filter(|(c, l)| { l.fill.is_some() || l.cf.is_some() - || merges.iter().any(|m| m.start == *c) + || by_start.contains_key(c) || !sh.value(*c).is_empty() || l.style.borders != Borders::default() || l.style.num_fmt.as_str() == "checkbox" }) .collect(); let rect_of = |c: CellRef| -> (f32, f32, f32, f32, bool) { - match merges.iter().find(|m| m.start == c) { + match by_start.get(&c) { Some(m) => { let (x, y, w, h) = b.rect(*m); (x, y, w, h, true) @@ -302,17 +393,17 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt continue; } if let Some(f) = l.fill { - pg.fill_rect(x, y, w, h, f); + pg.fill_rect(b.text_x(x, w), y, w, h, f); } else if merged && gridlines { - pg.fill_rect(x, y, w, h, [255, 255, 255]); + pg.fill_rect(b.text_x(x, w), y, w, h, [255, 255, 255]); } if merged && gridlines { - pg.rect_stroke(x, y, w, h, (0.35 * b.scale).clamp(0.1, 0.5), [0xB0, 0xB0, 0xB0]); + pg.rect_stroke(b.text_x(x, w), y, w, h, (0.35 * b.scale).clamp(0.1, 0.5), [0xB0, 0xB0, 0xB0]); } if let Some((frac, rgb, _)) = l.cf.as_ref().and_then(|cf| cf.bar) { let pad = 1.5 * b.k; let bwid = ((w - 2.0 * pad) * frac.clamp(0.0, 1.0)).max(0.0); - pg.fill_rect(x + pad, y + pad, bwid, (h - 2.0 * pad).max(0.0), rgb); + pg.fill_rect(b.text_x(x + pad, bwid), y + pad, bwid, (h - 2.0 * pad).max(0.0), rgb); } } // Borders. @@ -322,19 +413,19 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt continue; } let bd = &l.style.borders; - let (right, bottom) = match merges.iter().find(|m| m.start == *c).filter(|_| merged) { + let (right, bottom) = match by_start.get(c).filter(|_| merged) { Some(m) => ( b.wb.styles.get(sh.style_id(CellRef::new(m.start.row, m.end.col))).borders.right, b.wb.styles.get(sh.style_id(CellRef::new(m.end.row, m.start.col))).borders.bottom, ), None => (bd.right, bd.bottom), }; - draw_border(pg, &bd.top, b.wb, Point::new(x, y), Point::new(x + w, y), b.k); - draw_border(pg, &bottom, b.wb, Point::new(x, y + h), Point::new(x + w, y + h), b.k); - draw_border(pg, &bd.left, b.wb, Point::new(x, y), Point::new(x, y + h), b.k); - draw_border(pg, &right, b.wb, Point::new(x + w, y), Point::new(x + w, y + h), b.k); - draw_border(pg, &bd.diag_down, b.wb, Point::new(x, y), Point::new(x + w, y + h), b.k); - draw_border(pg, &bd.diag_up, b.wb, Point::new(x, y + h), Point::new(x + w, y), b.k); + draw_border(pg, &bd.top, b.wb, Point::new(b.point_x(x), y), Point::new(b.point_x(x + w), y), b.k); + draw_border(pg, &bottom, b.wb, Point::new(b.point_x(x), y + h), Point::new(b.point_x(x + w), y + h), b.k); + draw_border(pg, &bd.left, b.wb, Point::new(b.point_x(x), y), Point::new(b.point_x(x), y + h), b.k); + draw_border(pg, &right, b.wb, Point::new(b.point_x(x + w), y), Point::new(b.point_x(x + w), y + h), b.k); + draw_border(pg, &bd.diag_down, b.wb, Point::new(b.point_x(x), y), Point::new(b.point_x(x + w), y + h), b.k); + draw_border(pg, &bd.diag_up, b.wb, Point::new(b.point_x(x), y + h), Point::new(b.point_x(x + w), y), b.k); } // Text. for (c, l) in &looks { @@ -343,9 +434,9 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt continue; } if let Some(picture) = sh.cell_pictures.get(c) { - draw_cell_picture(pg, &picture.data, (x, y, w, h), b.k); + draw_cell_picture(pg, &picture.data, (b.text_x(x, w), y, w, h), b.k); } else if let Some(on) = checkbox_state(l, &sh.value(*c)) { - draw_checkbox(pg, (x, y, w, h), b.k, on); + draw_checkbox(pg, (b.text_x(x, w), y, w, h), b.k, on); } else { draw_text(pg, b, TextPlacement { cell: *c, look: l, rect: (x, y, w, h), merged, merges: &merges, limits: (bx, bx + bw) }); } @@ -399,6 +490,8 @@ pub(super) struct PrintedBlock { pub(super) x: f32, pub(super) y: f32, pub(super) k: f32, + /// The page x that right-to-left sheets are mirrored about; `None` for left-to-right sheets. + pub(super) mirror_axis: Option, } /// PDF annotations are not affected by graphics clipping, so intersect each cell with its page @@ -407,17 +500,18 @@ pub(super) struct PrintedBlock { pub(super) fn draw_block_links(pg: &mut Page, b: &Block, page: usize, pending: &mut Vec) { let range = b.range(); let (Some(&bx), Some(&by), Some(&right), Some(&bottom)) = (b.xs.first(), b.ys.first(), b.xs.last(), b.ys.last()) else { return }; + let by_start: std::collections::HashMap = block_merges(b.sh, range).into_iter().map(|m| (m.start, m)).collect(); for (cell, link) in b.sh.hyperlinks.range(range.start..=range.end) { if !range.contains(*cell) { continue; } - let cell_range = b.sh.merges.iter().find(|m| m.start == *cell).copied().unwrap_or(RangeRef::cell(*cell)); + let cell_range = by_start.get(cell).copied().unwrap_or(RangeRef::cell(*cell)); let (x, y, w, h) = b.rect(cell_range); let (x0, y0, x1, y1) = (x.max(bx), y.max(by), (x + w).min(right), (y + h).min(bottom)); if x1 <= x0 || y1 <= y0 { continue; } - let rect = (x0, y0, x1 - x0, y1 - y0); + let rect = (b.text_x(x0, x1 - x0), y0, x1 - x0, y1 - y0); match crate::links::classify(&link.target) { crate::links::LinkKind::External(uri) => pg.link_uri(rect.0, rect.1, rect.2, rect.3, &uri), crate::links::LinkKind::Internal(target) if !link.external => { @@ -436,6 +530,10 @@ pub(super) fn printed_target(wb: &Workbook, blocks: &[PrintedBlock], source_shee let sh = wb.sheet(sheet)?; let block = blocks.iter().find(|b| b.sheet == sheet && (b.rows.0..=b.rows.1).contains(&cell.row) && (b.cols.0..=b.cols.1).contains(&cell.col))?; let x = block.x + (sh.col_left(cell.col) - sh.col_left(block.cols.0)) as f32 * block.k; + let x = match block.mirror_axis { + Some(axis) => axis - x - (sh.col_left(cell.col.saturating_add(1)) - sh.col_left(cell.col)) as f32 * block.k, + None => x, + }; let y = block.y + (sh.row_top(cell.row) - sh.row_top(block.rows.0)) as f32 * block.k; Some((block.page, x, y)) } @@ -452,11 +550,11 @@ pub(super) fn draw_headings(pg: &mut Page, b: &Block, col_heads: bool, row_heads if x1 - x0 <= 0.0 { continue; } - pg.fill_rect(x0, y, x1 - x0, size.1, fill); - pg.rect_stroke(x0, y, x1 - x0, size.1, lw, line); + pg.fill_rect(b.text_x(x0, x1 - x0), y, x1 - x0, size.1, fill); + pg.rect_stroke(b.text_x(x0, x1 - x0), y, x1 - x0, size.1, lw, line); let t = gridcraft_core::col_to_letters(col); let tw = gridcraft_pdf::text_width(Font::Helvetica, fsize, &t); - pg.text(x0 + (x1 - x0 - tw) / 2.0, y + size.1 / 2.0 + fsize * 0.35, fsize, Font::Helvetica, [0x40, 0x40, 0x40], &t); + pg.text(b.text_x(x0 + (x1 - x0 - tw) / 2.0, tw), y + size.1 / 2.0 + fsize * 0.35, fsize, Font::Helvetica, [0x40, 0x40, 0x40], &t); } } if row_heads { @@ -466,11 +564,11 @@ pub(super) fn draw_headings(pg: &mut Page, b: &Block, col_heads: bool, row_heads if y1 - y0 <= 0.0 { continue; } - pg.fill_rect(x, y0, size.0, y1 - y0, fill); - pg.rect_stroke(x, y0, size.0, y1 - y0, lw, line); + pg.fill_rect(b.text_x(x, size.0), y0, size.0, y1 - y0, fill); + pg.rect_stroke(b.text_x(x, size.0), y0, size.0, y1 - y0, lw, line); let t = (row + 1).to_string(); let tw = gridcraft_pdf::text_width(Font::Helvetica, fsize, &t); - pg.text(x + (size.0 - tw) / 2.0, y0 + (y1 - y0) / 2.0 + fsize * 0.35, fsize, Font::Helvetica, [0x40, 0x40, 0x40], &t); + pg.text(b.text_x(x + (size.0 - tw) / 2.0, tw), y0 + (y1 - y0) / 2.0 + fsize * 0.35, fsize, Font::Helvetica, [0x40, 0x40, 0x40], &t); } } } @@ -501,5 +599,46 @@ pub(super) fn make_block<'a>( acc += sh.row_height(r) * k; ys.push(acc); } - Block { wb, si, sh, rows, cols, xs, ys, k, scale: lay.scale } + Block { wb, si, sh, rows, cols, xs, ys, k, scale: lay.scale, mirror_axis: lay.page_w } +} + +#[cfg(test)] +mod merge_tests { + use super::{CellRef, Covered, RangeRef, Sheet, block_merges}; + + fn r(a: &str) -> RangeRef { + RangeRef::parse(a).unwrap() + } + + #[test] + fn block_merges_drop_overlaps_and_cover_like_a_scan() { + let mut sh = Sheet::new("S"); + sh.merges = vec![r("B2:C3"), r("E1:F8"), r("B2:B5"), r("A10:D10"), r("H3:H4"), r("Z1:Z2"), r("C3:D4"), r("A1:A1048576")]; + let block = r("B1:H12"); + let merges = block_merges(&sh, block); + // B2:B5 and C3:D4 overlap B2:C3, which comes first; Z1:Z2 is outside the block. + assert_eq!(merges, vec![r("E1:F8"), r("B2:C3"), r("H3:H4"), r("A10:D10")]); + let mut covered = Covered::new(&merges, block); + for c in block.iter() { + assert_eq!(covered.covers(c), merges.iter().any(|m| m.contains(c)), "{}", c.a1()); + } + // Cells asked sparsely, still in order. + let mut covered = Covered::new(&merges, block); + for (a, want) in [("C1", false), ("F5", true), ("G5", false), ("D10", true), ("H12", false)] { + assert_eq!(covered.covers(CellRef::parse(a).unwrap()), want, "{a}"); + } + } + + #[test] + fn many_stacked_merges_are_swept_once() { + // 50,000 merges over the same cells: one is kept, and the block is checked against it + // without comparing every cell with every merge. + let mut sh = Sheet::new("S"); + sh.merges = (0..50_000u32).map(|i| RangeRef::new(CellRef::new(0, 0), CellRef::new(199, 1 + i % 50))).collect(); + let block = r("A1:AZ200"); + let merges = block_merges(&sh, block); + assert_eq!(merges, vec![r("A1:B200")]); + let mut covered = Covered::new(&merges, block); + assert_eq!(block.iter().filter(|c| covered.covers(*c)).count(), 400); + } } diff --git a/crates/engine/src/cmd/print/layout.rs b/crates/engine/src/cmd/print/layout.rs index 0f484f875..0d86de515 100644 --- a/crates/engine/src/cmd/print/layout.rs +++ b/crates/engine/src/cmd/print/layout.rs @@ -224,6 +224,12 @@ pub fn layout_sheet(wb: &Workbook, si: usize, opts: &PrintOptions, range_overrid Some(r) => trim_full(sh, r), None => print_range(sh).unwrap_or(RangeRef::cell(CellRef::new(0, 0))), }; + // A print area from a file or JSON can name coordinates past the grid; pagination walks every + // line of the area (zero-height rows never fill a band), so it is held to the grid first. + let area = RangeRef::new( + CellRef::new(area.start.row.min(MAX_ROWS - 1), area.start.col.min(MAX_COLS - 1)), + CellRef::new(area.end.row.min(MAX_ROWS - 1), area.end.col.min(MAX_COLS - 1)), + ); let title_rows = title_span(st.title_rows, MAX_ROWS); let title_cols = title_span(st.title_cols, MAX_COLS); let digits = format!("{}", area.end.row + 1).len() as f32; @@ -379,3 +385,19 @@ pub(super) fn segments(titles: Option<(u32, u32)>, band: (u32, u32)) -> Vec<(u32 pub(super) fn seg_size(sh: &Sheet, seg: (u32, u32), rows: bool) -> f32 { if rows { (seg.0..=seg.1).map(|r| sh.row_height(r)).sum() } else { (seg.0..=seg.1).map(|c| sh.col_width(c)).sum() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_print_area_past_the_grid_is_held_to_the_grid() { + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + sh.default_row_height = 0.0; + sh.print.print_area = Some(RangeRef { start: CellRef::new(0, 0), end: CellRef::new(u32::MAX, u32::MAX) }); + let lay = layout_sheet(&wb, 0, &PrintOptions::default(), None).unwrap(); + assert_eq!((lay.area.end.row, lay.area.end.col), (MAX_ROWS - 1, MAX_COLS - 1)); + assert!(lay.pages.len() <= MAX_PAGES); + } +} diff --git a/crates/engine/src/cmd/print/objects.rs b/crates/engine/src/cmd/print/objects.rs index faa855cb9..5b786b701 100644 --- a/crates/engine/src/cmd/print/objects.rs +++ b/crates/engine/src/cmd/print/objects.rs @@ -1,6 +1,6 @@ use super::draw::{Block, PdfMeasure, Xf, draw_prims, rgb_of}; use super::text::wrap_lines; -use super::{Anchor, Font, Page, RangeRef, Rect, ShapeKind, Sheet, Value, Workbook}; +use super::{Anchor, Font, Page, Point, RangeRef, Rect, ShapeKind, Sheet, TextDirection, Value, Workbook}; /// Cell pictures fit the current cell (or merged-cell) rectangle without changing aspect ratio. pub(super) fn draw_cell_picture(pg: &mut Page, data: &[u8], rect: (f32, f32, f32, f32), k: f32) { @@ -76,9 +76,11 @@ pub(super) fn draw_objects(pg: &mut Page, wb: &Workbook, si: usize, sh: &Sheet, let (bx, by) = (b.xs.first().copied().unwrap_or(0.0), b.ys.first().copied().unwrap_or(0.0)); let (bw, bh) = (b.xs.last().copied().unwrap_or(bx) - bx, b.ys.last().copied().unwrap_or(by) - by); let visible = |r: (f64, f64, f64, f64)| r.2 > 0.0 && r.3 > 0.0 && r.0 < sx1 && r.0 + r.2 > sx0 && r.1 < sy1 && r.1 + r.3 > sy0; - let place = |r: (f64, f64, f64, f64)| (bx + ((r.0 - sx0) as f32) * b.k, by + ((r.1 - sy0) as f32) * b.k, r.2 as f32 * b.k, r.3 as f32 * b.k); + let place = |r: (f64, f64, f64, f64)| { + (b.text_x(bx + ((r.0 - sx0) as f32) * b.k, r.2 as f32 * b.k), by + ((r.1 - sy0) as f32) * b.k, r.2 as f32 * b.k, r.3 as f32 * b.k) + }; pg.save(); - pg.clip_rect(bx, by, bw, bh); + pg.clip_rect(b.text_x(bx, bw), by, bw, bh); draw_sparklines(pg, wb, si, sh, b, block); for (imi, im) in sh.images.iter().enumerate() { let r = anchor_rect(sh, &im.anchor); @@ -88,9 +90,14 @@ pub(super) fn draw_objects(pg: &mut Page, wb: &Workbook, si: usize, sh: &Sheet, let (x, y, w, h) = place(r); // About 150 dpi is plenty for print. let (mw, mh) = (((w / 72.0) * 150.0).ceil().clamp(1.0, 4000.0) as u32, ((h / 72.0) * 150.0).ceil().clamp(1.0, 4000.0) as u32); - let decoded = pics.entry((si, imi)).or_insert_with(|| decode_image(&im.data, mw, mh)); + let decoded = pics.decoded.entry((si, imi)).or_insert_with(|| decode_image(&im.data, mw, mh)); if let Some((px, pw, ph)) = decoded.as_ref() { - pg.image_rgba(Rect::new(x, y, w, h), px, [*pw, *ph]); + // Every page the picture touches embeds its own copy: past the budget it is left out. + let bytes = px.len(); + if pics.image_bytes.saturating_add(bytes) <= super::MAX_EMBEDDED_IMAGE_BYTES { + pics.image_bytes += bytes; + pg.image_rgba(Rect::new(x, y, w, h), px, [*pw, *ph]); + } } } for shp in &sh.shapes { @@ -120,15 +127,20 @@ pub(super) fn draw_objects(pg: &mut Page, wb: &Workbook, si: usize, sh: &Sheet, } draw_shape(pg, wb, shp, b, (x, y, w, h)); } - for ch in &sh.charts { + for (chi, ch) in sh.charts.iter().enumerate() { let r = anchor_rect(sh, &ch.anchor); if !visible(r) { continue; } let (x, y, _, _) = place(r); - let data = gridcraft_chart::resolve(wb, si, ch); - let prims = gridcraft_chart::render(ch, &data, r.2 as f32, r.3 as f32, &PdfMeasure); - draw_prims(pg, &prims, Xf { ox: x, oy: y, k: b.k }); + // As for pictures, each page gets its own copy of the drawing commands. + let prims = pics.chart((si, chi), || { + let data = gridcraft_chart::resolve(wb, si, ch); + gridcraft_chart::render(ch, &data, r.2 as f32, r.3 as f32, &PdfMeasure) + }); + if let Some(prims) = prims { + draw_prims(pg, prims, Xf { ox: x, oy: y, k: b.k }); + } } pg.restore(); } @@ -143,9 +155,19 @@ fn draw_sparklines(pg: &mut Page, wb: &Workbook, si: usize, sh: &Sheet, b: &Bloc v => vec![v.as_f64()], }; let color = rgb_of(sp.color, wb).unwrap_or([0x15, 0x60, 0x82]); - let (x, y, w, h) = b.rect(RangeRef::cell(sp.cell)); + let (x, y, w, h) = b.drawn_rect(RangeRef::cell(sp.cell)); let (wpx, hpx) = (w / b.k, h / b.k); - let prims = gridcraft_chart::render_sparkline(sp.kind, &vals, [color[0], color[1], color[2], 255], sp.markers, wpx, hpx); + let mut prims = gridcraft_chart::render_sparkline(sp.kind, &vals, [color[0], color[1], color[2], 255], sp.markers, wpx, hpx); + if sh.right_to_left { + for prim in &mut prims { + match prim { + gridcraft_chart::Prim::Rect { x, w, .. } => *x = wpx - *x - *w, + gridcraft_chart::Prim::Line { pts, .. } => pts.iter_mut().for_each(|p| p[0] = wpx - p[0]), + gridcraft_chart::Prim::Circle { cx, .. } => *cx = wpx - *cx, + _ => {} + } + } + } draw_prims(pg, &prims, Xf { ox: x, oy: y, k: b.k }); } } @@ -189,7 +211,7 @@ fn draw_shape(pg: &mut Page, wb: &Workbook, shp: &gridcraft_model::Shape, b: &Bl let color = fm.text_color.and_then(|c| rgb_of(c, wb)).unwrap_or([0, 0, 0]); let align = fm.align.unwrap_or(if text_box { ShapeTextAlign::Left } else { ShapeTextAlign::Center }); let anchor = fm.anchor.unwrap_or(if text_box { ShapeTextAnchor::Top } else { ShapeTextAnchor::Middle }); - let lines = wrap_lines(&shp.text, font, size, (w - 4.0 * b.k).max(size)); + let lines = wrap_lines(&shp.text, font, size, (w - 4.0 * b.k).max(size), TextDirection::Auto); let block_h = lines.len().min(500) as f32 * size * 1.2; let mut yy = match anchor { ShapeTextAnchor::Top => y + 2.0 * b.k, @@ -197,13 +219,28 @@ fn draw_shape(pg: &mut Page, wb: &Workbook, shp: &gridcraft_model::Shape, b: &Bl ShapeTextAnchor::Bottom => y + h - 2.0 * b.k - block_h, } + size * 0.8; for l in lines.iter().take(500) { - let lw = gridcraft_pdf::text_width(font, size, l); + let lw = gridcraft_pdf::text_width_with_direction(font, size, &l.text, l.direction); let lx = match align { ShapeTextAlign::Left => x + 2.0 * b.k, ShapeTextAlign::Center => x + (w - lw) / 2.0, ShapeTextAlign::Right => x + w - 2.0 * b.k - lw, }; - pg.text(lx, yy, size, font, color, l); + // WordArt's drop shadow: the line again, offset, in the shadow colour (under the text). + if let Some(sc) = fm.text_shadow.and_then(|c| rgb_of(c, wb)) { + let off = (size * 0.06).max(0.5); + pg.text_with_direction(Point::new(lx + off, yy + off), size, font, sc, &l.text, l.direction); + } + // WordArt's text outline, as on screen: the line in the outline colour, offset in eight + // directions by half the outline width, under the filled text. + if let Some((c, ow)) = fm.text_outline + && let Some(oc) = rgb_of(c, wb) + { + let d = (ow * b.scale / 2.0).clamp(0.25, 20.0); + for (dx, dy) in [(-1.0, -1.0), (0.0, -1.0), (1.0, -1.0), (-1.0, 0.0), (1.0, 0.0), (-1.0, 1.0), (0.0, 1.0), (1.0, 1.0)] { + pg.text_with_direction(Point::new(lx + dx * d, yy + dy * d), size, font, oc, &l.text, l.direction); + } + } + pg.text_with_direction(Point::new(lx, yy), size, font, color, &l.text, l.direction); yy += size * 1.2; } } diff --git a/crates/engine/src/cmd/print/tests.rs b/crates/engine/src/cmd/print/tests.rs index 323060470..884432317 100644 --- a/crates/engine/src/cmd/print/tests.rs +++ b/crates/engine/src/cmd/print/tests.rs @@ -1,3 +1,4 @@ +use gridcraft_model::{Sparkline, SparklineKind}; use serde_json::json; use super::*; @@ -15,6 +16,35 @@ fn pdf_of(s: &mut Session, p: Json) -> (Vec, u64) { fn contains(hay: &[u8], needle: &str) -> bool { hay.windows(needle.len()).any(|w| w == needle.as_bytes()) } +fn arabic_face_available() -> bool { + let mut doc = gridcraft_pdf::PdfDoc::new(); + doc.add_page(100.0, 100.0).text(0.0, 50.0, 10.0, Font::Helvetica, [0, 0, 0], "مرحبا"); + !doc.missing_arabic_font() +} + +#[test] +fn right_to_left_sheets_mirror_link_rectangles_and_targets() { + fn numbers_after(pdf: &[u8], key: &str, n: usize) -> Vec { + let text = String::from_utf8_lossy(pdf); + let at = text.find(key).unwrap() + key.len(); + text[at..].split([' ', ']']).filter_map(|t| t.parse().ok()).take(n).collect() + } + let mut s = s(); + s.execute("insert.link", json!({"cell": "A1", "target": "https://example.org/report"})).unwrap(); + s.execute("insert.link", json!({"cell": "A2", "target": "Sheet1!B60"})).unwrap(); + s.execute("cell.set", json!({"cell": "B60", "input": "Destination"})).unwrap(); + let (ltr, _) = pdf_of(&mut s, json!({})); + s.execute("view.rightToLeft", json!({"on": true})).unwrap(); + let (rtl, _) = pdf_of(&mut s, json!({})); + let axis = numbers_after(<r, "/MediaBox [", 4)[2]; + let (l, r) = (numbers_after(<r, "/Rect [", 4), numbers_after(&rtl, "/Rect [", 4)); + assert!((r[0] - (axis - l[2])).abs() < 0.01 && (r[2] - (axis - l[0])).abs() < 0.01, "{l:?} mirrors to {r:?}"); + assert_eq!(r[1], l[1]); + let b_width = s.doc().unwrap().wb.active().unwrap().col_width(1); + let (lx, rx) = (numbers_after(<r, "/XYZ ", 1)[0], numbers_after(&rtl, "/XYZ ", 1)[0]); + let k = (l[2] - l[0]) / s.doc().unwrap().wb.active().unwrap().col_width(0); + assert!((rx - (axis - lx - b_width * k)).abs() < 0.01, "target {lx} mirrors to {rx}"); +} #[test] fn chart_data_table_prints_formatted_values_and_follows_recalculation() { @@ -58,6 +88,199 @@ fn pdf_export_keeps_safe_links_on_their_printed_pages() { assert!(!cropped.contains("/Dest [")); } +#[test] +fn arabic_with_characters_its_font_lacks_still_exports() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "مرحبا 😀"})).unwrap(); + s.execute("cell.set", json!({"cell": "A2", "input": "مرحبا Привет"})).unwrap(); + let out = s.execute("file.exportPdf", json!({})).unwrap(); + assert_eq!(out["pages"], 1); +} + +#[test] +fn wrapping_keeps_combining_marks_with_their_letters() { + let word = "a\u{301}a\u{301}a\u{301}a\u{301}"; + let lines = wrap_lines(word, Font::Helvetica, 10.0, 7.0, TextDirection::Auto); + assert_eq!(lines.iter().map(|line| line.text.as_str()).collect::(), word); + assert!(lines.iter().all(|line| line.text == "a\u{301}")); +} + +#[test] +fn arabic_cells_headers_and_charts_export_searchable_shaped_text() { + if !arabic_face_available() && std::env::var_os("CRAFT_FONTS_REQUIRED").is_none() { + return; + } + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [["مَرْحَبًا 123 (ABC)", "القيمة"], ["تفاح", 3], ["كمثرى", 5], ["العَرَبِيَّة العَرَبِيَّة العَرَبِيَّة العَرَبِيَّة العَرَبِيَّة", null], ["تقرير ١٢٣ (ABC)", null], ["مرحبا", null]]})).unwrap(); + s.execute("home.columnWidth", json!({"cols": "A:B", "chars": 32})).unwrap(); + s.execute("home.rowHeight", json!({"rows": "1:6", "height": 50})).unwrap(); + s.execute("home.fontSize", json!({"range": "A1:B6", "size": 20})).unwrap(); + s.execute("home.rowHeight", json!({"rows": "4:4", "height": 90})).unwrap(); + s.execute("home.bold", json!({"range": "A2"})).unwrap(); + s.execute("home.italic", json!({"range": "A3"})).unwrap(); + s.execute("home.wrapText", json!({"range": "A4"})).unwrap(); + s.execute("home.orientation", json!({"range": "A6", "angle": 30})).unwrap(); + s.execute("pageLayout.pageSetup", json!({"header": "&Cتقرير المبيعات 2026", "gridlines": true})).unwrap(); + s.execute("insert.chart", json!({"range": "A1:B3", "type": "column"})).unwrap(); + { + let d = s.doc_mut().unwrap(); + let wb = std::sync::Arc::make_mut(&mut d.wb); + for chart in &mut wb.sheet_mut(0).unwrap().charts { + chart.anchor.cell = CellRef::new(8, 0); + chart.anchor.width = 500.0; + chart.anchor.height = 280.0; + } + } + let (pdf, pages) = pdf_of(&mut s, json!({"fitToPage": true})); + assert_eq!(pages, 1); + assert!(contains(&pdf, "/Subtype /Type3")); + for text in ["مَرْحَبًا 123 (ABC)", "تقرير المبيعات 2026", "القيمة", "تفاح", "كمثرى"] { + let utf16 = text.encode_utf16().map(|unit| format!("{unit:04X}")).collect::(); + assert!(contains(&pdf, &format!("/ActualText ")), "missing text {text}"); + } + let lines = wrap_lines("العَرَبِيَّةالعَرَبِيَّة", Font::Helvetica, 20.0, 40.0, TextDirection::Auto); + assert_eq!(lines.iter().map(|line| line.text.as_str()).collect::(), "العَرَبِيَّةالعَرَبِيَّة"); + assert!(lines.iter().all(|line| !line.text.starts_with(['َ', 'ِ', 'ُ', 'ّ', 'ْ']))); + assert!(lines.iter().all(|line| gridcraft_pdf::text_width_with_direction(Font::Helvetica, 20.0, &line.text, line.direction) <= 40.0)); + let path = std::env::temp_dir().join(format!("gridcraft-arabic-export-{}.pdf", std::process::id())); + std::fs::write(&path, &pdf).unwrap(); + eprintln!("Arabic PDF artifact: {}", path.display()); +} + +#[test] +fn arabic_reading_order_export_artifact() { + if !arabic_face_available() && std::env::var_os("CRAFT_FONTS_REQUIRED").is_none() { + return; + } + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [["Context", "LeftToRight", "RightToLeft"], ["مرحبا ABC !", "مرحبا ABC !", "مرحبا ABC !"], ["123 مرحبا ABC", "123 مرحبا ABC", "123 مرحبا ABC"], ["(مرحبا) ABC", "(مرحبا) ABC", "(مرحبا) ABC"], ["مرحبا ABC ! DEF ? مرحبا", "مرحبا ABC ! DEF ? مرحبا", "مرحبا ABC ! DEF ? مرحبا"]]})).unwrap(); + s.execute("home.columnWidth", json!({"cols": "A:C", "chars": 24})).unwrap(); + s.execute("home.rowHeight", json!({"rows": "1:5", "height": 60})).unwrap(); + s.execute("home.fontSize", json!({"range": "A2:C5", "size": 18})).unwrap(); + for (range, order) in [("A2:A5", "Context"), ("B2:B5", "LeftToRight"), ("C2:C5", "RightToLeft")] { + s.execute("home.formatCells", json!({"range": range, "style": {"align": {"reading_order": order}}})).unwrap(); + } + s.execute("home.wrapText", json!({"range": "A5:C5"})).unwrap(); + s.execute("pageLayout.pageSetup", json!({"gridlines": true, "header": "&CArabic cell reading order"})).unwrap(); + let (pdf, _) = pdf_of(&mut s, json!({})); + for text in ["مرحبا ABC !", "123 مرحبا ABC", "(مرحبا) ABC"] { + let utf16 = text.encode_utf16().map(|unit| format!("{unit:04X}")).collect::(); + assert!(contains(&pdf, &format!("/ActualText "))); + } + let path = std::env::temp_dir().join(format!("gridcraft-arabic-reading-order-{}.pdf", std::process::id())); + std::fs::write(&path, &pdf).unwrap(); + eprintln!("Arabic reading-order PDF artifact: {}", path.display()); +} + +#[test] +fn wrapped_lines_keep_the_paragraph_direction() { + let lines = wrap_lines("مرحبا ABC ! DEF ?\nLatin مرحبا !", Font::Helvetica, 10.0, 30.0, TextDirection::Auto); + let split = lines.iter().position(|l| l.text.starts_with("Latin")).unwrap(); + assert!(split > 0); + assert!(lines[..split].iter().all(|l| l.direction == TextDirection::RightToLeft)); + assert!(lines[split..].iter().all(|l| l.direction == TextDirection::LeftToRight)); + let lines = wrap_lines("123 مرحبا ABC !", Font::Helvetica, 10.0, 30.0, TextDirection::Context.resolve("123 مرحبا ABC !")); + assert!(lines.iter().all(|l| l.direction == TextDirection::LeftToRight)); +} + +fn exported_text_x(pdf: &[u8], text: &str) -> f32 { + let pdf = String::from_utf8_lossy(pdf); + let marker = format!("({text}) Tj"); + let before = &pdf[..pdf.find(&marker).unwrap()]; + let matrix = before.lines().rev().find(|line| line.ends_with(" Tm")).unwrap(); + matrix.split_whitespace().nth(4).unwrap().parse().unwrap() +} + +#[test] +fn cell_reading_order_and_conditional_context_reach_pdf() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "ABC !"})).unwrap(); + s.execute("home.formatCells", json!({"range": "A1", "style": {"align": {"reading_order": "RightToLeft"}}})).unwrap(); + let (rtl, _) = pdf_of(&mut s, json!({})); + assert!(contains(&rtl, "(! ABC) Tj")); + assert!(!contains(&rtl, "/Subtype /Type3")); + s.execute("home.conditionalFormat", json!({"range": "A1", "rule": {"type": "noBlanks", "style": {"align": {"reading_order": "Context"}}}})) + .unwrap(); + let (context, _) = pdf_of(&mut s, json!({})); + assert!(contains(&context, "(ABC !) Tj")); + assert!(!contains(&context, "(! ABC) Tj")); + assert!(exported_text_x(&rtl, "! ABC") > exported_text_x(&context, "ABC !")); +} + +#[test] +fn rtl_sheet_sparkline_reflects_with_cell_geometry() { + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + sh.set_value(CellRef::new(0, 0), Value::Number(1.0)); + sh.set_value(CellRef::new(1, 0), Value::Number(3.0)); + sh.sparklines.push(Sparkline { + cell: CellRef::new(0, 1), + source: "A1:A2".into(), + kind: SparklineKind::Line, + color: Color::rgb(255, 0, 0), + markers: false, + }); + let options = PrintOptions { range: RangeRef::parse("A1:B2"), ..PrintOptions::default() }; + let ltr = render_pdf(&wb, &layouts(&wb, &options), "", "").unwrap().bytes; + wb.sheet_mut(0).unwrap().right_to_left = true; + let rtl = render_pdf(&wb, &layouts(&wb, &options), "", "").unwrap().bytes; + let points = |pdf: &[u8]| { + let pdf = String::from_utf8_lossy(pdf); + let after = pdf.split("1 0 0 RG\n").nth(1).unwrap(); + let line = after.lines().find(|line| line.ends_with(" l S")).unwrap(); + let values: Vec<_> = line.split_whitespace().collect(); + (values[0].parse::().unwrap(), values[3].parse::().unwrap()) + }; + let (left0, left1) = points(<r); + let (right0, right1) = points(&rtl); + assert!(left0 < left1 && right0 > right1); + assert!((left0 + right0 - 612.0).abs() < 0.001); + assert!((left1 + right1 - 612.0).abs() < 0.001); +} + +#[test] +fn explicit_left_and_right_keep_their_side_on_a_right_to_left_sheet() { + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [["LEFT"], ["RIGHT"]]})).unwrap(); + s.execute("home.alignLeft", json!({"range": "A1"})).unwrap(); + s.execute("home.alignRight", json!({"range": "A2"})).unwrap(); + let (ltr, _) = pdf_of(&mut s, json!({})); + s.execute("view.rightToLeft", json!({"on": true})).unwrap(); + let (rtl, _) = pdf_of(&mut s, json!({})); + let width = |label| gridcraft_pdf::text_width(Font::Helvetica, 11.0, label); + let (left, right) = (width("LEFT"), width("RIGHT")); + let ltr_gap = exported_text_x(<r, "RIGHT") + right - exported_text_x(<r, "LEFT"); + let rtl_gap = exported_text_x(&rtl, "RIGHT") + right - exported_text_x(&rtl, "LEFT"); + assert!((ltr_gap - rtl_gap).abs() < 0.01, "Left stays left of Right in the cell: {ltr_gap}, {rtl_gap}"); + assert!(exported_text_x(&rtl, "LEFT") + left < exported_text_x(&rtl, "RIGHT") + right); +} + +#[test] +fn rtl_sheet_pdf_mirrors_cells_merges_headings_and_object_anchors() { + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [["FIRST", "SECOND"], [null, "MERGED"]]})).unwrap(); + s.execute("home.mergeCenter", json!({"range": "B2:C2"})).unwrap(); + s.execute("pageLayout.pageSetup", json!({"printArea": "A1:D5", "headings": true, "gridlines": true, "left": 0.8, "right": 0.4})).unwrap(); + s.execute("insert.shape", json!({"kind": "rectangle", "at": "B4", "width": 128, "height": 30, "text": "OBJECT"})).unwrap(); + let (ltr, _) = pdf_of(&mut s, json!({})); + s.execute("view.rightToLeft", json!({"on": true})).unwrap(); + let (rtl, _) = pdf_of(&mut s, json!({})); + for label in ["FIRST", "SECOND", "MERGED", "A", "B", "C", "D", "1", "2", "OBJECT"] { + let left = exported_text_x(<r, label); + let right = exported_text_x(&rtl, label); + let font_size = if ["A", "B", "C", "D", "1", "2"].contains(&label) { 9.0 } else { 11.0 }; + let width = gridcraft_pdf::text_width(Font::Helvetica, font_size, label); + assert!((left + right + width - 612.0).abs() < 0.01, "{label}: {left}, {right}, {width}"); + } + assert!(exported_text_x(<r, "FIRST") < exported_text_x(<r, "SECOND")); + assert!(exported_text_x(&rtl, "FIRST") > exported_text_x(&rtl, "SECOND")); + for (name, pdf) in [("ltr", ltr), ("rtl", rtl)] { + let path = std::env::temp_dir().join(format!("gridcraft-print-direction-{name}-{}.pdf", std::process::id())); + std::fs::write(&path, pdf).unwrap(); + eprintln!("Print direction PDF artifact: {}", path.display()); + } +} + #[test] fn header_sections_split_join_and_field_names() { let spec = "&L&\"Arial,Bold\"Sales &&Co&CPage &P of &N&R&D"; @@ -694,11 +917,32 @@ fn bands_unit() { let b = bands(0, 2, |r| if r == 1 { 100.0 } else { 10.0 }, |_| 35.0, &[], 100); assert_eq!(b, vec![(0, 0), (1, 1), (2, 2)]); assert_eq!(bands(0, 1_000_000, |_| 10.0, |_| 5.0, &[], 3).len(), 3); - let w = wrap_lines("the quick brown fox jumps", Font::Helvetica, 10.0, 50.0); + let w = wrap_lines("the quick brown fox jumps", Font::Helvetica, 10.0, 50.0, TextDirection::Auto); assert!(w.len() >= 2); - assert!(w.iter().all(|l| gridcraft_pdf::text_width(Font::Helvetica, 10.0, l) <= 50.0 || !l.contains(' '))); - let w = wrap_lines("Supercalifragilistic", Font::Helvetica, 10.0, 20.0); + assert!(w.iter().all(|l| gridcraft_pdf::text_width_with_direction(Font::Helvetica, 10.0, &l.text, l.direction) <= 50.0 || !l.text.contains(' '))); + let w = wrap_lines("Supercalifragilistic", Font::Helvetica, 10.0, 20.0, TextDirection::Auto); assert!(w.len() > 2); + let long = "abcdefghij".repeat(300); + let w = wrap_lines(&long, Font::Helvetica, 10.0, 50.0, TextDirection::Auto); + assert_eq!(w.iter().map(|l| l.text.as_str()).collect::(), long); + assert!(w.iter().all(|l| gridcraft_pdf::text_width_with_direction(Font::Helvetica, 10.0, &l.text, l.direction) <= 50.0)); + let mut at = 0; + for l in &w[..w.len() - 1] { + let one_more = &long[at..at + l.text.len() + 1]; + assert!(gridcraft_pdf::text_width_with_direction(Font::Helvetica, 10.0, one_more, l.direction) > 50.0, "each split is as long as fits"); + at += l.text.len(); + } +} + +fn wrapped_text(text: &str, font: Font, size: f32, width: f32) -> Vec { + wrap_lines(text, font, size, width, TextDirection::Auto).into_iter().map(|line| line.text).collect() +} + +#[test] +fn wrapping_ignores_the_width_of_undrawn_bidi_marks() { + let word = "i\u{200e}".repeat(20); + assert!(gridcraft_pdf::text_width(Font::Helvetica, 10.0, &word) <= 50.0); + assert_eq!(wrapped_text(&word, Font::Helvetica, 10.0, 50.0), vec![word.clone()]); } /// The wrapper `wrap_lines` replaced, re-measuring strings, as the reference for its lines. @@ -755,7 +999,11 @@ fn wrap_lines_matches_the_remeasuring_wrapper() { for text in texts { for font in [Font::Helvetica, Font::HelveticaBold, Font::Courier] { for (size, width) in [(10.0, 50.0), (10.0, 20.0), (11.0, 1.0), (8.0, 300.0), (12.0, 0.0)] { - assert_eq!(wrap_lines(text, font, size, width), wrap_lines_remeasuring(text, font, size, width), "{text:?} {font:?} {size} {width}"); + assert_eq!( + wrapped_text(text, font, size, width), + wrap_lines_remeasuring(text, font, size, width), + "{text:?} {font:?} {size} {width}" + ); } } } @@ -766,12 +1014,24 @@ fn wrapping_a_long_word_in_a_narrow_box_stops_at_the_line_limit() { // One character per line: the old wrapper re-measured and copied the rest of the word for // each of the 100,000 lines before checking the limit. let word = "w".repeat(100_000); - let lines = wrap_lines(&word, Font::Helvetica, 10.0, 1.0); + let lines = wrapped_text(&word, Font::Helvetica, 10.0, 1.0); assert!(lines.len() <= 2002, "{}", lines.len()); assert!(lines.iter().all(|l| l == "w")); // A long paragraph in a very wide box is one line. let para = vec!["word"; 50_000].join(" "); - assert_eq!(wrap_lines(¶, Font::Helvetica, 10.0, 1.0e9), vec![para.clone()]); + assert_eq!(wrapped_text(¶, Font::Helvetica, 10.0, 1.0e9), vec![para.clone()]); +} + +#[test] +fn wrapping_a_long_arabic_word_measures_pieces_not_the_rest_of_the_word() { + let word = "مرحبا".repeat(10_000); + if !arabic_face_available() { + return; + } + let lines = wrapped_text(&word, Font::Helvetica, 10.0, 20.0); + assert!(lines.len() > 2000 && lines.len() <= 2002, "{}", lines.len()); + assert!(lines.iter().take(2000).all(|line| gridcraft_pdf::text_width(Font::Helvetica, 10.0, line) <= 20.0)); + assert!(word.starts_with(&lines.concat())); } #[test] @@ -938,3 +1198,27 @@ fn equations_print_with_the_symbol_font() { // A glyph no standard font has is a [?] box. assert!(contains(&pdf, "(?) Tj")); } + +#[test] +fn chart_budget_counts_path_points_and_drops_charts_that_do_not_fit() { + use gridcraft_chart::Prim; + let line = |n: usize| Prim::Line { pts: vec![[0.0, 0.0]; n], color: [0, 0, 0, 255], width: 1.0, dash: false }; + // A smoothed line is its points, not one unit. + assert_eq!(chart_cost(&[line(4000), Prim::Circle { cx: 0.0, cy: 0.0, r: 1.0, fill: None, stroke: None }]), 4002); + // Room for exactly two copies of a 1,000-point chart. + let mut pics = PicCache { chart_prims: MAX_CHART_PRIMS - 2002, ..Default::default() }; + let mut renders = 0; + for expect in [true, true, false, false] { + let got = pics.chart((0, 0), || { + renders += 1; + vec![line(1000)] + }); + assert_eq!(got.is_some(), expect); + } + assert_eq!(renders, 1, "a rejected chart is not rendered again"); + assert!(pics.charts.is_empty(), "and its primitives are not kept"); + // A chart too big for what is left is dropped without being cached at all. + let mut pics = PicCache { chart_prims: MAX_CHART_PRIMS - 10, ..Default::default() }; + assert!(pics.chart((0, 1), || vec![line(1000)]).is_none()); + assert!(pics.charts.is_empty() && pics.rejected_charts.contains(&(0, 1))); +} diff --git a/crates/engine/src/cmd/print/text.rs b/crates/engine/src/cmd/print/text.rs index 0523aa4ff..3c4c8374d 100644 --- a/crates/engine/src/cmd/print/text.rs +++ b/crates/engine/src/cmd/print/text.rs @@ -1,78 +1,176 @@ use super::draw::{Block, Look, rgb_of}; -use super::{CellRef, Color, Font, HAlign, MAX_COLS, Page, Point, RangeRef, Sheet, Style, StyleTable, Underline, VAlign, Value}; +use super::{ + Baseline, CellRef, Color, Font, HAlign, MAX_COLS, Page, Point, RangeRef, ReadingOrder, Sheet, Style, StyleTable, TextDirection, Underline, + VAlign, Value, +}; +use unicode_segmentation::UnicodeSegmentation; const PAD_PX: f32 = 2.0; /// Most lines [`wrap_lines`] returns (it may return one more, the rest of a paragraph). const MAX_WRAP_LINES: usize = 2000; -/// Word-wraps `text` to `width` points. +pub(super) struct TextLine { + pub(super) text: String, + pub(super) direction: TextDirection, +} + +/// Word-wraps `text` to `width` points, keeping each paragraph's base direction on every +/// continuation line. /// /// Widths are kept as running sums of glyph widths (the sum `text_width` makes), so each /// character is measured once: re-measuring the line for every word, or the rest of an /// over-long word for every piece cut from it, made a long paragraph or word quadratic. -pub(super) fn wrap_lines(text: &str, font: Font, size: f32, width: f32) -> Vec { +/// A paragraph whose width is not that sum is measured as drawn by [`wrap_measured_paragraph`]. +pub(super) fn wrap_lines(text: &str, font: Font, size: f32, width: f32, direction: TextDirection) -> Vec { + let mut out = Vec::new(); + for para in text.split('\n') { + let direction = direction.resolve(para); + if gridcraft_pdf::width_is_char_sum(font, para, direction) { + wrap_summed_paragraph(para, font, size, width, direction, &mut out); + } else { + let fits = |s: &str| gridcraft_pdf::text_width_with_direction(font, size, s, direction) <= width; + wrap_measured_paragraph(para, &fits, direction, &mut out); + } + if out.len() > MAX_WRAP_LINES { + return out; + } + } + out +} + +fn wrap_summed_paragraph(para: &str, font: Font, size: f32, width: f32, direction: TextDirection, out: &mut Vec) { let units_of = |s: &str| -> u64 { s.chars().map(|c| u64::from(font.char_width(c))).sum() }; let fits = |units: u64| units as f32 * size / 1000.0 <= width; let space = units_of(" "); - let mut out = Vec::new(); - for para in text.split('\n') { - let mut line = String::new(); - let mut line_units: u64 = 0; - for word in para.split(' ') { - let word_units = units_of(word); - if line.is_empty() { - line.push_str(word); - line_units = word_units; - } else if fits(line_units + space + word_units) { - line.push(' '); - line.push_str(word); - line_units += space + word_units; - } else { - out.push(std::mem::take(&mut line)); - line.push_str(word); - line_units = word_units; - } - // Break a single over-long word by characters: each piece is the longest that fits - // (at least one character). - if !fits(line_units) && line.chars().nth(1).is_some() { - let mut piece = String::new(); - let mut units: u64 = 0; - for c in line.chars() { - let w = u64::from(font.char_width(c)); - if !piece.is_empty() && !fits(units + w) { - out.push(std::mem::take(&mut piece)); - units = 0; - if out.len() > MAX_WRAP_LINES { - return out; - } + let mut line = String::new(); + let mut line_units: u64 = 0; + for word in para.split(' ') { + let word_units = units_of(word); + if line.is_empty() { + line.push_str(word); + line_units = word_units; + } else if fits(line_units + space + word_units) { + line.push(' '); + line.push_str(word); + line_units += space + word_units; + } else { + out.push(TextLine { text: std::mem::take(&mut line), direction }); + line.push_str(word); + line_units = word_units; + } + if !fits(line_units) && line.graphemes(true).nth(1).is_some() { + let mut piece = String::new(); + let mut units: u64 = 0; + for g in line.graphemes(true) { + let w = units_of(g); + if !piece.is_empty() && !fits(units + w) { + out.push(TextLine { text: std::mem::take(&mut piece), direction }); + units = 0; + if out.len() > MAX_WRAP_LINES { + return; } - piece.push(c); - units += w; } - line = piece; - line_units = units; + piece.push_str(g); + units += w; } - if out.len() > MAX_WRAP_LINES { - return out; + line = piece; + line_units = units; + } + if out.len() > MAX_WRAP_LINES { + return; + } + } + out.push(TextLine { text: line, direction }); +} + +/// One paragraph of [`wrap_lines`] whose width is not a sum of character widths (shaped Arabic, +/// right-to-left reordering or bidi controls), measuring each candidate line as drawn. +fn wrap_measured_paragraph(para: &str, fits: &impl Fn(&str) -> bool, direction: TextDirection, out: &mut Vec) { + let mut line = String::new(); + for word in para.split(' ') { + if !line.is_empty() { + let candidate = format!("{line} {word}"); + if fits(&candidate) { + line = candidate; + continue; + } + out.push(TextLine { text: std::mem::take(&mut line), direction }); + } + line = if fits(word) { word.to_string() } else { break_measured_word(word, fits, direction, out) }; + if out.len() > MAX_WRAP_LINES { + return; + } + } + out.push(TextLine { text: line, direction }); +} + +/// Cuts `word` into the longest grapheme prefixes that fit (at least one grapheme each), pushes +/// all but the last to `out` and returns the last. Each cut gallops from one grapheme, so it +/// measures prefixes about as long as the piece rather than the rest of the word. +fn break_measured_word(word: &str, fits: &impl Fn(&str) -> bool, direction: TextDirection, out: &mut Vec) -> String { + let ends: Vec = word.grapheme_indices(true).map(|(i, g)| i + g.len()).collect(); + let (mut start, mut first) = (0, 0); + loop { + let len = ends.len().saturating_sub(first); + let end = |count: usize| ends.get(first + count.saturating_sub(1)).copied().unwrap_or(word.len()); + let fits_prefix = |count: usize| fits(word.get(start..end(count)).unwrap_or_default()); + let (mut fit, mut over, mut probe) = (1, len + 1, 2); + while fit < len { + let count = probe.min(len); + if fits_prefix(count) { + fit = count; + probe = count.saturating_mul(2); + } else { + over = count; + break; } } - out.push(line); + if fit >= len { + return word.get(start..).unwrap_or_default().to_string(); + } + while over - fit > 1 { + let mid = fit + (over - fit) / 2; + if fits_prefix(mid) { + fit = mid; + } else { + over = mid; + } + } + let cut = end(fit); + out.push(TextLine { text: word.get(start..cut).unwrap_or_default().to_string(), direction }); + start = cut; + first += fit; + if out.len() > MAX_WRAP_LINES { + return word.get(start..).unwrap_or_default().to_string(); + } } - out } fn font_for(style: &Style) -> Font { Font::styled(Font::family_for(&style.font.name), style.font.bold, style.font.italic) } -fn text_align(align: HAlign, value: &Value) -> HAlign { +fn text_direction(order: ReadingOrder) -> TextDirection { + match order { + ReadingOrder::Context => TextDirection::Context, + ReadingOrder::LeftToRight => TextDirection::LeftToRight, + ReadingOrder::RightToLeft => TextDirection::RightToLeft, + } +} + +/// The alignment a cell is laid out with before a right-to-left sheet is mirrored. Explicit Left +/// and Right are swapped there, as in the grid, so they print on the side the user chose. +fn text_align(align: HAlign, value: &Value, direction: TextDirection, right_to_left_sheet: bool) -> HAlign { match align { HAlign::General => match value { Value::Number(_) => HAlign::Right, Value::Bool(_) | Value::Error(_) => HAlign::Center, + _ if direction == TextDirection::RightToLeft && !right_to_left_sheet => HAlign::Right, _ => HAlign::Left, }, + HAlign::Left if right_to_left_sheet => HAlign::Right, + HAlign::Right if right_to_left_sheet => HAlign::Left, HAlign::CenterAcross | HAlign::Distributed => HAlign::Center, HAlign::Justify | HAlign::Fill => HAlign::Left, other => other, @@ -183,32 +281,46 @@ pub(super) fn draw_text(pg: &mut Page, b: &Block, placement: TextPlacement<'_>) if text.is_empty() { return; } - let mut tw = gridcraft_pdf::text_width(font, size, &text); + let direction = text_direction(st.align.reading_order.unwrap_or_default()).resolve(&text); + let mut tw = gridcraft_pdf::text_width_with_direction(font, size, &text, direction); if st.align.shrink && !wrap && tw > avail && tw > 0.0 { size = (size * avail / tw).max(1.0); - tw = gridcraft_pdf::text_width(font, size, &text); + tw = gridcraft_pdf::text_width_with_direction(font, size, &text, direction); } if is_num && !wrap && tw > avail + 0.01 && !sh.show_formulas { let hash = gridcraft_pdf::text_width(font, size, "#").max(0.1); text = "#".repeat(((avail / hash).floor().max(1.0) as usize).min(255)); - tw = gridcraft_pdf::text_width(font, size, &text); + tw = gridcraft_pdf::text_width_with_direction(font, size, &text, direction); } - let h_align = text_align(st.align.h, &v); + let h_align = text_align(st.align.h, &v, direction, sh.right_to_left); let rot = st.align.rotation; let rotated = rot != 0 && rot != 255; // Rotated text wraps along its own direction: upright (±90°) text to the row height. - let lines: Vec = match (wrap, rotated) { - (true, false) => wrap_lines(&text, font, size, avail.max(size)), - (true, true) if rot.abs() == 90 => wrap_lines(&text, font, size, (h - 2.0 * pad).max(size)), - (true, true) => vec![text.lines().next().unwrap_or_default().to_string()], - (false, _) => vec![text], + let lines: Vec = match (wrap, rotated) { + (true, false) => wrap_lines(&text, font, size, avail.max(size), direction), + (true, true) if rot.abs() == 90 => wrap_lines(&text, font, size, (h - 2.0 * pad).max(size), direction), + (true, true) => vec![TextLine { text: text.lines().next().unwrap_or_default().to_string(), direction }], + (false, _) => vec![TextLine { text, direction }], }; - let line_h = size * 1.2; + let extents = lines + .iter() + .filter_map(|line| pg.text_vertical_extents_with_direction(font, size, &line.text, line.direction)) + .reduce(|a, b| (a.0.max(b.0), a.1.max(b.1))) + .map(|(ascent, descent)| (ascent.max(size * 0.8), descent.max(size * 0.22))); + let line_h = extents.map_or(size * 1.2, |(ascent, descent)| (ascent + descent + size * 0.2).max(size * 1.2)); let n = lines.len().max(1) as f32; - let first_baseline = match st.align.v { - VAlign::Top => y + pad + size * 0.8, - VAlign::Center | VAlign::Justify | VAlign::Distributed => y + (h - n * line_h) / 2.0 + line_h * 0.5 + size * 0.33, - VAlign::Bottom => y + h - pad - size * 0.22 - (n - 1.0) * line_h, + let first_baseline = if let Some((ascent, descent)) = extents { + match st.align.v { + VAlign::Top => y + pad + ascent, + VAlign::Center | VAlign::Justify | VAlign::Distributed => y + (h - (n - 1.0) * line_h - ascent - descent) / 2.0 + ascent, + VAlign::Bottom => y + h - pad - descent - (n - 1.0) * line_h, + } + } else { + match st.align.v { + VAlign::Top => y + pad + size * 0.8, + VAlign::Center | VAlign::Justify | VAlign::Distributed => y + (h - n * line_h) / 2.0 + line_h * 0.5 + size * 0.33, + VAlign::Bottom => y + h - pad - size * 0.22 - (n - 1.0) * line_h, + } }; // Overflow into empty neighbours (single-line text, not merged). // Center Across Selection spans the empty Center Across cells to its right; the text stays within that run. @@ -222,7 +334,7 @@ pub(super) fn draw_text(pg: &mut Page, b: &Block, placement: TextPlacement<'_>) let clip_l = clip_l.max(limits.0); let clip_r = clip_r.min(limits.1); pg.save(); - pg.clip_rect(clip_l, y, (clip_r - clip_l).max(0.0), h); + pg.clip_rect(b.text_x(clip_l, clip_r - clip_l), y, (clip_r - clip_l).max(0.0), h); if rotated { // Rotated text: anchored at the bottom-left (upward) or top-left (downward) of the cell. // Wrapped lines stack rightward for upward text and leftward for downward text, so @@ -233,23 +345,27 @@ pub(super) fn draw_text(pg: &mut Page, b: &Block, placement: TextPlacement<'_>) for (i, t) in lines.iter().enumerate() { let i = i as f32; let (ox, oy) = if deg > 0.0 { - (x + pad + size * 0.8 * deg.to_radians().sin() + i * line_h, y + h - pad) + (x + pad + extents.map_or(size * 0.8, |e| e.0) * deg.to_radians().sin() + i * line_h, y + h - pad) } else { - (x + pad + (last - i) * line_h, y + pad + size * 0.8) + (x + pad + (last - i) * line_h, y + pad + extents.map_or(size * 0.8, |e| e.0)) }; - pg.text_rotated(Point::new(ox, oy), size, font, color, t, angle); + let tw = gridcraft_pdf::text_width_with_direction(font, size, &t.text, t.direction); + let (sn, cs) = angle.sin_cos(); + let (ascent, descent) = extents.unwrap_or((size * 0.8, size * 0.22)); + let ox = if sh.right_to_left { b.mirror_axis - ox - tw * cs - (ascent - descent) * sn } else { ox }; + pg.text_rotated_with_direction(Baseline { origin: Point::new(ox, oy), angle }, size, font, color, &t.text, t.direction); } pg.restore(); return; } if rot == 255 { // Vertical stacked text. - let chars: Vec = lines.concat().chars().take(500).collect(); - let mut yy = y + pad + size * 0.8; + let stacked: String = lines.iter().map(|line| line.text.as_str()).collect(); + let chars: Vec<&str> = stacked.graphemes(true).take(500).collect(); + let mut yy = y + pad + extents.map_or(size * 0.8, |e| e.0); for ch in chars { - let s = ch.to_string(); - let cw = gridcraft_pdf::text_width(font, size, &s); - pg.text(x + (w - cw) / 2.0, yy, size, font, color, &s); + let cw = gridcraft_pdf::text_width(font, size, ch); + pg.text_with_direction(Point::new(b.text_x(x + (w - cw) / 2.0, cw), yy), size, font, color, ch, direction); yy += line_h; if yy > y + h + line_h { break; @@ -259,7 +375,7 @@ pub(super) fn draw_text(pg: &mut Page, b: &Block, placement: TextPlacement<'_>) return; } for (i, line) in lines.iter().enumerate() { - let lw = gridcraft_pdf::text_width(font, size, line); + let lw = gridcraft_pdf::text_width_with_direction(font, size, &line.text, line.direction); let lx = match h_align { HAlign::Right => x + w - pad - lw - indent, HAlign::Center => x + (w + across - lw) / 2.0, @@ -269,7 +385,8 @@ pub(super) fn draw_text(pg: &mut Page, b: &Block, placement: TextPlacement<'_>) if ly - size > y + h || ly < y - line_h { continue; } - pg.text(lx, ly, size, font, color, line); + let lx = b.text_x(lx, lw); + pg.text_with_direction(Point::new(lx, ly), size, font, color, &line.text, line.direction); if st.font.underline != Underline::None { let uw = (size * 0.06).max(0.2); pg.stroke_line(lx, ly + size * 0.12, lx + lw, ly + size * 0.12, uw, color); diff --git a/crates/engine/src/cmd/review.rs b/crates/engine/src/cmd/review.rs index de9c98ecd..7eafa26db 100644 --- a/crates/engine/src/cmd/review.rs +++ b/crates/engine/src/cmd/review.rs @@ -163,6 +163,8 @@ fn protect_sheet(s: &mut Session, p: &Json) -> Result { let what = d.wb.sheet(sheet).map(|sh| sh.name.clone()).unwrap_or_default(); let prot = SheetProtection { password: str_param(p, "password").filter(|x| !x.is_empty()).map(|pw| new_password_hash(d, &what, pw)), ..prot }; edit(s, |cx| { + // FORMULATEXT reads a sheet's protection: its results are recalculated. + cx.recalc_all = true; cx.sheet_mut(sheet)?.protection = Some(prot.clone()); Ok(Json::Null) }) @@ -182,7 +184,10 @@ fn unprotect_sheet(s: &mut Session, p: &Json) -> Result { } } edit(s, |cx| { - cx.sheet_mut(sheet)?.protection = None; + // Only removing protection changes what FORMULATEXT reads; a no-op unprotect skips the full recalculation. + if cx.sheet_mut(sheet)?.protection.take().is_some() { + cx.recalc_all = true; + } Ok(Json::Null) }) } @@ -404,6 +409,21 @@ mod tests { assert_eq!(at(&s), (0, "A1".to_string())); } + #[test] + fn protecting_and_unprotecting_leave_table_headers_alone() { + let mut s = Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [["a", "b"], [1, 2]]})).unwrap(); + s.execute("insert.table", json!({"range": "A1:B2", "header": true})).unwrap(); + // An imported header can hold a number: only a structural edit would turn it back into text. + let d = s.doc_mut().unwrap(); + std::sync::Arc::make_mut(&mut d.wb).sheet_mut(0).unwrap().set_value(gridcraft_core::CellRef::new(0, 0), gridcraft_core::Value::Number(7.0)); + s.execute("review.protectSheet", json!({})).unwrap(); + s.execute("review.unprotectSheet", json!({})).unwrap(); + let v = s.doc().unwrap().wb.sheet(0).unwrap().value(gridcraft_core::CellRef::new(0, 0)); + assert_eq!(v, gridcraft_core::Value::Number(7.0)); + } + #[test] fn protection_that_forbids_selecting_locked_cells_is_enforced_and_kept() { let mut s = book(); diff --git a/crates/engine/src/cmd/solver/reports.rs b/crates/engine/src/cmd/solver/reports.rs index 19500cab0..c2a5c6d84 100644 --- a/crates/engine/src/cmd/solver/reports.rs +++ b/crates/engine/src/cmd/solver/reports.rs @@ -52,14 +52,19 @@ pub(super) fn abs_ref(c: CellRef) -> String { format!("${}${}", gridcraft_core::col_to_letters(c.col), u64::from(c.row) + 1) } +/// How many cells to the left and above a cell its name is searched for. +const NAME_REACH: u32 = 1_000; + /// A cell's name in the reports: the nearest text to its left and the nearest text above it. pub(super) fn cell_name(sh: &Sheet, c: CellRef) -> String { let text = |c: CellRef| match sh.value(c) { Value::Text(t) if !t.trim().is_empty() => Some(t.to_string()), _ => None, }; - let left = (0..c.col).rev().find_map(|col| text(CellRef::new(c.row, col))); - let top = (0..c.row).rev().find_map(|row| text(CellRef::new(row, c.col))); + // Labels are next to their cells: looking further than this on a sparse sheet only costs time + // (a thousand cells near the last row would each probe a million empty coordinates). + let left = (c.col.saturating_sub(NAME_REACH)..c.col).rev().find_map(|col| text(CellRef::new(c.row, col))); + let top = (c.row.saturating_sub(NAME_REACH)..c.row).rev().find_map(|row| text(CellRef::new(row, c.col))); [left, top].into_iter().flatten().collect::>().join(" ") } @@ -438,3 +443,17 @@ pub(super) fn report(s: &mut Session, p: &Json) -> Result { } Ok(out) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_name_is_searched_only_a_bounded_distance() { + let mut sh = Sheet::new("S".to_string()); + sh.set_cell(CellRef::new(0, 0), gridcraft_model::Cell::value(Value::text("Far"))); + sh.set_cell(CellRef::new(5, 4), gridcraft_model::Cell::value(Value::text("Near"))); + assert_eq!(cell_name(&sh, CellRef::new(5, 5)), "Near"); + assert_eq!(cell_name(&sh, CellRef::new(0, NAME_REACH + 1)), ""); + } +} diff --git a/crates/engine/src/cmd/whatif/advanced_filter.rs b/crates/engine/src/cmd/whatif/advanced_filter.rs index 52630a3ef..21d518a7b 100644 --- a/crates/engine/src/cmd/whatif/advanced_filter.rs +++ b/crates/engine/src/cmd/whatif/advanced_filter.rs @@ -83,6 +83,9 @@ pub(crate) fn criterion_matches(v: &Value, text: &str, crit: &Value) -> bool { } } +/// Most (list row, criteria row) pairs one Advanced Filter tests. +const MAX_FILTER_TESTS: u64 = 50_000_000; + pub(super) fn advanced_filter(s: &mut Session, p: &Json) -> Result { const CMD: &str = "data.advancedFilter"; let d = s.doc()?; @@ -133,6 +136,10 @@ pub(super) fn advanced_filter(s: &mut Session, p: &Json) -> Result { rows.push(conds); } } + // Each list row is tried against every criteria row, so bound their product too. + if u64::from(list.height()) * rows.len() as u64 > MAX_FILTER_TESTS { + return Err(bad(CMD, "the list and criteria ranges together are too large")); + } let matches_row = |row: u32| -> bool { if rows.is_empty() { return true; diff --git a/crates/engine/src/cmd/whatif/consolidate.rs b/crates/engine/src/cmd/whatif/consolidate.rs index 149b72bc7..702baa312 100644 --- a/crates/engine/src/cmd/whatif/consolidate.rs +++ b/crates/engine/src/cmd/whatif/consolidate.rs @@ -61,6 +61,24 @@ enum Key { const MAX_CONSOLIDATE_CELLS: u64 = 5_000_000; +/// The cells a consolidated table writes: every row key meets every column key, plus a label for +/// each key on the sides that have labels. +fn output_cells(rows: usize, cols: usize, top: bool, left: bool) -> u64 { + let (r, c) = (rows as u64, cols as u64); + r.saturating_mul(c).saturating_add(if top { c } else { 0 }).saturating_add(if left { r } else { 0 }) +} + +#[cfg(test)] +#[test] +fn the_output_limit_counts_label_cells() { + // A million row keys and four column keys are 4 million cells, 5 million with the row labels. + assert_eq!(output_cells(1_000_000, 4, false, false), 4_000_000); + assert_eq!(output_cells(1_000_000, 4, false, true), 5_000_000); + assert!(output_cells(1_000_000, 5, false, true) > MAX_CONSOLIDATE_CELLS); + assert_eq!(output_cells(3, 2, true, true), 6 + 2 + 3); + assert_eq!(output_cells(usize::MAX, usize::MAX, true, true), u64::MAX); +} + pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { const CMD: &str = "data.consolidate"; let func = str_param(p, "function").unwrap_or("sum").to_string(); @@ -89,20 +107,20 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { } None => (d.wb.active_sheet(), d.selection.active), }; + // Keys in first-seen order, each with its index: a lookup is one probe, not a scan. let mut row_keys: Vec = Vec::new(); let mut col_keys: Vec = Vec::new(); + let mut row_index: HashMap = HashMap::new(); + let mut col_index: HashMap = HashMap::new(); let mut row_label: HashMap = HashMap::new(); let mut col_label: HashMap = HashMap::new(); let mut aggs: HashMap<(usize, usize), Agg> = HashMap::new(); let mut total: u64 = 0; - let index_of = |keys: &mut Vec, k: Key| -> usize { - match keys.iter().position(|x| *x == k) { - Some(i) => i, - None => { - keys.push(k); - keys.len() - 1 - } - } + let index_of = |keys: &mut Vec, index: &mut HashMap, k: Key| -> usize { + *index.entry(k).or_insert_with_key(|k| { + keys.push(k.clone()); + keys.len() - 1 + }) }; for t in &srcs { let (si, r) = parse_range(&d.wb, t, CMD)?; @@ -131,7 +149,7 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { } else { Key::Pos(col - c0) }; - cols.push((col, index_of(&mut col_keys, k))); + cols.push((col, index_of(&mut col_keys, &mut col_index, k))); } for row in r0..=r.end.row { let k = if left { @@ -145,7 +163,7 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { } else { Key::Pos(row - r0) }; - let ri = index_of(&mut row_keys, k); + let ri = index_of(&mut row_keys, &mut row_index, k); for (col, ci) in &cols { aggs.entry((ri, *ci)).or_default().add(&sh.value(CellRef::new(row, *col))); } @@ -157,6 +175,9 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { { return Err(EngineError::Other("The consolidated data doesn't fit below and right of the destination.".into())); } + if output_cells(row_keys.len(), col_keys.len(), top, left) > MAX_CONSOLIDATE_CELLS { + return Err(bad(CMD, "the consolidated table would be too large")); + } let mut cells: Vec<(CellRef, Value)> = Vec::new(); if top { for (ci, k) in col_keys.iter().enumerate() { diff --git a/crates/engine/src/cmd/whatif/scenarios.rs b/crates/engine/src/cmd/whatif/scenarios.rs index 75bf147a0..9ebb1efb5 100644 --- a/crates/engine/src/cmd/whatif/scenarios.rs +++ b/crates/engine/src/cmd/whatif/scenarios.rs @@ -189,6 +189,9 @@ pub(super) fn scenario_delete(s: &mut Session, p: &Json) -> Result { Ok(json!({"deleted": removed.name})) } +/// Most changing-cell-by-scenario cells one Scenario Summary writes. +const MAX_SUMMARY_CELLS: usize = 1_000_000; + pub(super) fn scenario_summary(s: &mut Session, p: &Json) -> Result { const CMD: &str = "data.scenarioSummary"; let d = s.doc()?; @@ -211,17 +214,24 @@ pub(super) fn scenario_summary(s: &mut Session, p: &Json) -> Result { // Changing cells of every scenario, first-seen order; per scenario its values. let mut changing: Vec = Vec::new(); let mut per: Vec> = Vec::new(); + let mut seen: std::collections::HashSet = Default::default(); for sc in &scenarios { let (_, cells) = cell_list(&d.wb, &sc.changing, CMD, MAX_SCENARIO_CELLS)?; let mut m = HashMap::new(); for (c, v) in cells.iter().zip(sc.values.iter()) { - if !changing.contains(c) { + if seen.insert(*c) { changing.push(*c); } m.insert(*c, json_value(v, &d.wb)); } per.push(m); } + // The summary has a cell for every changing cell in every scenario, filled from the current + // value where a scenario doesn't set it, so small disjoint scenarios multiply. + // Each result cell is a row of the summary too, with a cell per scenario. + if changing.len().saturating_add(results.len()).saturating_mul(per.len()) > MAX_SUMMARY_CELLS { + return Err(bad(CMD, "the scenarios together change too many cells for a summary")); + } let current: Vec = changing.iter().map(|c| sh.value(*c)).collect(); let current_results: Vec = results.iter().map(|c| sh.value(*c)).collect(); let mut trial = Trial::new(d); diff --git a/crates/engine/src/cmd/whatif/tests.rs b/crates/engine/src/cmd/whatif/tests.rs index c1476e401..a0a3eda32 100644 --- a/crates/engine/src/cmd/whatif/tests.rs +++ b/crates/engine/src/cmd/whatif/tests.rs @@ -297,3 +297,47 @@ fn consolidate_by_position_and_label() { assert!(s.execute("data.consolidate", json!({"sources": []})).is_err()); assert!(s.execute("data.consolidate", json!({"sources": ["A1"], "function": "median"})).is_err()); } + +#[test] +fn advanced_filter_refuses_a_list_times_criteria_product_over_the_work_limit() { + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [["Name"], ["A"]]})).unwrap(); + s.execute("range.setValues", json!({"range": "H1", "values": [["Name"], ["A"]]})).unwrap(); + // A sparse million-row list against 9,999 blank criteria rows would be ~10 billion tests. + assert!(s.execute("data.advancedFilter", json!({"range": "A2:A1048576", "criteria": "H1:H10000"})).is_err()); +} + +/// A hundred thousand positional rows take one key lookup each; a tall source and a wide one +/// whose keys would meet in a table of 160 million cells are refused before it is built. +#[test] +fn consolidate_keys_are_indexed_and_the_output_is_bounded() { + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1], [2]]})).unwrap(); + let r = s.execute("data.consolidate", json!({"sources": ["A1:A100000"], "destination": "C1"})).unwrap(); + assert_eq!(r["rows"], 100_000); + assert_eq!(n(&s, "Sheet1!C2"), 2.0); + let r = s.execute("data.consolidate", json!({"sources": ["A1:A10000", "A1:XFC1"], "destination": "A1"})); + assert!(r.unwrap_err().to_string().contains("too large")); +} + +#[test] +fn scenario_summary_refuses_a_matrix_over_its_cell_limit() { + let mut s = s(); + // 200 scenarios of 32 disjoint cells: 6,400 changing cells x 200 scenarios = 1.28 million cells. + for i in 0..200 { + let (a, b) = (i * 32 + 1, i * 32 + 32); + s.execute("data.scenarioAdd", json!({"name": format!("S{i}"), "changing": format!("A{a}:A{b}")})).unwrap(); + } + assert!(s.execute("data.scenarioSummary", json!({})).is_err()); +} + +#[test] +fn scenario_summary_counts_result_rows_in_its_cell_limit() { + let mut s = s(); + // 900 scenarios of one distinct cell each are 810,000 cells; 255 result rows take it past the limit. + for i in 0..900 { + s.execute("data.scenarioAdd", json!({"name": format!("S{i}"), "changing": format!("A{}", i + 1)})).unwrap(); + } + let err = s.execute("data.scenarioSummary", json!({"resultCells": "C1:C255"})).unwrap_err().to_string(); + assert!(err.contains("too many cells"), "{err}"); +} diff --git a/crates/engine/src/cmd/wordart.rs b/crates/engine/src/cmd/wordart.rs new file mode 100644 index 000000000..3577591a6 --- /dev/null +++ b/crates/engine/src/cmd/wordart.rs @@ -0,0 +1,241 @@ +//! Insert › Text › WordArt: decorative text as a floating shape (`ShapeKind::WordArt`) built on +//! the text-box machinery (anchor, move/resize/delete, undo, XLSX shape text body, printing). +//! The style palette below is original to GridCraft. + +use gridcraft_model::{Anchor, Color, Shape, ShapeFormat, ShapeKind, ShapeTextAlign, ShapeTextAnchor}; +use serde_json::{Value as Json, json}; + +use super::{ + CommandSpec, EngineError, PROTECTED, Result, Session, anchor_mode_param, bad, bool_param, cell_param, cmd, edit, f64_param, has_doc, ok, + u32_param, +}; + +/// One entry of the style gallery. +pub struct WordArtStyle { + pub name: &'static str, + pub fill: Color, + pub outline: Option<(Color, f32)>, + pub shadow: Option, + pub bold: bool, + pub italic: bool, + /// Font size in points. + pub size: f32, +} + +const fn rgb(v: u32) -> Color { + Color::Rgb(v) +} + +/// GridCraft's own WordArt presets. Theme colours where it makes sense (they follow the theme), +/// fixed colours for the rest. +pub const STYLES: [WordArtStyle; 10] = [ + WordArtStyle { name: "Classic", fill: Color::Theme(4, 0), outline: None, shadow: None, bold: true, italic: false, size: 36.0 }, + WordArtStyle { name: "Midnight", fill: rgb(0x1B2A49), outline: None, shadow: Some(rgb(0x8A94A6)), bold: true, italic: false, size: 40.0 }, + WordArtStyle { name: "Sunrise", fill: rgb(0xF2A33A), outline: Some((rgb(0xB5451B), 1.5)), shadow: None, bold: true, italic: false, size: 40.0 }, + WordArtStyle { + name: "Lagoon", + fill: rgb(0x1F9E9A), + outline: Some((rgb(0x0B4F5C), 1.0)), + shadow: Some(rgb(0x9CC9C7)), + bold: true, + italic: true, + size: 36.0, + }, + WordArtStyle { name: "Ember", fill: rgb(0xD64527), outline: None, shadow: Some(rgb(0x5A1A0E)), bold: true, italic: false, size: 44.0 }, + WordArtStyle { + name: "Mint Outline", + fill: rgb(0xFFFFFF), + outline: Some((rgb(0x2E9B6A), 2.0)), + shadow: None, + bold: true, + italic: false, + size: 40.0, + }, + WordArtStyle { name: "Ink Stamp", fill: rgb(0x2F3A8F), outline: Some((rgb(0x2F3A8F), 0.5)), shadow: None, bold: true, italic: false, size: 32.0 }, + WordArtStyle { name: "Orchid", fill: rgb(0x8E4FB5), outline: None, shadow: Some(rgb(0xD9C2E8)), bold: false, italic: true, size: 40.0 }, + WordArtStyle { + name: "Graphite", + fill: rgb(0x4A4F57), + outline: Some((rgb(0xFFFFFF), 1.0)), + shadow: Some(rgb(0x9AA0A8)), + bold: true, + italic: false, + size: 36.0, + }, + WordArtStyle { + name: "Lime Pop", + fill: rgb(0xA6D61C), + outline: Some((rgb(0x3F5A00), 1.5)), + shadow: Some(rgb(0x6E8B2C)), + bold: true, + italic: true, + size: 40.0, + }, +]; + +/// The text of a WordArt inserted without `text`. +pub const DEFAULT_TEXT: &str = "Your text here"; +/// Longest WordArt text (characters). +pub const MAX_CHARS: usize = 1000; +const MAX_POINTS: f64 = 400.0; +const MAX_SIDE: f64 = 4000.0; + +pub fn specs() -> Vec { + vec![ + cmd!( + "insert.wordArt", + "WordArt", + ["Insert", "Text"], + None, + "{style?: preset name or 0-based index (Classic, Midnight, Sunrise, Lagoon, Ember, Mint Outline, Ink Stamp, Orchid, Graphite, Lime Pop), text?, at?, width?, height?, fontSize?, bold?, italic?, fill?: hex, outline?: hex|false, outlineWidth?, shadow?: hex|bool}", + has_doc, + insert_word_art + ), + cmd!( + "wordArt.set", + "Format WordArt", + [], + None, + "{id: WordArt id, style?, text?, fontSize?, bold?, italic?, fill?: hex, outline?: hex|false, outlineWidth?, shadow?: hex|bool}", + has_doc, + set_word_art + ), + ] +} + +fn style_param(command: &str, p: &Json) -> Result> { + match p.get("style") { + None | Some(Json::Null) => Ok(None), + Some(Json::Number(n)) => n + .as_u64() + .and_then(|i| STYLES.get(usize::try_from(i).ok()?)) + .map(Some) + .ok_or_else(|| bad(command, format!("`style` index must be 0..{}", STYLES.len() - 1))), + Some(Json::String(name)) => STYLES + .iter() + .find(|s| s.name.eq_ignore_ascii_case(name.trim())) + .map(Some) + .ok_or_else(|| bad(command, format!("unknown WordArt style `{name}`"))), + Some(_) => Err(bad(command, "`style` must be a preset name or index")), + } +} + +fn text_param(command: &str, p: &Json) -> Result> { + match p.get("text") { + None | Some(Json::Null) => Ok(None), + Some(Json::String(t)) if t.is_empty() => Err(bad(command, "`text` must not be empty")), + Some(Json::String(t)) if t.chars().nth(MAX_CHARS).is_some() => Err(bad(command, format!("`text` is longer than {MAX_CHARS} characters"))), + Some(Json::String(t)) => Ok(Some(t.clone())), + Some(_) => Err(bad(command, "`text` must be a string")), + } +} + +/// Applies a preset, then explicit overrides, to a shape's format and fill. +fn apply(command: &str, shape: &mut Shape, style: Option<&WordArtStyle>, p: &Json) -> Result<()> { + if let Some(st) = style { + shape.format.text_color = Some(st.fill); + shape.format.text_outline = st.outline; + shape.format.text_shadow = st.shadow; + shape.format.bold = st.bold; + shape.format.italic = st.italic; + shape.format.font_size = Some(st.size); + } + let f = &mut shape.format; + if let Some(c) = p.get("fill").filter(|v| !v.is_null()) { + f.text_color = Some(super::format::color_param(Some(c)).ok_or_else(|| bad(command, "`fill` must be a hex colour"))?); + } + match p.get("outline") { + None | Some(Json::Null) => {} + Some(Json::Bool(false)) => f.text_outline = None, + Some(v) => { + let c = super::format::color_param(Some(v)).ok_or_else(|| bad(command, "`outline` must be a hex colour or false"))?; + let w = f64_param(p, "outlineWidth").map_or(f.text_outline.map_or(1.5, |o| o.1), |w| w.clamp(0.25, 20.0) as f32); + f.text_outline = Some((c, w)); + } + } + if let (Some(w), Some((c, _))) = (f64_param(p, "outlineWidth"), f.text_outline) { + f.text_outline = Some((c, w.clamp(0.25, 20.0) as f32)); + } + match p.get("shadow") { + None | Some(Json::Null) => {} + Some(Json::Bool(false)) => f.text_shadow = None, + Some(Json::Bool(true)) => f.text_shadow = Some(f.text_shadow.unwrap_or(rgb(0x808080))), + Some(v) => { + f.text_shadow = Some(super::format::color_param(Some(v)).ok_or_else(|| bad(command, "`shadow` must be a hex colour or a boolean"))?) + } + } + if let Some(b) = bool_param(p, "bold") { + f.bold = b; + } + if let Some(b) = bool_param(p, "italic") { + f.italic = b; + } + if let Some(sz) = f64_param(p, "fontSize") { + f.font_size = Some(sz.clamp(1.0, MAX_POINTS) as f32); + } + Ok(()) +} + +fn insert_word_art(s: &mut Session, p: &Json) -> Result { + let style = style_param("insert.wordArt", p)?.unwrap_or(&STYLES[0]); + let text = text_param("insert.wordArt", p)?.unwrap_or_else(|| DEFAULT_TEXT.to_string()); + let d = s.doc()?; + if d.wb.active().is_some_and(|sh| sh.is_protected()) { + return Err(EngineError::Other(PROTECTED.into())); + } + let id = d.wb.next_object_id(); + let at = cell_param(s, p, "at")?.unwrap_or(d.selection.active); + let mut shape = Shape { + id, + kind: ShapeKind::WordArt, + anchor: Anchor { cell: at, dx: 0.0, dy: 0.0, width: 0.0, height: 0.0, mode: anchor_mode_param(p).unwrap_or_default() }, + fill: Color::Auto, + line: Color::Auto, + text: text.clone(), + format: ShapeFormat { no_line: true, align: Some(ShapeTextAlign::Center), anchor: Some(ShapeTextAnchor::Middle), ..Default::default() }, + math: Default::default(), + }; + apply("insert.wordArt", &mut shape, Some(style), p)?; + // Default box: the longest line at the font size (about 0.62 em per character) plus padding. + let pt = f64::from(shape.format.font_size.unwrap_or(36.0)); + let longest = text.split('\n').map(|l| l.chars().count()).max().unwrap_or(1) as f64; + let lines = text.split('\n').count() as f64; + let width = f64_param(p, "width").unwrap_or(longest * pt * 0.62 * 96.0 / 72.0 + 24.0).clamp(8.0, MAX_SIDE); + let height = f64_param(p, "height").unwrap_or(lines * pt * 1.35 * 96.0 / 72.0 + 12.0).clamp(8.0, MAX_SIDE); + shape.anchor.width = width as f32; + shape.anchor.height = height as f32; + let sheet = d.wb.active_sheet(); + edit(s, |cx| { + cx.sheet_mut(sheet)?.shapes.push(shape); + Ok(json!({"shape": id})) + }) +} + +fn set_word_art(s: &mut Session, p: &Json) -> Result { + let id = u32_param(p, "id").ok_or_else(|| bad("wordArt.set", "`id` must be a WordArt id"))?; + let style = style_param("wordArt.set", p)?; + let text = text_param("wordArt.set", p)?; + let d = s.doc()?; + let sheet = d.wb.active_sheet(); + let sh = d.wb.active().ok_or_else(|| bad("wordArt.set", "no such WordArt"))?; + let current = sh.shapes.iter().find(|x| x.id == id).ok_or_else(|| bad("wordArt.set", "no such WordArt"))?; + if current.kind != ShapeKind::WordArt { + return Err(bad("wordArt.set", "the shape is not WordArt")); + } + if sh.is_protected() { + return Err(EngineError::Other(PROTECTED.into())); + } + let mut updated = current.clone(); + apply("wordArt.set", &mut updated, style, p)?; + if let Some(t) = text { + updated.text = t; + } + if &updated == current { + return ok(); + } + edit(s, |cx| { + let shape = cx.sheet_mut(sheet)?.shapes.iter_mut().find(|x| x.id == id).ok_or_else(|| bad("wordArt.set", "no such WordArt"))?; + *shape = updated; + Ok(Json::Null) + }) +} diff --git a/crates/engine/src/display.rs b/crates/engine/src/display.rs index 06a0fa587..4dbdf46e8 100644 --- a/crates/engine/src/display.rs +++ b/crates/engine/src/display.rs @@ -8,8 +8,13 @@ use gridcraft_model::{Sheet, Workbook}; use gridcraft_numfmt::{Formatted, NumberFormat, format_value}; /// Parsed number formats are cached by code. -fn parsed(code: &str) -> NumberFormat { +pub(crate) fn parsed(code: &str) -> NumberFormat { static CACHE: std::sync::LazyLock>> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + // A code of more bytes than 4 × the character limit can't be within it: it shows as General + // without being hashed or scanned again for every cell that shares it. + if code.len() > 4 * gridcraft_numfmt::MAX_FORMAT_CODE_CHARS { + return NumberFormat::parse("General"); + } let lock = || CACHE.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if let Some(f) = lock().get(code) { return f.clone(); @@ -24,7 +29,12 @@ fn parsed(code: &str) -> NumberFormat { f } +/// The parsed format of `code`; one longer than the character limit is a General format that keeps +/// the code it was made from. The engine itself formats through `parsed`, which doesn't copy it. pub fn number_format(code: &str) -> NumberFormat { + if code.len() > 4 * gridcraft_numfmt::MAX_FORMAT_CODE_CHARS { + return NumberFormat::parse(code); + } parsed(code) } @@ -228,3 +238,31 @@ pub fn stats(sheet: &Sheet, ranges: &[RangeRef]) -> Stats { } s } + +#[cfg(test)] +mod tests { + use super::*; + + /// A format code far past Excel's 255 characters shows its values as General, for every cell + /// that shares it, without being hashed into the cache each time. + #[test] + fn an_overlong_code_formats_as_general() { + let wb = Workbook::new(); + let code = format!("0.00{}", "\"x\"".repeat(400_000)); + for _ in 0..1_000 { + assert_eq!(format(&Value::Number(1.5), &code, &wb).text, "1.5"); + } + assert_eq!(format(&Value::Number(1.5), "0.00", &wb).text, "1.50"); + } + + /// The public `number_format` keeps the code of an overlong format; it is General, but not the + /// same format as a real `General`. + #[test] + fn number_format_keeps_an_overlong_code() { + let code = format!("0.00{}", "\"x\"".repeat(2_000)); + let f = number_format(&code); + assert!(f.is_general()); + assert_eq!(f.code(), code); + assert_ne!(f, number_format("General")); + } +} diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 690efec1d..6dd96b042 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -3,7 +3,7 @@ use std::fmt::Write as _; use gridcraft_core::{CellRef, RangeRef}; -use gridcraft_model::{Sheet, Workbook}; +use gridcraft_model::{LineIndex, Sheet, Workbook}; use crate::{EngineError, Result}; @@ -100,14 +100,18 @@ pub fn open_bytes_with(name: &str, bytes: &[u8], password: Option<&str>, csv_as_ /// line-height estimate can differ from Excel's, and shrinking files that already look /// right would be a visible regression. Custom-height rows are left alone entirely. fn fit_imported_rows(wb: &mut Workbook) { + // The whole pass shares one budget of merge comparisons: later sheets keep their heights once it is spent. + let mut left = crate::cmd::MERGE_FIT_WORK; for si in 0..wb.sheets.len() { let Some(sh) = wb.sheet(si) else { continue }; let mut rows: Vec = sh.cells.iter().map(|(c, _)| c.row).collect(); rows.sort_unstable(); rows.dedup(); + // One sweep over the sheet's merges for all of its rows, in ascending order. + let mut sweep = crate::cmd::MergeSweep::new(left); for row in rows { let before = wb.sheet(si).and_then(|sh| sh.rows.get(&row)).and_then(|i| i.size); - crate::cmd::auto_row_height(wb, si, row); + crate::cmd::fit_row_height(wb, si, row, &mut sweep); let Some(before) = before else { continue }; let after = wb.sheet(si).and_then(|sh| sh.rows.get(&row)).and_then(|i| i.size); if after.is_none_or(|a| before > a + 0.5) @@ -116,6 +120,7 @@ fn fit_imported_rows(wb: &mut Workbook) { sh.rows.entry(row).or_default().size = Some(before); } } + left = sweep.left(); } } @@ -576,7 +581,7 @@ pub fn csv_values_bytes(wb: &Workbook) -> Result> { let Some(sh) = wb.sheet(wb.active_sheet()) else { return Ok(Vec::new()) }; gridcraft_xlsx::write_csv(sh, wb.csv_delimiter.unwrap_or(b','), |c| match sh.value(c) { gridcraft_core::Value::Number(n) if n.is_finite() => { - let shape = crate::display::number_format(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).date_shape(n); + let shape = crate::display::parsed(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).date_shape(n); match shape.zip(gridcraft_core::date::datetime_from_serial(wb.date_system, n)) { Some(((date, time), d)) => iso(&d, date, time), None => n.to_string(), @@ -606,6 +611,24 @@ pub(crate) fn esc(s: &str) -> String { s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """) } +/// [`esc`], or `None` once the escaped text would pass `limit` bytes (nothing larger is built). +fn esc_within(s: &str, limit: usize) -> Option { + let mut out = String::with_capacity(s.len().min(limit)); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + c => out.push(c), + } + if out.len() > limit { + return None; + } + } + Some(out) +} + fn html_cell_css(wb: &Workbook, sh: &Sheet, c: CellRef, dir: CellDir) -> String { let st = wb.styles.get(sh.style_id(c)); let mut css = String::new(); @@ -681,6 +704,30 @@ fn table_dir(sh: &Sheet) -> &'static str { const CLIPBOARD_HTML_MAX_BYTES: usize = 4 * 1024 * 1024; const CLIPBOARD_HTML_MAX_CELLS: u64 = 10_000; +/// Most cells the merge index holds: one entry per merged cell inside the exported range. +const MERGE_INDEX_MAX_ENTRIES: usize = 1_000_000; + +/// The merge covering each cell of `rect`, as `Sheet::merge_at` would find it (the first in the +/// sheet's list), built once so a table lookup is one probe instead of a scan of every merge. +/// `None` when overlapping merges would cost more than a few visits per cell of `rect`, or merged +/// cells fill more than [`MERGE_INDEX_MAX_ENTRIES`]: a hostile file's merge list can't multiply the +/// work or the memory of a bounded export. +fn merge_index(sh: &Sheet, rect: RangeRef) -> Option> { + let mut map = std::collections::HashMap::new(); + let mut left = rect.count().saturating_mul(4); + for m in &sh.merges { + let Some(part) = m.intersection(&rect) else { continue }; + left = left.checked_sub(part.count())?; + for c in part.iter() { + map.entry(c).or_insert(*m); + } + if map.len() > MERGE_INDEX_MAX_ENTRIES { + return None; + } + } + Some(map) +} + /// An HTML fragment for the same range as `plain_text`, previously produced by /// `display::range_text`. Large copies keep plain text only; never return a partial table. pub(crate) fn range_html(wb: &Workbook, sheet: usize, r: RangeRef, plain_text: &str) -> Option { @@ -689,6 +736,7 @@ pub(crate) fn range_html(wb: &Workbook, sheet: usize, r: RangeRef, plain_text: & return None; } let sh = wb.sheet(sheet)?; + let merges = merge_index(sh, r)?; let mut out = ClipboardHtml(String::new()); write!(out, "", table_dir(sh)).ok()?; for row in r.start.row..=r.end.row { @@ -698,7 +746,7 @@ pub(crate) fn range_html(wb: &Workbook, sheet: usize, r: RangeRef, plain_text: & out.write_str("").ok()?; for col in r.start.col..=r.end.col { let c = CellRef::new(row, col); - let span = if let Some(merged) = sh.merges.iter().find(|m| m.contains(c)).and_then(|m| m.intersection(&r)) { + let span = if let Some(merged) = merges.get(&c).and_then(|m| m.intersection(&r)) { let first_row = (merged.start.row..=merged.end.row).find(|&row| !sh.is_row_hidden(row))?; if c != CellRef::new(first_row, merged.start.col) { continue; @@ -761,6 +809,14 @@ const HTML_EXPORT_MAX_ROWS: u32 = 100_001; /// tall, wide one) would otherwise be gigabytes of `` before finishing (#2387). const HTML_EXPORT_MAX_CELLS: u64 = 4_000_000; +/// Most bytes of one web page. Every cell may hold long text (shared across many cells in the +/// file), so the cell count alone doesn't bound the page; past this the export fails instead. +const HTML_EXPORT_MAX_BYTES: usize = 256 * 1024 * 1024; + +fn html_too_big() -> EngineError { + EngineError::Other(format!("the web page would be over {} MiB. Save as .xlsx or CSV instead.", HTML_EXPORT_MAX_BYTES / (1024 * 1024))) +} + /// A sheet as a standalone HTML table (Save as Web Page). A sheet with pictures gets the sheet's /// own column widths and row heights so each picture and chart (inline SVG) can sit at its anchor. pub fn to_html(wb: &Workbook, sheet: usize) -> Result { @@ -803,6 +859,24 @@ pub fn to_html(wb: &Workbook, sheet: usize) -> Result { r.width() ))); } + let merges = merge_index(sh, r) + .ok_or_else(|| EngineError::Other("the sheet's merged cells overlap too much, or cover too many cells, for a web page.".into()))?; + // Visible rows and columns of the table before each line, so a merge's span is counted + // in constant time whatever the file says its size is. + let visible_before = |hidden: &dyn Fn(u32) -> bool, from: u32, to: u32| { + let mut v = Vec::with_capacity((to - from) as usize + 2); + v.push(0u32); + for line in from..=to { + v.push(v.last().copied().unwrap_or(0) + u32::from(!hidden(line))); + } + v + }; + let rows_before = visible_before(&|y| sh.is_row_hidden(y), r.start.row, r.end.row); + let cols_before = visible_before(&|x| sh.is_col_hidden(x), r.start.col, r.end.col); + let visible = |before: &[u32], from: u32, a: u32, b: u32| { + let at = |line: u32| before.get((line - from) as usize).copied().unwrap_or(0); + at(b + 1) - at(a) + }; if fixed { out.push_str(""); for col in (r.start.col..=r.end.col).filter(|&c| !sh.is_col_hidden(c)) { @@ -821,27 +895,31 @@ pub fn to_html(wb: &Workbook, sheet: usize) -> Result { } for col in r.start.col..=r.end.col { let c = CellRef::new(row, col); - if sh.merges.iter().any(|m| m.contains(c) && m.start != c) || (fixed && sh.is_col_hidden(col)) { + let merge = merges.get(&c); + if merge.is_some_and(|m| m.start != c) || (fixed && sh.is_col_hidden(col)) { continue; } let text = crate::display::cell_text(wb, sh, c); let dir = CellDir::of(sh, c, &text); let css = html_cell_css(wb, sh, c, dir); - let span = sh - .merges - .iter() - .find(|m| m.start == c) + let span = merge .map(|m| { if fixed { - let rows = (m.start.row..=m.end.row).filter(|&y| !sh.is_row_hidden(y)).count(); - let cols = (m.start.col..=m.end.col).filter(|&x| !sh.is_col_hidden(x)).count(); + // Spans count the table's visible lines: the merge is clipped to the table. + let rows = visible(&rows_before, r.start.row, m.start.row, m.end.row.min(r.end.row)); + let cols = visible(&cols_before, r.start.col, m.start.col, m.end.col.min(r.end.col)); format!(" rowspan=\"{rows}\" colspan=\"{cols}\"") } else { format!(" rowspan=\"{}\" colspan=\"{}\"", m.height(), m.width()) } }) .unwrap_or_default(); - let _ = write!(out, "{}", dir.attr(), esc(&text)); + // The text is escaped only as far as the page still has room for it. + let Some(escaped) = esc_within(&text, HTML_EXPORT_MAX_BYTES.saturating_sub(out.len())) else { return Err(html_too_big()) }; + let _ = write!(out, "{escaped}", dir.attr()); + if out.len() > HTML_EXPORT_MAX_BYTES { + return Err(html_too_big()); + } } out.push_str(""); } @@ -849,34 +927,48 @@ pub fn to_html(wb: &Workbook, sheet: usize) -> Result { out.push_str(""); if fixed { // Sheet units are CSS pixels; the table starts at the used range, not at A1. - let (left, top) = used.map_or((0.0, 0.0), |r| (sh.col_left(r.start.col), sh.row_top(r.start.row))); + // Indexed positions: each anchor would otherwise sum every custom row and column before it. + let ri = LineIndex::new(&sh.rows, sh.default_row_height, gridcraft_core::MAX_ROWS); + let ci = LineIndex::new(&sh.cols, sh.default_col_width, gridcraft_core::MAX_COLS); + let (left, top) = used.map_or((0.0, 0.0), |r| (ci.start(r.start.col), ri.start(r.start.row))); let place = |a: &gridcraft_model::Anchor| { format!( "position:absolute;left:{}px;top:{}px;width:{}px;height:{}px", - sh.col_left(a.cell.col) - left + f64::from(a.dx), - sh.row_top(a.cell.row) - top + f64::from(a.dy), + ci.start(a.cell.col) - left + f64::from(a.dx), + ri.start(a.cell.row) - top + f64::from(a.dy), a.width.max(0.0), a.height.max(0.0) ) }; for im in &sh.images { - let _ = write!( - out, - "\"{}\"", - esc(&im.mime), - base64_encode(&im.data), - esc(&im.alt), - place(&im.anchor) - ); + // The base64 text is a third longer than the picture: refused before it is built. + if out.len().saturating_add(im.data.len().div_ceil(3).saturating_mul(4)) > HTML_EXPORT_MAX_BYTES { + return Err(html_too_big()); + } + // The attributes are escaped only as far as the page has room for them. + let room = HTML_EXPORT_MAX_BYTES.saturating_sub(out.len()); + let (Some(mime), Some(alt)) = (esc_within(&im.mime, room), esc_within(&im.alt, room)) else { return Err(html_too_big()) }; + let _ = write!(out, "\"{alt}\"", base64_encode(&im.data), place(&im.anchor)); + if out.len() > HTML_EXPORT_MAX_BYTES { + return Err(html_too_big()); + } } + let mut svg_budget = crate::chart_svg::MAX_PAGE_SVG_BYTES; for ch in &sh.charts { - if let Some(svg) = crate::chart_svg::chart_svg(wb, sheet, ch, &place(&ch.anchor)) { + if let Some(svg) = crate::chart_svg::chart_svg(wb, sheet, ch, &place(&ch.anchor), svg_budget)? { + svg_budget -= svg.len(); out.push_str(&svg); + if out.len() > HTML_EXPORT_MAX_BYTES { + return Err(html_too_big()); + } } } out.push_str(""); } out.push_str(""); + if out.len() > HTML_EXPORT_MAX_BYTES { + return Err(html_too_big()); + } Ok(out) } @@ -888,12 +980,12 @@ thread_local! { /// While one is alive, [`read_file`] and [`write_file`] on this thread refuse every path: a /// stored script's steps run inside one (#2178), so no command reaches the file system from a /// script whatever its parameters are called. Dropping it (also on unwind) lifts the refusal. -pub(crate) struct ScriptSandbox(()); +pub struct ScriptSandbox(std::marker::PhantomData<*const ()>); impl ScriptSandbox { - pub(crate) fn enter() -> Self { + pub fn enter() -> Self { let _ = SCRIPT_SANDBOX.try_with(|c| c.set(c.get().saturating_add(1))); - ScriptSandbox(()) + ScriptSandbox(std::marker::PhantomData) } } @@ -921,6 +1013,12 @@ const MAX_READ_BYTES: u64 = 2 * 1024 * 1024 * 1024; pub fn read_file(path: &str) -> Result> { use std::io::Read; script_sandbox_check(path)?; + // Checked before opening: opening a FIFO nobody writes to blocks the calling (GUI) thread. + // shortcut: a FIFO swapped in between this check and the open still blocks; close that with + // O_NONBLOCK when the crate takes a libc dependency. + if !std::fs::metadata(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?.is_file() { + return Err(EngineError::Other(format!("{path}: only regular files can be opened"))); + } let f = std::fs::File::open(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; // A symlink to an endless device reports length 0 but streams forever, so metadata // alone never bounds the read: require a real file and cap the bytes actually taken. @@ -952,18 +1050,37 @@ pub fn write_file(path: &str, bytes: &[u8]) -> Result<()> { // link's target; and it inherits the destination's access restrictions, so saving a // private workbook doesn't widen who can read it. let tmp = format!("{path}.sctmp"); - let prior = std::fs::metadata(path).ok().map(|m| m.permissions()); + let prior_meta = std::fs::metadata(path).ok(); + let prior = prior_meta.as_ref().map(std::fs::Metadata::permissions); let mut opts = std::fs::OpenOptions::new(); opts.write(true).create_new(true); + // Over an existing file the replacement starts owner-only, whatever the umask: its group is + // still the saving user's primary one until it is changed below, and a member of that group + // must not be able to open it (and keep the descriptor) before then. #[cfg(unix)] - if let Some(p) = &prior { - use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; - opts.mode(p.mode()); + if prior.is_some() { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); } let mut f = opts.open(&tmp).map_err(|e| { let _ = std::fs::remove_file(&tmp); EngineError::Other(format!("{path}: {e}")) })?; + // The replacement belongs to the saving user's primary group: restore the old file's group and + // then its mode. When the group can't be restored its access is dropped rather than handed to a + // different group; failing to read that back is a failed save. + #[cfg(unix)] + if let Some(m) = &prior_meta { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let _ = std::os::unix::fs::fchown(&f, None, Some(m.gid())); + let group_kept = f.metadata().map(|now| now.gid() == m.gid()).map_err(|e| { + let _ = std::fs::remove_file(&tmp); + EngineError::Other(format!("{path}: {e}")) + })?; + // Best effort: a file system that can't change modes keeps the owner-only one. Without the + // group, the old owner and other bits are kept and the group, setuid and setgid bits dropped. + let _ = f.set_permissions(std::fs::Permissions::from_mode(restored_mode(m.mode(), group_kept))); + } #[cfg(not(unix))] if let Some(p) = &prior { let _ = f.set_permissions(p.clone()); @@ -980,6 +1097,13 @@ pub fn write_file(path: &str, bytes: &[u8]) -> Result<()> { }) } +/// The mode a saved file gets from the file it replaces (`old`): all of it when its group was +/// restored; without the group, the owner and other bits only. +#[cfg(unix)] +fn restored_mode(old: u32, group_kept: bool) -> u32 { + if group_kept { old & 0o7777 } else { old & 0o707 } +} + #[cfg(target_arch = "wasm32")] pub fn write_file(path: &str, _bytes: &[u8]) -> Result<()> { script_sandbox_check(path)?; diff --git a/crates/engine/src/io/sylk.rs b/crates/engine/src/io/sylk.rs index 8eded1042..114574e9f 100644 --- a/crates/engine/src/io/sylk.rs +++ b/crates/engine/src/io/sylk.rs @@ -1,156 +1,57 @@ -//! SYLK (`.slk`) import: the symbolic link text format older spreadsheets and some legacy systems -//! export. One record per line, fields separated by `;` (`;;` is a literal semicolon): -//! -//! - `ID` starts the file; `E` ends it. -//! - `C;Y;X;K;E` is a cell. `Y` and `X` may be left out to stay on the -//! current row or column. `K` is a number, `"text"`, `TRUE`/`FALSE` or an error; `E` is a -//! formula in R1C1 notation; `S;R;C` shares the formula of another cell. -//! - `P;P` adds a number format to the file's list; `F;P;Y;X` gives a cell the -//! n-th format (0-based), and `F;W ` sets column widths in characters. -//! -//! Other records (fonts, options, window settings) are skipped. Formulas are converted to A1 with -//! the same relative and absolute references, as Excel does when it opens the file. +//! SYLK (`.slk`) import. `craft_csv::sylk` reads the file (values, formulas rewritten from R1C1 to +//! A1, number formats, column widths); this module makes it a one-sheet workbook: formulas parsed +//! (a formula GridCraft can't parse is skipped and counted), error values recognised, formats +//! interned as styles and widths converted from characters. use std::collections::HashMap; use std::sync::Arc; -use gridcraft_core::{CellError, CellRef, MAX_COLS, MAX_ROWS, Value, col_to_letters}; +use craft_csv::sylk; +use gridcraft_core::{CellError, CellRef, Value}; use gridcraft_model::{Cell, Formula, NumFmt, Sheet, Style, Workbook}; use crate::{EngineError, Result}; -/// The largest SYLK file read, as for other text imports: past it the file is refused. -const MAX_BYTES: usize = 512 << 20; -/// Column-width records kept from one file; each can cover every column. -const MAX_WIDTH_RECORDS: usize = 1024; - /// Reads a SYLK file into a one-sheet workbook. Returns the workbook and its warnings. pub fn read(name: &str, bytes: &[u8]) -> Result<(Workbook, Vec)> { - if bytes.len() > MAX_BYTES { - return Err(EngineError::Other(format!("We can't open '{name}': the SYLK file is larger than {} MB.", MAX_BYTES >> 20))); - } - let text = match std::str::from_utf8(bytes) { - Ok(s) => s.to_string(), - // Older exports are in a single-byte code page: keep each byte as its Latin-1 character. - Err(_) => bytes.iter().map(|&b| char::from(b)).collect(), - }; - let text = text.strip_prefix('\u{feff}').unwrap_or(&text); - if !text.starts_with("ID") { - return Err(EngineError::Other(format!("We can't open '{name}': it isn't a SYLK file (it doesn't start with an ID record)."))); - } + let file = sylk::read(bytes).map_err(|e| match e { + sylk::Error::TooLarge => EngineError::Other(format!("We can't open '{name}': the SYLK file is larger than {} MB.", sylk::MAX_BYTES >> 20)), + _ => EngineError::Other(format!("We can't open '{name}': it isn't a SYLK file (it doesn't start with an ID record).")), + })?; + let mut wb = Workbook::new(); let mut sheet = Sheet::new("Sheet1"); - let mut formats: Vec = Vec::new(); - // Cells' number formats by index into `formats`, applied once every format is known. - let mut cell_formats: Vec<(CellRef, usize)> = Vec::new(); - let mut formulas: HashMap = HashMap::new(); - let mut widths: Vec<(u32, u32, f64)> = Vec::new(); - let (mut row, mut col) = (0u32, 0u32); - let mut skipped = 0usize; - for line in text.lines() { - let fields = split_fields(line); - let Some((kind, rest)) = fields.split_first() else { continue }; - match kind.as_str() { - "C" => { - let (mut value, mut formula, mut shared) = (None, None, None::<(Option, Option)>); - let mut bad = false; - for f in rest { - let (tag, arg) = f.split_at(f.chars().next().map_or(0, char::len_utf8)); - match tag { - "Y" => match position(arg, MAX_ROWS) { - Some(r) => row = r, - None => bad = true, - }, - "X" => match position(arg, MAX_COLS) { - Some(c) => col = c, - None => bad = true, - }, - "K" => value = Some(parse_value(arg)), - "E" => formula = Some(arg.to_string()), - "S" => shared = Some((None, None)), - // In a shared-formula cell, R and C name the cell whose formula it shares. - "R" if shared.is_some() => shared = shared.map(|(_, c)| (position(arg, MAX_ROWS), c)), - "C" if shared.is_some() => shared = shared.map(|(r, _)| (r, position(arg, MAX_COLS))), - _ => {} - } - } - if bad { - skipped += 1; - continue; - } - let at = CellRef::new(row, col); - if let Some((Some(r), Some(c))) = shared { - formula = formulas.get(&CellRef::new(r, c)).cloned(); - } - let mut cell = sheet.cell(at).cloned().unwrap_or_else(|| Cell::value(Value::Empty)); - if let Some(v) = value { - cell.value = v; - } - if let Some(r1c1) = formula { - let a1 = r1c1_formula_to_a1(&r1c1, at); - let f = Formula::new(&a1); - if f.expr.is_some() { - cell.formula = Some(Arc::new(f)); - formulas.insert(at, r1c1); - } else { - skipped += 1; - } - } - sheet.set_cell(at, cell); + let mut skipped = file.skipped; + let mut styles = HashMap::new(); + for (&(row, col), c) in &file.cells { + let value = match &c.value { + sylk::Value::Number(n) => Value::Number(*n), + sylk::Value::Text(s) => Value::Text(Arc::from(s.as_str())), + sylk::Value::Bool(b) => Value::Bool(*b), + sylk::Value::Error(s) => CellError::parse(s).map_or_else(|| Value::Text(Arc::from(s.as_str())), Value::Error), + _ => Value::Empty, + }; + let mut cell = Cell::value(value); + if let Some(a1) = &c.formula { + let f = Formula::new(a1); + if f.expr.is_some() { + cell.formula = Some(Arc::new(f)); + } else { + skipped += 1; } - "F" => { - let mut format = None; - for f in rest { - let (tag, arg) = f.split_at(f.chars().next().map_or(0, char::len_utf8)); - match tag { - "Y" => row = position(arg, MAX_ROWS).unwrap_or(row), - "X" => col = position(arg, MAX_COLS).unwrap_or(col), - "P" => format = arg.parse::().ok(), - "W" => { - let mut parts = arg.split_whitespace(); - if let (Some(first), Some(last), Some(width)) = (parts.next(), parts.next(), parts.next()) - && let (Some(first), Some(last), Ok(width)) = - (position(first, MAX_COLS), position(last, MAX_COLS), width.parse::()) - && first <= last - && (0.0..=255.0).contains(&width) - // A file of width records can't make the import loop over every column - // again and again. - && widths.len() < MAX_WIDTH_RECORDS - { - widths.push((first, last, width)); - } - } - _ => {} - } - } - if let Some(format) = format { - cell_formats.push((CellRef::new(row, col), format)); - } - } - // Number formats only: `P;F…` and `P;E…` are fonts. - "P" => { - if let Some(code) = rest.first().and_then(|f| f.strip_prefix('P')) { - formats.push(code.to_string()); - } - } - "E" => break, - _ => {} } + if let Some(code) = &c.format { + cell.style = *styles + .entry(code.as_str()) + .or_insert_with(|| wb.styles.intern(Style { num_fmt: NumFmt(Arc::from(code.as_str())), ..Style::default() })); + } + sheet.set_cell(CellRef::new(row, col), cell); } - for (first, last, width) in widths { - let size = crate::cmd::format::chars_to_points(width) as f32; - for c in first..=last { + for w in &file.column_widths { + let size = crate::cmd::format::chars_to_points(w.width) as f32; + for c in w.first..=w.last { sheet.cols.entry(c).or_default().size = Some(size); } } - let mut wb = Workbook::new(); - let mut styles = HashMap::new(); - for (at, index) in cell_formats { - let Some(code) = formats.get(index).filter(|code| !code.is_empty() && *code != "General") else { continue }; - let style = *styles.entry(index).or_insert_with(|| wb.styles.intern(Style { num_fmt: NumFmt(Arc::from(code.as_str())), ..Style::default() })); - let mut cell = sheet.cell(at).cloned().unwrap_or_else(|| Cell::value(Value::Empty)); - cell.style = style; - sheet.set_cell(at, cell); - } if let Some(first) = wb.sheet_mut(0) { sheet.name = std::mem::take(&mut first.name); *first = sheet; @@ -161,168 +62,3 @@ pub fn read(name: &str, bytes: &[u8]) -> Result<(Workbook, Vec)> { } Ok((wb, warnings)) } - -/// A record's fields: `;` separates them and `;;` is a literal semicolon. -fn split_fields(line: &str) -> Vec { - let mut fields = vec![String::new()]; - let mut chars = line.trim_end_matches('\r').chars().peekable(); - while let Some(c) = chars.next() { - if c == ';' { - if chars.peek() == Some(&';') { - chars.next(); - if let Some(f) = fields.last_mut() { - f.push(';'); - } - } else { - fields.push(String::new()); - } - } else if let Some(f) = fields.last_mut() { - f.push(c); - } - } - fields -} - -/// A 1-based row or column number → 0-based, within `max`. -fn position(arg: &str, max: u32) -> Option { - arg.trim().parse::().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1) -} - -/// A `K` value: `"text"` (everything between the first and the last quote, as written), a -/// boolean, an error or a number. -fn parse_value(arg: &str) -> Value { - if let Some(text) = arg.strip_prefix('"') { - let text = text.strip_suffix('"').unwrap_or(text); - return Value::Text(Arc::from(text)); - } - if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") { - return Value::Bool(arg.eq_ignore_ascii_case("TRUE")); - } - if let Some(e) = CellError::parse(arg) { - return Value::Error(e); - } - match arg.trim().parse::() { - Ok(n) if n.is_finite() => Value::Number(n), - _ => Value::Text(Arc::from(arg)), - } -} - -/// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or -/// nothing (the cell's own row or column). -#[derive(Clone, Copy, Debug, PartialEq)] -enum Part { - Absolute(u32), - Relative(i64), -} - -impl Part { - /// The 0-based index and whether it is absolute, or `None` when it falls off the sheet. - fn resolve(self, base: u32, max: u32) -> Option<(u32, bool)> { - match self { - Part::Absolute(n) => (1..=max).contains(&n).then(|| (n - 1, true)), - Part::Relative(k) => i64::from(base).checked_add(k).and_then(|v| u32::try_from(v).ok()).filter(|v| *v < max).map(|v| (v, false)), - } - } -} - -/// Parses the number after `R` or `C` at `i`: digits, `[offset]` or nothing. Returns the part and -/// the index after it, or `None` when the text isn't a reference part. -fn parse_part(chars: &[char], i: usize) -> Option<(Part, usize)> { - match chars.get(i) { - Some('[') => { - let close = chars[i..].iter().position(|&c| c == ']')? + i; - let inner: String = chars[i + 1..close].iter().collect(); - Some((Part::Relative(inner.parse().ok()?), close + 1)) - } - Some(c) if c.is_ascii_digit() => { - let end = chars[i..].iter().position(|c| !c.is_ascii_digit()).map_or(chars.len(), |p| p + i); - let n: String = chars[i..end].iter().collect(); - Some((Part::Absolute(n.parse().unwrap_or(u32::MAX)), end)) - } - _ => Some((Part::Relative(0), i)), - } -} - -fn is_word_char(c: char) -> bool { - c.is_alphanumeric() || matches!(c, '_' | '.' | '\\' | '?' | '$') -} - -/// Rewrites the R1C1 references in a SYLK formula as A1 references for the cell at `at`: -/// `RC[-1]` → `B2`, `R1C1` → `$A$1`, `R[-2]C:R[-1]C` → `D2:D3`, a whole column `C4` → `$D:$D` and -/// a whole row `R3` → `$3:$3`. Text in quotes and quoted sheet names are left alone; a reference -/// off the sheet becomes `#REF!`. -pub(crate) fn r1c1_formula_to_a1(formula: &str, at: CellRef) -> String { - let chars: Vec = formula.chars().collect(); - let mut out = String::with_capacity(formula.len()); - let mut i = 0; - while i < chars.len() { - let c = chars[i]; - if c == '"' || c == '\'' { - // Copy the quoted run as it is; a doubled quote stays inside it. - let mut j = i + 1; - while j < chars.len() && !(chars[j] == c && chars.get(j + 1) != Some(&c)) { - j += if chars[j] == c { 2 } else { 1 }; - } - let stop = (j + 1).min(chars.len()); - out.extend(&chars[i..stop]); - i = stop; - continue; - } - let starts_word = i == 0 || !is_word_char(chars[i - 1]); - if starts_word - && matches!(c, 'R' | 'r' | 'C' | 'c') - && let Some((text, next)) = reference_at(&chars, i, at) - { - out.push_str(&text); - i = next; - continue; - } - // Copy the rest of a word as it is, so `ROUND` or `Sheet2` isn't read as a reference. - if is_word_char(c) { - let end = (i..chars.len()).find(|&j| !is_word_char(chars[j])).unwrap_or(chars.len()); - out.extend(&chars[i..end]); - i = end; - } else { - out.push(c); - i += 1; - } - } - out -} - -/// The A1 text of the R1C1 reference starting at `i`, and the index after it, or `None` when the -/// word there isn't a reference (a function or a name). -fn reference_at(chars: &[char], i: usize, at: CellRef) -> Option<(String, usize)> { - let mut j = i; - let mut row = None; - let mut col = None; - if matches!(chars.get(j), Some('R' | 'r')) { - let (part, next) = parse_part(chars, j + 1)?; - row = Some(part); - j = next; - } - if matches!(chars.get(j), Some('C' | 'c')) { - let (part, next) = parse_part(chars, j + 1)?; - col = Some(part); - j = next; - } - if row.is_none() && col.is_none() { - return None; - } - // A reference ends the word: `ROUND(`, `RATE` or `R2D2` are names. - if chars.get(j).is_some_and(|&c| is_word_char(c) || c == '(' || c == '[' || c == '!') { - return None; - } - let row_text = |p: Part| p.resolve(at.row, MAX_ROWS).map(|(r, abs)| format!("{}{}", if abs { "$" } else { "" }, r + 1)); - let col_text = |p: Part| p.resolve(at.col, MAX_COLS).map(|(c, abs)| format!("{}{}", if abs { "$" } else { "" }, col_to_letters(c))); - // One side of a range (`R2:R4`, `C1:C3`) is written alone; a lone row or column is the whole of it. - let prev = chars[..i].iter().rev().find(|c| !c.is_whitespace()); - let in_range = prev == Some(&':') || chars[j..].iter().find(|c| !c.is_whitespace()) == Some(&':'); - let text = match (row, col) { - (Some(r), Some(c)) => row_text(r).zip(col_text(c)).map(|(r, c)| format!("{c}{r}")), - (Some(r), None) => row_text(r).map(|r| if in_range { r } else { format!("{r}:{r}") }), - (None, Some(c)) => col_text(c).map(|c| if in_range { c } else { format!("{c}:{c}") }), - (None, None) => None, - }; - Some((text.unwrap_or_else(|| "#REF!".into()), j)) -} diff --git a/crates/engine/src/io/tests.rs b/crates/engine/src/io/tests.rs index ae5ea8832..ad8ca40f2 100644 --- a/crates/engine/src/io/tests.rs +++ b/crates/engine/src/io/tests.rs @@ -51,6 +51,14 @@ fn save_keeps_the_destination_access_restrictions() { write_file(dest.to_str().unwrap(), b"v2").unwrap(); let mode = std::fs::metadata(&dest).unwrap().permissions().mode() & 0o777; assert_eq!(mode, 0o600); + // The mode is set outright from the old file's, not through the umask: group and other access + // it had stay as they were. + for old in [0o640, 0o664] { + std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(old)).unwrap(); + write_file(dest.to_str().unwrap(), b"v3").unwrap(); + assert_eq!(std::fs::metadata(&dest).unwrap().permissions().mode() & 0o777, old); + } + assert!(!dir.join("private.xlsx.sctmp").exists()); // And a brand-new file is created with the restrictive, not permissive, default. let fresh = dir.join("new.xlsx"); write_file(fresh.to_str().unwrap(), b"v").unwrap(); @@ -59,6 +67,17 @@ fn save_keeps_the_destination_access_restrictions() { let _ = std::fs::remove_dir_all(&dir); } +/// Without the old group, a saved file keeps the old owner and other bits and nothing for a group. +#[cfg(not(target_arch = "wasm32"))] +#[cfg(unix)] +#[test] +fn restored_mode_drops_the_group_and_special_bits_when_the_group_is_lost() { + assert_eq!(super::restored_mode(0o100_664, true), 0o664); + assert_eq!(super::restored_mode(0o100_664, false), 0o604); + assert_eq!(super::restored_mode(0o102_444, false), 0o404, "an original read-only file stays read-only, setgid is dropped"); + assert_eq!(super::restored_mode(0o104_755, false), 0o705, "setuid is dropped"); +} + #[cfg(not(target_arch = "wasm32"))] #[test] fn open_refuses_directories_and_unknowable_lengths() { @@ -351,6 +370,19 @@ fn script_sandbox_refuses_file_access_until_dropped() { let _ = std::fs::remove_dir_all(&dir); } +/// The guard's drop lowers the counter of the thread it runs on, so it must not move between threads: +/// naming the trait's method is ambiguous (a compile error) if `ScriptSandbox` is `Send`. +#[test] +fn script_sandbox_is_not_send() { + trait AmbiguousIfSend { + fn check() {} + } + impl AmbiguousIfSend<()> for T {} + struct IsSend; + impl AmbiguousIfSend for T {} + >::check(); +} + /// A minimal compound (CFB/OLE2) file — FAT in sector 0, directory in sector 1 — holding a /// root entry and the given stream names. fn cfb(streams: &[&str]) -> Vec { @@ -731,7 +763,7 @@ fn sylk_r1c1_formulas_become_a1() { ("R[-5]C", "#REF!"), ("\"RC\"&R1C1", "\"RC\"&$A$1"), ] { - assert_eq!(sylk::r1c1_formula_to_a1(r1c1, b2), a1, "{r1c1}"); + assert_eq!(craft_csv::sylk::r1c1_formula_to_a1(r1c1, (b2.row, b2.col)), a1, "{r1c1}"); } } @@ -761,3 +793,91 @@ fn sylk_survives_huge_relative_offsets_and_width_floods() { let (wb, _) = open_bytes("widths.slk", &flood).unwrap(); assert_eq!(wb.sheet(0).unwrap().value(CellRef::parse("A1").unwrap()), gridcraft_core::Value::Number(1.0)); } + +/// A FIFO with no writer would block an open forever: it is refused before it is opened. +#[cfg(unix)] +#[test] +fn reading_a_fifo_is_refused_without_blocking() { + let dir = std::env::temp_dir().join(format!("gridcraft-fifo-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let fifo = dir.join("pipe.xlsx"); + if !std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()) { + let _ = std::fs::remove_dir_all(&dir); + return; // no mkfifo here + } + let err = read_file(fifo.to_str().unwrap()).unwrap_err().to_string(); + assert!(err.contains("only regular files"), "{err}"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Merges far from what is copied or exported, in a hostile number, are indexed once instead of +/// being scanned for every cell; a merge running past the table is clipped to it. +#[test] +fn html_merge_lookups_and_spans_stay_bounded() { + let mut wb = book_with_picture("A1", (0, 0)); + let sh = wb.sheet_mut(0).unwrap(); + for i in 0..200_000u32 { + sh.merges.push(RangeRef::new(CellRef::new(500_000 + i * 2, 5), CellRef::new(500_000 + i * 2, 6))); + } + sh.merges.push(RangeRef::new(CellRef::new(0, 0), CellRef::new(gridcraft_core::MAX_ROWS - 1, 1))); + sh.set_value(CellRef::new(2, 1), gridcraft_core::Value::Number(2.0)); + let html = copied(&wb, "A1:C50"); + assert!(html.contains("rowspan=\"50\" colspan=\"2\""), "{html}"); + let page = to_html(&wb, 0).unwrap(); + // The table is A1:B3 (the used range): the whole-height merge spans its three rows only. + assert!(page.contains("rowspan=\"3\" colspan=\"2\""), "{page}"); + // Merges stacked over one another past a few per cell are refused, not walked. + let sh = wb.sheet_mut(0).unwrap(); + sh.merges = vec![RangeRef::new(CellRef::new(0, 0), CellRef::new(2, 1)); 100]; + assert!(to_html(&wb, 0).is_err()); + assert!(range_html(&wb, 0, RangeRef::parse("A1:B3").unwrap(), "").is_none()); +} + +/// Thousands of pictures over thousands of custom rows: each is placed through a row index. +#[test] +fn html_export_places_many_pictures_over_many_custom_rows() { + let mut wb = book_with_picture("A1", (0, 0)); + let sh = wb.sheet_mut(0).unwrap(); + for row in 0..50_000 { + sh.rows.insert(row, gridcraft_model::LineInfo { size: Some(30.0), ..Default::default() }); + } + let image = sh.images[0].clone(); + for i in 0..5_000u32 { + let mut im = image.clone(); + im.anchor.cell = CellRef::new(40_000 + i, 0); + sh.images.push(im); + } + let page = to_html(&wb, 0).unwrap(); + assert!(page.contains(&format!("top:{}px", 30 * 44_999)), "last picture placed by the index"); +} + +/// One long text shown by many cells: the page fails at its byte limit instead of growing on. +#[test] +fn html_export_refuses_a_page_past_its_byte_limit() { + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + let text = gridcraft_core::Value::text("x".repeat(32_000)); + for row in 0..10_000 { + sh.set_value(CellRef::new(row, 0), text.clone()); + } + let err = to_html(&wb, 0).unwrap_err(); + assert!(err.to_string().contains("MiB"), "{err}"); +} + +#[test] +fn escaping_stops_at_the_room_left() { + assert_eq!(esc_within("a&b\"d", 100).as_deref(), Some(esc("a&b\"d").as_str())); + assert_eq!(esc_within("&&&", 14), None, "three `&` are 15 bytes"); + assert_eq!(esc_within("&&&", 15).as_deref(), Some("&&&")); +} + +#[test] +fn the_merge_index_holds_a_bounded_number_of_cells() { + let mut sh = Sheet::new("S"); + let rect = RangeRef::new(CellRef::new(0, 0), CellRef::new(1_100_000, 0)); + sh.merges.push(RangeRef::new(CellRef::new(0, 0), CellRef::new(999_999, 0))); + assert_eq!(merge_index(&sh, rect).map(|m| m.len()), Some(1_000_000)); + sh.merges = vec![RangeRef::new(CellRef::new(0, 0), CellRef::new(1_000_000, 0))]; + assert!(merge_index(&sh, rect).is_none()); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index c608a8e39..e068f7a1e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -237,6 +237,22 @@ const JOURNAL_MAX_ENTRIES: usize = 10_000; pub(crate) const JOURNAL_MAX_BYTES: usize = 64 * 1024 * 1024; const UNDO_BYTES_LIMIT: usize = 128 * 1024 * 1024; +/// Comment and hyperlink text of a sheet: a changed sheet's snapshot keeps its own copy of these +/// strings (an edit deep-copies them), so they count against the history budget with the cells. +fn sheet_text_bytes(sheet: &gridcraft_model::Sheet) -> usize { + let comments = sheet.comments.values().fold(0usize, |sum, c| { + // The vectors' own storage counts too: an empty reply still takes an element. + let replies = c.replies.iter().fold(0usize, |n, (a, t)| n.saturating_add(a.len()).saturating_add(t.len())); + let replies = replies.saturating_add(c.replies.capacity().saturating_mul(std::mem::size_of::<(String, String)>())); + let dates = c.dates.iter().fold(0usize, |n, d| n.saturating_add(d.len())); + let dates = dates.saturating_add(c.dates.capacity().saturating_mul(std::mem::size_of::())); + sum.saturating_add(std::mem::size_of::() + c.author.len() + c.text.len() + replies + dates) + }); + sheet.hyperlinks.values().fold(comments, |sum, h| { + sum.saturating_add(std::mem::size_of::() + h.target.len() + h.tooltip.as_ref().map_or(0, String::len)) + }) +} + fn history_retained_bytes(older: &Workbook, newer: &Workbook) -> usize { let mut bytes = std::mem::size_of::(); for (index, sheet) in older.sheets.iter().enumerate() { @@ -248,6 +264,7 @@ fn history_retained_bytes(older: &Workbook, newer: &Workbook) -> usize { let empty = gridcraft_model::CellStore::new(); let next = newer.sheets.get(index).map_or(&empty, |next| &next.cells); bytes = bytes.saturating_add(sheet.cells.retained_bytes_against(next)); + bytes = bytes.saturating_add(sheet_text_bytes(sheet)); } // Snapshots share the macro list until a change to it; a snapshot with its own list keeps it. if !older.macros.shares(&newer.macros) { @@ -288,6 +305,19 @@ fn trim_to_budget(history: &mut Vec, byte_budget: usize) { } } +/// A writer that only counts the bytes written to it. +struct ByteCounter(usize); + +impl std::io::Write for ByteCounter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 = self.0.saturating_add(buf.len()); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + impl DocState { /// A document for `wb`, which came from where `trust` says: [`Trust::FromFile`] for anything /// read from bytes (a file, a download, a recovery copy), [`Trust::Local`] only for a workbook @@ -621,7 +651,11 @@ impl Session { /// Journals a command, then trims the oldest entries past either cap. A recording in progress /// keeps pointing at the same entry (the steps the trim dropped are lost). fn push_journal(&mut self, id: &str, params: Json) { - let size = |e: &(String, Json)| serde_json::to_vec(e).map_or(0, |v| v.len()); + // Counted as it is written, without building the text: a paste's params can be MBs. + let size = |e: &(String, Json)| { + let mut counter = ByteCounter(0); + serde_json::to_writer(&mut counter, e).map_or(0, |()| counter.0) + }; // `journal` is public: if it was changed from outside, measure it again. if self.journal_sizes.len() != self.journal.len() { self.journal_sizes = self.journal.iter().map(size).collect(); diff --git a/crates/engine/src/macros/recorder.rs b/crates/engine/src/macros/recorder.rs index 337c41661..ef9be781f 100644 --- a/crates/engine/src/macros/recorder.rs +++ b/crates/engine/src/macros/recorder.rs @@ -440,7 +440,7 @@ pub fn infer_input(wb: &Workbook, sheet: usize, r: RangeRef) -> NamedInput { Some(match v { Value::Empty => return None, Value::Text(t) if t.is_empty() => return None, - Value::Number(_) if crate::display::number_format(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).is_date() => ColType::Date, + Value::Number(_) if crate::display::parsed(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).is_date() => ColType::Date, Value::Number(_) => ColType::Number, Value::Text(_) => ColType::Text, Value::Bool(_) => ColType::Bool, diff --git a/crates/engine/src/macros/run.rs b/crates/engine/src/macros/run.rs index 9e31e3ca9..1247b702c 100644 --- a/crates/engine/src/macros/run.rs +++ b/crates/engine/src/macros/run.rs @@ -335,7 +335,7 @@ fn check_input(wb: &Workbook, input: &gridcraft_model::NamedInput, place: Place) gridcraft_model::ColType::Number => v.is_number(), gridcraft_model::ColType::Text => v.is_text(), gridcraft_model::ColType::Bool => matches!(v, Value::Bool(_)), - gridcraft_model::ColType::Date => v.is_number() && crate::display::number_format(&date()).is_date(), + gridcraft_model::ColType::Date => v.is_number() && crate::display::parsed(&date()).is_date(), }; if !ok { wrong.push((c, holds(&v, ty))); @@ -1281,9 +1281,18 @@ fn content_differs(b: Option<&Cell>, a: Option<&Cell>) -> bool { /// Objects added, changed and removed between two versions of a sheet. fn objects(b: &Sheet, a: &Sheet) -> (usize, usize, usize) { fn count(b: &[T], a: &[T], id: impl Fn(&T) -> u32) -> (usize, usize, usize) { - let added = a.iter().filter(|x| !b.iter().any(|y| id(y) == id(x))).count(); - let removed = b.iter().filter(|x| !a.iter().any(|y| id(y) == id(x))).count(); - let changed = b.iter().filter(|x| a.iter().any(|y| id(y) == id(x) && y != *x)).count(); + // Objects grouped by id, so each lookup is a map probe instead of a scan of the list. + fn group<'a, T>(v: &'a [T], id: &impl Fn(&T) -> u32) -> std::collections::HashMap> { + let mut m: std::collections::HashMap> = std::collections::HashMap::new(); + for x in v { + m.entry(id(x)).or_default().push(x); + } + m + } + let (gb, ga) = (group(b, &id), group(a, &id)); + let added = a.iter().filter(|x| !gb.contains_key(&id(x))).count(); + let removed = b.iter().filter(|x| !ga.contains_key(&id(x))).count(); + let changed = b.iter().filter(|x| ga.get(&id(x)).is_some_and(|ys| ys.iter().any(|y| *y != *x))).count(); (added, changed, removed) } let parts = [ diff --git a/crates/engine/src/pivot.rs b/crates/engine/src/pivot.rs index bfaccace4..1b5d48155 100644 --- a/crates/engine/src/pivot.rs +++ b/crates/engine/src/pivot.rs @@ -182,7 +182,7 @@ pub fn read_source(wb: &Workbook, text: &str, default_sheet: usize) -> Result Option Vec { @@ -64,6 +68,7 @@ pub fn chart_series(sheet: &str, origin: CellRef, out: &Output) -> Vec { let categories = out.label_col.and_then(|c| rows_ref(sheet, origin, c, &rows)); out.chart_cols .iter() + .take(MAX_CHART_SERIES) .filter_map(|(c, name)| { Some(Series { name: Some(format!("\"{}\"", name.replace('"', "\"\""))), @@ -106,3 +111,22 @@ pub fn clear_output(wb: &mut Workbook, sheet: usize, idx: usize) { p.last_range = None; } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_pivot_chart_gets_at_most_the_series_the_renderer_reads() { + let out = Output { + grid: vec![], + row_kinds: vec![], + body_offset: 0, + width: 1000, + chart_rows: vec![1, 2, 3], + chart_cols: (1..1000).map(|c| (c, format!("S{c}"))).collect(), + label_col: Some(0), + }; + assert_eq!(chart_series("Sheet1", CellRef::new(0, 0), &out).len(), MAX_CHART_SERIES); + } +} diff --git a/crates/engine/src/pivot/compute.rs b/crates/engine/src/pivot/compute.rs index 90e81408d..2bb6bdbc6 100644 --- a/crates/engine/src/pivot/compute.rs +++ b/crates/engine/src/pivot/compute.rs @@ -1,5 +1,7 @@ use super::items::{Acc, FieldItems, build_tree, collect_siblings, is_collapsed, item_label}; -use super::output::{AggMap, CellKind, ColKind, ColSpec, Ctx, Grid, Line, LineKind, Output, RowKind, ValSrc, data_cols_exceed_sheet, flatten_cols}; +use super::output::{ + AggMap, CellKind, ColKind, ColSpec, Ctx, Grid, Line, LineKind, Output, RowKind, ValSrc, caption, data_cols_exceed_sheet, flatten_cols, +}; use super::{ CalcSet, CellError, CellRef, HashMap, HashSet, MAX_AXIS_FIELDS, MAX_COLS, MAX_VALUE_FIELDS, PivotDateGroup, PivotField, PivotFilter, PivotFunc, PivotLayout, PivotShowAs, PivotTable, PivotValue, Source, Value, Workbook, read_source, @@ -94,6 +96,10 @@ struct AggregateData { col_leaves: HashSet>, } +/// Most accumulators (groups times value columns) one report may hold: twice what the 5 million +/// group limit allowed at one value field. +const MAX_ACCUMULATORS: usize = 16_000_000; + fn aggregate_record( input: &AggregateInput<'_>, data: &mut AggregateData, @@ -152,7 +158,9 @@ fn aggregate_record( if !input.cols.is_empty() && data_cols_exceed_sheet(data.col_leaves.len(), input.pt.grand_totals_rows, input.vals.len()) { return Err("The PivotTable report will not fit on the sheet.".into()); } - if data.aggs.len() > 5_000_000 { + // Each entry holds one accumulator per value field and calculated-field source column, so the + // width counts too: a handful of groups over hundreds of fields is large as well. + if data.aggs.len() > 5_000_000 || data.aggs.len().saturating_mul(input.vals.len() + input.extra.len()) > MAX_ACCUMULATORS { return Err("The PivotTable report is too large.".into()); } Ok(()) @@ -259,8 +267,8 @@ fn write_headers(g: &mut Grid, input: HeaderInput<'_>) -> Result<(), String> { let lv = spec.path.len().saturating_sub(1); let label = spec.path.last().and_then(|it| col_items.get(lv).map(|fi| fi.label(*it))).unwrap_or_default(); let text = match vi { - Some(vi) if k > 1 => format!("{label} {}", val_name(*vi)), - _ => format!("{label} Total"), + Some(vi) if k > 1 => caption(&label, &val_name(*vi)), + _ => caption(&label, "Total"), }; g.set(first_level + lv, c, Value::text(text), CellKind::ColLabel, 0, None); } @@ -318,6 +326,7 @@ struct ShowContext<'a> { value_index: usize, level0: &'a [u32], siblings: &'a HashMap, Vec>, + cache: &'a ShowCache, } fn number(value: &Option) -> Option { @@ -334,42 +343,89 @@ fn ratio(a: Value, b: Option) -> Value { } } +/// What Running Total and Rank need from other cells, computed once per sibling set instead of +/// once per displayed value (which rescanned the whole set every time). +type ShowKey = (Vec, Vec, usize); +/// Cumulative values by top-level item, and the total of all of them. +type RunningTotals = (HashMap, f64); +/// Top-level items by (rest of a row path, column path). +type ItemsByRest = HashMap<(Vec, Vec), Vec>; + +#[derive(Default)] +struct ShowCache { + /// Ranked values' sibling numbers, sorted ascending without repeats, by (parent, column, value). + ranks: std::cell::RefCell>>, + /// Running totals by top-level item, with the total of all of them, by (rest of path, column, value). + running: std::cell::RefCell>, + /// The top-level items that have a row path ending in `rest` (with column path `col`), in + /// top-level order, by (`rest`, `col`): built once from the report's cells, so a running total + /// visits the items that can hold a number, not every top-level item per distinct `rest`. + items_by_rest: std::cell::RefCell>, +} + fn running_total(raw: Value, lookup: &ValueLookup<'_>, context: &ShowContext<'_>) -> Value { - let Some((first, _)) = context.row_path.split_first() else { return raw }; + let Some((first, rest)) = context.row_path.split_first() else { return raw }; if !matches!(raw, Value::Number(_)) { return raw; } - let mut total = 0.0; - for item in context.level0 { - let mut path = context.row_path.to_vec(); - if let Some(first_item) = path.first_mut() { - *first_item = *item; + let mut index = context.cache.items_by_rest.borrow_mut(); + let items_by_rest = index.get_or_insert_with(|| { + let position: HashMap = context.level0.iter().enumerate().rev().map(|(at, item)| (*item, at)).collect(); + let mut map = ItemsByRest::new(); + for (row, col) in lookup.aggs.keys() { + if let Some((item, rest)) = row.split_first() + && position.contains_key(item) + { + map.entry((rest.to_vec(), col.clone())).or_default().push(*item); + } } - total += number(&lookup.get(&path, context.col_path, context.value_index)).unwrap_or(0.0); - if item == first { - break; + for items in map.values_mut() { + items.sort_by_key(|item| position.get(item).copied().unwrap_or(usize::MAX)); } - } - Value::number(total) + map + }); + let mut cache = context.cache.running.borrow_mut(); + let (by_item, all) = cache.entry((rest.to_vec(), context.col_path.to_vec(), context.value_index)).or_insert_with(|| { + let mut total = 0.0; + let mut by_item = HashMap::new(); + for item in items_by_rest.get(&(rest.to_vec(), context.col_path.to_vec())).into_iter().flatten() { + let mut path = Vec::with_capacity(rest.len() + 1); + path.push(*item); + path.extend_from_slice(rest); + // An item with no number here adds nothing and is not stored (the shown cell always has one). + let Some(value) = number(&lookup.get(&path, context.col_path, context.value_index)) else { continue }; + total += value; + by_item.entry(*item).or_insert(total); + } + (by_item, total) + }); + Value::number(by_item.get(first).copied().unwrap_or(*all)) } fn ranked_value(raw: Value, lookup: &ValueLookup<'_>, context: &ShowContext<'_>) -> Value { let Some((_, parent)) = context.row_path.split_last() else { return Value::Empty }; let Some(me) = raw.as_f64() else { return raw }; - let siblings = context.siblings.get(parent).cloned().unwrap_or_default(); + let mut cache = context.cache.ranks.borrow_mut(); + let sorted = cache.entry((parent.to_vec(), context.col_path.to_vec(), context.value_index)).or_insert_with(|| { + let mut all: Vec = context + .siblings + .get(parent) + .into_iter() + .flatten() + .filter_map(|sibling| { + let mut path = parent.to_vec(); + path.push(*sibling); + number(&lookup.get(&path, context.col_path, context.value_index)) + }) + .collect(); + all.sort_by(f64::total_cmp); + all.dedup(); + all + }); // Excel's ranks are dense: ties share a rank and the next value takes the next one // (5, 4, 3, 3, 4 rank 1, 2, 3, 3, 2), so count the distinct larger values. - let mut larger: Vec = siblings - .into_iter() - .filter_map(|sibling| { - let mut path = parent.to_vec(); - path.push(sibling); - number(&lookup.get(&path, context.col_path, context.value_index)).filter(|value| *value > me) - }) - .collect(); - larger.sort_by(f64::total_cmp); - larger.dedup(); - Value::number(larger.len() as f64 + 1.0) + let larger = sorted.len() - sorted.partition_point(|value| *value <= me); + Value::number(larger as f64 + 1.0) } fn show_value(definition: &PivotValue, raw: Value, lookup: &ValueLookup<'_>, context: &ShowContext<'_>) -> Value { @@ -642,6 +698,7 @@ pub(super) fn compute_from(wb: &Workbook, pt: &PivotTable, src: &Source, mut pro }) .collect(); let lookup = ValueLookup { wb, source: src, calc: &calc, vals: &vals, extra: &extra, aggs: &aggs, value_count: k }; + let show_cache = ShowCache::default(); let mut chart_rows: Vec = Vec::new(); let chart_cols: Vec<(u32, String)> = data_cols .iter() @@ -699,7 +756,7 @@ pub(super) fn compute_from(wb: &Workbook, pt: &PivotTable, src: &Source, mut pro let Some((_, vdef)) = vals.get(*vi) else { continue }; let cp = spec.path.as_slice(); let Some(raw) = lookup.get(rp, cp, *vi) else { continue }; - let context = ShowContext { row_path: rp, col_path: cp, value_index: *vi, level0: &level0, siblings: &siblings }; + let context = ShowContext { row_path: rp, col_path: cp, value_index: *vi, level0: &level0, siblings: &siblings, cache: &show_cache }; let shown = show_value(vdef, raw, &lookup, &context); if let Some(pr) = probe.as_deref_mut() { if pr.at == Some((r, l + j)) { diff --git a/crates/engine/src/pivot/output.rs b/crates/engine/src/pivot/output.rs index c3d6ee88d..ca680335b 100644 --- a/crates/engine/src/pivot/output.rs +++ b/crates/engine/src/pivot/output.rs @@ -121,6 +121,18 @@ pub(super) enum ColKind { Grand, } +/// Longest item caption Excel keeps; a subtotal caption repeats its parent's label for every value +/// field at every level, so a long source label is cut rather than multiplied. +const MAX_CAPTION_LABEL_CHARS: usize = 255; + +/// `label` (cut to [`MAX_CAPTION_LABEL_CHARS`]) and `suffix`, as in "East Total". +pub(super) fn caption(label: &str, suffix: &str) -> String { + let mut out: String = label.chars().take(MAX_CAPTION_LABEL_CHARS).collect(); + out.push(' '); + out.push_str(suffix); + out +} + pub(super) struct ColSpec { pub(super) kind: ColKind, pub(super) path: Vec, @@ -162,7 +174,7 @@ impl Ctx<'_> { leaf: false, path: prefix.to_vec(), values: true, - labels: vec![(col, Value::text(format!("{label} {name}")), indent, None)], + labels: vec![(col, Value::text(caption(label, name)), indent, None)], vi: Some(vi), }); } @@ -249,7 +261,7 @@ impl Ctx<'_> { leaf: false, path: prefix.clone(), values: true, - labels: vec![(col, Value::text(format!("{label} Total")), indent, None)], + labels: vec![(col, Value::text(caption(&label, "Total")), indent, None)], vi, }); } @@ -316,7 +328,7 @@ impl Ctx<'_> { leaf: false, path: prefix.clone(), values: true, - labels: vec![(col, Value::text(format!("{label} Total")), 0, None)], + labels: vec![(col, Value::text(caption(&label, "Total")), 0, None)], vi, }); } @@ -438,3 +450,17 @@ pub(super) fn cell_style(lk: &Look, rk: RowKind, cell: &OutCell, last_header: bo } s } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn captions_cut_a_long_parent_label() { + assert_eq!(caption("East", "Total"), "East Total"); + let long = "é".repeat(100_000); + let c = caption(&long, "Sum of Sales"); + assert_eq!(c.chars().count(), MAX_CAPTION_LABEL_CHARS + " Sum of Sales".len()); + assert!(c.ends_with(" Sum of Sales")); + } +} diff --git a/crates/engine/src/pivot_calc.rs b/crates/engine/src/pivot_calc.rs index 014ad2db8..e8b4df271 100644 --- a/crates/engine/src/pivot_calc.rs +++ b/crates/engine/src/pivot_calc.rs @@ -153,6 +153,11 @@ pub fn compile(formula: &str, field: &dyn Fn(&str) -> Option) -> Result< Expr::Name(n) if n.starts_with(PREFIX) => {} Expr::Name(n) => problem = Some(format!("There's no field named '{n}'.")), Expr::Ref(_) | Expr::Struct(_) => problem = Some("A calculated field can't refer to cells or tables; use field names.".into()), + // Evaluated once per group and per report, with only the first value kept: Excel does not + // allow functions that need arrays in a calculated field, and these build one from scalars. + Expr::Call(f, _) if matches!(f.as_str(), "SEQUENCE" | "RANDARRAY" | "MAKEARRAY" | "EXPAND" | "MUNIT") => { + problem = Some(format!("A calculated field can't use {f}: it builds an array.")) + } _ => {} } }); @@ -332,6 +337,16 @@ mod tests { assert!(compile(&format!("={}", "1+".repeat(MAX_FORMULA_LEN)), &fields).is_err()); } + #[test] + fn a_calculated_field_cannot_build_arrays_from_scalars() { + for f in + ["=SEQUENCE(5000,1000)", "=SUM(RANDARRAY(1000,1000))", "=MAKEARRAY(9,9,LAMBDA(r,c,r*c))+Sales", "=expand(Sales,1000,1000)", "=MUNIT(500)"] + { + assert!(compile(f, &fields).unwrap_err().contains("builds an array"), "{f}"); + } + assert!(compile("=SUM({1,2,3})*Sales", &fields).is_ok(), "an array constant is bounded by the formula's own length"); + } + #[test] fn deep_and_cyclic_chains_are_bounded() { // 300 calculated fields, each using the previous one, and a two-field cycle. diff --git a/crates/engine/src/selection.rs b/crates/engine/src/selection.rs index 3f3feee9d..c0c6959cd 100644 --- a/crates/engine/src/selection.rs +++ b/crates/engine/src/selection.rs @@ -43,6 +43,10 @@ impl Selection { pub fn is_single_cell(&self) -> bool { self.ranges.len() == 1 && self.current().is_single() } + /// Whether the selection is exactly one merged cell, which Enter and Tab treat as one cell. + pub fn is_one_merged_cell(&self, sheet: &Sheet) -> bool { + self.ranges.len() == 1 && sheet.merge_at(self.active).is_some_and(|m| m == self.current()) + } /// The corner of the current area opposite the anchor: the end that Shift+arrows move. pub fn moving_end(&self) -> CellRef { let r = self.current(); @@ -67,23 +71,28 @@ impl Selection { /// they are: clicking column B's header selects just B even when B1:C1 is merged, as in Excel. pub fn expand_merges(&mut self, sheet: &Sheet) { for r in self.ranges.iter_mut() { - if r.is_full_cols() || r.is_full_rows() { - continue; - } - let mut cur = *r; - loop { - let mut grown = cur; - for m in &sheet.merges { - if m.intersects(&grown) { - grown = grown.union(m); - } - } - if grown == cur { - break; - } - cur = grown; - } - *r = cur; + *r = with_merges(sheet, *r); + } + } + /// Shift+arrows: moves the edge of the current area that the anchor doesn't hold by `dr` + /// rows and `dc` columns, keeping merged cells whole, as Excel does. The anchor is the whole + /// merged cell it is in, so from a selected merged cell Shift+Left and Shift+Up widen the + /// selection past its left and top edges. A step that would shrink the area onto the inside + /// of a merged cell (where keeping it whole would undo the step) goes on to the next line + /// that does shrink it, and when the area can't shrink any more it grows in that direction. + pub fn extend_by(&mut self, sheet: &Sheet, dr: i64, dc: i64) { + let anchor = sheet.merge_at(self.anchor).unwrap_or(RangeRef::cell(self.anchor)); + let mut r = self.current(); + if dc != 0 { + r = step_edge(sheet, r, anchor, dc, true); + } + if dr != 0 { + r = step_edge(sheet, r, anchor, dr, false); + } + if let Some(last) = self.ranges.last_mut() { + *last = r; + } else { + self.ranges.push(r); } } /// Moves the active cell within the selection (Enter/Tab inside a multi-cell selection). @@ -128,6 +137,91 @@ impl Selection { } } +/// `r` grown until every merged cell it touches is inside it. Whole columns and whole rows stay +/// as they are. +fn with_merges(sheet: &Sheet, r: RangeRef) -> RangeRef { + if r.is_full_cols() || r.is_full_rows() { + return r; + } + let mut cur = r; + loop { + let mut grown = cur; + for m in &sheet.merges { + if m.intersects(&grown) { + grown = grown.union(m); + } + } + if grown == cur { + return cur; + } + cur = grown; + } +} + +/// One Shift+arrow step of [`Selection::extend_by`] along one axis (`cols`: columns, else rows). +fn step_edge(sheet: &Sheet, r: RangeRef, anchor: RangeRef, d: i64, cols: bool) -> RangeRef { + let max = i64::from(if cols { MAX_COLS } else { MAX_ROWS }) - 1; + let hidden = |i: i64| if cols { sheet.is_col_hidden(i as u32) } else { sheet.is_row_hidden(i as u32) }; + // `from` moved by `by`, then on past hidden lines in the same direction (not off the sheet). + let land = |from: i64, by: i64| { + let mut t = from.saturating_add(by).clamp(0, max); + // Bounded like the arrow keys' walk over hidden lines. + for _ in 0..100_000 { + if !hidden(t) { + break; + } + let n = (t + by.signum()).clamp(0, max); + if n == t { + break; + } + t = n; + } + t + }; + let span = |x: &RangeRef| { + if cols { (i64::from(x.start.col), i64::from(x.end.col)) } else { (i64::from(x.start.row), i64::from(x.end.row)) } + }; + let with = |lo: i64, hi: i64| { + let (lo, hi) = (lo.clamp(0, max) as u32, hi.clamp(0, max) as u32); + let (s, e) = if cols { + (CellRef::new(r.start.row, lo), CellRef::new(r.end.row, hi)) + } else { + (CellRef::new(lo, r.start.col), CellRef::new(hi, r.end.col)) + }; + with_merges(sheet, RangeRef::new(s, e)) + }; + let (lo, hi) = span(&r); + let (alo, ahi) = span(&anchor); + // The anchor holds the low edge unless the area reaches past it on that side. + let moving_hi = lo >= alo; + let shrinking = if moving_hi { d < 0 && hi > ahi } else { d > 0 && lo < alo }; + if shrinking { + let mut t = if moving_hi { hi } else { lo }; + let mut by = d; + loop { + t = if moving_hi { land(t, by).max(ahi) } else { land(t, by).min(alo) }; + let cand = if moving_hi { with(lo, t) } else { with(t, hi) }; + if cand != r { + return cand; + } + if t == if moving_hi { ahi } else { alo } { + break; + } + // The step was undone because a merged cell crosses the new edge, and so is every + // edge up to where that merged cell starts (ends, for the low edge): go on from there + // so a tall or wide merged cell costs one step, not one per line it covers. + let across = sheet.merges.iter().filter(|m| m.intersects(&r)).filter_map(|m| { + let (ms, me) = span(m); + if moving_hi { (ms <= t && me > t).then_some(ms) } else { (ms < t && me >= t).then_some(me) } + }); + t = if moving_hi { across.min().map_or(t, |ms| ms.min(t)) } else { across.max().map_or(t, |me| me.max(t)) }; + by = d.signum(); + } + } + // Grow the area on the side the arrow points to. + if d > 0 { with(lo, land(hi, d)) } else { with(land(lo, d), hi) } +} + impl Selection { /// Ctrl+. (Next Corner of Selection): the active cell moves clockwise to the next corner of /// the current area (top-left, top-right, bottom-right, bottom-left) and becomes the anchor, @@ -287,6 +381,36 @@ mod tests { assert_eq!(current_region(&s, CellRef::new(5, 0)).a1(), "A1:B20000"); } + #[test] + fn shift_arrows_from_a_merged_cell() { + let mut s = Sheet::new("S"); + s.merges.push(RangeRef::parse("B2:C3").unwrap()); + let at = |s: &Sheet, a1: &str| { + let mut sel = Selection::at(CellRef::parse(a1).unwrap()); + sel.expand_merges(s); + sel + }; + for ((dr, dc), want) in [((0, -1), "A2:C3"), ((-1, 0), "B1:C3"), ((0, 1), "B2:D3"), ((1, 0), "B2:C4")] { + let mut sel = at(&s, "B2"); + sel.extend_by(&s, dr, dc); + assert_eq!(sel.a1(), want); + } + // Shrinking onto the inside of the merged cell goes on past it. + let mut sel = at(&s, "A1"); + sel.extend_by(&s, 1, 1); + assert_eq!(sel.a1(), "A1:C3"); + sel.extend_by(&s, 0, -1); + assert_eq!(sel.a1(), "A1:A3"); + // A hidden column is stepped over, and the sheet edge stops the area. + s.cols.entry(3).or_default().hidden = true; + let mut sel = at(&s, "B2"); + sel.extend_by(&s, 0, 1); + assert_eq!(sel.a1(), "B2:E3"); + let mut sel = at(&s, "A1"); + sel.extend_by(&s, -1, -1); + assert_eq!(sel.a1(), "A1"); + } + #[test] fn cycling() { let mut sel = Selection::range(RangeRef::parse("A1:B2").unwrap()); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 08fc9119f..747c0cd52 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -532,6 +532,118 @@ fn shape_text_edit_survives_xlsx_roundtrip() { assert_eq!(shape.text, text); } +#[test] +fn word_art_insert_respects_sheet_protection() { + let mut s = s(); + s.execute("review.protectSheet", json!({})).unwrap(); + let before = s.doc().unwrap().wb.clone(); + let undo_len = s.doc().unwrap().undo.len(); + assert!(s.execute("insert.wordArt", json!({})).is_err()); + assert!(std::sync::Arc::ptr_eq(&s.doc().unwrap().wb, &before)); + assert_eq!(s.doc().unwrap().undo.len(), undo_len); + s.execute("review.unprotectSheet", json!({})).unwrap(); + s.execute("insert.wordArt", json!({})).unwrap(); + assert_eq!(s.doc().unwrap().wb.active().unwrap().shapes.len(), 1); +} + +#[test] +fn word_art_inserts_with_defaults_and_undoes() { + use gridcraft_model::ShapeKind; + let mut s = s(); + // Empty params: default style and text, no dialog. + let r = s.execute("insert.wordArt", json!({})).unwrap(); + assert!(s.take_ui_requests().is_empty()); + let shape = s.doc().unwrap().wb.active().unwrap().shapes[0].clone(); + assert_eq!(shape.id, r["shape"].as_u64().unwrap() as u32); + assert_eq!(shape.kind, ShapeKind::WordArt); + assert_eq!(shape.text, crate::cmd::wordart::DEFAULT_TEXT); + assert!(shape.anchor.width > 8.0 && shape.format.text_color.is_some()); + // Named style plus overrides; restyle and edit the text through wordArt.set. + s.execute("insert.wordArt", json!({"style": "ember", "text": "Q3", "fontSize": 60, "shadow": false, "outline": "#112233", "outlineWidth": 3})) + .unwrap(); + let ember = s.doc().unwrap().wb.active().unwrap().shapes[1].clone(); + assert_eq!(ember.format.font_size, Some(60.0)); + assert_eq!((ember.format.text_shadow, ember.format.text_outline.map(|o| o.1)), (None, Some(3.0))); + s.execute("wordArt.set", json!({"id": ember.id, "style": 3, "text": "Q4"})).unwrap(); + let styled = s.doc().unwrap().wb.active().unwrap().shapes[1].clone(); + assert_eq!((styled.text.as_str(), styled.format.italic), ("Q4", true)); + s.execute("shape.setText", json!({"id": ember.id, "text": "Q5"})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(s.doc().unwrap().wb.active().unwrap().shapes[1], ember); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(s.doc().unwrap().wb.active().unwrap().shapes[1].text, "Q4"); + // Moves and deletes like any floating shape. + s.execute("object.move", json!({"kind": "shape", "id": ember.id, "at": "D4"})).unwrap(); + s.execute("object.delete", json!({"kind": "shape", "id": ember.id})).unwrap(); + assert_eq!(s.doc().unwrap().wb.active().unwrap().shapes.len(), 1); +} + +#[test] +fn word_art_rejects_hostile_params() { + let mut s = s(); + let id = s.execute("insert.wordArt", json!({})).unwrap()["shape"].clone(); + let rectangle = s.execute("insert.shape", json!({"kind": "rectangle"})).unwrap()["shape"].clone(); + let before = s.doc().unwrap().wb.clone(); + for (cmd, params) in [ + ("insert.wordArt", json!({"style": "No Such Style"})), + ("insert.wordArt", json!({"style": 99})), + ("insert.wordArt", json!({"style": -1})), + ("insert.wordArt", json!({"text": 5})), + ("insert.wordArt", json!({"text": ""})), + ("insert.wordArt", json!({"text": "x".repeat(crate::cmd::wordart::MAX_CHARS + 1)})), + ("insert.wordArt", json!({"fill": "not a colour"})), + ("wordArt.set", json!({})), + ("wordArt.set", json!({"id": 999})), + ("wordArt.set", json!({"id": rectangle})), + ("shape.setText", json!({"id": id, "text": ""})), + ] { + assert!(matches!(s.execute(cmd, params), Err(crate::EngineError::BadParams { .. })), "{cmd}"); + assert!(std::sync::Arc::ptr_eq(&s.doc().unwrap().wb, &before)); + } + // Absurd sizes are clamped, not trusted. + s.execute("insert.wordArt", json!({"fontSize": 1e12, "width": 1e300, "height": -5})).unwrap(); + let big = &s.doc().unwrap().wb.active().unwrap().shapes[2]; + assert!(big.anchor.width <= 4000.0 && big.anchor.height >= 8.0 && big.format.font_size == Some(400.0)); +} + +#[test] +fn word_art_survives_xlsx_roundtrip() { + use gridcraft_model::ShapeKind; + let mut s = s(); + s.execute("insert.wordArt", json!({"style": "Lagoon", "text": "Sales <2026>\nOutlook & more"})).unwrap(); + let before = s.doc().unwrap().wb.active().unwrap().shapes[0].clone(); + let saved = s.execute("file.saveBytes", json!({"format": "xlsx"})).unwrap(); + s.execute("file.open", json!({"name": "wordart.xlsx", "base64": saved["base64"]})).unwrap(); + let after = &s.doc().unwrap().wb.active().unwrap().shapes[0]; + assert_eq!(after.kind, ShapeKind::WordArt); + assert_eq!(after.text, before.text); + assert_eq!(after.format.text_color, before.format.text_color); + assert_eq!(after.format.text_outline, before.format.text_outline); + assert_eq!(after.format.text_shadow, before.format.text_shadow); + assert_eq!((after.format.bold, after.format.italic, after.format.font_size), (true, true, before.format.font_size)); + assert!(after.format.no_line); +} + +/// White text with a green outline (Mint Outline) prints its outline, so it is not invisible on a +/// white page. +#[test] +fn word_art_outline_prints() { + let mut s = s(); + s.execute("insert.wordArt", json!({"style": "Mint Outline", "text": "VISIBLE", "at": "A1"})).unwrap(); + let pdf = s.execute("file.exportPdf", json!({"range": "A1:G10"})).unwrap(); + let data = crate::io::base64_decode(pdf["base64"].as_str().unwrap()).unwrap(); + let text = String::from_utf8_lossy(&data); + let outline = text.matches("0.18 0.608 0.416 rg").count(); + assert!(outline >= 1, "the outline colour (2E9B6A) is drawn"); + assert_eq!(text.matches("(VISIBLE) Tj").count(), 9, "eight outline copies under the white text"); + assert!(text.contains("1 1 1 rg"), "the white fill is drawn on top"); + // The last copy drawn is the filled text, in white; every outline copy comes before it. + let before_fill = &text[..text.rfind("(VISIBLE) Tj").unwrap()]; + assert!(before_fill.rfind("1 1 1 rg") > before_fill.rfind("0.18 0.608 0.416 rg"), "the white fill is drawn last"); + assert_eq!(before_fill.matches("(VISIBLE) Tj").count(), 8); +} + #[test] fn no_border_clears_only_shared_edges_and_undo_restores_them() { use gridcraft_core::RangeRef; @@ -1350,6 +1462,18 @@ fn scripts_leave_file_access_working_after_they_run() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn get_data_refuses_json_with_too_many_fields_or_rows_before_building_rows() { + let mut s = s(); + let wide = format!("[{}]", (0..=gridcraft_core::MAX_COLS).map(|i| format!("{{\"k{i}\":1}}")).collect::>().join(",")); + let err = s.execute("data.fromTextCsv", json!({"text": wide})).unwrap_err().to_string(); + assert!(err.contains("more fields"), "{err}"); + // MAX_ROWS objects plus the header row are one too many. + let tall = format!("[{}]", vec!["{}"; gridcraft_core::MAX_ROWS as usize].join(",")); + let err = s.execute("data.fromTextCsv", json!({"text": tall})).unwrap_err().to_string(); + assert!(err.contains("more rows"), "{err}"); +} + #[test] fn scripts_cannot_reach_outside_the_workbook_even_when_trusted() { let dir = std::env::temp_dir().join(format!("gridcraft-script-deny-{}", std::process::id())); @@ -2816,6 +2940,127 @@ fn validation_formulas_are_relative_and_see_the_new_value() { assert!(!ok(&mut s, "C1", "50")); } +/// Every value of the active sheet as the incremental recalculation left it must equal a full one. +fn assert_matches_full_recalc(s: &Session) { + let wb = &s.doc().unwrap().wb; + let mut fresh = (**wb).clone(); + gridcraft_calc::Calc::new().recalc_all(&mut fresh); + let (a, b) = (wb.active().unwrap(), fresh.active().unwrap()); + for row in 0..30 { + for col in 0..12 { + let cell = CellRef { row, col }; + assert_eq!(a.value(cell), b.value(cell), "{cell} in memory vs after a full recalc"); + } + } +} + +#[test] +fn filling_a_formula_next_to_a_spill_matches_a_full_recalc() { + // Found by the command-sequence fuzzer: C14 spills along row 14; filling `=D14:E18` over + // E13:F13 puts two array formulas whose areas run into that spill and into each other. The + // incremental pass left F13 as #SPILL! while a full recalc gave F13 its array and C14 + // #SPILL!. The spill that already exists keeps its cells (as in Excel), in both. + let mut s = s(); + s.execute("range.setValues", json!({"range": "B18", "values": [[1], [2], [3], [4], [5], [6], [7]]})).unwrap(); + s.execute("cell.set", json!({"cell": "C14", "input": "=TRANSPOSE(B18:B24)"})).unwrap(); + s.execute("selection.set", json!({"range": "E13"})).unwrap(); + s.execute("range.fill", json!({"range": "E13:F13", "input": "=D14:E18"})).unwrap(); + assert_eq!(v(&s, "C14"), Value::Number(1.0)); + assert_eq!(v(&s, "I14"), Value::Number(7.0)); + assert_eq!(v(&s, "F13"), Value::Error(gridcraft_core::CellError::Spill)); + assert_matches_full_recalc(&s); +} + +#[test] +fn overlapping_spills_keep_the_one_that_exists_whichever_anchor_is_first() { + use gridcraft_core::CellError::Spill; + // E10 spills along row 10, F8 down over F10: F8 is first in reading order, E10 may be first + // to spill. Excel keeps whichever spilled first. + for (first, second) in [(("E10", "=SEQUENCE(1,3)"), ("F8", "=SEQUENCE(5)")), (("F8", "=SEQUENCE(5)"), ("E10", "=SEQUENCE(1,3)"))] { + let mut s = s(); + s.execute("cell.set", json!({"cell": first.0, "input": first.1})).unwrap(); + s.execute("cell.set", json!({"cell": second.0, "input": second.1})).unwrap(); + assert_ne!(v(&s, first.0), Value::Error(Spill), "{} keeps its cells", first.0); + assert_eq!(v(&s, second.0), Value::Error(Spill), "{} is blocked", second.0); + assert_matches_full_recalc(&s); + // Clearing the winner lets the other one spill. + s.execute("edit.clearContents", json!({"range": first.0})).unwrap(); + assert_ne!(v(&s, second.0), Value::Error(Spill), "{} spills once {} is cleared", second.0, first.0); + assert_matches_full_recalc(&s); + } +} + +#[test] +fn a_spill_growing_into_another_and_back_matches_a_full_recalc_at_every_step() { + use gridcraft_core::CellError::Spill; + // Both arrays already spill; F8 (first in reading order) grows down over E10's spill, takes + // its cells, shrinks back and lets it spill again, then the steps are undone, redone, edited + // inside the contested area and cleared. Every state must match a full recalc. + let mut s = s(); + s.execute("cell.set", json!({"cell": "H1", "input": "1"})).unwrap(); + s.execute("cell.set", json!({"cell": "F8", "input": "=SEQUENCE(H1)"})).unwrap(); + s.execute("cell.set", json!({"cell": "E10", "input": "=SEQUENCE(1,3)*10"})).unwrap(); + assert_eq!(v(&s, "G10"), Value::Number(30.0)); + assert_matches_full_recalc(&s); + let steps: [(&str, serde_json::Value); 9] = [ + ("cell.set", json!({"cell": "H1", "input": "5"})), + ("cell.set", json!({"cell": "E10", "input": "=SEQUENCE(1,2)*10"})), + ("cell.set", json!({"cell": "H1", "input": "1"})), + ("edit.undo", json!({})), + ("edit.undo", json!({})), + ("edit.redo", json!({})), + ("cell.set", json!({"cell": "F11", "input": "x"})), + ("cell.set", json!({"cell": "H1", "input": "2"})), + ("edit.clearContents", json!({"range": "E8:F10"})), + ]; + for (i, (id, params)) in steps.iter().enumerate() { + s.execute(id, params.clone()).unwrap(); + assert_matches_full_recalc(&s); + if i == 0 { + // Both held a spill: reading order gives F8 the cells. + assert_eq!(v(&s, "F12"), Value::Number(5.0)); + assert_eq!(v(&s, "E10"), Value::Error(Spill)); + } + if i == 2 { + assert_eq!(v(&s, "E10"), Value::Number(10.0)); + assert_eq!(v(&s, "F10"), Value::Number(20.0)); + } + if i == 3 { + // Undone back to the grown F8. + assert_eq!(v(&s, "F12"), Value::Number(5.0)); + assert_eq!(v(&s, "E10"), Value::Error(Spill)); + } + if i == 8 { + assert_eq!((v(&s, "F8"), v(&s, "E10"), v(&s, "F9")), (Value::Empty, Value::Empty, Value::Empty)); + } + } +} + +#[test] +fn overlapping_spills_with_no_history_go_to_the_earlier_anchor() { + use gridcraft_core::CellError::Spill; + // A workbook calculated for the first time (a file that carries no layout) has no entry + // order: the anchor first in reading order wins, whatever order the formulas were set in. + for order in [["E10", "F8"], ["F8", "E10"]] { + let mut s = s(); + for a in order { + let input = if a == "E10" { "=SEQUENCE(1,3)" } else { "=SEQUENCE(5)" }; + s.execute("cell.set", json!({"cell": a, "input": input})).unwrap(); + } + let mut fresh = (*s.doc().unwrap().wb).clone(); + for i in 0..fresh.sheets.len() { + let sh = fresh.sheet_mut(i).unwrap(); + sh.spill.clear(); + sh.spill_ranges.clear(); + } + gridcraft_calc::Calc::new().recalc_all(&mut fresh); + let a = fresh.active().unwrap(); + let cell = |r: &str| a.value(CellRef::parse(r).unwrap()); + assert_ne!(cell("F8"), Value::Error(Spill), "order {order:?}"); + assert_eq!(cell("E10"), Value::Error(Spill), "order {order:?}"); + } +} + #[test] fn multi_area_rule_formulas_are_relative_to_the_top_left_cell_of_all_areas() { // Issue #3152: Excel writes the formula for the top-left cell of all the areas (H26), whatever @@ -3600,6 +3845,8 @@ fn xls_fixture() -> Vec { let mut put = |at: usize, v: &[u8]| b[at..at + v.len()].copy_from_slice(v); put(0, &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]); put(26, &3u16.to_le_bytes()); + // The byte order mark [MS-CFB] 2.2 requires; craft-cfb refuses a header without it. + put(28, &0xFFFEu16.to_le_bytes()); put(30, &9u16.to_le_bytes()); put(32, &6u16.to_le_bytes()); put(44, &1u32.to_le_bytes()); @@ -6252,6 +6499,16 @@ mod objects; mod python_cells; mod python_commands; mod python_files; + +#[test] +fn text_to_columns_keeps_a_mid_field_quote() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "'5\" pipe,abc"})).unwrap(); + s.execute("data.textToColumns", json!({"range": "A1", "delimiters": [","]})).unwrap(); + assert_eq!(v(&s, "A1"), Value::text("5\" pipe")); + assert_eq!(v(&s, "B1"), Value::text("abc")); +} + mod seams; #[test] @@ -6468,6 +6725,36 @@ fn clearing_links_and_notes_keeps_allow_edit_ranges() { assert!(sh.hyperlinks.is_empty() && sh.comments.is_empty()); } +#[test] +fn replace_does_not_search_hidden_formula_source() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "=SEQUENCE(3)*7"})).unwrap(); + s.execute("home.formatCells", json!({"range": "A1", "style": {"protection": {"locked": true, "hidden": true}}})).unwrap(); + s.execute("review.protectSheet", json!({})).unwrap(); + // Both give the same answer, and it is not an error: a hidden formula's source is not searched. + let hit = s.execute("edit.replace", json!({"what": "SEQUENCE", "with": "X"})).unwrap(); + let miss = s.execute("edit.replace", json!({"what": "NOSUCHTEXT", "with": "X"})).unwrap(); + assert_eq!(hit, miss); + assert_eq!(hit["replaced"], json!(0), "{hit}"); +} + +#[test] +fn formulatext_does_not_reveal_a_hidden_formula() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "=1+1"})).unwrap(); + s.execute("home.formatCells", json!({"range": "A1", "style": {"protection": {"locked": false, "hidden": true}}})).unwrap(); + s.execute("home.formatCells", json!({"range": "B1", "style": {"protection": {"locked": false}}})).unwrap(); + s.execute("cell.set", json!({"cell": "B1", "input": "=FORMULATEXT(A1)"})).unwrap(); + assert_eq!(v(&s, "B1"), Value::from("=1+1"), "not protected yet"); + s.execute("review.protectSheet", json!({})).unwrap(); + // Without entering the formula again: protecting recalculates what reads the protection. + assert_eq!(v(&s, "B1"), Value::Error(gridcraft_core::CellError::NA)); + s.execute("cell.set", json!({"cell": "B1", "input": "=FORMULATEXT(A1)"})).unwrap(); + assert_eq!(v(&s, "B1"), Value::Error(gridcraft_core::CellError::NA)); + s.execute("review.unprotectSheet", json!({})).unwrap(); + assert_eq!(v(&s, "B1"), Value::from("=1+1"), "unprotecting shows it again without re-entering"); +} + #[test] fn protected_sheet_refuses_layout_commands_its_permissions_do_not_allow() { let mut s = s(); @@ -6526,3 +6813,39 @@ fn pivot_new_sheet_destination_respects_workbook_structure_protection() { } assert!(s.execute("insert.pivotTable", json!({"source": "A1:B3", "location": "New"})).is_err()); } + +#[test] +fn circle_invalid_says_when_its_scan_allowance_ended_early() { + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [[-1]]})).unwrap(); + s.execute("range.setValues", json!({"range": "A200000", "values": [[-1]]})).unwrap(); + s.execute("data.validation", json!({"range": "A1:A5", "type": "whole", "operator": "greater", "formula1": "0"})).unwrap(); + let out = s.execute("data.circleInvalid", json!({})).unwrap(); + assert_eq!((out["invalid"].as_array().map(Vec::len), out["truncated"].as_bool()), (Some(1), Some(false)), "{out}"); + s.execute("data.validation", json!({"range": "A1:A200000", "type": "whole", "operator": "greater", "formula1": "0"})).unwrap(); + let out = s.execute("data.circleInvalid", json!({})).unwrap(); + assert_eq!(out["truncated"].as_bool(), Some(true), "{out}"); +} + +#[test] +fn history_charges_a_changed_sheets_comment_text() { + let mut s = s(); + s.execute("review.newNote", json!({"cell": "A1", "text": "n".repeat(30_000), "author": "Example"})).unwrap(); + let older = (*s.doc().unwrap().wb).clone(); + s.execute("cell.set", json!({"cell": "B2", "input": "x"})).unwrap(); + let newer = (*s.doc().unwrap().wb).clone(); + // The older snapshot owns the comment text now that the sheet changed under it. + assert!(crate::history_retained_bytes(&older, &newer) >= 30_000); +} + +#[test] +fn history_charges_the_storage_of_empty_comment_replies() { + let mut s = s(); + s.execute("review.newComment", json!({"cell": "A1", "text": "c", "author": "Ann"})).unwrap(); + for _ in 0..500 { + s.execute("review.replyComment", json!({"cell": "A1", "text": "", "author": ""})).unwrap(); + } + let sheet = s.doc().unwrap().wb.sheet(0).unwrap().clone(); + // 500 replies of no text are still 500 elements. + assert!(crate::sheet_text_bytes(&sheet) >= 500 * std::mem::size_of::<(String, String)>()); +} diff --git a/crates/engine/src/tests/python_cells.rs b/crates/engine/src/tests/python_cells.rs index 63cf1d92b..8c7805dc4 100644 --- a/crates/engine/src/tests/python_cells.rs +++ b/crates/engine/src/tests/python_cells.rs @@ -175,8 +175,8 @@ fn text_to_columns_keeps_a_py_piece_as_text() { s.execute("cell.set", json!({"cell": "A1", "input": "'=PY(\"x\");7"})).unwrap(); assert_eq!(super::v(&s, "A1"), Value::text("=PY(\"x\");7")); s.execute("data.textToColumns", json!({"range": "A1", "delimiters": [";"]})).unwrap(); - // The default text qualifier takes the quotes away, as in Excel. - assert_eq!(super::v(&s, "A1"), Value::text("=PY(x)"), "the piece is kept, not cleared"); + // A quote in the middle of a field is kept, as in Excel. + assert_eq!(super::v(&s, "A1"), Value::text("=PY(\"x\")"), "the piece is kept, not cleared"); assert_eq!(code(&s, 0, "A1"), None, "and it is not a Python cell"); assert_eq!(super::v(&s, "B1"), Value::Number(7.0)); } diff --git a/crates/engine/tests/it/main.rs b/crates/engine/tests/it/main.rs index ca960d759..d73a70235 100644 --- a/crates/engine/tests/it/main.rs +++ b/crates/engine/tests/it/main.rs @@ -18,6 +18,7 @@ mod hidden_sheet_navigation; mod ink_roundtrip; mod line_count_overflow; mod long_operator_chains; +mod merged_selection_keys; mod offset_overflow; mod percent_entry; mod pivot_unknown_func; diff --git a/crates/engine/tests/it/merged_selection_keys.rs b/crates/engine/tests/it/merged_selection_keys.rs new file mode 100644 index 000000000..dcb689843 --- /dev/null +++ b/crates/engine/tests/it/merged_selection_keys.rs @@ -0,0 +1,162 @@ +//! Shift+arrows and Enter/Tab on merged cells, as Excel does them. + +use gridcraft_engine::Session; +use serde_json::json; + +/// A new workbook with B2:C3 merged and nothing selected but A1. +fn merged_book() -> Session { + let mut s = Session::new(); + s.new_workbook(); + s.execute("selection.set", json!({"range": "B2:C3"})).unwrap(); + s.execute("home.mergeCells", json!({})).unwrap(); + s.execute("selection.set", json!({"cell": "A1"})).unwrap(); + s +} + +fn shift(s: &mut Session, dr: i64, dc: i64) -> String { + s.execute("selection.move", json!({"dr": dr, "dc": dc, "extend": true})).unwrap(); + s.doc().unwrap().selection.a1() +} + +fn enter(s: &mut Session, forward: bool, by_row: bool) -> (String, String) { + s.execute("selection.next", json!({"forward": forward, "byRow": by_row})).unwrap(); + let sel = &s.doc().unwrap().selection; + (sel.active.a1(), sel.a1()) +} + +#[test] +fn shift_arrows_from_a_merged_cell_widen_past_each_edge() { + let mut s = merged_book(); + for ((dr, dc), want) in [((0, -1), "A2:C3"), ((-1, 0), "B1:C3"), ((0, 1), "B2:D3"), ((1, 0), "B2:C4")] { + s.execute("selection.set", json!({"cell": "B2"})).unwrap(); + assert_eq!(s.doc().unwrap().selection.a1(), "B2:C3"); + assert_eq!(shift(&mut s, dr, dc), want, "Shift+({dr},{dc}) from the merged cell B2:C3"); + assert_eq!(s.doc().unwrap().selection.active.a1(), "B2", "the active cell stays"); + } +} + +#[test] +fn shift_arrows_keep_going_and_come_back_to_the_merged_cell() { + let mut s = merged_book(); + s.execute("selection.set", json!({"cell": "B2"})).unwrap(); + assert_eq!(shift(&mut s, 1, 0), "B2:C4"); + assert_eq!(shift(&mut s, 1, 0), "B2:C5"); + assert_eq!(shift(&mut s, 1, 0), "B2:C6"); + assert_eq!(shift(&mut s, -1, 0), "B2:C5"); + s.execute("selection.set", json!({"cell": "B2"})).unwrap(); + assert_eq!(shift(&mut s, 0, -1), "A2:C3"); + assert_eq!(shift(&mut s, 0, 1), "B2:C3", "back to just the merged cell"); + assert_eq!(shift(&mut s, 0, 1), "B2:D3"); + assert_eq!(shift(&mut s, -1, 0), "B1:D3"); + assert_eq!(shift(&mut s, 1, 0), "B2:D3"); +} + +#[test] +fn shift_arrows_shrink_past_a_merged_cell_they_pulled_in() { + let mut s = merged_book(); + assert_eq!(shift(&mut s, 0, 1), "A1:B1"); + assert_eq!(shift(&mut s, 1, 0), "A1:C3", "B2 pulls in the whole merged cell"); + assert_eq!(shift(&mut s, 0, -1), "A1:A3", "one step back leaves the merged cell out"); + assert_eq!(shift(&mut s, 0, 1), "A1:C3"); + assert_eq!(shift(&mut s, -1, 0), "A1:C1"); + assert_eq!(shift(&mut s, 1, 0), "A1:C3"); + assert_eq!(shift(&mut s, 1, 0), "A1:C4"); +} + +#[test] +fn enter_and_tab_leave_a_selected_merged_cell() { + let mut s = merged_book(); + for (forward, by_row, want) in [(true, false, "B4"), (false, false, "B1"), (true, true, "D2"), (false, true, "A2")] { + s.execute("selection.set", json!({"cell": "B2"})).unwrap(); + assert_eq!(enter(&mut s, forward, by_row), (want.to_string(), want.to_string()), "forward {forward} byRow {by_row}"); + } +} + +#[test] +fn enter_and_tab_out_of_a_merged_cell_step_over_hidden_lines() { + let mut s = merged_book(); + s.execute("selection.set", json!({"range": "A4"})).unwrap(); + s.execute("home.hideRows", json!({})).unwrap(); + s.execute("selection.set", json!({"range": "D1"})).unwrap(); + s.execute("home.hideColumns", json!({})).unwrap(); + s.execute("selection.set", json!({"cell": "B2"})).unwrap(); + assert_eq!(enter(&mut s, true, false).0, "B5", "row 4 is hidden"); + s.execute("selection.set", json!({"cell": "B2"})).unwrap(); + assert_eq!(enter(&mut s, true, true).0, "E2", "column D is hidden"); + s.execute("selection.set", json!({"cell": "B2"})).unwrap(); + s.execute("selection.move", json!({"dr": 1})).unwrap(); + assert_eq!(s.doc().unwrap().selection.a1(), "B5", "the Down arrow too"); +} + +#[test] +fn enter_inside_a_selection_visits_a_merged_cell_once() { + let mut s = merged_book(); + s.execute("selection.set", json!({"range": "A1:C3", "active": "A1"})).unwrap(); + let seen: Vec = (0..6).map(|_| enter(&mut s, true, false).0).collect(); + assert_eq!(seen, ["A2", "A3", "B1", "B2", "C1", "A1"]); + assert_eq!(s.doc().unwrap().selection.a1(), "A1:C3", "the selection stays"); + let back: Vec = (0..2).map(|_| enter(&mut s, false, false).0).collect(); + assert_eq!(back, ["C1", "B2"]); +} + +#[test] +fn shift_arrow_that_cannot_shrink_the_area_grows_it() { + // A2:B3 merged, anchor B1: Shift+Down pulls the area out to column A, past the anchor. + // Shift+Right can't shrink it from the left (the merged cell would come back), so it grows. + let mut s = Session::new(); + s.new_workbook(); + s.execute("selection.set", json!({"range": "A2:B3"})).unwrap(); + s.execute("home.mergeCells", json!({})).unwrap(); + s.execute("selection.set", json!({"cell": "B1"})).unwrap(); + assert_eq!(shift(&mut s, 1, 0), "A1:B3"); + assert_eq!(shift(&mut s, 0, 1), "A1:C3"); + assert_eq!(s.doc().unwrap().selection.active.a1(), "B1"); +} + +#[test] +fn shift_left_shrinks_past_a_chain_of_merged_cells_in_one_step() { + // B2:C3 and D3:E4 are both inside A1:E4. Shift+Left from E can't stop in column D (D3:E4 + // would come back) and stops in C, where B2:C3 is whole. + let mut s = merged_book(); + s.execute("selection.set", json!({"range": "D3:E4"})).unwrap(); + s.execute("home.mergeCells", json!({})).unwrap(); + s.execute("selection.set", json!({"cell": "A1"})).unwrap(); + assert_eq!(shift(&mut s, 3, 4), "A1:E4"); + assert_eq!(shift(&mut s, 0, -1), "A1:C4"); + assert_eq!(shift(&mut s, -1, 0), "A1:C3", "row 4 holds nothing merged once column D is out"); + assert_eq!(shift(&mut s, 0, -1), "A1:A3", "C can't stop inside B2:C3"); +} + +#[test] +fn shift_up_over_a_tall_merged_cell_takes_one_step() { + let mut s = Session::new(); + s.new_workbook(); + s.execute("selection.set", json!({"range": "B2:C500000"})).unwrap(); + s.execute("home.mergeCells", json!({})).unwrap(); + s.execute("selection.set", json!({"cell": "A1"})).unwrap(); + assert_eq!(shift(&mut s, 0, 1), "A1:B1"); + assert_eq!(shift(&mut s, 1, 0), "A1:C500000"); + assert_eq!(shift(&mut s, -1, 0), "A1:C1", "row 1 is the last row above the merged cell"); + assert_eq!(shift(&mut s, 1, 0), "A1:C500000"); + assert_eq!(shift(&mut s, 0, -1), "A1:A500000"); +} + +#[test] +fn shift_arrows_change_only_the_last_area() { + let mut s = merged_book(); + s.execute("selection.set", json!({"range": "E5:F6,B2"})).unwrap(); + assert_eq!(s.doc().unwrap().selection.a1(), "E5:F6,B2:C3"); + assert_eq!(shift(&mut s, 0, -1), "E5:F6,A2:C3"); +} + +#[test] +fn enter_skips_a_merged_cell_whose_start_is_outside_the_selection() { + // A whole column isn't widened to the merged cells it touches. B1 is covered by A1:B1, whose + // top-left cell is outside column B, so Enter skips it. + let mut s = Session::new(); + s.new_workbook(); + s.execute("selection.set", json!({"range": "A1:B1"})).unwrap(); + s.execute("home.mergeCells", json!({})).unwrap(); + s.execute("selection.set", json!({"range": "B:B", "active": "B2"})).unwrap(); + assert_eq!(enter(&mut s, false, false).0, "B1048576", "B1 is skipped, going up wraps to the bottom"); +} diff --git a/crates/engine/tests/it/row_autofit.rs b/crates/engine/tests/it/row_autofit.rs index 5bea5abca..438a39870 100644 --- a/crates/engine/tests/it/row_autofit.rs +++ b/crates/engine/tests/it/row_autofit.rs @@ -59,3 +59,31 @@ fn whole_row_format_keeps_hidden_rows_hidden() { assert!(info.size.unwrap_or(before) > before + 10.0, "hidden row still grows to its font: before={before}, size={:?}", info.size); assert!(info.hidden, "formatting a hidden row must not unhide it"); } + +#[test] +fn many_merges_fit_rows_the_same_and_skip_merged_cells() { + // Rows are fitted with one sweep over the sheet's merges; a merged cell still never grows + // its row, and rows next to thousands of merges still fit. + let mut s = gridcraft_engine::Session::new(); + s.new_workbook(); + let values: Vec<_> = (0..2_000).map(|i| json!([format!("v{i}")])).collect(); + s.execute("range.setValues", json!({"range": "A1", "values": values})).unwrap(); + { + let sh = sheet_mut(&mut s); + sh.merges.push(gridcraft_engine::core::RangeRef::parse("A3:B4").unwrap()); + for i in 0..20_000u32 { + let row = 4 + i % 1_990; + sh.merges.push(gridcraft_engine::core::RangeRef::new( + gridcraft_engine::core::CellRef::new(row, 3 + (i % 7) * 2), + gridcraft_engine::core::CellRef::new(row + 3, 4 + (i % 7) * 2), + )); + } + } + s.execute("home.fontSize", json!({"range": "A:A", "size": 20})).unwrap(); + let sh = s.doc().unwrap().wb.active().unwrap(); + let default = sh.default_row_height; + assert!(sh.row_height(0) > 30.0 && sh.row_height(1_999) > 30.0, "rows with big text grow"); + assert_eq!(sh.row_height(2), default, "a merged cell does not grow its row"); + assert_eq!(sh.row_height(3), default, "nor any row of the merge"); + assert!(sh.row_height(4) > 30.0 && sh.row_height(1_000) > 30.0, "rows beside other merges grow"); +} diff --git a/crates/formula/src/ast.rs b/crates/formula/src/ast.rs index 94756c09f..3c784d2f4 100644 --- a/crates/formula/src/ast.rs +++ b/crates/formula/src/ast.rs @@ -215,7 +215,9 @@ impl Expr { /// Visits every node depth-first, parents before children, left to right. pub fn walk<'a>(&'a self, f: &mut dyn FnMut(&'a Expr)) { // An explicit stack: a long operator chain must not recurse once per operand. - let mut pending = vec![self]; + // Room for a typical formula up front, so walking one costs one allocation, not several. + let mut pending = Vec::with_capacity(8); + pending.push(self); while let Some(e) = pending.pop() { f(e); match e { @@ -236,6 +238,19 @@ impl Expr { } /// Rewrites every node bottom-up, children left to right before their parent. pub fn map(self, f: &mut dyn FnMut(Expr) -> Expr) -> Expr { + // One operator on a non-chain operand (`A1*2`), the common case, needs no spine: rewrite + // the operands in their boxes. + if let Expr::Binary(op, mut a, mut b) = self { + if !matches!(*a, Expr::Binary(..)) { + *a = std::mem::replace(&mut *a, Expr::Missing).map(f); + *b = std::mem::replace(&mut *b, Expr::Missing).map(f); + return f(Expr::Binary(op, a, b)); + } + return Expr::Binary(op, a, b).map_chain(f); + } + self.map_chain(f) + } + fn map_chain(self, f: &mut dyn FnMut(Expr) -> Expr) -> Expr { // Unwind the left spine of a long operator chain into a list, then rebuild it from the // innermost operand outwards, in the order a recursive rewrite would visit the nodes. let mut spine = Vec::new(); @@ -283,6 +298,12 @@ impl Expr { impl Clone for Expr { fn clone(&self) -> Self { + // One operator on a non-chain operand needs no spine. + if let Expr::Binary(op, a, b) = self + && !matches!(**a, Expr::Binary(..)) + { + return Expr::Binary(*op, a.clone(), b.clone()); + } // Copy the left spine of an operator chain in a loop (see [`Expr`]). let mut spine = Vec::new(); let mut node = self; diff --git a/crates/formula/src/parser.rs b/crates/formula/src/parser.rs index 2b6f6d408..f8d73744e 100644 --- a/crates/formula/src/parser.rs +++ b/crates/formula/src/parser.rs @@ -69,11 +69,35 @@ fn parse_top(src: &str, locale: crate::FormulaLocale, allow_union: bool) -> Resu let mut p = Parser { toks, i: 0, depth: 0, input: locale == crate::FormulaLocale::Es }; let e = p.expr(0, allow_union)?; match p.peek() { + Tok::Eof if wrapper_height(&e) > MAX_DEPTH => Err(ParseError { msg: "formula is nested too deeply".into(), pos: 0 }), Tok::Eof => Ok(e), t => Err(p.err(format!("unexpected {}", describe(t)))), } } +/// The most unary operators, parentheses and invocations on one path of `e`. The parser bounds +/// each level's postfix wrappers by its own depth, so levels inside levels can still stack them +/// far past [`MAX_DEPTH`]; code that maps over the tree recurses once per wrapper. +fn wrapper_height(e: &Expr) -> usize { + let mut most = 0; + let mut pending = vec![(e, 0usize)]; + while let Some((e, h)) = pending.pop() { + most = most.max(h); + match e { + Expr::Unary(_, a) | Expr::Paren(a) => pending.push((a, h + 1)), + Expr::Invoke(callee, args) => { + pending.push((callee, h + 1)); + pending.extend(args.iter().map(|a| (a, h + 1))); + } + Expr::Binary(_, a, b) => pending.extend([(&**a, h), (&**b, h)]), + Expr::Call(_, args) => pending.extend(args.iter().map(|a| (a, h))), + Expr::Array(rows) => pending.extend(rows.iter().flatten().map(|a| (a, h))), + _ => {} + } + } + most +} + fn describe(t: &Tok) -> String { match t { Tok::RParen => "`)`".into(), diff --git a/crates/formula/src/tests.rs b/crates/formula/src/tests.rs index 6e3829925..274a66667 100644 --- a/crates/formula/src/tests.rs +++ b/crates/formula/src/tests.rs @@ -576,3 +576,19 @@ fn column_starting_with_at_round_trips() { assert_eq!(parse(&text).ok(), Some(Expr::Struct(s)), "printed as {text}"); } } + +#[test] +fn wrappers_stacked_across_levels_are_bounded_as_a_whole() { + // Each level adds 40 percents inside the parser's per-level bound, so no level is too deep alone. + let stacked = |levels: usize| { + let mut f = "1".to_string(); + for _ in 0..levels { + f = format!("({f}){}", "%".repeat(40)); + } + f + }; + assert!(parse(&stacked(5)).is_ok()); + assert!(parse(&stacked(7)).is_err(), "7 x 41 wrappers on one path"); + assert!(parse(&format!("{}1{}", "(".repeat(200), ")".repeat(200))).is_ok()); + assert!(parse(&format!("={}", vec!["A1%"; 100].join("+"))).is_ok(), "siblings are not stacked"); +} diff --git a/crates/functions/src/database.rs b/crates/functions/src/database.rs index 37289d9fb..1ef713cdc 100644 --- a/crates/functions/src/database.rs +++ b/crates/functions/src/database.rs @@ -70,39 +70,59 @@ fn cell_criterion(v: &Value) -> Option { /// when the label is unknown, criterion). type CriteriaRow = (usize, Vec<(usize, Option, Criterion)>); +/// Most condition tests one selection may make (records times conditions): every record can be +/// tested against every criteria row when none matches, so a modest database and criteria table +/// multiply into billions of tests. Past it the function is `#NUM!`, as lifted lookups are; 5e8 +/// is a few seconds of matching. +const SELECTION_WORK_BUDGET: usize = 500_000_000; + /// Rows (1-based within the database, excluding the header) of matching records. -fn matching_records(db: &Array, crit: &Array, ctx: &mut dyn Ctx) -> Vec { +fn matching_records(db: &Array, crit: &Array, ctx: &mut dyn Ctx) -> R> { + // The database column under each criteria label, found once per label rather than per cell. + let columns: Vec> = (0..crit.cols) + .map(|c| match crit.get(0, c) { + Some(Value::Text(t)) => find_column(db, t), + Some(Value::Number(n)) => find_column(db, &gridcraft_core::number_to_text(*n)), + _ => None, + }) + .collect(); // Per criteria row: its index and a list of (criteria column, database column or None when // the label is unknown, criterion). let mut rows: Vec = Vec::new(); + let mut tests = 0usize; for r in 1..crit.rows { let mut conds = Vec::new(); for c in 0..crit.cols { let Some(cell) = crit.get(r, c) else { continue }; let Some(cr) = cell_criterion(cell) else { continue }; - let col = match crit.get(0, c) { - Some(Value::Text(t)) => find_column(db, t), - Some(Value::Number(n)) => find_column(db, &gridcraft_core::number_to_text(*n)), - _ => None, - }; - conds.push((c, col, cr)); + conds.push((c, columns.get(c).copied().flatten(), cr)); } + tests = tests.saturating_add(conds.len().max(1)); rows.push((r, conds)); } - let all_match = rows.is_empty(); - (1..db.rows) + // An empty criteria row matches every record without testing a condition, so nothing is charged. + let all_match = rows.is_empty() || rows.iter().any(|(_, conds)| conds.is_empty()); + // Computed criteria (a formula evaluated per record) count like any other test. + if !all_match && db.rows.saturating_sub(1).saturating_mul(tests) > SELECTION_WORK_BUDGET { + return Err(CellError::Num); + } + Ok((1..db.rows) .filter(|&rec| { all_match - || rows.iter().any(|(r, conds)| { - conds.iter().all(|(c, col, cr)| match col { - Some(f) => cr.matches(db.get(rec, *f).unwrap_or(&Value::Empty)), - // Under a label that names no field, only a formula (a computed - // criterion) can hold. - None => ctx.computed_criterion(*r, *c, db, rec).unwrap_or(false), - }) - }) + // Out of work: a computed criterion would fail at once, so records are left out + // without scanning the criteria rows; the per-row check still stops mid-scan. + || (!ctx.exhausted() + && rows.iter().any(|(r, conds)| { + !ctx.exhausted() + && conds.iter().all(|(c, col, cr)| !ctx.exhausted() && match col { + Some(f) => cr.matches(db.get(rec, *f).unwrap_or(&Value::Empty)), + // Under a label that names no field, only a formula (a computed + // criterion) can hold. + None => ctx.computed_criterion(*r, *c, db, rec).unwrap_or(false), + }) + })) }) - .collect() + .collect()) } struct Selection<'a> { @@ -123,7 +143,7 @@ fn select<'a>(a: &'a [Arg], ctx: &mut dyn Ctx) -> R> { let db = as_array(dbv); let crit = as_array(critv); let field = field_index(&db, a)?; - let records = matching_records(&db, &crit, ctx); + let records = matching_records(&db, &crit, ctx)?; Ok(Selection { db, field, records }) } @@ -352,6 +372,63 @@ mod tests { is_err(d("DSUM", n(9.0)), CellError::Value); } + /// Computed criteria are no longer asked for once the host is out of work. + #[test] + fn matching_stops_asking_computed_criteria_when_exhausted() { + struct Spent { + asked: usize, + } + impl crate::Ctx for Spent { + fn date_system(&self) -> gridcraft_core::DateSystem { + gridcraft_core::DateSystem::D1900 + } + fn now_serial(&self) -> f64 { + 0.0 + } + fn random(&mut self) -> f64 { + 0.0 + } + fn computed_criterion(&mut self, _: usize, _: usize, _: &gridcraft_core::Array, _: usize) -> Option { + self.asked += 1; + Some(true) + } + fn exhausted(&self) -> bool { + self.asked > 0 + } + } + let criteria = arr(vec![vec![tv("Formula")], vec![nv(1.0)]]); + let mut ctx = Spent { asked: 0 }; + let spec = crate::lookup("DCOUNT").unwrap(); + crate::call(spec, &[rf(db()), t("Age"), rf(criteria)], &mut ctx); + assert_eq!(ctx.asked, 1); + // Within one criteria row, the conditions after the one that spent the work are skipped too. + let criteria = arr(vec![vec![tv("Formula"), tv("Other")], vec![nv(1.0), nv(1.0)]]); + let mut ctx = Spent { asked: 0 }; + crate::call(spec, &[rf(db()), t("Age"), rf(criteria)], &mut ctx); + assert_eq!(ctx.asked, 1); + } + + /// Security: records times criteria conditions past the selection budget is `#NUM!` before + /// any record is tested; a nonmatching criteria table used to test every record against + /// every criteria row. + #[test] + fn selection_work_is_bounded() { + use gridcraft_core::Array; + let column = |label: &str, len: usize, v: Value| { + let mut data = vec![tv(label)]; + data.extend(std::iter::repeat_n(v, len)); + Value::from(Array::column(data)) + }; + let db = column("Tree", 100_000, tv("Apple")); + let wide = column("Tree", 5_001, tv("=Plum")); + is_err(ev("DCOUNTA", vec![rf(db.clone()), t("Tree"), rf(wide)]), CellError::Num); + let narrow = column("Tree", 2, tv("=Apple")); + close(ev("DCOUNTA", vec![rf(db.clone()), t("Tree"), rf(narrow)]), 100_000.0); + // Empty criteria rows test no condition: the first one matches every record, so they cost nothing. + let empty_rows = column("Tree", 5_001, Value::Empty); + close(ev("DCOUNTA", vec![rf(db), t("Tree"), rf(empty_rows)]), 100_000.0); + } + #[test] fn simple_criteria() { let c = |cells: Vec| arr(vec![vec![tv("Tree"), tv("Height")], cells]); diff --git a/crates/functions/src/date.rs b/crates/functions/src/date.rs index 21d2a49c1..2060267c2 100644 --- a/crates/functions/src/date.rs +++ b/crates/functions/src/date.rs @@ -416,13 +416,17 @@ fn workday_core(a: &[Arg], c: &mut dyn Ctx, mask: [bool; 7], hol: Option<&Arg>) if days == 0 { return num_val(start as f64); } + // The holidays on working days, still sorted (`holidays` sorts them), so each step counts + // the ones it passed with two binary searches rather than a scan of every holiday: a run of + // consecutive holidays takes one step per holiday. + let hs: Vec = hs.into_iter().filter(|&h| !mask[weekday0(sys, h)]).collect(); let dir = days.signum(); let mut cur = start; let mut rem = days.abs(); while rem > 0 { let next = advance(sys, cur, rem, dir, &mask, per_week); let (lo, hi) = if dir > 0 { (cur + 1, next) } else { (next, cur - 1) }; - rem = hs.iter().filter(|&&h| h >= lo && h <= hi && !mask[weekday0(sys, h)]).count() as i64; + rem = hs.partition_point(|&h| h <= hi).saturating_sub(hs.partition_point(|&h| h < lo)) as i64; cur = next; if !(0..=MAX_SERIAL as i64).contains(&cur) { return Err(CellError::Num); @@ -755,6 +759,23 @@ mod tests { is_err(ev("TIMEVALUE", vec![t("x")]), CellError::Value); } + /// Security: WORKDAY counts the holidays each step passes by binary search, so a long run of + /// consecutive holidays is linear work, not one scan of every holiday per holiday. + #[test] + fn workday_over_many_consecutive_holidays() { + let start = d(2000.0, 1.0, 3.0); + let hol: Vec = (1..=200_000).map(|i| start + f64::from(i)).collect(); + let out = ev("WORKDAY.INTL", vec![n(start), n(1.0), n(11.0), av(col(&hol))]); + // Sundays are the only weekend (11): 200,000 days of holidays are skipped, landing on the + // first working day after them. + let after = start + 200_001.0; + let sunday = ev("WEEKDAY", vec![n(after), n(2.0)]) == nv(7.0); + close(out, if sunday { after + 1.0 } else { after }); + // Holidays on a weekend don't push the result out. + let fri = d(2026.0, 10.0, 9.0); + close(ev("WORKDAY", vec![n(fri), n(1.0), av(col(&[fri + 1.0, fri + 3.0]))]), fri + 4.0); + } + #[test] fn day_counts() { close(ev("DAYS", vec![n(d(2021.0, 3.0, 15.0)), n(d(2021.0, 2.0, 1.0))]), 42.0); diff --git a/crates/functions/src/dynamic.rs b/crates/functions/src/dynamic.rs index 816af8ea2..15e9ff7c9 100644 --- a/crates/functions/src/dynamic.rs +++ b/crates/functions/src/dynamic.rs @@ -171,13 +171,17 @@ fn sort(args: &[Arg], _c: &mut dyn Ctx) -> R { let a = if by_col { a.transpose() } else { a }; let idx = if has(args, 1) { int_list(&arg(args, 1)?.value)? } else { vec![1] }; let ord = if has(args, 2) { int_list(&arg(args, 2)?.value)? } else { vec![1] }; - if idx.is_empty() || idx.iter().any(|&i| i < 1 || i as usize > a.cols) || ord.iter().any(|&o| o != 1 && o != -1) { + if idx.is_empty() || idx.iter().any(|&i| i < 1 || i > a.cols as i64) || ord.iter().any(|&o| o != 1 && o != -1) { return Err(CellError::Value); } if ord.len() > 1 && ord.len() != idx.len() { return Err(CellError::Value); } let orders: Vec = (0..idx.len()).map(|i| *ord.get(i).or(ord.first()).unwrap_or(&1)).collect(); + // A column repeated in sort_index can't break a tie its first use left, so only first uses + // stay: the keys copied per row are then at most the row itself, however long sort_index is. + let mut seen = std::collections::HashSet::new(); + let (idx, orders): (Vec, Vec) = idx.into_iter().zip(orders).filter(|(i, _)| seen.insert(*i)).unzip(); let mut lines: Vec<(Vec, Vec)> = rows_of(&a).into_iter().map(|r| (idx.iter().map(|&i| r.get(i as usize - 1).cloned().unwrap_or(Value::Empty)).collect(), r)).collect(); lines.sort_by(|x, y| cmp_keys(&x.0, &y.0, &orders)); @@ -928,9 +932,16 @@ fn pivotby(args: &[Arg], ctx: &mut dyn Ctx) -> R { let ckeys: Vec> = p.rows.iter().map(|&r| key_row(&cf, r)).collect(); let rk = distinct_keys(&rkeys, if row_sort.unsigned_abs() as usize <= rf.cols { row_sort } else { 0 }); let ck = distinct_keys(&ckeys, if col_sort.unsigned_abs() as usize <= cf.cols { col_sort } else { 0 }); + // Source rows of every row key, column key and pair, built in one pass, so a total reads + // its rows instead of rescanning the whole source once per total. let mut cell: HashMap<(String, String), Vec> = HashMap::new(); + let mut by_row: HashMap> = HashMap::new(); + let mut by_col: HashMap> = HashMap::new(); for ((r, c), &i) in rkeys.iter().zip(ckeys.iter()).zip(p.rows.iter()) { - cell.entry((row_key(r), row_key(c))).or_default().push(i); + let (r, c) = (row_key(r), row_key(c)); + by_row.entry(r.clone()).or_default().push(i); + by_col.entry(c.clone()).or_default().push(i); + cell.entry((r, c)).or_default().push(i); } let mut agg = |rows: &[usize], c: usize| -> Value { if rows.is_empty() { @@ -940,14 +951,12 @@ fn pivotby(args: &[Arg], ctx: &mut dyn Ctx) -> R { aggregate_with(f, &vs, ctx) }; let rows_for = |r: Option<&Vec>, c: Option<&Vec>| -> Vec { - let rkey = r.map(|k| row_key(k)); - let ckey = c.map(|k| row_key(k)); - p.rows - .iter() - .zip(rkeys.iter().zip(ckeys.iter())) - .filter(|(_, (a, b))| rkey.as_ref().is_none_or(|k| *k == row_key(a)) && ckey.as_ref().is_none_or(|k| *k == row_key(b))) - .map(|(&i, _)| i) - .collect() + match (r, c) { + (Some(r), Some(c)) => cell.get(&(row_key(r), row_key(c))).cloned().unwrap_or_default(), + (Some(r), None) => by_row.get(&row_key(r)).cloned().unwrap_or_default(), + (None, Some(c)) => by_col.get(&row_key(c)).cloned().unwrap_or_default(), + (None, None) => p.rows.clone(), + } }; let mut cols: Vec>> = ck.iter().map(Some).collect(); if col_total > 0 { @@ -997,10 +1006,7 @@ fn pivotby(args: &[Arg], ctx: &mut dyn Ctx) -> R { } }; for c in &cols { - let rows = match (r, c) { - (Some(rk), Some(ck)) => cell.get(&(row_key(rk), row_key(ck))).cloned().unwrap_or_default(), - _ => rows_for(r, *c), - }; + let rows = rows_for(r, *c); for vc in 0..vals.cols { line.push(agg(&rows, vc)); } @@ -1204,6 +1210,21 @@ mod tests { assert_eq!(rows_of(&r), vec![vec![nv(0.0), nv(5.0)], vec![nv(1.0), nv(2.0)], vec![nv(1.0), nv(1.0)]]); } + /// Security: a column repeated in sort_index keys the sort once. The keys copied per row stay + /// within the row, however long sort_index is, and a repeat (even with the other order) can't + /// change the result, since its first use already ordered the rows. + #[test] + fn sort_drops_repeated_sort_indices() { + let m = || av(arr(vec![vec![nv(1.0), nv(2.0)], vec![nv(1.0), nv(1.0)], vec![nv(0.0), nv(5.0)]])); + let want = vec![vec![nv(0.0), nv(5.0)], vec![nv(1.0), nv(2.0)], vec![nv(1.0), nv(1.0)]]; + let r = ev("SORT", vec![m(), av(row(&[1.0, 1.0, 2.0, 1.0])), av(row(&[1.0, -1.0, -1.0, 1.0]))]); + assert_eq!(rows_of(&r), want); + let idx = vec![1.0; 1_000_000]; + let tall = av(gridcraft_core::Array::filled(2_000, 1, nv(1.0)).into()); + let r = ev("SORT", vec![tall, av(row(&idx))]); + assert_eq!(rows_of(&r).len(), 2_000); + } + #[test] fn sortby() { let r = ev("SORTBY", vec![av(col(&[1.0, 2.0, 3.0])), av(col(&[30.0, 10.0, 20.0]))]); @@ -1447,6 +1468,16 @@ mod tests { ); } + /// Security: PIVOTBY totals read their source rows from indexes built in the grouping pass, + /// rather than rescanning the source once per total, so many distinct keys stay linear. + #[test] + fn pivotby_totals_with_many_groups() { + let keys: Vec = (0..20_000).map(f64::from).collect(); + let r = rows_of(&ev("PIVOTBY", vec![av(col(&keys)), av(col(&vec![1.0; 20_000])), av(col(&vec![2.0; 20_000])), t("SUM")])); + assert_eq!(r.get(1), Some(&vec![nv(0.0), nv(2.0), nv(2.0)])); + assert_eq!(r.last(), Some(&vec![tv("Total"), nv(40_000.0), nv(40_000.0)])); + } + #[test] fn pivotby_size_check_counts_column_header_rows() { // One row, n column fields and n value columns: n header rows + value-name row + body row + total row, diff --git a/crates/functions/src/financial.rs b/crates/functions/src/financial.rs index b569c1617..75b6fbc78 100644 --- a/crates/functions/src/financial.rs +++ b/crates/functions/src/financial.rs @@ -13,6 +13,7 @@ mod day_count; mod depreciation; mod securities; mod tvm; +pub(crate) use tvm::cumulative_args; use day_count::{coupdaybs, coupdays, coupdaysnc, coupncd, coupnum, couppcd}; use depreciation::{amordegrc, amorlinc, db, ddb, sln, syd, vdb}; diff --git a/crates/functions/src/financial/securities.rs b/crates/functions/src/financial/securities.rs index ea696c8a6..4deb579bd 100644 --- a/crates/functions/src/financial/securities.rs +++ b/crates/functions/src/financial/securities.rs @@ -71,7 +71,8 @@ pub(super) fn accrintm(a: &[Arg], c: &mut dyn Ctx) -> R { let (issue, settle, rate) = (date(a, 0)?, date(a, 1)?, num(a, 2)?); let par = if has(a, 3) { num(a, 3)? } else { 1000.0 }; let basis = basis_arg(a, 4)?; - if issue >= settle || rate <= 0.0 || par <= 0.0 { + // Excel accepts equal (whole-day) dates and accrues nothing; only issue after settlement is #NUM!. + if issue > settle || rate <= 0.0 || par <= 0.0 { return Err(CellError::Num); } num_val(par * rate * year_frac(c.date_system(), issue, settle, basis)?) diff --git a/crates/functions/src/financial/tests.rs b/crates/functions/src/financial/tests.rs index 2788da94a..a12e31022 100644 --- a/crates/functions/src/financial/tests.rs +++ b/crates/functions/src/financial/tests.rs @@ -317,3 +317,28 @@ fn securities_read_dates_in_the_1904_system() { let odd = vec![d4(2008, 11, 11), d4(2021, 3, 1), d4(2008, 10, 15), d4(2009, 3, 1), n(0.0785), n(0.0625), n(100.0), n(2.0), n(1.0)]; close_tol(ev4("ODDFPRICE", odd), 113.597717474079, 1e-9); } + +/// Excel truncates ACCRINTM's dates to whole days and accrues nothing when issue equals +/// settlement, in every basis and both date systems; only issue after settlement is #NUM! (#4096). +#[test] +fn accrintm_equal_dates_accrue_zero() { + let day = serial_from_ymd(DateSystem::D1900, 2025, 1, 15).unwrap(); + for basis in 0..=4 { + close(ev("ACCRINTM", vec![n(day), n(day), n(0.05), n(1000.0), n(f64::from(basis))]), 0.0); + } + // Distinct times on the same day truncate to equal dates. + close(ev("ACCRINTM", vec![n(day + 0.1), n(day + 0.9), n(0.05), n(1000.0), n(2.0)]), 0.0); + close_tol(ev("ACCRINTM", vec![n(day), n(day + 1.0), n(0.05), n(1000.0), n(2.0)]), 0.1388888888888889, 1e-15); + is_err(ev("ACCRINTM", vec![n(day + 1.0), n(day), n(0.05), n(1000.0), n(2.0)]), CellError::Num); + // Equal dates do not excuse a zero rate or par, or an invalid basis. + is_err(ev("ACCRINTM", vec![n(day), n(day), n(0.0), n(1000.0), n(2.0)]), CellError::Num); + is_err(ev("ACCRINTM", vec![n(day), n(day), n(0.05), n(0.0), n(2.0)]), CellError::Num); + is_err(ev("ACCRINTM", vec![n(day), n(day), n(0.05), n(1000.0), n(5.0)]), CellError::Num); + + let day4 = serial_from_ymd(DateSystem::D1904, 2025, 1, 15).unwrap(); + for basis in 0..=4 { + let mut ctx = TestCtx { sys: DateSystem::D1904, ..TestCtx::default() }; + let args = vec![n(day4), n(day4), n(0.05), n(1000.0), n(f64::from(basis))]; + close(crate::call(crate::lookup("ACCRINTM").unwrap(), &args, &mut ctx), 0.0); + } +} diff --git a/crates/functions/src/financial/tvm.rs b/crates/functions/src/financial/tvm.rs index 61710f0bd..872e7cbeb 100644 --- a/crates/functions/src/financial/tvm.rs +++ b/crates/functions/src/financial/tvm.rs @@ -116,7 +116,9 @@ pub(super) fn ppmt(a: &[Arg], _c: &mut dyn Ctx) -> R { num_val(pmt_calc(rate, np, pv, fv, typ) - ipmt_calc(rate, per, np, pv, fv, typ)) } -fn cumulative(a: &[Arg], principal: bool) -> R { +/// The validated (rate, periods, pv, start, end, type) of CUMIPMT/CUMPRINC: every check that +/// refuses a call before its period loop, shared with the lifted-call work preflight. +pub(crate) fn cumulative_args(a: &[Arg]) -> R<(f64, f64, f64, f64, f64, f64)> { let (rate, np, pv) = (num(a, 0)?, num(a, 1)?, num(a, 2)?); let (start, end, typ) = (num(a, 3)?.trunc(), num(a, 4)?.trunc(), num(a, 5)?); let np_i = np.trunc(); @@ -126,6 +128,11 @@ fn cumulative(a: &[Arg], principal: bool) -> R { if end - start > 1e7 || np_i > 1e15 { return Err(CellError::Num); } + Ok((rate, np_i, pv, start, end, typ)) +} + +fn cumulative(a: &[Arg], principal: bool) -> R { + let (rate, np_i, pv, start, end, typ) = cumulative_args(a)?; let p = pmt_calc(rate, np_i, pv, 0.0, typ); let mut sum = 0.0; let mut per = start; diff --git a/crates/functions/src/info.rs b/crates/functions/src/info.rs index 309f0323e..82eda5828 100644 --- a/crates/functions/src/info.rs +++ b/crates/functions/src/info.rs @@ -91,7 +91,7 @@ fn na(_a: &[Arg], _c: &mut dyn Ctx) -> R { fn n_fn(a: &[Arg], _c: &mut dyn Ctx) -> R { // N(range) reads the range's first cell, but a computed array (N(A1:A3>0), N({...})) is // converted element by element, so N can stand in for `--` inside SUMPRODUCT. - if let Some(Arg { value: Value::Array(arr), from_ref: false }) = a.first() { + if let Some(Arg { value: Value::Array(arr), from_ref: false, .. }) = a.first() { let data = arr.data.iter().map(|v| n_of(v.clone()).unwrap_or_else(Value::Error)).collect(); return Array::new(arr.rows, arr.cols, data).map(Value::from).ok_or(CellError::Calc); } diff --git a/crates/functions/src/lib.rs b/crates/functions/src/lib.rs index ccc61b026..dfca164a2 100644 --- a/crates/functions/src/lib.rs +++ b/crates/functions/src/lib.rs @@ -87,16 +87,19 @@ pub use stat::{aggregate_values, aggregate_values_k}; pub struct Arg { pub value: Value, pub from_ref: bool, + /// A written argument whose value came out blank (a lookup of an empty cell): given, not + /// omitted, although its value is `Value::Empty`. + pub blank_value: bool, } impl Arg { /// A literal (non-reference) argument. pub fn val(value: impl Into) -> Arg { - Arg { value: value.into(), from_ref: false } + Arg { value: value.into(), from_ref: false, blank_value: false } } /// An argument that came from a cell or range reference. pub fn reference(value: impl Into) -> Arg { - Arg { value: value.into(), from_ref: true } + Arg { value: value.into(), from_ref: true, blank_value: false } } } @@ -113,6 +116,11 @@ pub trait Ctx { fn computed_criterion(&mut self, _row: usize, _col: usize, _db: &Array, _record: usize) -> Option { None } + /// Whether the cell's work budget has run out: the database functions stop asking for computed + /// criteria then, as every one would fail at once. + fn exhausted(&self) -> bool { + false + } /// For GROUPBY/PIVOTBY: whether their function argument is a LAMBDA (its value is then not used). fn has_lambda(&self) -> bool { false @@ -308,6 +316,12 @@ mod tests { assert_eq!(ev("REGEXREPLACE", vec![t("2026-10-22"), t("(\\d+)-(\\d+)-(\\d+)"), t("$3/$2/$1")]), Value::text("22/10/2026")); assert_eq!(ev("REGEXREPLACE", vec![t("abc"), t("x*"), t("-")]), Value::text("-a-b-c-")); assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(2.0)]), Value::text("a1b#")); + assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(-2.0)]), Value::text("a#b2")); + assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(-3.0)]), Value::text("a1b2")); + // 2^32 is no occurrence the text has (and on 32-bit targets it must not wrap to 0). + for occurrence in [4_294_967_296.0, -4_294_967_296.0] { + assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(occurrence)]), Value::text("a1b2")); + } // One match whose replacement repeats it 10,000 times: refused before it is built. assert_eq!(ev("REGEXREPLACE", vec![t(&many), t("^.*$"), t(&"$0".repeat(10_000))]), Value::Error(CellError::Value)); assert_eq!(ev("REGEXREPLACE", vec![t(&many), t("^.*$"), t(&"$0".repeat(10_000)), n(1.0)]), Value::Error(CellError::Value)); diff --git a/crates/functions/src/lift.rs b/crates/functions/src/lift.rs index ea7f07fd9..81d9e353e 100644 --- a/crates/functions/src/lift.rs +++ b/crates/functions/src/lift.rs @@ -50,12 +50,7 @@ fn reject_multi_cell_refs(spec: &FnSpec, args: &[Arg]) -> Option> { if !args.iter().enumerate().any(|(i, a)| multi(i, a)) { return None; } - Some( - args.iter() - .enumerate() - .map(|(i, a)| if multi(i, a) { Arg { value: Value::Error(CellError::Value), from_ref: false } } else { a.clone() }) - .collect(), - ) + Some(args.iter().enumerate().map(|(i, a)| if multi(i, a) { Arg::val(Value::Error(CellError::Value)) } else { a.clone() }).collect()) } /// Lookups whose every call searches a lookup vector from its start, so a call lifted over `n` @@ -68,26 +63,86 @@ const SCANS: &[&str] = &["VLOOKUP", "HLOOKUP", "LOOKUP", "MATCH", "XLOOKUP", "XM /// `XMATCH(SEQUENCE(1048576), -SEQUENCE(1048576))` runs for hours and hangs opening the file. /// Past the budget the call is `#NUM!`, as MMULT is past 2e9 multiply-adds. 5e8 keeps the worst /// case to a few seconds and still admits 5,000 needles over a 100,000-row table. +/// +/// The same budget covers all the work of a lifted call added up across its elements: criteria +/// aggregates rescanning their ranges, period loops, and array results built per element. const LIFTED_SCAN_BUDGET: usize = 500_000_000; -/// The length of the longest vector a lifted call of `spec` searches per needle: 0 unless `spec` -/// is one of the [`SCANS`] lookups and a non-lifted argument is an array. +/// Criteria aggregates: every call tests each cell of its ranges against its criteria, so a call +/// lifted over `n` criteria does `n` x (range cells) work, like the [`SCANS`] lookups. +const CRITERIA: &[&str] = &["COUNTIF", "COUNTIFS", "SUMIF", "SUMIFS", "AVERAGEIF", "AVERAGEIFS", "MAXIFS", "MINIFS"]; + +/// Cumulative TVM functions that step through every period from `start_period` (argument 3) to +/// `end_period` (argument 4) on each call. +const PERIOD_LOOPS: &[&str] = &["CUMIPMT", "CUMPRINC"]; + +/// Budget units one period of a [`PERIOD_LOOPS`] call costs: each period computes a payment and a +/// balance with powers, many times a scanned cell. +const PERIOD_COST: usize = 16; + +/// The work every lifted call of `spec` repeats over its non-lifted arguments: the longest vector +/// a [`SCANS`] lookup searches, or every cell a [`CRITERIA`] aggregate tests. 0 for other +/// functions. fn scan_len(spec: &FnSpec, args: &[Arg]) -> usize { - if !SCANS.contains(&spec.name) { + let scans = SCANS.contains(&spec.name); + if !scans && !CRITERIA.contains(&spec.name) { return 0; } - args.iter() - .enumerate() - .filter(|(i, _)| !spec.is_scalar_arg(*i)) - .filter_map(|(_, a)| match &a.value { - Value::Array(arr) => Some(arr.rows.max(arr.cols)), - _ => None, - }) - .max() - .unwrap_or(0) + // SUMIF and AVERAGEIF read the optional third range only at the coordinates of the first. + let first = args.first().and_then(|a| if let Value::Array(arr) = &a.value { Some(arr.data.len()) } else { None }); + let lens = args.iter().enumerate().filter(|(i, _)| !spec.is_scalar_arg(*i)).filter_map(|(i, a)| match &a.value { + Value::Array(arr) if scans => Some(arr.rows.max(arr.cols)), + Value::Array(arr) if i == 2 && matches!(spec.name, "SUMIF" | "AVERAGEIF") => Some(arr.data.len().min(first.unwrap_or(usize::MAX))), + Value::Array(arr) => Some(arr.data.len()), + _ => None, + }); + if scans { lens.max().unwrap_or(0) } else { lens.fold(0, usize::saturating_add) } +} + +/// The work of one lifted call that depends on its own (lifted) scalar arguments: the number of +/// periods a [`PERIOD_LOOPS`] function steps through. 0 for other functions. +fn call_cost(spec: &FnSpec, cell_args: &[Arg]) -> usize { + if !PERIOD_LOOPS.contains(&spec.name) { + return 0; + } + // Coercing numeric text reads all of it, even in a call the function then refuses, so each + // character is charged as well: of the arguments it coerces in order, up to the first that fails. + let mut parse = 0usize; + for (i, a) in cell_args.iter().enumerate().take(6) { + if let Value::Text(t) = &a.value { + parse = parse.saturating_add(t.len()); + } + if crate::util::num(cell_args, i).is_err() { + break; + } + } + // The function's own validation and coercion (text like "10000000" counts), so a call it + // refuses before its loop costs nothing and no loop goes uncharged. + let periods = crate::financial::cumulative_args(cell_args).map_or(0, |(.., start, end, _)| (end - start + 1.0) as usize); + parse.saturating_add(periods.saturating_mul(PERIOD_COST)) } pub(crate) fn call_lifted(spec: &FnSpec, args: &[Arg], ctx: &mut dyn Ctx) -> Value { + call_lifted_within(spec, args, ctx, LIFTED_SCAN_BUDGET) +} + +/// Sets the lifted arguments of `cell_args` to their elements at (`r`, `c`). +fn element_args(spec: &FnSpec, args: &[Arg], r: usize, c: usize, cell_args: &mut [Arg]) { + for (i, a) in args.iter().enumerate() { + if spec.is_scalar_arg(i) + && let Value::Array(arr) = &a.value + && let Some(slot) = cell_args.get_mut(i) + { + let v = arr.get_broadcast(r, c); + slot.value = if let Value::Array(inner) = &v { inner.data.first().cloned().unwrap_or(Value::Empty) } else { v }; + // A blank element of a written array argument is given, as a scalar blank is. + slot.blank_value = !slot.from_ref && matches!(slot.value, Value::Empty); + } + } +} + +/// [`call_lifted`] with every call's work added up against `budget` (see [`LIFTED_SCAN_BUDGET`]). +fn call_lifted_within(spec: &FnSpec, args: &[Arg], ctx: &mut dyn Ctx, budget: usize) -> Value { let rejected = if spec.value_args.is_empty() { None } else { reject_multi_cell_refs(spec, args) }; let args = rejected.as_deref().unwrap_or(args); let mut rows = 0usize; @@ -106,25 +161,42 @@ pub(crate) fn call_lifted(spec: &FnSpec, args: &[Arg], ctx: &mut dyn Ctx) -> Val return (spec.imp)(args, ctx); } let needles = rows.saturating_mul(cols); - if needles > MAX_CELLS || needles.saturating_mul(scan_len(spec, args)) > LIFTED_SCAN_BUDGET { + // Every call is charged its repeated scan (at least 1), its own period loop, and the cells of + // an array result (built in full, though only its first cell is kept), so lifting a scalar + // argument can't multiply what one call may do into hours of work: + // `COUNTIF(big_range, SEQUENCE(1e4))`, `CUMIPMT(0.1, 1e7, 1, 1, SEQUENCE(1e4,,1e7,0), 0)`, + // `MUNIT(SEQUENCE(1e4,,3000,0))`. + let per_call = scan_len(spec, args).max(1); + let mut cell_args: Vec = args.to_vec(); + let mut spent = needles.saturating_mul(per_call); + if needles > MAX_CELLS || spent > budget { return Value::Error(CellError::Num); } - let mut cell_args: Vec = args.to_vec(); + // Period loops are known before any call runs, so a call over budget does none of them. + if PERIOD_LOOPS.contains(&spec.name) { + for r in 0..rows { + for c in 0..cols { + element_args(spec, args, r, c, &mut cell_args); + spent = spent.saturating_add(call_cost(spec, &cell_args)); + if spent > budget { + return Value::Error(CellError::Num); + } + } + } + } let mut data = Vec::with_capacity(rows * cols); for r in 0..rows { for c in 0..cols { - for (i, a) in args.iter().enumerate() { - if spec.is_scalar_arg(i) - && let Value::Array(arr) = &a.value - && let Some(slot) = cell_args.get_mut(i) - { - let v = arr.get_broadcast(r, c); - slot.value = if let Value::Array(inner) = &v { inner.data.first().cloned().unwrap_or(Value::Empty) } else { v }; - } - } + element_args(spec, args, r, c, &mut cell_args); let out = (spec.imp)(&cell_args, ctx); data.push(match out { - Value::Array(a) => a.data.first().cloned().unwrap_or(Value::Empty), + Value::Array(a) => { + spent = spent.saturating_add(a.data.len()); + if spent > budget { + return Value::Error(CellError::Num); + } + a.data.first().cloned().unwrap_or(Value::Empty) + } v => v, }); } @@ -209,6 +281,100 @@ mod tests { assert_eq!(rows_of(&out), vec![vec![nv(-1.0)]; 1000]); } + /// Security: lifting a criteria aggregate rescans its range for every criterion, so criteria + /// x range cells past the budget is `#NUM!` before any scan runs. + #[test] + fn lifted_criteria_aggregates_stop_past_their_work_budget() { + use gridcraft_core::Array; + let tall = |n: usize| av(Array::filled(n, 1, nv(-1.0)).into()); + is_err(ev("COUNTIF", vec![tall(1_000_000), tall(501)]), CellError::Num); + is_err(ev("SUMIF", vec![tall(1_000_000), tall(501)]), CellError::Num); + is_err(ev("COUNTIFS", vec![tall(1_000_000), tall(501)]), CellError::Num); + // Two ranges of 250,000 cells each count together. + is_err(ev("SUMIFS", vec![tall(250_000), tall(250_000), tall(1001)]), CellError::Num); + // SUMIF reads its sum range only where the criteria range has cells: a one-cell range over a + // million-cell sum range is one cell of work per criterion. + let one = av(Array::filled(1, 1, nv(-1.0)).into()); + assert_eq!(rows_of(&ev("SUMIF", vec![one, tall(501), tall(1_000_000)])).len(), 501); + let counts = ev("COUNTIF", vec![av(col(&[1.0, 2.0, 2.0])), av(col(&[2.0, 1.0, 5.0]))]); + assert_eq!(rows_of(&counts), vec![vec![nv(2.0)], vec![nv(1.0)], vec![nv(0.0)]]); + for name in super::CRITERIA.iter().chain(super::PERIOD_LOOPS) { + assert!(crate::lookup(name).is_some(), "{name} isn't registered"); + } + } + + /// Security: CUMIPMT and CUMPRINC loop over their periods once per lifted element; the + /// periods of all elements are added up first, so an over-budget call runs none of them. + #[test] + fn lifted_period_loops_stop_past_their_work_budget() { + use gridcraft_core::Array; + let ends = |len: usize, end: f64| av(Array::filled(len, 1, nv(end)).into()); + let args = |len: usize, end: f64| vec![n(0.01), n(1e7), n(1000.0), n(1.0), ends(len, end), n(0.0)]; + // 51 elements of 1e7 periods each is past 5e8. + is_err(ev("CUMIPMT", args(51, 1e7)), CellError::Num); + is_err(ev("CUMPRINC", args(51, 1e7)), CellError::Num); + // A period costs more than a scanned cell, so fewer elements are enough. + is_err(ev("CUMIPMT", args(49, 1e7)), CellError::Num); + // Numeric text is coerced by the function, so its periods are charged the same. + let text_ends = av(Array::filled(51, 1, Value::text("10000000")).into()); + is_err(ev("CUMIPMT", vec![n(0.01), n(1e7), n(1000.0), n(1.0), text_ends, n(0.0)]), CellError::Num); + let out = ev("CUMIPMT", args(2, 12.0)); + assert_eq!(rows_of(&out).len(), 2); + assert!(matches!(rows_of(&out).first().and_then(|r| r.first()), Some(Value::Number(x)) if *x < 0.0)); + } + + /// Security: numeric text is parsed once per lifted element even when the call is refused; the + /// characters read are charged, so a long text repeated down a column can't cost gigabytes of parsing. + #[test] + fn lifted_period_loops_charge_the_text_they_parse() { + use gridcraft_core::Array; + // 20,000 elements x 32,000 characters is past 5e8 though no call reaches its loop (rate is negative). + let long = Value::text(format!("{}5", "0".repeat(32_000))); + let ends = av(Array::filled(20_000, 1, long).into()); + is_err(ev("CUMIPMT", vec![n(-0.01), n(10.0), n(1000.0), n(1.0), ends, n(0.0)]), CellError::Num); + // A short text period costs next to nothing. + let ends = av(Array::filled(3, 1, Value::text("12")).into()); + assert_eq!(rows_of(&ev("CUMIPMT", vec![n(0.01), n(20.0), n(1000.0), n(1.0), ends, n(0.0)])).len(), 3); + } + + /// Text after the first argument that fails to coerce is never read, so it is not charged: the + /// elements keep their own errors instead of one budget `#NUM!`. + #[test] + fn lifted_period_loops_charge_only_the_text_they_reach() { + use gridcraft_core::Array; + let rates = av(Array::filled(20_000, 1, Value::text("x")).into()); + let long = Value::text(format!("{}5", "0".repeat(32_000))); + let out = ev("CUMIPMT", vec![rates, n(10.0), n(1000.0), n(1.0), av(long), n(0.0)]); + assert!(rows_of(&out).iter().all(|r| r.first() == Some(&Value::Error(CellError::Value)))); + } + + /// A call the function refuses before its loop (here: end period past the period count) is + /// not charged, so rejected elements can't push a valid one over the budget. + #[test] + fn lifted_period_loops_do_not_charge_rejected_calls() { + use gridcraft_core::Array; + let mut ends = vec![nv(1e7 + 1.0); 60]; + ends.push(nv(12.0)); + let ends = av(Array::new(ends.len(), 1, ends).unwrap_or_else(|| panic!("shape")).into()); + let out = ev("CUMIPMT", vec![n(0.01), n(1e7), n(1000.0), n(1.0), ends, n(0.0)]); + let rows = rows_of(&out); + assert_eq!(rows.len(), 61); + assert!(matches!(rows.last().and_then(|r| r.first()), Some(Value::Number(x)) if *x < 0.0)); + } + + /// Security: an array-producing function lifted over a computed array builds each element's + /// whole result before keeping one cell; their cells count against the budget. + #[test] + fn lifted_array_results_count_against_the_budget() { + use gridcraft_core::Array; + let spec = crate::lookup("MUNIT").unwrap_or_else(|| panic!("MUNIT isn't registered")); + let dims = |n: usize| [av(Array::filled(n, 1, nv(10.0)).into())]; + // Ten 10x10 identity matrices are 1,000 cells. + let out = super::call_lifted_within(spec, &dims(10), &mut TestCtx::default(), 1_010); + assert_eq!(rows_of(&out), vec![vec![nv(1.0)]; 10]); + is_err(super::call_lifted_within(spec, &dims(11), &mut TestCtx::default(), 1_010), CellError::Num); + } + #[test] fn value_params_reject_multi_cell_references() { let s = FnSpec { value_args: &[true, false], ..spec() }; diff --git a/crates/functions/src/lookup.rs b/crates/functions/src/lookup.rs index 17c177d15..6652e00b1 100644 --- a/crates/functions/src/lookup.rs +++ b/crates/functions/src/lookup.rs @@ -132,11 +132,6 @@ fn row(a: &Array, r: usize) -> Vec { (0..a.cols).map(|c| a.get(r, c).cloned().unwrap_or(Value::Empty)).collect() } -/// A looked-up cell value: blanks read as 0 like a reference to an empty cell. -fn cell_result(v: Value) -> Value { - if matches!(v, Value::Empty) { Value::Number(0.0) } else { v } -} - // --------------------------------------------------------------------------------------------- // VLOOKUP / HLOOKUP / LOOKUP / MATCH @@ -160,7 +155,9 @@ fn vh_lookup(args: &[Arg], vertical: bool) -> R { let pos = if approx { bsearch_last(&x, &keys, false) } else { find_exact(&x, &keys, true, false) }; let pos = pos.ok_or(CellError::NA)?; let v = if vertical { table.get(pos, idx) } else { table.get(idx, pos) }; - Ok(cell_result(v.cloned().unwrap_or(Value::Empty))) + // An empty target stays blank, as in Excel: ISBLANK sees it as blank and `&` / + // CONCATENATE treat it as ""; only a formula's final result displays 0 (#374). + Ok(v.cloned().unwrap_or(Value::Empty)) } fn vlookup(a: &[Arg], _c: &mut dyn Ctx) -> R { @@ -193,18 +190,18 @@ fn lookup(args: &[Arg], _c: &mut dyn Ctx) -> R { } else { None }; - return v.map(cell_result).ok_or(CellError::NA); + return v.ok_or(CellError::NA); } // Array form: search the first row of a wide array, else the first column; return the // matching entry from the last row / column. if la.cols > la.rows { let keys = row(&la, 0); let pos = bsearch_last(&x, &keys, false).ok_or(CellError::NA)?; - Ok(cell_result(la.get(la.rows - 1, pos).cloned().unwrap_or(Value::Empty))) + Ok(la.get(la.rows - 1, pos).cloned().unwrap_or(Value::Empty)) } else { let keys = column(&la, 0); let pos = bsearch_last(&x, &keys, false).ok_or(CellError::NA)?; - Ok(cell_result(la.get(pos, la.cols - 1).cloned().unwrap_or(Value::Empty))) + Ok(la.get(pos, la.cols - 1).cloned().unwrap_or(Value::Empty)) } } @@ -325,13 +322,13 @@ fn xlookup(args: &[Arg], _c: &mut dyn Ctx) -> R { if vertical { let r = row(&ra, p); if r.len() == 1 { - return Ok(cell_result(r.into_iter().next().unwrap_or(Value::Empty))); + return Ok(r.into_iter().next().unwrap_or(Value::Empty)); } array_val(1, r.len(), r) } else { let c = column(&ra, p); if c.len() == 1 { - return Ok(cell_result(c.into_iter().next().unwrap_or(Value::Empty))); + return Ok(c.into_iter().next().unwrap_or(Value::Empty)); } array_val(c.len(), 1, c) } @@ -595,7 +592,8 @@ mod tests { is_err(ev("VLOOKUP", vec![n(4.0), rf(table()), n(2.0), b(false)]), CellError::NA); is_err(ev("VLOOKUP", vec![n(4.0), rf(table()), n(2.0), empty()]), CellError::NA); close(ev("VLOOKUP", vec![n(3.0), rf(table()), n(3.0), b(false)]), 30.0); - close(ev("VLOOKUP", vec![n(5.0), rf(table()), n(3.0), b(false)]), 0.0); + // An empty target stays blank (the cell displays 0 only as a final result) (#374). + assert_eq!(ev("VLOOKUP", vec![n(5.0), rf(table()), n(3.0), b(false)]), Value::Empty); is_err(ev("VLOOKUP", vec![n(3.0), rf(table()), n(4.0)]), CellError::Ref); is_err(ev("VLOOKUP", vec![n(3.0), rf(table()), n(0.0)]), CellError::Value); close(ev("VLOOKUP", vec![t("b*"), rf(names()), n(2.0), b(false)]), 2.0); diff --git a/crates/functions/src/math.rs b/crates/functions/src/math.rs index 7c84d553b..37c617fc0 100644 --- a/crates/functions/src/math.rs +++ b/crates/functions/src/math.rs @@ -520,7 +520,7 @@ fn integer_args(a: &[Arg]) -> R> { v.iter() .map(|x| { let t = x.trunc(); - if !(0.0..9.007_199_254_740_992e15).contains(&t) { Err(CellError::Num) } else { Ok(t as u64) } + if !(0.0..9.007_199_254_740_992e15).contains(x) { Err(CellError::Num) } else { Ok(t as u64) } }) .collect() } diff --git a/crates/functions/src/math/tests.rs b/crates/functions/src/math/tests.rs index 92449eb26..b6143b4f6 100644 --- a/crates/functions/src/math/tests.rs +++ b/crates/functions/src/math/tests.rs @@ -229,6 +229,20 @@ fn combinatorics() { is_err(ev("MULTINOMIAL", vec![n(-1.0), n(2.0)]), Num); } +#[test] +fn gcd_and_lcm_reject_negative_fractions_before_truncation() { + for name in ["GCD", "LCM"] { + for x in [-0.1, -0.5, -0.999] { + is_err(ev2(name, x, 2.0), Num); + is_err(ev(name, vec![rf(row(&[2.0, x]))]), Num); + // A zero elsewhere must not hide an invalid negative argument. + is_err(ev2(name, 0.0, x), Num); + } + close(ev2(name, -0.0, 2.0), if name == "GCD" { 2.0 } else { 0.0 }); + close(ev2(name, 12.9, 18.9), if name == "GCD" { 6.0 } else { 36.0 }); + } +} + #[test] fn trig() { close(ev1("SIN", PI / 2.0), 1.0); diff --git a/crates/functions/src/stat/regression.rs b/crates/functions/src/stat/regression.rs index 55bf0aec7..622022ee9 100644 --- a/crates/functions/src/stat/regression.rs +++ b/crates/functions/src/stat/regression.rs @@ -55,12 +55,12 @@ fn strict_num(v: &Value) -> R { } } -/// Most predictor columns LINEST, LOGEST, TREND and GROWTH accept. The normal equations are a -/// k×k matrix inverted in O(k³), so an unbounded k (one `SEQUENCE(1,16384)` argument) allocated +/// Most predictor columns LINEST, LOGEST, TREND and GROWTH accept. The fit works on a k×k +/// triangle in O(k³), so an unbounded k (one `SEQUENCE(1,16384)` argument) allocated /// gigabytes and could abort the process. pub(super) const MAX_REGRESSORS: usize = 255; -/// Most multiply-adds (rows × k²) a regression may spend building its normal equations. +/// Most multiply-adds (rows × k²) a regression may spend on its factorisation. const MAX_REGRESSION_WORK: usize = 1_000_000_000; /// Rejects regressions too large to fit: more than [`MAX_REGRESSORS`] predictors is `#VALUE!`, @@ -121,59 +121,136 @@ struct Fit { ssresid: f64, } -/// Inverts a symmetric positive semi-definite matrix by Gauss–Jordan on the diagonal, dropping -/// (zeroing) collinear variables. Returns the inverse and the dropped flags. -fn sym_inverse(m: &[Vec]) -> (Vec>, Vec) { - let p = m.len(); - let mut a: Vec> = m.to_vec(); - let mut inv: Vec> = (0..p).map(|i| (0..p).map(|j| if i == j { 1.0 } else { 0.0 }).collect()).collect(); - let mut dropped = vec![false; p]; - let diag: Vec = (0..p).map(|i| m.get(i).and_then(|r| r.get(i)).copied().unwrap_or(0.0).abs()).collect(); - for j in 0..p { - let piv = a.get(j).and_then(|r| r.get(j)).copied().unwrap_or(0.0); - let scale = diag.get(j).copied().unwrap_or(0.0); - if piv.abs() <= 1e-12 * scale || piv.abs() < 1e-300 { - if let Some(d) = dropped.get_mut(j) { - *d = true; - } - for row in a.iter_mut() { - if let Some(v) = row.get_mut(j) { - *v = 0.0; - } - } - if let Some(r) = a.get_mut(j) { - r.iter_mut().for_each(|v| *v = 0.0); - } - if let Some(r) = inv.get_mut(j) { - r.iter_mut().for_each(|v| *v = 0.0); - } +/// Euclidean norm without intermediate overflow or underflow. +fn norm2(v: &[f64]) -> f64 { + let m = v.iter().fold(0.0f64, |m, x| m.max(x.abs())); + if m == 0.0 || !m.is_finite() { + return m; + } + m * v.iter().map(|x| (x / m) * (x / m)).sum::().sqrt() +} + +/// Result of [`qr_solve`]: the least-squares solution on the kept columns, in the caller's +/// column scale, and what the standard errors need. +struct QrFit { + coef: Vec, + dropped: Vec, + /// Residual sum of squares of the fit. + ssresid: f64, + /// Diagonal of `(X'X)^-1` over the kept columns (0 for dropped ones). + diag: Vec, + /// `xm' (X'X)^-1 xm` over the kept columns. + quad: f64, +} + +/// Least squares by Householder QR on unit-norm columns, instead of the normal equations, whose +/// condition number is the square of the data's. Columns are taken left to right; one whose part +/// outside the span of the kept columns is under 1e-6 of its own length (the 1e-12 relative +/// pivot of the earlier normal-equation solver, on squares) is collinear: it is dropped, with +/// coefficient and standard error 0, as Excel does. +/// +/// `constant[j]` says column `j` was constant before centring. Centring it leaves rounding +/// residue, not 0 (`[0.1; 3]` centres to about -1.4e-17): it is collinear with the intercept, so +/// it is dropped instead of being scaled up to unit length. Whether a column is constant is +/// decided on its raw values, not by comparing the centred length with the offset: a predictor +/// like `[1e13, 1e13 + 1, 1e13 + 2]` is far smaller than its offset and still varies. +fn qr_solve(mut cols: Vec>, mut y: Vec, xm: &[f64], constant: &[bool]) -> QrFit { + let n = y.len(); + let k = cols.len(); + let mut scale = vec![1.0; k]; + let mut dropped = vec![false; k]; + // Householder vectors (unit norm, acting on rows `at..n`) and the columns of R. + let mut refl: Vec<(usize, Vec)> = Vec::new(); + let mut r_cols: Vec> = Vec::new(); + let mut kept: Vec = Vec::new(); + for (j, col) in cols.iter_mut().enumerate() { + let len = norm2(col); + if constant.get(j).copied().unwrap_or(false) || !(len > 1e-300 && len.is_finite()) { + dropped[j] = true; continue; } - let arow: Vec = a.get(j).map(|r| r.iter().map(|v| v / piv).collect()).unwrap_or_default(); - let irow: Vec = inv.get(j).map(|r| r.iter().map(|v| v / piv).collect()).unwrap_or_default(); - for i in 0..p { - if i == j { - continue; - } - let f = a.get(i).and_then(|r| r.get(j)).copied().unwrap_or(0.0); - if f == 0.0 { - continue; - } - if let Some(r) = a.get_mut(i) { - r.iter_mut().zip(&arow).for_each(|(v, w)| *v -= f * w); - } - if let Some(r) = inv.get_mut(i) { - r.iter_mut().zip(&irow).for_each(|(v, w)| *v -= f * w); - } + col.iter_mut().for_each(|v| *v /= len); + scale[j] = len; + for (at, v) in &refl { + apply_reflector(*at, v, col); } - if let Some(r) = a.get_mut(j) { - *r = arow; + let at = refl.len(); + let tail = col.get(at..).unwrap_or(&[]); + let alpha = norm2(tail); + if alpha * alpha <= 1e-12 { + dropped[j] = true; + continue; } - if let Some(r) = inv.get_mut(j) { - *r = irow; + let sign = if tail.first().is_some_and(|t| *t < 0.0) { -1.0 } else { 1.0 }; + let mut v: Vec = tail.to_vec(); + if let Some(h) = v.first_mut() { + *h += sign * alpha; + } + let vn = norm2(&v); + v.iter_mut().for_each(|x| *x /= vn); + let mut rc: Vec = col.get(..at).unwrap_or(&[]).to_vec(); + rc.push(-sign * alpha); + r_cols.push(rc); + refl.push((at, v)); + kept.push(j); + } + for (at, v) in &refl { + apply_reflector(*at, v, &mut y); + } + let r = kept.len(); + // Back substitution R c = (Q'y)[..r], then undo the column scaling. + let mut c = vec![0.0; r]; + for i in (0..r).rev() { + let mut t = y.get(i).copied().unwrap_or(0.0); + for jj in i + 1..r { + t -= r_cols.get(jj).and_then(|rc| rc.get(i)).copied().unwrap_or(0.0) * c.get(jj).copied().unwrap_or(0.0); + } + if let Some(ci) = c.get_mut(i) { + *ci = t / r_cols.get(i).and_then(|rc| rc.get(i)).copied().unwrap_or(f64::NAN); } } - (inv, dropped) + let ssresid = y.get(r..n).unwrap_or(&[]).iter().map(|v| v * v).sum::(); + // R^-1 (upper triangular), row i of the inverse at rinv[i]. + let rij = |i: usize, j: usize| r_cols.get(j).and_then(|rc| rc.get(i)).copied().unwrap_or(0.0); + let mut rinv = vec![vec![0.0; r]; r]; + for t in 0..r { + for i in (0..=t).rev() { + let mut v = if i == t { 1.0 } else { 0.0 }; + for jj in i + 1..=t { + v -= rij(i, jj) * rinv.get(jj).and_then(|row| row.get(t)).copied().unwrap_or(0.0); + } + if let Some(cell) = rinv.get_mut(i).and_then(|row| row.get_mut(t)) { + *cell = v / rij(i, i); + } + } + } + let mut coef = vec![0.0; k]; + let mut diag = vec![0.0; k]; + for (i, &j) in kept.iter().enumerate() { + let s = scale.get(j).copied().unwrap_or(1.0); + if let Some(v) = coef.get_mut(j) { + *v = c.get(i).copied().unwrap_or(0.0) / s; + } + let row = rinv.get(i); + if let Some(v) = diag.get_mut(j) { + *v = row.map_or(0.0, |rw| rw.iter().map(|x| x * x).sum::()) / (s * s); + } + } + // xm' (S R'R S)^-1 xm = |R^-T S^-1 xm|². + let w: Vec = kept.iter().map(|&j| xm.get(j).copied().unwrap_or(0.0) / scale.get(j).copied().unwrap_or(1.0)).collect(); + let quad = (0..r) + .map(|t| { + (0..=t).map(|i| rinv.get(i).and_then(|row| row.get(t)).copied().unwrap_or(0.0) * w.get(i).copied().unwrap_or(0.0)).sum::().powi(2) + }) + .sum::(); + QrFit { coef, dropped, ssresid, diag, quad } +} + +/// Applies the Householder reflection `I - 2vv'` on rows `at..` of `x`. +fn apply_reflector(at: usize, v: &[f64], x: &mut [f64]) { + let Some(tail) = x.get_mut(at..) else { return }; + let dot: f64 = v.iter().zip(tail.iter()).map(|(a, b)| a * b).sum(); + tail.iter_mut().zip(v).for_each(|(t, a)| *t -= 2.0 * dot * a); } fn regress(y: &[f64], x: &[Vec], k: usize, constant: bool) -> R { @@ -189,35 +266,22 @@ fn regress(y: &[f64], x: &[Vec], k: usize, constant: bool) -> R { } else { (vec![0.0; k], 0.0) }; - // Normal equations on (centred) data. - let mut xtx = vec![vec![0.0; k]; k]; - let mut xty = vec![0.0; k]; - for (row, yv) in x.iter().zip(y) { - let d: Vec = (0..k).map(|j| row.get(j).copied().unwrap_or(0.0) - xm.get(j).copied().unwrap_or(0.0)).collect(); - let yd = yv - ym; - for (i, di) in d.iter().enumerate() { - if let Some(t) = xty.get_mut(i) { - *t += di * yd; - } - if let Some(r) = xtx.get_mut(i) { - for (j, dj) in d.iter().enumerate() { - if let Some(v) = r.get_mut(j) { - *v += di * dj; - } - } - } - } - } - let (inv, dropped) = sym_inverse(&xtx); - let coef: Vec = (0..k).map(|i| inv.get(i).map(|r| r.iter().zip(&xty).map(|(a, b)| a * b).sum()).unwrap_or(0.0)).collect(); + // (Centred) predictor columns and response. + let cols: Vec> = + (0..k).map(|j| x.iter().map(|r| r.get(j).copied().unwrap_or(0.0) - xm.get(j).copied().unwrap_or(0.0)).collect()).collect(); + let yc: Vec = y.iter().map(|v| v - ym).collect(); + let sstot: f64 = if constant { yc.iter().map(|v| v * v).sum() } else { y.iter().map(|v| v * v).sum() }; + // Only centring leaves rounding residue behind; without an intercept a constant column is data. + // Constant means every raw value is the same double: distinct values, however close, vary. + let constant_cols: Vec = (0..k) + .map(|j| { + let mut col = x.iter().map(|r| r.get(j).copied().unwrap_or(0.0)); + let first = col.next(); + constant && first.is_some_and(|f| col.all(|v| v == f)) + }) + .collect(); + let QrFit { coef, dropped, ssresid, diag, quad } = qr_solve(cols, yc, &xm, &constant_cols); let b = if constant { ym - coef.iter().zip(&xm).map(|(c, m)| c * m).sum::() } else { 0.0 }; - let mut ssresid = 0.0; - let mut sstot = 0.0; - for (row, yv) in x.iter().zip(y) { - let pred = b + coef.iter().zip(row).map(|(c, v)| c * v).sum::(); - ssresid += (yv - pred).powi(2); - sstot += if constant { (yv - ym).powi(2) } else { yv * yv }; - } let used = dropped.iter().filter(|d| !**d).count() + usize::from(constant); let df = nf - used as f64; let ssreg = sstot - ssresid; @@ -230,20 +294,9 @@ fn regress(y: &[f64], x: &[Vec], k: usize, constant: bool) -> R { _ => Err(CellError::Num), }; let s2 = sey2.unwrap_or(f64::NAN); - let se: Vec = (0..k) - .map(|i| if dropped.get(i).copied().unwrap_or(true) { 0.0 } else { (inv.get(i).and_then(|r| r.get(i)).copied().unwrap_or(0.0) * s2).sqrt() }) - .collect(); - let se_b = if constant { - let mut q = 0.0; - for (i, mi) in xm.iter().enumerate() { - for (j, mj) in xm.iter().enumerate() { - q += mi * mj * inv.get(i).and_then(|r| r.get(j)).copied().unwrap_or(0.0); - } - } - Some((s2 * (1.0 / nf + q)).sqrt()) - } else { - None - }; + let se: Vec = + (0..k).map(|i| if dropped.get(i).copied().unwrap_or(true) { 0.0 } else { (diag.get(i).copied().unwrap_or(0.0) * s2).sqrt() }).collect(); + let se_b = if constant { Some((s2 * (1.0 / nf + quad)).sqrt()) } else { None }; Ok(Fit { coef, b, se, se_b, r2, sey, f, df, ssreg, ssresid }) } diff --git a/crates/functions/src/stat/tests.rs b/crates/functions/src/stat/tests.rs index fb67a2abb..28fed967e 100644 --- a/crates/functions/src/stat/tests.rs +++ b/crates/functions/src/stat/tests.rs @@ -433,6 +433,185 @@ fn linest_family() { is_err(ev("LINEST", vec![c(&[1.0, 2.0]), c(&[1.0, 2.0, 3.0])]), CellError::Ref); } +/// LINEST on columns of predictors, `stats` on, as rows of numbers (`None` for an error cell). +fn linest_rows(y: &[f64], xs: &[Vec], constant: bool) -> Vec>> { + let x = arr((0..y.len()).map(|i| xs.iter().map(|col| nv(col[i])).collect()).collect()); + let r = rows_of(&ev("LINEST", vec![c(y), av(x), b(constant), b(true)])); + r.iter().map(|row| row.iter().map(Value::as_f64).collect()).collect() +} + +fn rel_close(got: Option, want: f64, tol: f64, what: &str) { + let got = got.unwrap_or(f64::NAN); + assert!((got - want).abs() <= tol * want.abs().max(1e-300), "{what}: got {got:e}, want {want:e}, rel tol {tol:e}"); +} + +// B2: LINEST solved the normal equations, whose condition number is the square of the data's. +// Expected values below come from exact rational arithmetic: the normal equations solved with +// Python `fractions.Fraction` (no rounding at all), rounded once to f64 (script in the PR text). +#[test] +fn linest_ill_conditioned_polynomial_is_exact() { + // y = 1 + x + x² + x³ + x⁴ + x⁵ on x = 1..12: every coefficient is exactly 1. The normal + // equations were off by up to 2.6e-6. + let xs: Vec = (1..=12).map(f64::from).collect(); + let ys: Vec = xs.iter().map(|x| (0..=5).map(|j| x.powi(j)).sum()).collect(); + let cols: Vec> = (1..=5).map(|j| xs.iter().map(|x| x.powi(j)).collect()).collect(); + let r = linest_rows(&ys, &cols, true); + for (i, v) in r[0].iter().enumerate() { + rel_close(*v, 1.0, 1e-9, &format!("coefficient {i}")); + } +} + +#[test] +fn linest_large_offset_regressors_recover_the_line() { + // y = 3x + 7 exactly with x = 1e6 + i and regressors [x, x²]: the true fit has x² coefficient + // 0, x coefficient 3, intercept 7. The normal equations returned 2.99976 and 251.14. + let xs: Vec = (0..12).map(|i| 1_000_000.0 + f64::from(i)).collect(); + let ys: Vec = xs.iter().map(|x| 3.0 * x + 7.0).collect(); + let r = linest_rows(&ys, &[xs.clone(), xs.iter().map(|x| x * x).collect()], true); + assert!(r[0][0].unwrap().abs() < 1e-12, "x² coefficient {:?}", r[0][0]); + rel_close(r[0][1], 3.0, 1e-9, "x coefficient"); + // The intercept subtracts 3e6 from 3e6 + 7 and carries the x² coefficient times 1e12, so + // it is good to about 1e-4 in absolute terms, not better. + assert!((r[0][2].unwrap() - 7.0).abs() < 1e-3, "intercept {:?}", r[0][2]); + // One regressor: exact. + let one = linest_rows(&ys, std::slice::from_ref(&xs), true); + rel_close(one[0][0], 3.0, 1e-12, "slope"); + rel_close(one[0][1], 7.0, 1e-9, "intercept"); +} + +#[test] +fn linest_noisy_fits_match_exact_rational_least_squares() { + // Two predictors with noise, constant fitted. Order of each row: x2, x1, intercept. + let x1 = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; + let x2 = [2.0, 1.0, 4.0, 3.0, 7.0, 5.0, 9.0, 6.0]; + let y = [5.0, 6.0, 11.0, 12.0, 20.0, 19.0, 30.0, 28.0]; + let r = linest_rows(&y, &[x1.to_vec(), x2.to_vec()], true); + let (coef, se) = ([1.4270152505446623, 2.452069716775599, -1.2592592592592593], [0.3376442444771937, 0.3679392851360571, 1.0847489958915801]); + for i in 0..3 { + rel_close(r[0][i], coef[i], 1e-12, "coef"); + rel_close(r[1][i], se[i], 1e-12, "se"); + } + rel_close(r[2][0], 0.985070139277778, 1e-12, "r2"); + rel_close(r[2][1], 1.367057173604725, 1e-12, "sey"); + rel_close(r[3][0], 164.94965318256004, 1e-11, "F"); + assert_eq!(r[3][1], Some(5.0)); + rel_close(r[4][0], 616.5307734204792, 1e-12, "ssreg"); + rel_close(r[4][1], 9.344226579520697, 1e-11, "ssresid"); + assert_eq!(r[2][2], None); + // const = FALSE: through the origin, R² and F from the uncentred sums. + let r = linest_rows(&y, &[x1.to_vec(), x2.to_vec()], false); + rel_close(r[0][0], 1.3529411764705883, 1e-12, "coef"); + rel_close(r[0][1], 2.303921568627451, 1e-12, "coef"); + assert_eq!(r[0][2], Some(0.0)); + rel_close(r[1][0], 0.34102994374364876, 1e-12, "se"); + rel_close(r[1][1], 0.35495521934550084, 1e-12, "se"); + assert_eq!(r[1][2], None); + rel_close(r[2][0], 0.995718966041848, 1e-12, "r2"); + rel_close(r[2][1], 1.4061024795535124, 1e-12, "sey"); + rel_close(r[3][0], 697.7652892561983, 1e-11, "F"); + assert_eq!(r[3][1], Some(6.0)); + rel_close(r[4][0], 2759.1372549019607, 1e-12, "ssreg"); + rel_close(r[4][1], 11.862745098039216, 1e-11, "ssresid"); + // Cubic and quintic with noise on x = 1..12; order x^k .. x, intercept. + let xs: Vec = (1..=12).map(f64::from).collect(); + let noise = [3.0, -2.0, 4.0, -1.0, 2.0, -3.0, 1.0, -4.0, 2.0, 3.0, -2.0, 1.0]; + let cub: Vec = xs.iter().zip(noise).map(|(x, e)| 1.0 + x + x * x + x.powi(3) + e).collect(); + let r = linest_rows(&cub, &(1..=3).map(|j| xs.iter().map(|x| x.powi(j)).collect()).collect::>>(), true); + let coef = [0.9955969955969955, 1.1452991452991452, -0.33514633514633513, 4.373737373737374]; + let se = [0.02755463331301788, 0.5434124199147191, 3.102549978874372, 4.851885058051787]; + for i in 0..4 { + rel_close(r[0][i], coef[i], 1e-11, "cubic coef"); + rel_close(r[1][i], se[i], 1e-10, "cubic se"); + } + rel_close(r[2][0], 0.9999837292113533, 1e-12, "cubic r2"); + rel_close(r[4][1], 70.35586635586635, 1e-9, "cubic ssresid"); + let quint: Vec = xs.iter().zip(noise).map(|(x, e)| (0..=5).map(|j| x.powi(j)).sum::() + 50.0 * e).collect(); + let r = linest_rows(&quint, &(1..=5).map(|j| xs.iter().map(|x| x.powi(j)).collect()).collect::>>(), true); + let coef = [0.9330693815987934, 3.022273070067188, -20.97398190045249, 108.20489681886741, -250.24940010969422, 271.45454545454544]; + let se = [0.20046539191103213, 6.538253746916523, 78.39635356777087, 421.2608646012634, 976.3504976093745, 736.5866411764556]; + for i in 0..6 { + rel_close(r[0][i], coef[i], 1e-8, "quintic coef"); + rel_close(r[1][i], se[i], 1e-8, "quintic se"); + } + rel_close(r[2][1], 168.58174051770763, 1e-9, "quintic sey"); + rel_close(r[4][1], 170518.81941587824, 1e-7, "quintic ssresid"); +} + +#[test] +fn linest_drops_collinear_columns() { + // x2 = 2·x1: the later column is dropped (coefficient and standard error 0), the fit is the + // one on x1 alone, and the degrees of freedom count only the kept column. + let x1 = [1.0, 2.0, 3.0, 4.0, 5.0]; + let y = [2.0, 4.0, 5.0, 4.0, 5.0]; + let x2: Vec = x1.iter().map(|v| 2.0 * v).collect(); + let r = linest_rows(&y, &[x1.to_vec(), x2], true); + // Exact: slope 0.6, intercept 2.2, ssresid 2.4, ssreg 3.6, df 3 (Fraction arithmetic). + assert_eq!(r[0][0], Some(0.0)); + rel_close(r[0][1], 0.6, 1e-12, "slope"); + rel_close(r[0][2], 2.2, 1e-12, "intercept"); + assert_eq!(r[1][0], Some(0.0)); + rel_close(r[1][1], (0.8f64 / 10.0).sqrt(), 1e-12, "se slope"); + assert_eq!(r[3][1], Some(3.0)); + rel_close(r[4][0], 3.6, 1e-12, "ssreg"); + rel_close(r[4][1], 2.4, 1e-12, "ssresid"); + // A constant column is collinear with the intercept and dropped as well. + let r = linest_rows(&y, &[vec![7.0; 5], x1.to_vec()], true); + assert_eq!(r[0][1], Some(0.0)); + assert_eq!(r[1][1], Some(0.0)); + rel_close(r[0][0], 0.6, 1e-12, "slope"); + rel_close(r[0][2], 2.2, 1e-12, "intercept"); + // A constant column whose mean does not centre to exactly 0 (the mean of three 0.1 is + // 0.10000000000000002) is just as collinear with the intercept: dropped, not scaled up to a + // unit vector. Some length hits a rounding mean for each constant. + for c in [0.1, 1.1, 1e6 + 0.3] { + for n in 3..=12 { + let xs: Vec = (1..=n).map(f64::from).collect(); + let ys: Vec = (1..=n).map(|i| f64::from(i * i % 7) + 0.5 * f64::from(i)).collect(); + let r = linest_rows(&ys, &[vec![c; n as usize], xs], true); + assert_eq!(r[0][1], Some(0.0), "coefficient of the constant {c}, n={n}"); + assert_eq!(r[1][1], Some(0.0), "standard error of the constant {c}, n={n}"); + assert_eq!(r[3][1], Some(f64::from(n) - 2.0), "df with the constant {c}, n={n}"); + } + } + // A column that merely sits far from zero is not noise and is kept. + let off: Vec = x1.iter().map(|v| 1e9 + v).collect(); + let r = linest_rows(&y, &[off], true); + rel_close(r[0][0], 0.6, 1e-6, "slope on an offset column"); + assert_eq!(r[3][1], Some(3.0)); + // Distinct values far from zero, whose spread is below 1e-12 of the offset, still vary. + let r = linest_rows(&[0.0, 1.0, 2.0], &[vec![1e13, 1e13 + 1.0, 1e13 + 2.0]], true); + rel_close(r[0][0], 1.0, 1e-9, "slope on a predictor with a 1e13 offset"); + assert_eq!(r[3][1], Some(1.0), "df: the predictor is kept"); + // Same at 1e15, where the spread is only a few ulps of the offset: still distinct doubles. + let x15 = [1e15, 1e15 + 1.0, 1e15 + 2.0]; + let r = linest_rows(&[0.0, 1.0, 2.0], &[x15.to_vec()], true); + rel_close(r[0][0], 1.0, 1e-9, "slope on a predictor with a 1e15 offset"); + rel_close(r[0][1], -1e15, 1e-9, "intercept on a predictor with a 1e15 offset"); + assert_eq!(r[3][1], Some(1.0), "df: the 1e15 predictor is kept"); + assert!(r[4][1].is_some_and(|v| v.abs() < 1e-9), "ssresid: the 1e15 line is exact, got {:?}", r[4][1]); + let fitted = rows_of(&ev("TREND", vec![c(&[0.0, 1.0, 2.0]), c(&x15)])); + close(fitted[2][0].clone(), 2.0); + // More predictors than observations: the surplus columns drop, no panic. + let r = linest_rows(&[1.0, 2.0], &[vec![1.0, 2.0], vec![2.0, 1.0], vec![0.0, 5.0]], true); + assert_eq!(r[0].len(), 4); +} + +#[test] +fn logest_is_linest_of_ln_y() { + // y = 5·2^x: m = 2, b = 5. + let x = [0.0, 1.0, 2.0, 3.0, 4.0]; + let y: Vec = x.iter().map(|v| 5.0 * 2f64.powf(*v)).collect(); + let l = rows_of(&ev("LOGEST", vec![c(&y), c(&x)])); + close_tol(l[0][0].clone(), 2.0, 1e-12); + close_tol(l[0][1].clone(), 5.0, 1e-12); + // The statistics are those of the fit on ln y, untransformed. + let ln: Vec = y.iter().map(|v| v.ln()).collect(); + let ls = rows_of(&ev("LOGEST", vec![c(&y), c(&x), b(true), b(true)])); + let ns = rows_of(&ev("LINEST", vec![c(&ln), c(&x), b(true), b(true)])); + assert_eq!(ls[1..], ns[1..]); + close_tol(ls[0][0].clone(), ns[0][0].as_f64().unwrap().exp(), 1e-14); +} + // #2177: one predictor row of 16,384 columns built three 16,384² matrices (6 GB) and an O(k³) // inversion; a wider TOROW aborted the process. It is now refused before allocating. #[test] diff --git a/crates/functions/src/text.rs b/crates/functions/src/text.rs index 24674a93c..53186bb0e 100644 --- a/crates/functions/src/text.rs +++ b/crates/functions/src/text.rs @@ -30,17 +30,27 @@ fn check_len(s: String) -> R { /// Pushes `piece` onto `out`, failing once the result exceeds the cell limit. fn push_capped(out: &mut String, piece: &str) -> R<()> { - out.push_str(piece); - if out.len() > MAX_TEXT * 4 { + // Checked before copying, so one oversized piece is refused without being appended. + if out.len().saturating_add(piece.len()) > MAX_TEXT * 4 { return Err(CellError::Value); } + out.push_str(piece); Ok(()) } +/// `v.to_text()` without copying text values. +fn piece(v: &Value) -> R> { + match v { + Value::Text(t) => Ok(std::borrow::Cow::Borrowed(t)), + Value::Array(a) => a.data.first().map_or(Ok(std::borrow::Cow::Borrowed("")), piece), + _ => v.to_text().map(std::borrow::Cow::Owned), + } +} + fn concatenate(a: &[Arg], _c: &mut dyn Ctx) -> R { let mut out = String::new(); for x in a { - push_capped(&mut out, &scalar(x).to_text()?)?; + push_capped(&mut out, &piece(&scalar(x))?)?; } check_len(out) } @@ -49,31 +59,49 @@ fn concat(a: &[Arg], _c: &mut dyn Ctx) -> R { let mut out = String::new(); for x in a { for v in as_array(&x.value).iter() { - push_capped(&mut out, &v.to_text()?)?; + push_capped(&mut out, &piece(v)?)?; } } check_len(out) } +/// The error `v.to_text()` would give, found without building any text. +fn text_error(v: &Value) -> Option { + match v { + Value::Error(e) => Some(*e), + Value::Array(a) => a.data.first().and_then(text_error), + _ => None, + } +} + fn textjoin(a: &[Arg], _c: &mut dyn Ctx) -> R { - let delims: Vec = as_array(&arg(a, 0)?.value).iter().map(|v| v.to_text()).collect::>()?; + // Errors are found first, in argument order, so they win as before; text is then built one + // piece at a time into the capped output, so many references to one long text (or a long + // delimiter list) stop at the cap instead of first copying every piece. + let delims = as_array(&arg(a, 0)?.value); + if let Some(e) = delims.iter().find_map(text_error) { + return Err(e); + } let ignore = boolean(a, 1)?; - let mut items = Vec::new(); - for x in a.iter().skip(2) { - for v in as_array(&x.value).iter() { - let s = v.to_text()?; - if ignore && s.is_empty() { - continue; - } - items.push(s); - } + let items: Vec<_> = a.iter().skip(2).map(|x| as_array(&x.value)).collect(); + if let Some(e) = items.iter().flat_map(|x| x.iter()).find_map(text_error) { + return Err(e); } let mut out = String::new(); - for (i, s) in items.iter().enumerate() { - if i > 0 && !delims.is_empty() { - push_capped(&mut out, &delims[(i - 1) % delims.len()])?; + let mut joined = 0usize; + for v in items.iter().flat_map(|x| x.iter()) { + let s = piece(v)?; + if ignore && s.is_empty() { + continue; } - push_capped(&mut out, s)?; + if joined > 0 + && !delims.data.is_empty() + && let Some(d) = delims.data.get((joined - 1) % delims.data.len()) + { + push_capped(&mut out, &piece(d)?)?; + } + push_capped(&mut out, &s)?; + joined += 1; } check_len(out) } @@ -914,13 +942,25 @@ fn regexreplace(a: &[Arg], _c: &mut dyn Ctx) -> R { out.push_str(s.get(last..).unwrap_or_default()); return check_len(out); } - let caps: Vec = re.captures_iter(&s).collect(); - let k = occ.unsigned_abs() as usize; - let idx = if occ > 0 { k.checked_sub(1) } else { caps.len().checked_sub(k) }; - let Some(cap) = idx.and_then(|i| caps.get(i)) else { return text_val(s) }; + // An occurrence past what `usize` holds (32-bit targets) names no match: the text is returned as it was. + let Ok(k) = usize::try_from(occ.unsigned_abs()) else { return text_val(s) }; + // Keeps only the wanted match's captures: the nth from the front, or a window of the last `k`. + let cap = if occ > 0 { + re.captures_iter(&s).nth(k - 1) + } else { + let mut last = std::collections::VecDeque::new(); + for cap in re.captures_iter(&s) { + if last.len() == k { + last.pop_front(); + } + last.push_back(cap); + } + if last.len() == k { last.pop_front() } else { None } + }; + let Some(cap) = cap else { return text_val(s) }; let Some(m) = cap.get(0) else { return text_val(s) }; out.push_str(s.get(..m.start()).unwrap_or_default()); - expand_capped(cap, &rep, &mut out, limit).ok_or(CellError::Value)?; + expand_capped(&cap, &rep, &mut out, limit).ok_or(CellError::Value)?; out.push_str(s.get(m.end()..).unwrap_or_default()); check_len(out) } @@ -1171,6 +1211,22 @@ mod tests { assert_eq!(rows_of(&ev("CONCATENATE", vec![av(row_t(&["a", "b"])), t("x")])), vec![vec![tv("ax"), tv("bx")]]); } + /// Security: TEXTJOIN builds its result piece by piece and stops at the text cap, rather + /// than first copying every input (100,000 references to one 32,767-character text were + /// 3 GB of copies). Errors among the inputs still win, in argument order. + #[test] + fn textjoin_stops_at_the_text_cap() { + use gridcraft_core::Array; + let long = tv(&"x".repeat(32_767)); + let many = || av(Array::filled(100_000, 1, long.clone()).into()); + is_err(ev("TEXTJOIN", vec![t(","), b(true), many()]), CellError::Value); + is_err(ev("TEXTJOIN", vec![av(Array::filled(1, 100_000, long.clone()).into()), b(false), t("a"), t("b")]), CellError::Value); + is_err(ev("TEXTJOIN", vec![t(","), b(true), many(), e(CellError::Div0)]), CellError::Div0); + is_err(ev("TEXTJOIN", vec![av(arr(vec![vec![tv(","), Value::Error(CellError::NA)]])), b(true), t("a")]), CellError::NA); + // Delimiters cycle and are converted only where used. + is_text(ev("TEXTJOIN", vec![av(arr(vec![vec![nv(1.0), tv(";")]])), b(false), t("a"), t("b"), t("c"), t("d")]), "a1b;c1d"); + } + fn row_t(xs: &[&str]) -> Value { arr(vec![xs.iter().map(|x| tv(x)).collect()]) } diff --git a/crates/functions/src/util.rs b/crates/functions/src/util.rs index 14c627937..4e7853d2c 100644 --- a/crates/functions/src/util.rs +++ b/crates/functions/src/util.rs @@ -51,7 +51,7 @@ pub(crate) fn arg(args: &[Arg], i: usize) -> R<&Arg> { /// Whether argument `i` was supplied (present and not an empty literal like `F(1,)`). pub(crate) fn has(args: &[Arg], i: usize) -> bool { - matches!(args.get(i), Some(a) if !matches!(a.value, Value::Empty) || a.from_ref) + matches!(args.get(i), Some(a) if !matches!(a.value, Value::Empty) || a.from_ref || a.blank_value) } /// Number coercion for a scalar parameter: empty = 0, TRUE = 1, numeric text parses. diff --git a/crates/mcp/src/backend.rs b/crates/mcp/src/backend.rs index c77b612f7..e345ee0ee 100644 --- a/crates/mcp/src/backend.rs +++ b/crates/mcp/src/backend.rs @@ -53,6 +53,15 @@ impl Remote { self.conn = None; let mut last = std::io::Error::new(std::io::ErrorKind::NotFound, format!("cannot resolve {}", self.addr)); for sa in self.addr.to_socket_addrs()? { + // The token goes over plain TCP, so only to this machine: a network peer could read it, + // and the app's control port listens on loopback only. + if !sa.ip().is_loopback() { + last = std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!("{sa} is not a loopback address: the control token is sent only to 127.0.0.1 or ::1"), + ); + continue; + } match TcpStream::connect_timeout(&sa, Duration::from_secs(2)) { Ok(s) => { s.set_nodelay(true).ok(); diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs index 75b82d8d8..0d262dbe7 100644 --- a/crates/mcp/src/server.rs +++ b/crates/mcp/src/server.rs @@ -1,6 +1,6 @@ //! JSON-RPC 2.0 framing and the MCP lifecycle / tools / resources / prompts methods. -use std::io::{BufRead, Write}; +use std::io::{BufRead, Read, Write}; use serde_json::{Value, json}; @@ -45,6 +45,45 @@ fn response(id: Value, result: Value) -> Value { json!({"jsonrpc": "2.0", "id": id, "result": result}) } +/// Reads the next newline-delimited message into `buf`, keeping at most `MAX_LINE` bytes of it: +/// a client sending no newline can't make the server allocate more (#2164). `None` at the end of +/// `input`; `Some(false)` for a line over the limit, whose rest is read and dropped so the next +/// message is read whole. +pub(crate) fn read_message(input: &mut impl BufRead, buf: &mut Vec) -> std::io::Result> { + buf.clear(); + // One byte past the limit tells an over-long line from one of exactly `MAX_LINE` bytes. + let limit = u64::try_from(MAX_LINE).unwrap_or(u64::MAX).saturating_add(1); + if input.take(limit).read_until(b'\n', buf)? == 0 { + return Ok(None); + } + if buf.len() > MAX_LINE && buf.last() != Some(&b'\n') { + skip_line(input)?; + buf.clear(); + return Ok(Some(false)); + } + Ok(Some(true)) +} + +/// Reads up to and including the next newline (or to the end of `input`) without keeping it. +fn skip_line(input: &mut impl BufRead) -> std::io::Result<()> { + loop { + let chunk = input.fill_buf()?; + if chunk.is_empty() { + return Ok(()); + } + match chunk.iter().position(|&b| b == b'\n') { + Some(i) => { + input.consume(i + 1); + return Ok(()); + } + None => { + let n = chunk.len(); + input.consume(n); + } + } + } +} + fn error(id: Value, code: i64, message: impl Into) -> Value { json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message.into()}}) } @@ -68,14 +107,13 @@ impl Server { pub fn serve(&mut self, mut input: impl BufRead, mut output: impl Write) -> std::io::Result<()> { let mut buf = Vec::new(); loop { - buf.clear(); - let n = input.read_until(b'\n', &mut buf)?; - if n == 0 { - return Ok(()); - } - let reply = match std::str::from_utf8(&buf) { - Ok(line) => self.handle_line(line), - Err(_) => Some(error(Value::Null, PARSE_ERROR, "message is not valid UTF-8").to_string()), + let reply = match read_message(&mut input, &mut buf)? { + None => return Ok(()), + Some(false) => Some(error(Value::Null, INVALID_REQUEST, "message too large (16 MB max)").to_string()), + Some(true) => match std::str::from_utf8(&buf) { + Ok(line) => self.handle_line(line), + Err(_) => Some(error(Value::Null, PARSE_ERROR, "message is not valid UTF-8").to_string()), + }, }; if let Some(reply) = reply { output.write_all(reply.as_bytes())?; diff --git a/crates/mcp/src/tests.rs b/crates/mcp/src/tests.rs index 71631b5e8..0f9dd665f 100644 --- a/crates/mcp/src/tests.rs +++ b/crates/mcp/src/tests.rs @@ -359,6 +359,71 @@ fn serve_loop_over_buffers() { assert!(lines[2]["result"]["content"][0]["text"].as_str().unwrap().contains("42")); } +/// Streams `len` bytes of `a` from one reused buffer, then `tail`: a huge line costs no memory to +/// produce, so only the server can make it expensive. +struct HugeLine { + left: u64, + tail: &'static [u8], + chunk: Vec, +} + +impl std::io::Read for HugeLine { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + let buf = std::io::BufRead::fill_buf(self)?; + let n = buf.len().min(out.len()); + out[..n].copy_from_slice(&buf[..n]); + std::io::BufRead::consume(self, n); + Ok(n) + } +} + +impl std::io::BufRead for HugeLine { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + if self.left > 0 { + let n = self.chunk.len().min(usize::try_from(self.left).unwrap_or(usize::MAX)); + return Ok(&self.chunk[..n]); + } + Ok(self.tail) + } + fn consume(&mut self, n: usize) { + if self.left > 0 { + self.left -= n as u64; + } else { + self.tail = &self.tail[n..]; + } + } +} + +#[test] +fn a_line_without_newline_keeps_at_most_the_limit_in_memory() { + // #2164: the whole line was buffered before its size was checked (4 GiB here; any size a + // client cares to send). + let mut input = + HugeLine { left: 4 << 30, tail: concat!("\n", r#"{"jsonrpc":"2.0","id":7,"method":"ping"}"#, "\n").as_bytes(), chunk: vec![b'a'; 1 << 20] }; + let mut buf = Vec::new(); + assert_eq!(crate::server::read_message(&mut input, &mut buf).unwrap(), Some(false)); + assert!(buf.capacity() <= 2 * (16 << 20) + 1, "kept {} bytes", buf.capacity()); + assert_eq!(crate::server::read_message(&mut input, &mut buf).unwrap(), Some(true)); + assert!(buf.starts_with(br#"{"jsonrpc""#)); + assert_eq!(crate::server::read_message(&mut input, &mut buf).unwrap(), None); +} + +#[test] +fn serve_loop_bounds_a_line_without_newline() { + // #2164: a 4 GiB line was read whole before its size was checked. Now the server keeps at + // most 16 MB of it, answers that it is too large, and reads the next message normally. + let input = + HugeLine { left: 4 << 30, tail: concat!("\n", r#"{"jsonrpc":"2.0","id":7,"method":"ping"}"#, "\n").as_bytes(), chunk: vec![b'a'; 1 << 20] }; + let mut out = Vec::new(); + server().serve(input, &mut out).unwrap(); + let lines: Vec = String::from_utf8(out).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0]["error"]["code"], -32600); + assert!(lines[0]["error"]["message"].as_str().unwrap().contains("too large")); + assert_eq!(lines[1]["id"], 7); + assert!(lines[1].get("result").is_some(), "{}", lines[1]); +} + #[test] fn cell_set_without_input_is_error() { let mut s = server(); @@ -417,6 +482,9 @@ fn remote_backend_over_tcp() { drop(s); handle.join().unwrap(); assert!(Remote::connect_with_token("127.0.0.1:1", "x".into()).is_err()); + // A network address is refused before anything, the token included, is sent. + let err = Remote::connect_with_token("192.0.2.1:7979", "x".into()).err().unwrap(); + assert_eq!(err.kind(), std::io::ErrorKind::PermissionDenied, "{err}"); } #[test] diff --git a/crates/model/src/features.rs b/crates/model/src/features.rs index 8ab74606a..5c9f961d7 100644 --- a/crates/model/src/features.rs +++ b/crates/model/src/features.rs @@ -668,6 +668,9 @@ pub enum ShapeKind { /// A freehand ink stroke; `Shape::text` holds the points as `x,y;x,y;…` relative to the /// anchor (sheet points), `Shape::line` the ink colour and `fill` unused. Ink, + /// Decorative text (`insert.wordArt`): a transparent shape whose `Shape::text` is drawn with + /// the large, filled/outlined/shadowed text properties of `Shape::format`. + WordArt, /// An equation: `Shape::text` holds its LaTeX source, `Shape::line` its ink colour /// (`Color::Auto` for the sheet's text colour) and `Shape::math` its cached layout. Equation, @@ -1026,6 +1029,8 @@ pub struct ShapeFormat { pub text_color: Option, /// Text outline (`a:rPr/a:ln`): colour and width in points. pub text_outline: Option<(Color, f32)>, + /// Text drop shadow colour (`a:rPr/a:effectLst/a:outerShdw`); `None` is no shadow. + pub text_shadow: Option, /// `None`: the shape's default (left for text boxes, centred for other shapes). pub align: Option, /// `None`: the shape's default (top for text boxes, middle for other shapes). diff --git a/crates/model/src/store.rs b/crates/model/src/store.rs index e2da31e43..21406efa7 100644 --- a/crates/model/src/store.rs +++ b/crates/model/src/store.rs @@ -235,11 +235,12 @@ impl CellStore { bytes = bytes.saturating_add(size_of::()); for (row, cols) in &band.rows { bytes = bytes.saturating_add(size_of::() + 3 * size_of::()); + let next_row = next.and_then(|b| b.rows.get(row)); for (col, cell) in cols { // Cells and BTreeMap nodes are duplicated with a cloned band even when // their formula/value Arcs are still shared with the next snapshot. bytes = bytes.saturating_add(size_of::() + size_of::() + 3 * size_of::()); - let next_cell = next.and_then(|b| b.rows.get(row)).and_then(|r| r.get(col)); + let next_cell = next_row.and_then(|r| r.get(col)); if let Some(formula) = &cell.formula && !next_cell.and_then(|c| c.formula.as_ref()).is_some_and(|f| Arc::ptr_eq(f, formula)) { diff --git a/crates/numfmt/src/builtin.rs b/crates/numfmt/src/builtin.rs deleted file mode 100644 index ed03627f7..000000000 --- a/crates/numfmt/src/builtin.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! Built-in number formats (ECMA-376 §18.8.30, en-US). - -const BUILTINS: [(u32, &str); 36] = [ - (0, "General"), - (1, "0"), - (2, "0.00"), - (3, "#,##0"), - (4, "#,##0.00"), - (5, "\"$\"#,##0_);\\(\"$\"#,##0\\)"), - (6, "\"$\"#,##0_);[Red]\\(\"$\"#,##0\\)"), - (7, "\"$\"#,##0.00_);\\(\"$\"#,##0.00\\)"), - (8, "\"$\"#,##0.00_);[Red]\\(\"$\"#,##0.00\\)"), - (9, "0%"), - (10, "0.00%"), - (11, "0.00E+00"), - (12, "# ?/?"), - (13, "# ??/??"), - (14, "m/d/yyyy"), - (15, "d-mmm-yy"), - (16, "d-mmm"), - (17, "mmm-yy"), - (18, "h:mm AM/PM"), - (19, "h:mm:ss AM/PM"), - (20, "h:mm"), - (21, "h:mm:ss"), - (22, "m/d/yyyy h:mm"), - (37, "#,##0 ;(#,##0)"), - (38, "#,##0 ;[Red](#,##0)"), - (39, "#,##0.00;(#,##0.00)"), - (40, "#,##0.00;[Red](#,##0.00)"), - (41, "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"_);_(@_)"), - (42, "_(\"$\"* #,##0_);_(\"$\"* \\(#,##0\\);_(\"$\"* \"-\"_);_(@_)"), - (43, "_(* #,##0.00_);_(* \\(#,##0.00\\);_(* \"-\"??_);_(@_)"), - (44, "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)"), - (45, "mm:ss"), - (46, "[h]:mm:ss"), - (47, "mmss.0"), - (48, "##0.0E+0"), - (49, "@"), -]; - -/// Built-in format code for an id (0..=49, en-US), e.g. 14 => "m/d/yyyy". Locale-specific ids -/// (23–36) and unknown ids return `None`. -pub fn builtin_format(id: u32) -> Option<&'static str> { - BUILTINS.iter().find(|(i, _)| *i == id).map(|(_, c)| *c) -} - -/// Id of a built-in format code. Matching ignores case and backslash escapes. -pub fn builtin_id(code: &str) -> Option { - if let Some((id, _)) = BUILTINS.iter().find(|(_, c)| *c == code) { - return Some(*id); - } - let norm = |s: &str| s.chars().filter(|&c| c != '\\').flat_map(char::to_lowercase).collect::(); - let n = norm(code); - BUILTINS.iter().find(|(_, c)| norm(c) == n).map(|(id, _)| *id) -} diff --git a/crates/numfmt/src/decimal.rs b/crates/numfmt/src/decimal.rs index b6c831f71..80fb23e32 100644 --- a/crates/numfmt/src/decimal.rs +++ b/crates/numfmt/src/decimal.rs @@ -4,6 +4,8 @@ //! digits and then rounded half away from zero on that decimal string. That makes 2.675 with //! "0.00" show 2.68 and 1.005 show 1.01, as in Excel, even though neither is exact in binary. +use std::fmt::Write as _; + /// Most decimals we ever compute; placeholders beyond this are padded with zeros. pub(crate) const MAX_DECIMALS: usize = 400; @@ -21,7 +23,8 @@ pub(crate) fn decimal_digits(a: f64) -> ([u8; 15], i32) { } // 16 digits, rounded. Unless the 16th is a 5 it decides the 15th exactly (4 or less rounds // down, 6 or more up), so one format serves almost every number. - let (d16, exp) = significant_digits(a, 16); + let mut d16 = [0u8; 16]; + let exp = significant_digits(a, &mut d16); for (slot, d) in digits.iter_mut().zip(&d16) { *slot = *d; } @@ -30,11 +33,8 @@ pub(crate) fn decimal_digits(a: f64) -> ([u8; 15], i32) { 5 if is_exact_tie(a) => (digits, exp), 5 => { // Not a tie: round the exact value (the 16-digit rounding above may be a double rounding). - let (d15, exp15) = significant_digits(a, 15); let mut out = [0u8; 15]; - for (slot, d) in out.iter_mut().zip(&d15) { - *slot = *d; - } + let exp15 = significant_digits(a, &mut out); (out, exp15) } _ => { @@ -59,12 +59,54 @@ pub(crate) fn decimal_digits(a: f64) -> ([u8; 15], i32) { } } -/// `n` significant digits of `a` (rounded half to even on the exact binary value) and the power of -/// ten of the first. -fn significant_digits(a: f64, n: usize) -> (Vec, i32) { - let s = format!("{:.*e}", n.saturating_sub(1), a); - let (mant, exp) = s.split_once('e').unwrap_or((s.as_str(), "0")); - (mant.bytes().filter(u8::is_ascii_digit).map(|b| b - b'0').collect(), exp.parse().unwrap_or(0)) +/// Writes `out.len()` significant digits of `a` (rounded half to even on the exact binary value) +/// into `out` and returns the power of ten of the first. +/// +/// Up to 32 digits are formatted in a stack buffer; more (the 800 of the tie check) use a string. +fn significant_digits(a: f64, out: &mut [u8]) -> i32 { + fn parse(s: &str, out: &mut [u8]) -> i32 { + let (mant, exp) = s.split_once('e').unwrap_or((s, "0")); + let digits = mant.bytes().filter(u8::is_ascii_digit).map(|b| b - b'0'); + for (slot, d) in out.iter_mut().zip(digits) { + *slot = d; + } + exp.parse().unwrap_or(0) + } + let prec = out.len().saturating_sub(1); + if out.len() <= 32 { + let mut buf = StackStr::default(); + if write!(buf, "{:.*e}", prec, a).is_ok() { + return parse(buf.as_str(), out); + } + } + parse(&format!("{:.*e}", prec, a), out) +} + +/// A fixed-size string buffer for short number formatting; a write that does not fit fails. +struct StackStr { + buf: [u8; 64], + len: usize, +} + +impl Default for StackStr { + fn default() -> Self { + StackStr { buf: [0; 64], len: 0 } + } +} + +impl StackStr { + fn as_str(&self) -> &str { + self.buf.get(..self.len).and_then(|b| std::str::from_utf8(b).ok()).unwrap_or("") + } +} + +impl std::fmt::Write for StackStr { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + let end = self.len.checked_add(s.len()).ok_or(std::fmt::Error)?; + self.buf.get_mut(self.len..end).ok_or(std::fmt::Error)?.copy_from_slice(s.as_bytes()); + self.len = end; + Ok(()) + } } /// True when `a`'s exact decimal expansion has a 5 as its 16th significant digit and only zeros after. @@ -72,27 +114,59 @@ fn is_exact_tie(a: f64) -> bool { let is_tie = |ds: &[u8]| ds.get(15) == Some(&5) && ds.get(16..).is_some_and(|t| t.iter().all(|&d| d == 0)); // A cheap screen first (30 digits), then the exact expansion: a double has at most 767 // significant digits and Rust prints every one of them. - is_tie(&significant_digits(a, 30).0) && is_tie(&significant_digits(a, 800).0) + let mut screen = [0u8; 30]; + significant_digits(a, &mut screen); + if !is_tie(&screen) { + return false; + } + let mut exact = vec![0u8; 800]; + significant_digits(a, &mut exact); + is_tie(&exact) } /// Rounds the number `digits × 10^exp` (see [`decimal_digits`]) to `decimals` places, half away /// from zero. Returns the integer digits ("" when the integer part is zero) and exactly /// `decimals` fraction digits. pub(crate) fn fixed_from_digits(digits: &[u8; 15], exp: i32, decimals: usize) -> (String, String) { + let (mut int, mut frac) = (String::new(), String::new()); + fixed_into(digits, exp, decimals, &mut int, &mut frac); + (int, frac) +} + +/// [`fixed_from_digits`] into caller-owned strings (cleared first), so a loop can reuse them. +/// +/// The rounded digits are `lead` (at most 16, a carry can add a leading 1) followed by `zeros` +/// zero digits (placeholders beyond the 15 known digits), which is all a `Vec` of them held. +fn fixed_into(digits: &[u8; 15], exp: i32, decimals: usize, int: &mut String, frac: &mut String) { + int.clear(); + frac.clear(); let decimals_calc = decimals.min(MAX_DECIMALS); let keep = exp as i64 + decimals_calc as i64 + 1; - let mut m: Vec = if keep <= 0 { - if keep == 0 && digits.first().copied().unwrap_or(0) >= 5 { vec![1] } else { Vec::new() } + let mut head = [0u8; 16]; + let mut head_len = 0usize; + let mut zeros = 0usize; + if keep <= 0 { + if keep == 0 && digits.first().copied().unwrap_or(0) >= 5 { + if let Some(h) = head.first_mut() { + *h = 1; + } + head_len = 1; + } } else if keep as usize >= digits.len() { - let mut v = digits.to_vec(); - v.resize(keep as usize, 0); - v + for (h, d) in head.iter_mut().zip(digits) { + *h = *d; + } + head_len = digits.len(); + zeros = keep as usize - digits.len(); } else { let keep = keep as usize; - let mut v = digits.get(..keep).map(<[u8]>::to_vec).unwrap_or_default(); + for (h, d) in head.iter_mut().zip(digits.iter().take(keep)) { + *h = *d; + } + head_len = keep; if digits.get(keep).copied().unwrap_or(0) >= 5 { let mut carry = true; - for d in v.iter_mut().rev() { + for d in head.iter_mut().take(keep).rev() { if *d == 9 { *d = 0; } else { @@ -102,26 +176,36 @@ pub(crate) fn fixed_from_digits(digits: &[u8; 15], exp: i32, decimals: usize) -> } } if carry { - v.insert(0, 1); + head.copy_within(0..keep, 1); + if let Some(h) = head.first_mut() { + *h = 1; + } + head_len += 1; } } - v - }; - let lead = m.iter().take_while(|&&d| d == 0).count(); - m.drain(..lead); - let to_s = |ds: &[u8]| ds.iter().map(|d| char::from(b'0' + d)).collect::(); - let (int, mut frac) = if m.len() <= decimals_calc { - let mut f = "0".repeat(decimals_calc - m.len()); - f.push_str(&to_s(&m)); - (String::new(), f) - } else { - let split = m.len() - decimals_calc; - (to_s(m.get(..split).unwrap_or(&[])), to_s(m.get(split..).unwrap_or(&[]))) - }; - if decimals > decimals_calc { - frac.push_str(&"0".repeat(decimals - decimals_calc)); } - (int, frac) + let total = head_len + zeros; + let mut lead = head.iter().take(head_len).take_while(|&&d| d == 0).count(); + if lead == head_len { + lead = total; // all zeros + } + let len = total - lead; + // Digit `k` of the rounded number after its leading zeros. + let digit_at = |k: usize| { + let pos = lead + k; + if pos < head_len { head.get(pos).map_or('0', |&d| char::from(b'0' + d)) } else { '0' } + }; + if len <= decimals_calc { + frac.extend(std::iter::repeat_n('0', decimals_calc - len)); + frac.extend((0..len).map(digit_at)); + } else { + let split = len - decimals_calc; + int.extend((0..split).map(digit_at)); + frac.extend((split..len).map(digit_at)); + } + if decimals > decimals_calc { + frac.extend(std::iter::repeat_n('0', decimals - decimals_calc)); + } } /// Rounds a non-negative number to `decimals` places (see [`fixed_from_digits`]). @@ -148,21 +232,25 @@ pub fn format_general_fit(n: f64, max_chars: usize) -> Option { let sign = if n < 0.0 { "-" } else { "" }; let (digits, x) = decimal_digits(n.abs()); let int_len = sign.len() + (x.max(0) as usize) + 1; - // Fixed notation for magnitudes from 1E-4 up to what fits. + // Fixed notation for magnitudes from 1E-4 up to what fits. The 15 digits are computed once; + // each width only re-rounds them, into strings that are reused. + let (mut i, mut f, mut s) = (String::new(), String::new(), String::new()); if x >= -4 && int_len <= max_chars { let avail = max_chars - int_len; - let mut decimals = if avail >= 2 { avail - 1 } else { 0 }; + // Zeros past the 400th decimal are trimmed below anyway, so a huge width needs no more. + let mut decimals = (if avail >= 2 { avail - 1 } else { 0 }).min(MAX_DECIMALS); loop { - let (i, f) = fixed_from_digits(&digits, x, decimals); - let f = f.trim_end_matches('0'); - if i.is_empty() && f.is_empty() { + fixed_into(&digits, x, decimals, &mut i, &mut f); + let ft = f.trim_end_matches('0'); + if i.is_empty() && ft.is_empty() { break; // rounded away to zero: use scientific } - let mut s = String::from(sign); + s.clear(); + s.push_str(sign); s.push_str(if i.is_empty() { "0" } else { &i }); - if !f.is_empty() { + if !ft.is_empty() { s.push('.'); - s.push_str(f); + s.push_str(ft); } if s.chars().count() <= max_chars { return Some(s); @@ -175,19 +263,20 @@ pub fn format_general_fit(n: f64, max_chars: usize) -> Option { } for md in (0..=max_chars.min(14)).rev() { let mut e = x; - let (mut mi, mut mf) = fixed_from_digits(&digits, 0, md); - if mi.len() > 1 { + fixed_into(&digits, 0, md, &mut i, &mut f); + if i.len() > 1 { e += 1; - (mi, mf) = fixed_from_digits(&digits, -1, md); + fixed_into(&digits, -1, md, &mut i, &mut f); } - let mf = mf.trim_end_matches('0'); - let mut s = String::from(sign); - s.push_str(if mi.is_empty() { "0" } else { &mi }); - if !mf.is_empty() { + let ft = f.trim_end_matches('0'); + s.clear(); + s.push_str(sign); + s.push_str(if i.is_empty() { "0" } else { &i }); + if !ft.is_empty() { s.push('.'); - s.push_str(mf); + s.push_str(ft); } - s.push_str(&format!("E{}{:02}", if e < 0 { '-' } else { '+' }, e.unsigned_abs())); + let _ = write!(s, "E{}{:02}", if e < 0 { '-' } else { '+' }, e.unsigned_abs()); if s.chars().count() <= max_chars { return Some(s); } @@ -211,6 +300,10 @@ mod tests { assert_eq!(fixed(0.0005, 3), ("".into(), "001".into())); assert_eq!(fixed(1e20, 0).0, "100000000000000000000"); assert_eq!(fixed(0.0, 2), ("".into(), "00".into())); + // Past 400 decimals only zeros are added; huge exponents keep all their digits. + assert_eq!(fixed(1.5, 450), ("1".into(), format!("5{}", "0".repeat(449)))); + assert_eq!(fixed(1e300, 1), (format!("1{}", "0".repeat(300)), "0".into())); + assert_eq!(fixed(123456789012345678.0, 0).0, "123456789012346000"); } #[test] diff --git a/crates/numfmt/src/lib.rs b/crates/numfmt/src/lib.rs index 57384cc89..524ff5d47 100644 --- a/crates/numfmt/src/lib.rs +++ b/crates/numfmt/src/lib.rs @@ -3,7 +3,6 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] #![forbid(unsafe_code)] -mod builtin; mod calendar; mod decimal; mod parse; @@ -13,7 +12,6 @@ use std::sync::Arc; use gridcraft_core::{CellError, DateSystem, Value}; -pub use builtin::{builtin_format, builtin_id}; pub use decimal::format_general_fit; use parse::{DatePart, SecKind, Section, Tok}; @@ -395,6 +393,14 @@ impl NumberFormat { None => Formatted { text: t.to_string(), color: None, fill: None, numeric: false }, } } + + /// [`Self::format_text`]'s text, or `None` once it passes `max_units` UTF-16 code units. + fn format_text_within(&self, t: &str, max_units: usize) -> Option { + match self.inner.text_section.and_then(|i| self.inner.sections.get(i)) { + Some(sec) => render::render_text_within(sec, t, max_units).map(|out| out.text), + None => (t.encode_utf16().count() <= max_units).then(|| t.to_string()), + } + } } /// Formats a cell value for display. @@ -488,6 +494,10 @@ impl NumberFormat { /// The `TEXT()` worksheet function: formats a value with a format code. Numeric text is /// converted to a number first; other text and booleans (as `TRUE`/`FALSE`) go through the text /// section, or are returned as is. +/// Longest text a cell holds, in UTF-16 code units (Excel's 32,767 characters, which XLSB also +/// stores that way), and so the longest `TEXT` result. +const MAX_CELL_TEXT_CHARS: usize = 32_767; + pub fn text_function(v: &Value, code: &str, sys: DateSystem) -> Result { if code.is_empty() { return Ok(String::new()); @@ -506,7 +516,9 @@ pub fn text_function(v: &Value, code: &str, sys: DateSystem) -> Result return Ok(fmt.format_text(if b { "TRUE" } else { "FALSE" }).text), Value::Text(t) => match gridcraft_core::parse::parse_number_text(&t) { Some(n) => n, - None => return Ok(fmt.format_text(&t).text), + // Every `@` repeats the text: a result past a cell's limit is `#VALUE!`, so nested + // TEXT calls can't multiply it further. + None => return fmt.format_text_within(&t, MAX_CELL_TEXT_CHARS).ok_or(CellError::Value), }, Value::Array(_) => return Err(CellError::Value), }; diff --git a/crates/numfmt/src/parse.rs b/crates/numfmt/src/parse.rs index c2b121940..e9804b5c7 100644 --- a/crates/numfmt/src/parse.rs +++ b/crates/numfmt/src/parse.rs @@ -60,18 +60,28 @@ pub(crate) enum Tok { } impl Tok { + /// Appends the text this token shows when it has no special meaning in its section. + pub(crate) fn push_literal(&self, out: &mut String) { + match self { + Tok::Lit(s) | Tok::FixedDenom(s) => out.push_str(s), + Tok::Digit(c) => out.push(*c), + Tok::Point => out.push('.'), + Tok::Comma => out.push(','), + Tok::Percent => out.push('%'), + Tok::Exp { upper, plus } => { + out.push(if *upper { 'E' } else { 'e' }); + out.push(if *plus { '+' } else { '-' }); + } + Tok::Slash => out.push('/'), + Tok::Fill(_) | Tok::At | Tok::General | Tok::Date(_) => {} + } + } + /// The text this token shows when it has no special meaning in its section. pub(crate) fn literal_text(&self) -> String { - match self { - Tok::Lit(s) | Tok::FixedDenom(s) => s.clone(), - Tok::Digit(c) => c.to_string(), - Tok::Point => ".".into(), - Tok::Comma => ",".into(), - Tok::Percent => "%".into(), - Tok::Exp { upper, plus } => format!("{}{}", if *upper { 'E' } else { 'e' }, if *plus { '+' } else { '-' }), - Tok::Slash => "/".into(), - Tok::Fill(_) | Tok::At | Tok::General | Tok::Date(_) => String::new(), - } + let mut s = String::new(); + self.push_literal(&mut s); + s } } @@ -128,6 +138,13 @@ pub(crate) struct Section { pub fraction: bool, pub currency: bool, pub fill: bool, + /// Date sections: the most digits of any `.0`/`.00`/`.000` part (at most 3), which decides how + /// the time of day is rounded. + pub sub_digits: u8, + /// Date sections: has a year, month or day part, so the serial's date is needed. + pub needs_date: bool, + /// Date sections: has `AM/PM` or `A/P`, so hours show on the 12-hour clock. + pub ampm: bool, pub calendar: Calendar, /// Excel rejects the whole code because of this section: a `B` and digit that is not a /// calendar prefix, or a prefix before a number format (`B20.00`). @@ -515,6 +532,9 @@ fn analyze(raw: Raw) -> Section { fraction: false, currency, fill: toks.iter().any(|t| matches!(t, Tok::Fill(_))), + sub_digits: 0, + needs_date: false, + ampm: false, calendar, invalid, }; @@ -573,6 +593,9 @@ fn analyze_date(mut toks: Vec, mut sec: Section) -> Section { } i += 1; } + sec.sub_digits = out.iter().filter_map(|t| if let Tok::Date(DatePart::SubSec(n)) = t { Some(*n) } else { None }).max().unwrap_or(0).min(3); + sec.needs_date = out.iter().any(|t| matches!(t, Tok::Date(d) if d.is_calendar())); + sec.ampm = out.iter().any(|t| matches!(t, Tok::Date(DatePart::AmPm | DatePart::AP(..)))); sec.toks = out; sec } diff --git a/crates/numfmt/src/render.rs b/crates/numfmt/src/render.rs index bfd31e56c..e086ecaa2 100644 --- a/crates/numfmt/src/render.rs +++ b/crates/numfmt/src/render.rs @@ -1,5 +1,7 @@ //! Rendering a value with a parsed section. +use std::fmt::Write as _; + use gridcraft_core::DateSystem; use gridcraft_core::date::{datetime_from_serial, max_serial}; @@ -25,6 +27,10 @@ impl Out { self.fill = Some((c, self.text.len())); } } + /// Appends what `t` shows when it has no special meaning (no temporary string). + fn push_tok(&mut self, t: &Tok) { + t.push_literal(&mut self.text); + } pub(crate) fn prepend(&mut self, s: &str) { self.text.insert_str(0, s); if let Some((_, p)) = &mut self.fill { @@ -40,12 +46,41 @@ pub(crate) fn render_text(sec: &Section, text: &str) -> Out { match t { Tok::At => out.push(text), Tok::Fill(c) => out.fill(*c), - other => out.push(&other.literal_text()), + other => out.push_tok(other), } } out } +/// [`render_text`], but `None` once the output passes `max_units` UTF-16 code units (a cell's +/// limit), checked before each piece is appended, so an oversized result is never built. +pub(crate) fn render_text_within(sec: &Section, text: &str, max_units: usize) -> Option { + let text_units = text.encode_utf16().count(); + let mut out = Out::default(); + let mut units = 0usize; + for t in &sec.toks { + match t { + Tok::At => { + units = units.saturating_add(text_units); + if units > max_units { + return None; + } + out.push(text); + } + Tok::Fill(c) => out.fill(*c), + other => { + let literal = other.literal_text(); + units = units.saturating_add(literal.encode_utf16().count()); + if units > max_units { + return None; + } + out.push(&literal); + } + } + } + Some(out) +} + /// Renders a number through a General section (literals around the General keyword). pub(crate) fn render_general(sec: &Section, v: f64, width: usize) -> Out { let mut out = Out::default(); @@ -53,7 +88,7 @@ pub(crate) fn render_general(sec: &Section, v: f64, width: usize) -> Out { match t { Tok::General => out.push(&format_general_fit(v, width).unwrap_or_else(|| "#".repeat(width.clamp(1, 11)))), Tok::Fill(c) => out.fill(*c), - other => out.push(&other.literal_text()), + other => out.push_tok(other), } } out @@ -64,8 +99,7 @@ pub(crate) fn render_date(sec: &Section, v: f64, sys: DateSystem) -> Option if !v.is_finite() || !(0.0..max_serial(sys) as f64 + 1.0).contains(&v) { return None; } - let sub_digits = sec.toks.iter().filter_map(|t| if let Tok::Date(DatePart::SubSec(n)) = t { Some(*n) } else { None }).max().unwrap_or(0).min(3); - let unit: i64 = 10i64.pow(3 - sub_digits as u32); + let unit: i64 = 10i64.pow(3 - u32::from(sec.sub_digits.min(3))); let mut day = v.floor() as i64; let mut ms = (((v - v.floor()) * 86_400_000.0) / unit as f64).round() as i64 * unit; if ms >= 86_400_000 { @@ -82,38 +116,33 @@ pub(crate) fn render_date(sec: &Section, v: f64, sys: DateSystem) -> Option { return None; } - let needs_date = sec.toks.iter().any(|t| matches!(t, Tok::Date(d) if d.is_calendar())); - let dt = if needs_date { datetime_from_serial(sys, day as f64)? } else { datetime_from_serial(sys, 0.0).unwrap_or_default() }; - let ampm = sec.toks.iter().any(|t| matches!(t, Tok::Date(DatePart::AmPm | DatePart::AP(..)))); + let dt = if sec.needs_date { datetime_from_serial(sys, day as f64)? } else { datetime_from_serial(sys, 0.0).unwrap_or_default() }; + let ampm = sec.ampm; let hour = ms / 3_600_000; let minute = (ms / 60_000) % 60; let second = (ms / 1000) % 60; let milli = ms % 1000; - let pad = |n: i64, w: u8| if w >= 2 { format!("{:0w$}", n, w = w.min(20) as usize) } else { n.to_string() }; // No date in the section's calendar (serial 60 under `B2`): shown like an out-of-range date. let date = sec.calendar.date(&dt)?; let mut out = Out::default(); for t in &sec.toks { match t { Tok::Date(d) => { - let s = match *d { + let text = &mut out.text; + match *d { DatePart::Year(n) => { - if n <= 2 { - format!("{:02}", date.year.rem_euclid(100)) - } else { - format!("{:04}", date.year) - } + let _ = if n <= 2 { write!(text, "{:02}", date.year.rem_euclid(100)) } else { write!(text, "{:04}", date.year) }; } DatePart::Month(n) => match n { - 1 | 2 => pad(date.month as i64, n), - 3 => date.month_short.to_string(), - 5 => date.month_name.chars().take(1).collect(), - _ => date.month_name.to_string(), + 1 | 2 => push_padded(text, date.month as i64, n), + 3 => text.push_str(date.month_short), + 5 => text.extend(date.month_name.chars().take(1)), + _ => text.push_str(date.month_name), }, DatePart::Day(n) => match n { - 1 | 2 => pad(date.day as i64, n), - 3 => date.day_short.to_string(), - _ => date.day_name.to_string(), + 1 | 2 => push_padded(text, date.day as i64, n), + 3 => text.push_str(date.day_short), + _ => text.push_str(date.day_name), }, DatePart::Hour(n) => { let h = if ampm { @@ -122,78 +151,168 @@ pub(crate) fn render_date(sec: &Section, v: f64, sys: DateSystem) -> Option } else { hour }; - pad(h, n) + push_padded(text, h, n); } - DatePart::Minute(n) => pad(minute, n), - DatePart::Second(n) => pad(second, n), + DatePart::Minute(n) => push_padded(text, minute, n), + DatePart::Second(n) => push_padded(text, second, n), DatePart::SubSec(n) => { - let mut s = format!(".{:03}", milli); - s.truncate(1 + (n as usize).min(3)); - if n > 3 { - s.push_str(&"0".repeat(n as usize - 3)); + // `.` and the first `n` (at most 3) of the three millisecond digits, then + // zeros for the digits beyond milliseconds. + text.push('.'); + for (k, div) in [100, 10, 1].into_iter().enumerate() { + if k < n as usize { + text.push(char::from(b'0' + (milli / div % 10) as u8)); + } + } + for _ in 3..n { + text.push('0'); } - s } - DatePart::ElapsedH(n) => pad(total_ms / 3_600_000, n), - DatePart::ElapsedM(n) => pad(total_ms / 60_000, n), - DatePart::ElapsedS(n) => pad(total_ms / 1000, n), - DatePart::AmPm => (if hour >= 12 { "PM" } else { "AM" }).to_string(), - DatePart::AP(a, p) => (if hour >= 12 { p } else { a }).to_string(), - }; - out.push(&s); + DatePart::ElapsedH(n) => push_padded(text, total_ms / 3_600_000, n), + DatePart::ElapsedM(n) => push_padded(text, total_ms / 60_000, n), + DatePart::ElapsedS(n) => push_padded(text, total_ms / 1000, n), + DatePart::AmPm => text.push_str(if hour >= 12 { "PM" } else { "AM" }), + DatePart::AP(a, p) => text.push(if hour >= 12 { p } else { a }), + } } Tok::Fill(c) => out.fill(*c), - other => out.push(&other.literal_text()), + other => out.push_tok(other), } } Some(out) } +/// Appends `n`, zero padded to `w` digits (at most 20) when `w` is 2 or more. +fn push_padded(out: &mut String, n: i64, w: u8) { + let _ = if w >= 2 { write!(out, "{:0w$}", n, w = w.min(20) as usize) } else { write!(out, "{n}") }; +} + +/// Scratch text of one number rendering: every token's own output (`piece`, replaces the token's +/// text), the text inserted ahead of a token (`before`) and the replacement of a literal +/// (`lit_spaces`: fractions blank the literals of a hidden part) live in ONE buffer, addressed +/// by byte ranges, instead of one string per token. Writing a piece again appends the new text +/// and points the slot at it. +struct Parts { + buf: String, + slots: Vec, +} + +#[derive(Clone, Copy, Default)] +struct Slot { + piece: (usize, usize), + before: (usize, usize), + /// `Some(k)`: show `k` spaces instead of the literal. + lit_spaces: Option, +} + +impl Parts { + fn new(n: usize) -> Parts { + Parts { buf: String::new(), slots: vec![Slot::default(); n + 1] } + } + /// Start of a piece written straight into the buffer; finish it with [`Parts::end_piece`]. + fn mark(&self) -> usize { + self.buf.len() + } + fn end_piece(&mut self, ti: usize, start: usize) { + let end = self.buf.len(); + if let Some(sl) = self.slots.get_mut(ti) { + sl.piece = (start, end); + } + } + fn set_piece(&mut self, ti: usize, text: &str) { + let start = self.mark(); + self.buf.push_str(text); + self.end_piece(ti, start); + } + fn set_spaces(&mut self, ti: usize, k: usize) { + let start = self.mark(); + self.buf.extend(std::iter::repeat_n(' ', k)); + self.end_piece(ti, start); + } + fn set_before(&mut self, ti: usize, text: &str) { + let start = self.mark(); + self.buf.push_str(text); + let end = self.buf.len(); + if let Some(sl) = self.slots.get_mut(ti) { + sl.before = (start, end); + } + } + fn set_lit_spaces(&mut self, ti: usize, k: usize) { + if let Some(sl) = self.slots.get_mut(ti) { + sl.lit_spaces = Some(k); + } + } + fn text(&self, (a, b): (usize, usize)) -> &str { + self.buf.get(a..b).unwrap_or("") + } + fn piece(&self, ti: usize) -> &str { + self.slots.get(ti).map_or("", |sl| self.text(sl.piece)) + } + fn before(&self, ti: usize) -> &str { + self.slots.get(ti).map_or("", |sl| self.text(sl.before)) + } +} + /// Assigns integer digits right-to-left to the placeholders at `idxs`; the leftmost placeholder /// takes all remaining digits. `force_zero` shows `0` in the units placeholder when there are no /// digits. A blank `?` pads with a space, and so does the thousands separator it would carry. -fn fill_int(toks: &[Tok], idxs: &[usize], digits: &str, thousands: bool, force_zero: bool, pieces: &mut [String]) { - let ds: Vec = digits.chars().collect(); +/// +/// `digits` is ASCII (digit strings made by this crate), so bytes are characters. Each +/// placeholder's text is written left to right: a digit, then the separator that follows it when +/// its position counted from the right is a multiple of 3. +fn fill_int(toks: &[Tok], idxs: &[usize], digits: &str, thousands: bool, force_zero: bool, parts: &mut Parts) { + let ds = digits.as_bytes(); let mut rem = ds.len(); let mut pos = 0usize; let count = idxs.len(); + let sep = |p: usize| thousands && p > 0 && p.is_multiple_of(3); for (k, &ti) in idxs.iter().rev().enumerate() { let leftmost = k + 1 == count; let ph = match toks.get(ti) { Some(Tok::Digit(c)) => *c, _ => '0', }; - let mut rev = String::new(); - let push_digit = |c: char, rev: &mut String, pos: &mut usize| { - if thousands && *pos > 0 && pos.is_multiple_of(3) { - rev.push(','); - } - rev.push(c); - *pos += 1; - }; + let start = parts.mark(); let take = if leftmost { rem } else { rem.min(1) }; if take == 0 { match ph { - '0' => push_digit('0', &mut rev, &mut pos), - _ if force_zero && k == 0 => push_digit('0', &mut rev, &mut pos), + '0' => { + parts.buf.push('0'); + if sep(pos) { + parts.buf.push(','); + } + pos += 1; + } + _ if force_zero && k == 0 => { + parts.buf.push('0'); + if sep(pos) { + parts.buf.push(','); + } + pos += 1; + } '?' => { - if thousands && pos > 0 && pos.is_multiple_of(3) { - rev.push(' '); + parts.buf.push(' '); + if sep(pos) { + parts.buf.push(' '); } - rev.push(' '); pos += 1; } _ => {} } } else { - for _ in 0..take { - rem -= 1; - push_digit(ds.get(rem).copied().unwrap_or('0'), &mut rev, &mut pos); + // The `take` digits just left of `rem`, most significant first; the digit `s` places + // from the right end of the run sits at position `pos + s`. + for s in (0..take).rev() { + let d = ds.get(rem.saturating_sub(1 + s)).copied().unwrap_or(b'0'); + parts.buf.push(char::from(d)); + if sep(pos + s) { + parts.buf.push(','); + } } + rem -= take; + pos += take; } - if let Some(p) = pieces.get_mut(ti) { - *p = rev.chars().rev().collect(); - } + parts.end_piece(ti, start); } } @@ -213,26 +332,26 @@ fn with_commas(digits: &str, thousands: bool) -> String { } /// Fraction-digit placeholders: trailing zeros show as `0`, nothing (`#`) or a space (`?`). -fn fill_frac(toks: &[Tok], idxs: &[usize], frac: &str, pieces: &mut [String]) { - let fd: Vec = frac.chars().collect(); - let last_sig = fd.iter().rposition(|&c| c != '0'); +/// `frac` is ASCII digits. +fn fill_frac(toks: &[Tok], idxs: &[usize], frac: &str, parts: &mut Parts) { + let fd = frac.as_bytes(); + let last_sig = fd.iter().rposition(|&c| c != b'0'); for (i, &ti) in idxs.iter().enumerate() { let ph = match toks.get(ti) { Some(Tok::Digit(c)) => *c, _ => '0', }; - let s = if last_sig.is_some_and(|l| i <= l) { - fd.get(i).copied().unwrap_or('0').to_string() + let start = parts.mark(); + if last_sig.is_some_and(|l| i <= l) { + parts.buf.push(char::from(fd.get(i).copied().unwrap_or(b'0'))); } else { match ph { - '0' => "0".into(), - '?' => " ".into(), - _ => String::new(), + '0' => parts.buf.push('0'), + '?' => parts.buf.push(' '), + _ => {} } - }; - if let Some(p) = pieces.get_mut(ti) { - *p = s; } + parts.end_piece(ti, start); } } @@ -347,15 +466,12 @@ pub(crate) fn render_number(sec: &Section, v: f64) -> Out { } let toks = &sec.toks; let n = toks.len(); - // Pieces replace the token's own output; `before` is inserted ahead of a token; `lits` - // replaces a literal's text (fractions blank the literals of a hidden part). - let mut pieces: Vec = vec![String::new(); n]; - let mut before: Vec = vec![String::new(); n + 1]; - let mut lits: Vec> = Vec::new(); + // Pieces replace the token's own output; `before` is inserted ahead of a token; literal + // replacements blank the literals of a hidden fraction part (see [`Parts`]). + let mut parts = Parts::new(n); let mut zero = false; if sec.fraction { - lits = vec![None; n]; - render_fraction(sec, x, &mut pieces, &mut lits); + render_fraction(sec, x, &mut parts); } else { let NumberLayout { point, exp, int_idx, frac_idx } = number_layout(toks); let (int_s, frac_s) = if let Some(e_at) = exp { @@ -369,7 +485,6 @@ pub(crate) fn render_number(sec: &Section, v: f64) -> Out { if int_s.is_empty() && frac_s.chars().all(|c| c == '0') { e = 0; } - let head = if upper { "E" } else { "e" }; let sign = if e < 0 { "-" } else if plus { @@ -382,18 +497,16 @@ pub(crate) fn render_number(sec: &Section, v: f64) -> Out { // `E` (observed: `|0|e+|0|` shows `|1|e|+5|`); `?` pads the digits with spaces. match exp_idx.first() { Some(&first) => { - if let Some(p) = pieces.get_mut(e_at) { - *p = head.to_string(); - } - if let Some(b) = before.get_mut(first) { - *b = sign.to_string(); - } - fill_int(toks, &exp_idx, &digits, false, false, &mut pieces); + parts.set_piece(e_at, if upper { "E" } else { "e" }); + parts.set_before(first, sign); + fill_int(toks, &exp_idx, &digits, false, false, &mut parts); } None => { - if let Some(p) = pieces.get_mut(e_at) { - *p = format!("{head}{sign}{digits}"); - } + let start = parts.mark(); + parts.buf.push(if upper { 'E' } else { 'e' }); + parts.buf.push_str(sign); + parts.buf.push_str(&digits); + parts.end_piece(e_at, start); } } (int_s, frac_s) @@ -401,32 +514,32 @@ pub(crate) fn render_number(sec: &Section, v: f64) -> Out { fixed(x, frac_idx.len()) }; zero = int_s.chars().chain(frac_s.chars()).all(|c| c == '0'); - fill_int(toks, &int_idx, &int_s, sec.thousands, false, &mut pieces); + fill_int(toks, &int_idx, &int_s, sec.thousands, false, &mut parts); if int_idx.is_empty() && !int_s.is_empty() && let Some(p) = point.or(exp) - && let Some(b) = before.get_mut(p) { - *b = with_commas(&int_s, sec.thousands); + parts.set_before(p, &with_commas(&int_s, sec.thousands)); } - fill_frac(toks, &frac_idx, &frac_s, &mut pieces); - if let Some(p) = point - && let Some(s) = pieces.get_mut(p) - { - *s = ".".into(); + fill_frac(toks, &frac_idx, &frac_s, &mut parts); + if let Some(p) = point { + parts.set_piece(p, "."); } } let mut out = Out::default(); for (i, t) in toks.iter().enumerate() { - if let Some(b) = before.get(i) { - out.push(b); - } + out.push(parts.before(i)); match t { - Tok::Lit(s) => out.push(lits.get(i).and_then(Option::as_deref).unwrap_or(s)), + Tok::Lit(s) => match parts.slots.get(i).and_then(|sl| sl.lit_spaces) { + Some(k) => out.text.extend(std::iter::repeat_n(' ', k)), + None => out.push(s), + }, Tok::Percent => out.push("%"), Tok::Fill(c) => out.fill(*c), - Tok::At | Tok::General | Tok::Date(_) | Tok::Comma => {} - _ => out.push(pieces.get(i).map(String::as_str).unwrap_or("")), + // Every other token shows its piece. `At` (`0@`) is reachable here, `Comma`, `General` + // and `Date` are not (analysis turns commas into literals and routes sections with + // dates or General elsewhere); none of them is ever given a piece, so they print "". + _ => out.push(parts.piece(i)), } } out.zero = zero; @@ -482,22 +595,14 @@ fn shown_width(t: &Tok) -> usize { } /// Replaces the literals in `range` with spaces (`spaces`) or nothing; placeholders and the bar -/// in it get the same treatment through `pieces`. -fn blank_range(toks: &[Tok], range: std::ops::Range, spaces: bool, pieces: &mut [String], lits: &mut [Option]) { +/// in it get the same treatment as pieces. +fn blank_range(toks: &[Tok], range: std::ops::Range, spaces: bool, parts: &mut Parts) { for i in range { let Some(t) = toks.get(i) else { break }; - let s = if spaces { " ".repeat(shown_width(t)) } else { String::new() }; + let width = if spaces { shown_width(t) } else { 0 }; match t { - Tok::Lit(_) => { - if let Some(l) = lits.get_mut(i) { - *l = Some(s); - } - } - Tok::Digit(_) | Tok::Slash | Tok::FixedDenom(_) => { - if let Some(p) = pieces.get_mut(i) { - *p = s; - } - } + Tok::Lit(_) => parts.set_lit_spaces(i, width), + Tok::Digit(_) | Tok::Slash | Tok::FixedDenom(_) => parts.set_spaces(i, width), _ => {} } } @@ -581,7 +686,7 @@ fn fraction_parts(lay: &FractionLayout, x: f64) -> (f64, f64, u64) { /// becomes spaces when the whole part or the numerator has a `?`); when the fraction is zero and /// the numerator has no `0` placeholder, the separator, numerator, bar and denominator disappear /// (or become spaces when any part has a `?`) and the whole part shows at least `0`. -fn render_fraction(sec: &Section, x: f64, pieces: &mut [String], lits: &mut [Option]) { +fn render_fraction(sec: &Section, x: f64, parts: &mut Parts) { let toks = &sec.toks; let Some(lay) = fraction_layout(toks) else { return }; let (whole, num, den) = fraction_parts(&lay, x); @@ -591,32 +696,28 @@ fn render_fraction(sec: &Section, x: f64, pieces: &mut [String], lits: &mut [Opt let whole_s = if whole >= 1.0 { fixed(whole, 0).0 } else { String::new() }; let num_zero_ph = num_idx.iter().any(|&i| toks.get(i) == Some(&Tok::Digit('0'))); if has_whole && num == 0.0 && !num_zero_ph { - fill_int(toks, &whole_idx, &whole_s, sec.thousands, true, pieces); + fill_int(toks, &whole_idx, &whole_s, sec.thousands, true, parts); let spaces = has_q(&whole_idx) || has_q(&num_idx) || has_q(&den_idx); - blank_range(toks, sep_start..den_end, spaces, pieces, lits); + blank_range(toks, sep_start..den_end, spaces, parts); return; } if has_whole { // A zero value shows `0` in a `?` or `0` units place, nothing in a `#` one. let units_hash = whole_idx.last().is_some_and(|&i| toks.get(i) == Some(&Tok::Digit('#'))); let force = whole_s.is_empty() && num == 0.0 && !units_hash; - fill_int(toks, &whole_idx, &whole_s, sec.thousands, force, pieces); - let whole_shown = whole_idx.iter().any(|&i| pieces.get(i).is_some_and(|p| p.chars().any(|c| c.is_ascii_digit()))); + fill_int(toks, &whole_idx, &whole_s, sec.thousands, force, parts); + let whole_shown = whole_idx.iter().any(|&i| parts.piece(i).chars().any(|c| c.is_ascii_digit())); if !whole_shown { let spaces = has_q(&whole_idx) || has_q(&num_idx); - blank_range(toks, sep_start..num_start, spaces, pieces, lits); + blank_range(toks, sep_start..num_start, spaces, parts); } } let num_s = if num < 1e15 { (num as u64).to_string() } else { fixed(num, 0).0 }; - fill_int(toks, &num_idx, &num_s, false, true, pieces); - if let Some(p) = pieces.get_mut(slash) { - *p = "/".into(); - } + fill_int(toks, &num_idx, &num_s, false, true, parts); + parts.set_piece(slash, "/"); let den_s = den.to_string(); if fixed_den.is_some() { - if let Some(p) = pieces.get_mut(den_start) { - *p = den_s; - } + parts.set_piece(den_start, &den_s); } else { // Denominator digits are zero padded to the `0` placeholders, then left aligned; `?` // pads with spaces after them. @@ -641,9 +742,7 @@ fn render_fraction(sec: &Section, x: f64, pieces: &mut [String], lits: &mut [Opt } else { s }; - if let Some(p) = pieces.get_mut(ti) { - *p = s; - } + parts.set_piece(ti, &s); } } } diff --git a/crates/numfmt/src/tests.rs b/crates/numfmt/src/tests.rs index 0e9a169a5..7bbfaf66f 100644 --- a/crates/numfmt/src/tests.rs +++ b/crates/numfmt/src/tests.rs @@ -15,6 +15,9 @@ fn format_colors_resolve_named_and_indexed_rgb_safely() { const S: DateSystem = DateSystem::D1900; +/// Built-in format 44 (accounting with a currency symbol, two decimals). +const ACCOUNTING_2DP: &str = "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)"; + fn f(code: &str, n: f64) -> String { format_value(&Value::Number(n), &NumberFormat::parse(code), S).text } @@ -256,7 +259,7 @@ fn literals_and_fill() -> Result<(), Box> { assert_eq!(f("(0)", 5.0), "(5)"); let r = full("*-0", Value::Number(5.0)); assert_eq!((r.text.as_str(), r.fill), ("5", Some(('-', 0)))); - let acct = builtin_format(44).ok_or("none")?; + let acct = ACCOUNTING_2DP; let r = full(acct, Value::Number(0.0)); assert_eq!((r.text.as_str(), r.fill), (" $- ", Some((' ', 2)))); let r = full(acct, Value::Number(1234.5)); @@ -356,7 +359,7 @@ fn kinds() -> Result<(), Box> { assert_eq!(NumberFormat::parse("0.00").kind(), FormatKind::Number); assert_eq!(NumberFormat::parse("#,##0.00;[Red](#,##0.00)").kind(), FormatKind::Number); assert_eq!(NumberFormat::parse("\"$\"#,##0.00").kind(), FormatKind::Currency); - assert_eq!(NumberFormat::parse(builtin_format(44).ok_or("none")?).kind(), FormatKind::Accounting); + assert_eq!(NumberFormat::parse(ACCOUNTING_2DP).kind(), FormatKind::Accounting); assert_eq!(NumberFormat::parse("m/d/yyyy").kind(), FormatKind::Date); assert_eq!(NumberFormat::parse("h:mm AM/PM").kind(), FormatKind::Time); assert_eq!(NumberFormat::parse("0%").kind(), FormatKind::Percentage); @@ -369,26 +372,11 @@ fn kinds() -> Result<(), Box> { } #[test] -fn builtins() -> Result<(), Box> { - assert_eq!(builtin_format(0), Some("General")); - assert_eq!(builtin_format(14), Some("m/d/yyyy")); - assert_eq!(builtin_format(49), Some("@")); - assert_eq!(builtin_format(30), None); - assert_eq!(builtin_format(50), None); - assert_eq!(builtin_id("m/d/yyyy"), Some(14)); - assert_eq!(builtin_id("0.00%"), Some(10)); - assert_eq!(builtin_id("GENERAL"), Some(0)); - assert_eq!(builtin_id("[h]:mm:ss"), Some(46)); - assert_eq!(builtin_id("#,##0.00_);(#,##0.00)"), None); - for id in 0..=60 { - if let Some(code) = builtin_format(id) { - assert_eq!(builtin_id(code), Some(id)); - } - } - assert_eq!(f(builtin_format(5).ok_or("none")?, -1234.0), "($1,234)"); - assert_eq!(f(builtin_format(12).ok_or("none")?, 0.5), " 1/2"); - assert_eq!(f(builtin_format(47).ok_or("none")?, 61.2 / 86400.0), "0101.2"); - Ok(()) +fn builtin_codes_render() { + // Built-in ids 5, 12 and 47 (the id tables live with the file formats, in gridcraft-xlsx). + assert_eq!(f("\"$\"#,##0_);\\(\"$\"#,##0\\)", -1234.0), "($1,234)"); + assert_eq!(f("# ?/?", 0.5), " 1/2"); + assert_eq!(f("mmss.0", 61.2 / 86400.0), "0101.2"); } #[test] @@ -396,6 +384,13 @@ fn text_fn() { assert_eq!(text_function(&Value::Number(1234.5), "$#,##0.00", S), Ok("$1,234.50".into())); assert_eq!(text_function(&Value::from("5"), "0.00", S), Ok("5.00".into())); assert_eq!(text_function(&Value::from("abc"), "0.00", S), Ok("abc".into())); + // Each `@` repeats the text: up to a cell's 32,767 characters, then #VALUE!. + let at = "@".repeat(255); + assert_eq!(text_function(&Value::from("x".repeat(128).as_str()), &at, S).map(|s| s.len()), Ok(128 * 255)); + assert_eq!(text_function(&Value::from("x".repeat(32_000).as_str()), &at, S), Err(CellError::Value)); + // The limit counts UTF-16 units: 16,384 emoji are 32,768 of them, one too many. + assert_eq!(text_function(&Value::from("\u{1F600}".repeat(16_383).as_str()), "@", S).map(|s| s.chars().count()), Ok(16_383)); + assert_eq!(text_function(&Value::from("\u{1F600}".repeat(16_384).as_str()), "@", S), Err(CellError::Value)); assert_eq!(text_function(&Value::Error(CellError::NA), "0", S), Err(CellError::NA)); assert_eq!(text_function(&Value::Number(46302.0), "yyyy", S), Ok("2026".into())); assert_eq!(text_function(&Value::Number(0.5), "General", S), Ok("0.5".into())); diff --git a/crates/numfmt/tests/golden/codes.txt b/crates/numfmt/tests/golden/codes.txt new file mode 100644 index 000000000..118f8a767 --- /dev/null +++ b/crates/numfmt/tests/golden/codes.txt @@ -0,0 +1,152 @@ +1427c0507a69d2eb 79 "0.0000000000000000000000000" +c37039701ea91f71 79 "#,###,###,###,###,###,###,###,##0.00" +c886ef7cc0381aca 79 "0.0000000000000000000000000E+00" +16878498a86d6270 79 "# ?????/?????" +25810fe3aa348757 79 "General" +637698d5ec4840e7 79 "0" +f480db9c3f30f4f9 79 "0.00" +233cef56f68009ce 79 "#,##0" +d2fae9f7b5756366 79 "#,##0.00" +fcf13b3494133da6 79 "#.##" +93b3291c9e9847f7 79 "0.0##" +20ca0c941ad5c11f 79 "00000" +db86a5c476fc29cd 79 "000.000" +7cc10690ed9769ab 79 "#,###" +ea3312bb2d5c1342 79 "#,##0," +d14a7d6e48db33f0 79 "#,##0,," +e44a9c0ed1f49c8a 79 "0.0,,\"M\"" +3c70c4c8ff8af422 79 "?.??" +93f48e4f2d83fd29 79 "##0.0" +a06b56173759b328 79 "# " +5a35d2ec11553fa4 79 "#" +f3495305635eab2d 79 ".00" +1f2ae975e66d556b 79 ",0" +253e097c9bdaf643 79 "0,0" +ae00b79ca5b8d288 79 "0.00," +1b989416a1ec6e7c 79 "\"$\"#,##0.00" +f82256c8ac29c8f3 79 "\"$\"#,##0_);\\(\"$\"#,##0\\)" +7fc54d574b4ee62c 79 "#,##0.00;[Red](#,##0.00)" +7a81a86d7a4d3c69 79 "#,##0 ;(#,##0)" +57a11793e92c9715 79 "0;-0;;@" +a3d6c52367c35259 79 "0.00;;" +b45d3fdec1e8ff9a 79 ";;;" +11a021e5408029bb 79 "0_);(0)" +99b2e56004ca9b4a 79 "0\" units\"" +c9aaa3ef8150f3a6 79 "\\A0" +641306b16f0036fe 79 "(0)" +19af7c406782b02c 79 "00-00" +c298712a7d3815bd 79 "000-00-0000" +28e5bc8fc3d88c8e 79 "[<=9999999]###-####;(###) ###-####" +f494c7ff5ff0fd7a 79 "[$€-407]#,##0.00" +4e11e82e14ea8f05 79 "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"_);_(@_)" +fc2d989610e136f3 79 "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)" +091f29e23e924f36 79 "*-0" +93217b43f4b7af91 79 "0*x" +6a7561a5ce77014d 79 "*0 0.0" +9a6620cdfa67c60b 79 "@*_" +0b8f6e85e6ff12f0 79 "0%" +d2f558040c30c4cc 79 "0.00%" +371df01b15432009 79 "#,##0%" +e5016314db39e07b 79 "0.0%%" +f603836d162018f0 79 "%0" +7d08aed42e8d71e0 79 "0%;[Red]-0%" +d7d44df9f81bff2e 79 "0.00E+00" +275678f83b4d4113 79 "##0.0E+0" +497033fffb9c0967 79 "0E+0" +9679af8d3aa45c26 79 "0.0e-0" +eafc341422b759ce 79 "0.00E+000" +67cf5d053575d3df 79 "E+0" +f9d5b40fa2875a1e 79 "0.00E+" +e633fc616f9a3ff3 79 "#.##E+00" +8ecbeef181fb381e 79 "0.0E+00 \"x\"" +7b74fb5636bda27e 79 "# ?/?" +aa283fbec589cdfe 79 "# ??/??" +9c9625146b582644 79 "# ???/???" +7d7dd7cefba77e8e 79 "?/?" +3babcf6e56373ec4 79 "# ?/8" +c2e84ddf50f1e21d 79 "# ??/16" +d5bbf7173f227e7a 79 "0/0" +6fe467468cd9a182 79 "# #/#" +a73c12b90cfc3656 79 "#/#" +d034fba50699f7ea 79 "?/100" +02ee892d135e01dc 79 "# ?/?;-# ?/?" +4bd7e5c1d9b3503f 79 "0 0/0" +2162452594679802 154 "m/d/yyyy" +d04b4c59e67bea74 154 "d-mmm-yy" +5dec91c67babe5f2 154 "d-mmm" +c384b64cedef18eb 154 "mmm-yy" +d7aa777afafbc668 154 "h:mm AM/PM" +b175c2057a22f6de 154 "h:mm:ss AM/PM" +aded84fe78136262 154 "h:mm" +5a79757ea5163c90 154 "h:mm:ss" +cd019c5ea6050d45 154 "m/d/yyyy h:mm" +54dbe04a04108479 154 "mm:ss" +e0718e196529a8f2 154 "[h]:mm:ss" +a38b480cf5c5a364 154 "[h]:mm" +2a913161d1313180 154 "[m]:ss" +76fdaf50d35fb2a2 154 "[s]" +ff9403ec80b0bc4f 154 "mmss.0" +425d4aa682fdc092 154 "yyyy-mm-dd" +058ce7638d05bbb9 154 "dddd, mmmm d, yyyy" +fe4bc250009a8329 154 "ddd mmm dd yy" +ccb6419ee78b2955 154 "mmmmm" +6c0289cc8797f513 154 "yy" +47cc3767e6229ec4 154 "h:mm:ss.000" +e6112ef1c8665ef8 154 "h:mm:ss.0000" +0bf1d0573ca9d704 154 "hh:mm:ss A/P" +e2a3d66078e1cf02 154 "[hh]:mm:ss" +8707505efd8a6284 154 "d" +7fa481a6092fa43c 154 "dd" +7d4b6020f781a3ba 154 "dddd" +949ed74d9b020535 154 "m" +0aa17a2f309cbecd 154 "mm" +c70a36183af563af 154 "s.00" +a8c8c009a7b0012f 154 "yyyy-mm-dd hh:mm:ss.00" +a70fc60b044d8b5e 154 "h AM/PM" +cf3e0c4fccbedb80 154 "[$-409]d-mmm-yy" +80b17497f3b9d5dd 154 "yyyy\"年\"m\"月\"" +e00bb34ae8360b4c 154 "[h]\" hrs\"" +8b75446f8310605f 154 "mmmm yyyy;@" +01123dabdab536a7 79 "@" +59fa7b60ca99b9b7 79 "\"x\"@" +b1eee7ca88961e38 79 "0;\"neg\"0;\"zero\";\"txt: \"@" +4d3f8a891532765e 79 "General;General;General;@" +59dc8b6bd14d719a 79 "[Red]0.0;[Blue]-0.0" +6a4d8666a78e97b3 79 "[Green]0;[Magenta]0;[Cyan]0;[Yellow]@" +15b794f754adc79d 79 "[Color3]0.00" +158526f0a5b736c5 79 "[>100]\"big\"0;[<-100]\"small\"0;0.0" +7a94ed36a6bb6573 79 "[=0]\"zero\";0.0" +30d048d3f229b1ba 79 "[>=1000]#,##0,\"K\";0" +2b2b1f13faeb468d 79 "[<1]0.00%;0.0" +b78758e73c416a1e 79 "[Red][>5]0;0.00" +8b2379e7de300595 79 "General\" kg\"" +73f3485fe971d2bd 79 "\"v\"General" +2d2ce0ee57d199a2 79 "[Red]General" +ea2f102aad8c3c1b 79 "0.0;[Red]General" +ef5fc7ae5e0cf36c 79 "[" +d80872ea795758e3 79 "0.0.0" +145e66eaa5b6a665 79 "0\"" +ddb909bb72f98e33 79 "\\" +cc44573b688554a4 154 "[h" +c78707c90ee7a1b6 79 "[$" +aaf7da8233ee4f96 154 "yyyy-mm-dd;" +b1bbc169427ccdf8 79 ";" +2cf377cde4d0586d 79 "0;0;0;0;0" +9293c3fbfd2434b1 79 "\"abc" +3a0ef807a097eb69 79 "[>x]0" +1ef168bf45be1582 79 "ggg" +b39c47241ef60000 79 "#?#0" +92199ef29ea54843 154 "0E" +fe0eba9c258b10e7 154 "E0" +c5adbc0bf74eded8 79 "/" +d77317d9cc005a8c 79 "0/" +86aefad6fd9ab034 79 "/0" +d08e2f5c1fb6d629 79 "*" +5257c26622f2cfe0 79 "_" +ef14d84a8bf4f1b4 79 "_0" +c1a24c12c4381ef2 154 "[Red" +1653b1ac403ed200 79 "0.00.00%" +aac2038aa4ae7651 154 "mm:ss.0.0" +00083314e9f89199 79 "[ABC]0" +a456ed739adcc87c 79 "[=1]0;[=2]0;0" +b0b3aa7b9f08690c 79 "0 ??/??;0" diff --git a/crates/numfmt/tests/golden/general.txt b/crates/numfmt/tests/golden/general.txt new file mode 100644 index 000000000..ab338965e --- /dev/null +++ b/crates/numfmt/tests/golden/general.txt @@ -0,0 +1,555 @@ +5fda7bf42a47c479 1 0e0 +e400479573b2baf2 1 -0e0 +57f99cd6daf9cc72 1 1e0 +0ddafbadc6bf1a04 1 -1e0 +882d8b786644b3c6 1 5e-1 +846783d06acc0a60 1 -5e-1 +eb4b115b5778e14e 1 2.5e-1 +d320cfc393d5b774 1 1.005e0 +e7497067a4de5fb0 1 2.675e0 +68aed91bbaac1534 1 9.999995e2 +96a3c4703fc3b01a 1 9.995e-1 +df98061474126a71 1 9.995e0 +5916478863984e9b 1 9.95e1 +13813099e41458c9 1 5e-2 +e6381aba4f09a980 1 5e-3 +6c09e99da5f3e27f 1 5e-4 +1e9840c7c9074f0a 1 1e-5 +b8e3e7ad44ea4900 1 1e-10 +a9d70b3df9c809d1 1 1e-300 +4c1d46df8304d5b7 1 5e-324 +c2a28fae0bcd7949 1 1.23456789e8 +c38f4a506cc3267c 1 1.2345678e3 +bde85249e1dcc206 1 1.2345678901e10 +f61e4f9f86db3eea 1 1e11 +72057cacab3997fa 1 1e15 +4183c6065332dc5d 1 1.23456789012345e14 +e1cb4037184e1fe4 1 1.234567890123455e15 +503e585193837caa 1 1e20 +a59688774ce6d2a1 1 1e100 +20a1bd96371a3de0 1 1.7976931348623157e308 +3b202f375efc1640 1 -1.2345e3 +816da9ca801294d6 1 -4e-3 +f8adad661b001d1f 1 -1e-10 +a8aaed4940499212 1 3.333333333333333e-1 +b146e4ee19146f95 1 6.666666666666666e-1 +7433fb99403da062 1 3.0000000000000004e-1 +1c0a0bcb4c0422b8 1 1e2 +4d9afc58f74b7ddb 1 1e3 +0d48b7819a8a4b8d 1 5.9e1 +01452d3bffb28025 1 6e1 +4921ebb52b8c73c1 1 6.1e1 +09ea6a1671f8c563 1 3.6526e4 +b63b4cad6f332f99 1 3.652675e4 +2bc41104f2e19271 1 4.38315e4 +4de1bb6ce4b9358b 1 4.5000123456e4 +ed7fda487b9bcf16 1 2.958465e6 +9059129e9a79299b 1 2.958465999999e6 +a32ca3ab1dbe969f 1 2.958466e6 +90b8f9a1e42bd0a4 1 -5e0 +7e564b2cabd8de16 1 9.9999999e-1 +8781fec461d58bfd 1 9.99e-4 +7c708dbcf3460659 1 4.199999999999e1 +20b2def65bc7174d 1 NaN +957f300224ed1cab 1 inf +4b2f7ba9aa3ce108 1 -inf +9ae51ec9ae4d5114 1 4.1666666666666664e-2 +2689dad0ffde687c 1 9.999999999e-1 +1599f472f24a81fa 1 1.0000000001e0 +5f1f3c268b6168ac 1 1.23456789e1 +c513d218d1599535 1 3.14159265358979e0 +8140c7ee3b5a5345 1 6.5535e4 +bd982ddec1e00aaa 1 1e5 +97614145ed7f215c 1 1.23456e5 +26cde2690b2bcbcc 1 4.5e0 +1a8add9df90d93c9 1 3.75e-1 +eef82337e19fa9f3 1 1e300 +884afff127bc9c8a 1 1.5e-8 +723565414e3a8eb5 1 9.99999999999999e14 +410dc1f39f0470e8 1 9.999999999999998e13 +abe0337311e85bc2 1 1.234e-5 +e05b983d81a775a8 1 1.2345678901234568e17 +f1b04c74292211ec 1 1e22 +8bd34401b876c52c 1 1e23 +8162c16a7c44dc93 1 1.234567890123456e-1 +5df098e551929df8 1 7.083333333333334e-4 +403536010a422332 1 -2.4456788999965284e-22 +5fda7bf42a47c479 1 0e0 +6f8d525947f4496b 1 5.078754219653861e-20 +5fda7bf42a47c479 1 0e0 +db9fc2cd4be272f9 1 4.762524884899572e-19 +5fda7bf42a47c479 1 0e0 +4e42d845bdfe1e05 1 -9.481878134872896e-18 +5fda7bf42a47c479 1 0e0 +e2b246d2b6df4843 1 4.677068358276083e-17 +5fda7bf42a47c479 1 0e0 +db3ef5385bc881e0 1 4.2817210912547903e-16 +5fda7bf42a47c479 1 0e0 +bbf8726d918fa10a 1 -1.2049587102547775e-16 +5fda7bf42a47c479 1 0e0 +c0efc68af4f6f086 1 1.3789513024013423e-14 +5fda7bf42a47c479 1 0e0 +790b21d90f49a509 1 6.059016384138212e-13 +5fda7bf42a47c479 1 0e0 +ce56d3a5c4486cd1 1 -2.0839028757975573e-12 +5fda7bf42a47c479 1 0e0 +975db8510513c040 1 8.895735060164848e-11 +5fda7bf42a47c479 1 0e0 +472bbe20b67f109c 1 2.924228743531876e-11 +5fda7bf42a47c479 1 0e0 +a9c44b1d793ee953 1 -2.790173620128004e-9 +5fda7bf42a47c479 1 0e0 +b65a4a2339333f83 1 3.047667157619239e-8 +5fda7bf42a47c479 1 0e0 +0c9b50a66593ab52 1 9.768724036381636e-7 +5fda7bf42a47c479 1 0e0 +deb83d6ed094b3b2 1 -6.624321624908681e-7 +5fda7bf42a47c479 1 0e0 +cfb05c2a70b35a2d 1 1.6894338933954122e-5 +5fda7bf42a47c479 1 0e0 +dfa7bf97c3a3f630 1 5.610159605226053e-4 +aa35c0cb97f7addc 1 1e-3 +6f2e8f1578d1b6f7 1 -4.5736943003751186e-3 +e6381aba4f09a980 1 5e-3 +618e160ad7d75e3f 1 1.2177853639931391e-2 +7b3b780b251205f7 1 1.2e-2 +bc71798a9a9fd283 1 5.376575791159676e-1 +552d6fe712db0701 1 5.38e-1 +30f24eae79fdbea7 1 -1.6906316761960716e0 +9a85be1470daf3e4 1 1.691e0 +428a08f2bbd5c129 1 6.804349057483373e1 +36f22379ff98da63 1 6.8043e1 +42c9ce8792ca41ca 1 5.180578200923046e2 +b753a84983c4b8fb 1 5.18058e2 +d48ddf21b575ae2e 1 -6.281034210276412e3 +335061600a21e480 1 6.281034e3 +93205c65ad863f72 1 3.435675535581831e4 +1abaaae29d8209f0 1 3.4356755e4 +d09ef10e029d4d19 1 6.613076865085881e5 +b1c40c1667e951b0 1 6.61307687e5 +8e6cdd6bc70fbd22 1 -4.48220238062414e6 +c083e5d00d97cef4 1 4.482202381e6 +e127f684d7698427 1 1.0000341372267152e8 +d4d94bdd00026ac9 1 1.00003413723e8 +0cab466bd5e1d8fa 1 1.5849160403504446e8 +d4c67910bde50f32 1 1.58491604035e8 +0e7f466ac9853aae 1 -1.4058601193934329e9 +a643a7677b4c5ac7 1 1.405860119393e9 +2776bab16cf335d4 1 2.8871069252525196e10 +fc7419911bfbaca8 1 2.8871069252525e10 +a76f1c3dce467478 1 9.690172366761895e11 +be689212e28414e9 1 9.6901723667619e11 +c4c5bd5055043ad5 1 -2.346417551608747e12 +93b654146869cfc6 1 2.346417551608747e12 +645dfd0b8d651920 1 6.228668439208695e12 +645dfd0b8d651920 1 6.228668439208695e12 +f147744d6ebe1c8c 1 2.2428112791069544e14 +f147744d6ebe1c8c 1 2.2428112791069544e14 +ba8da370ef0b6f07 1 -7.327413159633475e15 +76ca14ec7527c55d 1 7.327413159633475e15 +813a82002c70fe12 1 6.675488044574204e16 +813a82002c70fe12 1 6.675488044574204e16 +8a50e37fec07f962 1 6.706148330480582e17 +8a50e37fec07f962 1 6.706148330480582e17 +b2bcb6262b7ad869 1 -7.27445337914015e18 +a958885d3e964a90 1 7.27445337914015e18 +f8c71a4f984138a2 1 3.874586892728994e19 +f8c71a4f984138a2 1 3.874586892728994e19 +143942bc00828b3f 1 4.207699940806956e-21 +5fda7bf42a47c479 1 0e0 +cca09550d6d0af64 1 -6.321408874535118e-20 +5fda7bf42a47c479 1 0e0 +804ebfa29a23426e 1 3.689215801512328e-19 +5fda7bf42a47c479 1 0e0 +34d878721f2d74d3 1 9.392721997790782e-18 +5fda7bf42a47c479 1 0e0 +bf13de1377b6a335 1 -5.788399687409319e-17 +5fda7bf42a47c479 1 0e0 +9164e2fa13905b35 1 2.279056237090581e-16 +5fda7bf42a47c479 1 0e0 +4fc8a97fbf0232f5 1 3.463636269309186e-15 +5fda7bf42a47c479 1 0e0 +eb0a80785bd2f283 1 -4.489928391258454e-14 +5fda7bf42a47c479 1 0e0 +9f70d66705199764 1 7.703173974678871e-13 +5fda7bf42a47c479 1 0e0 +fb73018ae299a802 1 1.3090206448341722e-12 +5fda7bf42a47c479 1 0e0 +c064d37e034a6497 1 -6.338150789669849e-11 +5fda7bf42a47c479 1 0e0 +53be38791b9b3a71 1 3.809051908390393e-11 +5fda7bf42a47c479 1 0e0 +87e6cfab84040f20 1 1.0070431565221817e-9 +5fda7bf42a47c479 1 0e0 +5db583e96d56b61f 1 -5.266594942615038e-8 +5fda7bf42a47c479 1 0e0 +87730d5ab9b90358 1 2.039725677192573e-7 +5fda7bf42a47c479 1 0e0 +2c827aff0066cbce 1 4.851068050995432e-6 +5fda7bf42a47c479 1 0e0 +31b9628affbc1814 1 -7.75941473244553e-5 +5fda7bf42a47c479 1 0e0 +8be16fa85aa991e0 1 7.381056138192671e-4 +aa35c0cb97f7addc 1 1e-3 +6c3609330a25ae41 1 1.575376680923546e-4 +5fda7bf42a47c479 1 0e0 +02745e58c6fb95fe 1 -8.309622978077537e-2 +3ae24003b2396fd4 1 8.3e-2 +35cb86c8421094f5 1 5.938677033303984e-1 +cd17bcf30868b4d7 1 5.94e-1 +5fc071051c10dcf2 1 2.1055700940714903e0 +3301c223c3ee7087 1 2.106e0 +374453ca4e69570f 1 -4.979516792760796e0 +30ac305e01179f6c 1 4.98e0 +dc9118090ba75534 1 7.702392128251413e2 +5668bd8822fd37eb 1 7.70239e2 +bc0fea3db4020ccf 1 5.295197566581555e2 +6a1f8e82ef33f9d0 1 5.2952e2 +8f67ffd819348bfb 1 -9.175765278181084e4 +7431aae5707ad8ce 1 9.1757653e4 +e1050c33ba3c5491 1 9.482550236540656e5 +a129426c85bdbae0 1 9.48255024e5 +e6d83dc1bd22212c 1 5.347153721033019e6 +e026e88cae226949 1 5.347153721e6 +03a3405b7ce1bb74 1 -2.3568277599192787e7 +61da64c36fd8c5eb 1 2.3568277599e7 +b3b4116a1d7c332a 1 1.0018445295210177e8 +ad683901e4974657 1 1.00184452952e8 +0be5c818158d8aba 1 8.356382245461647e7 +9656312094a4767c 1 8.3563822455e7 +cf7cd1e0127e076f 1 -4.557698877252628e10 +d82db809145f4cb7 1 4.5576988772526e10 +e21e145b37008e47 1 5.270348510870472e11 +3eeab7eb66861c31 1 5.27034851087047e11 +d51733208c288c7c 1 5.782033227859766e12 +d51733208c288c7c 1 5.782033227859766e12 +f0e51f3bf67fc7a3 1 -5.931696066185405e13 +e15f77a9ff4fb907 1 5.931696066185405e13 +9a7a6b9b153d807e 1 5.1455458768478e14 +3cce9402217011dc 1 5.1455458768478006e14 +c9f98f1fa9b23bc5 1 1.3538071072562253e15 +c9f98f1fa9b23bc5 1 1.3538071072562253e15 +bbff8f21267563e7 1 -9.90253824456554e15 +1a63f8bd04651493 1 9.90253824456554e15 +64140d1f4b064459 1 8.527630873184316e17 +64140d1f4b064459 1 8.527630873184316e17 +c42d45691a199adf 1 3.2925484547629455e18 +c42d45691a199adf 1 3.2925484547629455e18 +6d966e6760bed43d 1 -3.91537728865695e19 +efdecd348456b99a 1 3.91537728865695e19 +51ff7d6fc5809c27 1 4.8757246809796105e-21 +5fda7bf42a47c479 1 0e0 +84bceb480eb2dcb6 1 2.3420748266718282e-20 +5fda7bf42a47c479 1 0e0 +de6b157cd8b3068b 1 -6.293170081812841e-19 +5fda7bf42a47c479 1 0e0 +fca52fc30bf77496 1 5.53513945740202e-18 +5fda7bf42a47c479 1 0e0 +fef0b8399d29f15b 1 9.77860691981301e-17 +5fda7bf42a47c479 1 0e0 +8956eb136a63fc19 1 -5.1566694357030435e-17 +5fda7bf42a47c479 1 0e0 +c8d0796283c1a680 1 5.364510629593392e-15 +5fda7bf42a47c479 1 0e0 +c10f295101314665 1 9.661665594611814e-14 +5fda7bf42a47c479 1 0e0 +71f69f42ed1abd5f 1 -3.925165717977516e-13 +5fda7bf42a47c479 1 0e0 +4b3b18313d353d45 1 4.634788713583143e-12 +5fda7bf42a47c479 1 0e0 +1714fa5b40a6cf7b 1 2.1289153323939538e-11 +5fda7bf42a47c479 1 0e0 +fbe658ebfc667eeb 1 -3.7731542867717375e-10 +5fda7bf42a47c479 1 0e0 +ed1059691b29437f 1 3.0793918014219435e-9 +5fda7bf42a47c479 1 0e0 +d68a671d72cbb7d4 1 1.398193907677646e-8 +5fda7bf42a47c479 1 0e0 +bf3150d59315bcd7 1 -5.243893844613249e-7 +5fda7bf42a47c479 1 0e0 +93b6b3ec8b06d2ff 1 9.40673096940851e-6 +5fda7bf42a47c479 1 0e0 +71583d853e2cc61b 1 9.755344390282753e-5 +5fda7bf42a47c479 1 0e0 +a2726686baf8eaa6 1 -7.322925001904205e-4 +aa35c0cb97f7addc 1 1e-3 +7b84f8a587116b61 1 2.20079478849153e-3 +97b75dab0686cb37 1 2e-3 +69efff5278fdc06b 1 5.4369720160556294e-2 +22dfdcd990abd9ff 1 5.4e-2 +bdd2e88716b365e0 1 -1.90566789746209e-1 +a419c77256cf893c 1 1.91e-1 +cfb6f41a39039f7f 1 1.1954616597037193e0 +8818e7a995d67595 1 1.195e0 +d936acae76796026 1 3.1198426146081694e1 +f7c4c11aa02f61b9 1 3.1198e1 +4d61587438c8539f 1 -1.7276546543447557e2 +31d3905e625db28b 1 1.72765e2 +592a0f53b05918db 1 3.716258271721481e3 +c5e0c130f048add8 1 3.716258e3 +9964fe1fd34a9e66 1 6.354064796316502e4 +a3273ac06e88377e 1 6.3540648e4 +9d1cc1cc40d1e375 1 -6.247169817552513e5 +6e32184b6303fecd 1 6.24716982e5 +a59dec2d400de042 1 9.672875989854455e6 +c6184456a5f3d6dc 1 9.67287599e6 +952a4c5d6d9f3cc7 1 5.101929774891923e7 +00398960ce567662 1 5.1019297749e7 +2985fb1731569c12 1 -7.868553457247695e8 +cbe14af9c8d6397f 1 7.86855345725e8 +6ddbb1af610f79cf 1 6.192364765951433e9 +e146c8288f6cd3d2 1 6.192364765951e9 +674dfcf3b4a569b6 1 5.0322395435787346e10 +4eaeab0011c8ca30 1 5.0322395435787e10 +2a2da0e805cf5409 1 -8.387392837998628e11 +6e45a8daf47f9904 1 8.38739283799863e11 +adea7a9a9dfe1129 1 3.475227374631468e12 +adea7a9a9dfe1129 1 3.475227374631468e12 +928198bdae5603d2 1 6.046260397573301e13 +928198bdae5603d2 1 6.046260397573301e13 +3b2d1a775477a31c 1 -9.366184270581025e14 +856529b379f79ef7 1 9.366184270581025e14 +ac839bfff6b8812f 1 9.330285659281354e15 +ac839bfff6b8812f 1 9.330285659281354e15 +73160e61af42cec9 1 3.539370193508612e16 +73160e61af42cec9 1 3.539370193508612e16 +dedb455241069fa2 1 -9.996650818086104e17 +ec679a20500b031d 1 9.996650818086104e17 +26215fcc4901f42b 1 7.905433521846371e18 +26215fcc4901f42b 1 7.905433521846371e18 +ec9afc963bc85c7b 1 2.961789170831489e19 +ec9afc963bc85c7b 1 2.961789170831489e19 +f2482a70bd7552a8 1 -4.148371208994727e-21 +5fda7bf42a47c479 1 0e0 +a55409a5a800406d 1 7.170663257742498e-20 +5fda7bf42a47c479 1 0e0 +17f938f9cc679cd3 1 3.9583585927085557e-20 +5fda7bf42a47c479 1 0e0 +9a113924c396bb44 1 -5.892919583042963e-18 +5fda7bf42a47c479 1 0e0 +f0c40ba08b4e8d1c 1 6.485392183833719e-17 +5fda7bf42a47c479 1 0e0 +da9719b7ced0671f 1 7.180575172217469e-16 +5fda7bf42a47c479 1 0e0 +90ac83c7b7ea92c2 1 -2.7801459081525537e-16 +5fda7bf42a47c479 1 0e0 +069d079990d92448 1 8.42511493280958e-14 +5fda7bf42a47c479 1 0e0 +1c765f9111408737 1 1.0841559011512436e-13 +5fda7bf42a47c479 1 0e0 +7530679d8e6abd87 1 -2.1480013377898877e-12 +5fda7bf42a47c479 1 0e0 +873ab178136be7f4 1 2.8019016665180154e-11 +5fda7bf42a47c479 1 0e0 +4f3329b0b1b9010d 1 4.740529408461166e-10 +5fda7bf42a47c479 1 0e0 +84f8fb1f92637dc1 1 -7.552388125782508e-9 +5fda7bf42a47c479 1 0e0 +3d7d836e4bdb867e 1 9.755292951606497e-9 +5fda7bf42a47c479 1 0e0 +3fa8b742b75bff54 1 3.8472751651678525e-7 +5fda7bf42a47c479 1 0e0 +d349c8520a8cc6de 1 -6.977907562349183e-6 +5fda7bf42a47c479 1 0e0 +e1df5d730caf3475 1 8.218407552134532e-5 +5fda7bf42a47c479 1 0e0 +3df6765577ebd93a 1 4.996897368127074e-4 +5fda7bf42a47c479 1 0e0 +81b26f24e3ef8f0d 1 -3.151243912693312e-3 +77147cf949266682 1 3e-3 +bd1c5f937c96ac6a 1 3.0347488452323525e-2 +fc87ec37c8733143 1 3e-2 +a73ab956230ebaef 1 6.889166586655288e-1 +5fa353279229150c 1 6.89e-1 +f568fd4cfe3142c6 1 -9.743654785321732e0 +1bc6257bb3848aa9 1 9.744e0 +0f5ba1866b7d2845 1 5.668388855766816e1 +ce72a327bacd2b00 1 5.6684e1 +b4efc6800c4b914d 1 2.628257089515055e2 +21784204a4be65c0 1 2.62826e2 +e67a2d6c6630643f 1 -1.6168861365096154e3 +47815c5434eded45 1 1.616886e3 +4e3b51240c986695 1 3.280045990086601e4 +6890cb1d3a7f9b75 1 3.280046e4 +cabedf8f491b936e 1 1.4504200063410145e5 +e3836c93597707d8 1 1.45042001e5 +4271771f51534d50 1 -7.313314211991392e6 +1fb3dc6157f32057 1 7.313314212e6 +46c2ec9b7d7dd831 1 2.618837245417135e7 +3b956a81d84ff17d 1 2.6188372454e7 +5a8bbd3fdc03e83f 1 2.2253015708835256e8 +394a2739b0947d1f 1 2.22530157088e8 +40691261266728fd 1 -9.870234060874973e9 +eb3d9a317a8874aa 1 9.870234060875e9 +6a990a269a5c74f0 1 1.867926582347318e10 +f2ca9284e3a4f179 1 1.8679265823473e10 +280ece39f3f50572 1 3.567370490275957e11 +c14e0f480d1a7640 1 3.56737049027596e11 +3789dd57fc5c815c 1 -7.912946695129458e12 +b5ae37bd691873c7 1 7.912946695129458e12 +9f1553246fdb40cb 1 4.524229340690999e12 +9f1553246fdb40cb 1 4.524229340690999e12 +b24a577eb86c3e84 1 2.3102331568926594e14 +b24a577eb86c3e84 1 2.3102331568926594e14 +cdce2c77867a3106 1 -4.547025311981324e15 +e947b96f57d17bec 1 4.547025311981324e15 +cf45a38c70fcf953 1 4.6288039353802424e16 +cf45a38c70fcf953 1 4.6288039353802424e16 +014ea8cd6a1290e2 1 6.162091464101659e17 +014ea8cd6a1290e2 1 6.162091464101659e17 +ce7e9262127eff44 1 -4.849757599353334e18 +e50f3fe044d209e7 1 4.849757599353334e18 +d85b0205e7941c69 1 6.452894445273737e19 +d85b0205e7941c69 1 6.452894445273737e19 +f2c90f259094e92e 1 4.7803298279157025e-21 +5fda7bf42a47c479 1 0e0 +9a93ee439db2dc35 1 -7.233406321955394e-20 +5fda7bf42a47c479 1 0e0 +d1e6a63ae1d3a9bb 1 2.950670879528398e-19 +5fda7bf42a47c479 1 0e0 +9230461e2e26d575 1 3.063434779812392e-18 +5fda7bf42a47c479 1 0e0 +314136843a1e646c 1 -5.4890433629545986e-17 +5fda7bf42a47c479 1 0e0 +6f3c47632e4d68b3 1 3.820800755734363e-16 +5fda7bf42a47c479 1 0e0 +6f9edd60cd32e0d5 1 5.829213571595438e-16 +5fda7bf42a47c479 1 0e0 +78eb51a008a64997 1 -2.4171609780651217e-14 +5fda7bf42a47c479 1 0e0 +44e17d26d950269c 1 1.1540630209261826e-13 +5fda7bf42a47c479 1 0e0 +8c1d1f942eaccbbf 1 1.84539975340409e-12 +5fda7bf42a47c479 1 0e0 +04d9cdb28aac62c3 1 -1.1081220338777577e-11 +5fda7bf42a47c479 1 0e0 +ed11aa9ba5e5353f 1 6.04223566612428e-10 +5fda7bf42a47c479 1 0e0 +f99d682b436ba383 1 5.35352701590592e-9 +5fda7bf42a47c479 1 0e0 +98b4918f9567bfd3 1 -8.711002748201745e-8 +5fda7bf42a47c479 1 0e0 +1ad36850fde48c38 1 6.114307847139126e-7 +5fda7bf42a47c479 1 0e0 +08f08db80439f666 1 7.209491487707519e-6 +5fda7bf42a47c479 1 0e0 +6a2bea77bad32684 1 -9.107581134395797e-5 +5fda7bf42a47c479 1 0e0 +a6e64a2db8e7c580 1 1.502958882607809e-4 +5fda7bf42a47c479 1 0e0 +bb4f7310114464cd 1 9.694573849985441e-3 +e40ccf007d65d503 1 1e-2 +99d400856d68de78 1 -6.734696325484274e-2 +6f6224f1a2f8d8f1 1 6.7e-2 +e1bf6652cb8fa67a 1 5.736804397820615e-1 +3503377f10431893 1 5.74e-1 +abde9a0944f0075c 1 8.385525171527297e-1 +bba484bf8ca102d3 1 8.39e-1 +f0cadc58f065b9d4 1 -7.281427491271538e1 +3c3889813ef35db4 1 7.2814e1 +977a2a510c565470 1 8.277766877236081e1 +1342b4ca96de7c68 1 8.2778e1 +7d0118272533a618 1 6.537925550437317e3 +eb730fb885709956 1 6.537926e3 +5cbc224dc75590af 1 -9.556341676004886e4 +651cd96eb2cd481d 1 9.5563417e4 +7454f12c9c69f37a 1 8.917218866870608e5 +66dff276a09c18ca 1 8.91721887e5 +6cd5d2549504ffe4 1 1.7117781599952288e6 +b1b8b8d63f96b6e3 1 1.71177816e6 +4a42c81491eef17c 1 -7.887393405242969e7 +84299b08c59543e1 1 7.8873934052e7 +e59890374c3b92fd 1 4.975792917240606e8 +744d7f2f33d43087 1 4.97579291724e8 +62bc9e9bf68188bf 1 2.1101063788838792e9 +d651f84aed764523 1 2.110106378884e9 +03d61121d089d8a5 1 -5.020627637227572e10 +e7dec7d4092686f3 1 5.0206276372276e10 +f17fe9bd3b512a80 1 6.810322205888846e11 +734531128a13110d 1 6.81032220588885e11 +9c1dec292ef7b37f 1 1.1358071611803188e12 +dd424abd6b3af684 1 1.135807161180319e12 +694f57c35b260310 1 -3.644307586777936e13 +13611f86ee78a0ed 1 3.644307586777936e13 +d2a2a06dea91c0a6 1 4.619231907602407e14 +d2a2a06dea91c0a6 1 4.619231907602407e14 +77901e2ac387e970 1 6.618779769599664e15 +77901e2ac387e970 1 6.618779769599664e15 +89f326d5337edbf9 1 -1.6799818086520645e15 +4522b28cd51ce9fc 1 1.6799818086520645e15 +f9e0155a978000b9 1 8.738754212610816e17 +f9e0155a978000b9 1 8.738754212610816e17 +4bd5f489ae1421a8 1 3.782517862984159e18 +4bd5f489ae1421a8 1 3.782517862984159e18 +40b6664a43480fc1 1 -2.4148772852069077e19 +9441359dbc20f6b6 1 2.4148772852069077e19 +76f2826b72e1fda6 1 8.877221241287098e-21 +5fda7bf42a47c479 1 0e0 +bb4a3ae1f5949c11 1 1.8341454532358146e-20 +5fda7bf42a47c479 1 0e0 +766ae042bb11d920 1 -9.887584766107361e-19 +5fda7bf42a47c479 1 0e0 +c959d5910bc866f9 1 9.166579699039185e-18 +5fda7bf42a47c479 1 0e0 +b2699aa224c131ce 1 3.2452783206646016e-18 +5fda7bf42a47c479 1 0e0 +e784da29c1b96c28 1 -4.798816460260819e-16 +5fda7bf42a47c479 1 0e0 +c3bc806511396a74 1 5.664578770036096e-15 +5fda7bf42a47c479 1 0e0 +6c133c6895703b73 1 8.829601615987267e-14 +5fda7bf42a47c479 1 0e0 +b7721097837aef20 1 -4.3573790293046475e-13 +5fda7bf42a47c479 1 0e0 +af58a97244a9c6c3 1 5.551460106733779e-12 +5fda7bf42a47c479 1 0e0 +4646a79cd8815ccf 1 6.050294303578548e-11 +5fda7bf42a47c479 1 0e0 +5a4ffdfaaac64359 1 -3.3879885578121597e-10 +5fda7bf42a47c479 1 0e0 +d30c3dba6f8b8d72 1 9.069160028722885e-9 +5fda7bf42a47c479 1 0e0 +35719d8fb5e8e209 1 3.1971165267170906e-8 +5fda7bf42a47c479 1 0e0 +8f3a36e216599613 1 -8.76974502693964e-7 +5fda7bf42a47c479 1 0e0 +4a7c13e5c2e949a0 1 4.680359748821511e-6 +5fda7bf42a47c479 1 0e0 +c58c1c5093cfd666 1 7.563237465046436e-5 +5fda7bf42a47c479 1 0e0 +71c0d2f679b366ab 1 -1.79232054129694e-4 +5fda7bf42a47c479 1 0e0 +cc39dc0b3f31ef1a 1 8.188147563049265e-3 +1e7c9f5936317e01 1 8e-3 +7810fc63819aa54f 1 4.8276882501172715e-2 +316f14fbacc0f6dc 1 4.8e-2 +91fcbed9b6c1e1eb 1 -4.4497532569195164e-1 +1ba135db6a0ed229 1 4.45e-1 +6a646bd946016bba 1 7.6485650387578605e0 +49f07016ba5d174a 1 7.649e0 +35aacb91d397e63c 1 6.863911531586618e1 +ad8b40222f9c0b67 1 6.8639e1 +b62458bc71ec92fd 1 -4.564364881857964e2 +d8bdf521d8767965 1 4.56436e2 +f0d014cf4e9edf7f 1 1.9163551023901937e3 +15c1eb9d0605d6b0 1 1.916355e3 +b051547190f1dead 1 1.850604973023119e4 +a8b5ae0e34ac6db2 1 1.850605e4 +57da0f4c75eb541f 1 -6.297725994049538e5 +41c4a242cd154e87 1 6.29772599e5 +3053c5c8720a172e 1 7.738395738921099e4 +aaeee63373ab5c58 1 7.7383957e4 +122c1b2c08853c0c 1 8.396373407043353e7 +7b7f558a837722ff 1 8.396373407e7 +ba34066cdc427569 1 -2.4288547122230375e8 +420f589afd6b0565 1 2.42885471222e8 +041703a36b497da8 1 2.812715976368404e9 +c889fbd6f49dfffc 1 2.812715976368e9 +72f6aaac4485ab61 1 5.522396329298781e10 +60ac6434d6875dde 1 5.5223963292988e10 +36803089910ba7e2 1 -7.073708200884093e11 +5afe826153ea2a19 1 7.07370820088409e11 +8a4f45dfac6a7733 1 3.73164417144063e12 +8a4f45dfac6a7733 1 3.73164417144063e12 +69bca16e7f65b43d 1 1.69380681630822e13 +69bca16e7f65b43d 1 1.69380681630822e13 diff --git a/crates/numfmt/tests/render_equality.rs b/crates/numfmt/tests/render_equality.rs new file mode 100644 index 000000000..12a3ab94d --- /dev/null +++ b/crates/numfmt/tests/render_equality.rs @@ -0,0 +1,225 @@ +//! Equality tests for the render/General cleanup: output over a large grid of format codes x +//! values (format_value and text_function in both date systems, then General at widths 1..=30 and a few larger ones) +//! is compared with golden digests recorded from the implementation BEFORE the cleanup +//! (`tests/golden/*.txt`, one FNV-1a digest per format code / per value, so the files stay +//! small). A failure names the code or value that differs; set `NUMFMT_DUMP_DIR` to write the +//! full text of the current output for diffing against a dump from the old code. To re-record +//! after an intended behaviour change run with `NUMFMT_BLESS=1`. + +#![allow(clippy::approx_constant, clippy::excessive_precision)] // values are test data, not constants + +use std::fmt::Write as _; + +use gridcraft_core::{DateSystem, Value}; +use gridcraft_numfmt::{NumberFormat, format_general_fit, format_value, text_function}; + +// Placeholders far beyond the 15 significant digits. +const LONG_DECIMALS: &str = "0.0000000000000000000000000"; +#[rustfmt::skip] +const CODES: &[&str] = &[ + LONG_DECIMALS, "#,###,###,###,###,###,###,###,##0.00", "0.0000000000000000000000000E+00", "# ?????/?????", + // numbers + "General", "0", "0.00", "#,##0", "#,##0.00", "#.##", "0.0##", "00000", "000.000", "#,###", "#,##0,", "#,##0,,", "0.0,,\"M\"", "?.??", "##0.0", "# ", "#", ".00", + ",0", "0,0", "0.00,", "\"$\"#,##0.00", "\"$\"#,##0_);\\(\"$\"#,##0\\)", "#,##0.00;[Red](#,##0.00)", "#,##0 ;(#,##0)", "0;-0;;@", "0.00;;", ";;;", + "0_);(0)", "0\" units\"", "\\A0", "(0)", "00-00", "000-00-0000", "[<=9999999]###-####;(###) ###-####", "[$€-407]#,##0.00", + // currency/accounting with fill + "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"_);_(@_)", "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)", "*-0", "0*x", "*0 0.0", "@*_", + // percent + "0%", "0.00%", "#,##0%", "0.0%%", "%0", "0%;[Red]-0%", + // scientific + "0.00E+00", "##0.0E+0", "0E+0", "0.0e-0", "0.00E+000", "E+0", "0.00E+", "#.##E+00", "0.0E+00 \"x\"", + // fractions + "# ?/?", "# ??/??", "# ???/???", "?/?", "# ?/8", "# ??/16", "0/0", "# #/#", "#/#", "?/100", "# ?/?;-# ?/?", "0 0/0", + // dates/times + "m/d/yyyy", "d-mmm-yy", "d-mmm", "mmm-yy", "h:mm AM/PM", "h:mm:ss AM/PM", "h:mm", "h:mm:ss", "m/d/yyyy h:mm", "mm:ss", "[h]:mm:ss", "[h]:mm", "[m]:ss", "[s]", "mmss.0", + "yyyy-mm-dd", "dddd, mmmm d, yyyy", "ddd mmm dd yy", "mmmmm", "yy", "h:mm:ss.000", "h:mm:ss.0000", "hh:mm:ss A/P", "[hh]:mm:ss", "d", "dd", "dddd", "m", "mm", "s.00", + "yyyy-mm-dd hh:mm:ss.00", "h AM/PM", "[$-409]d-mmm-yy", "yyyy\"年\"m\"月\"", "[h]\" hrs\"", "mmmm yyyy;@", + // text sections, conditions, colours + "@", "\"x\"@", "0;\"neg\"0;\"zero\";\"txt: \"@", "General;General;General;@", "[Red]0.0;[Blue]-0.0", "[Green]0;[Magenta]0;[Cyan]0;[Yellow]@", "[Color3]0.00", + "[>100]\"big\"0;[<-100]\"small\"0;0.0", "[=0]\"zero\";0.0", "[>=1000]#,##0,\"K\";0", "[<1]0.00%;0.0", "[Red][>5]0;0.00", "General\" kg\"", "\"v\"General", + "[Red]General", "0.0;[Red]General", + // invalid / odd + "[", "0.0.0", "0\"", "\\", "[h", "[$", "yyyy-mm-dd;", ";", "0;0;0;0;0", "\"abc", "[>x]0", "ggg", "#?#0", "0E", "E0", "/", "0/", "/0", "*", "_", "_0", "[Red", "0.00.00%", "mm:ss.0.0", + "[ABC]0", "[=1]0;[=2]0;0", "0 ??/??;0", +]; + +fn values() -> Vec { + let mut v = vec![ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + -0.5, + 0.25, + 1.005, + 2.675, + 999.9995, + 0.9995, + 9.995, + 99.5, + 0.05, + 0.005, + 0.0005, + 1e-5, + 1e-10, + 1e-300, + 5e-324, + 123456789.0, + 1234.5678, + 12345678901.0, + 1e11, + 1e15, + 123456789012345.0, + 1234567890123455.0, + 1e20, + 1e100, + 1.7976931348623157e308, + -1234.5, + -0.004, + -1e-10, + 1.0 / 3.0, + 2.0 / 3.0, + 0.1 + 0.2, + 100.0, + 1000.0, + 59.0, + 60.0, + 61.0, + 36526.0, + 36526.75, + 43831.5, + 45000.123456, + 2958465.0, + 2958465.999999, + 2958466.0, + -5.0, + 0.99999999, + 0.000999, + 41.99999999999, + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + 0.041666666666666664, + 0.9999999999, + 1.0000000001, + 12.3456789, + 3.14159265358979, + 65535.0, + 1e5, + 123456.0, + 4.5, + 0.375, + 1e300, + 1.5e-8, + 999999999999999.0, + 99999999999999.99, + 0.00001234, + 123456789012345678.0, + 1e22, + 9.999999999999999e22, + 0.1234567890123456, + ]; + v.push(61.2 / 86400.0); + v +} + +fn fmt_formatted(f: &gridcraft_numfmt::Formatted) -> String { + format!("{:?}|{:?}|{:?}|{}", f.text, f.color, f.fill, f.numeric) +} + +fn dump_codes() -> Vec<(String, String)> { + assert!(CODES.len() >= 80 && values().len() >= 40); + let mut rows = Vec::new(); + let others = [Value::Text("abc".into()), Value::Text("12.5".into()), Value::Bool(true), Value::Empty]; + for code in CODES { + let mut out = String::new(); + let nf = NumberFormat::parse(code); + let systems: &[(DateSystem, &str)] = + if nf.is_date() { &[(DateSystem::D1900, "1900"), (DateSystem::D1904, "1904")] } else { &[(DateSystem::D1900, "1900")] }; + for (sys, name) in systems { + for n in values() { + let v = Value::Number(n); + let _ = + writeln!(out, "{code:?} {name} {n:e} => {} || {:?}", fmt_formatted(&format_value(&v, &nf, *sys)), text_function(&v, code, *sys)); + } + } + for v in &others { + let _ = writeln!( + out, + "{code:?} {v:?} => {} || {:?}", + fmt_formatted(&format_value(v, &nf, DateSystem::D1900)), + text_function(v, code, DateSystem::D1900) + ); + } + rows.push((format!("{code:?}"), out)); + } + rows +} + +fn dump_general() -> Vec<(String, String)> { + let mut vals = values(); + let mut x = 0.7f64; + // Deterministic spread over many magnitudes and digit patterns (no RNG dependency). + for i in 0..240 { + x = (x * 997.0 + 0.123_456_789).fract() + 1e-3; + let e = (i % 41) - 20; + let m = x * 10f64.powi(e); + vals.push(if i % 3 == 0 { -m } else { m }); + vals.push((m * 1000.0).round() / 1000.0); + } + assert!(vals.len() >= 300); + let mut rows = Vec::new(); + for n in vals { + let mut out = String::new(); + let _ = write!(out, "{n:e} =>"); + for w in (1..=30usize).chain([31, 40, 100, 399, 400, 401, 500, 1000]) { + let _ = write!(out, " {:?}", format_general_fit(n, w)); + } + out.push('\n'); + rows.push((format!("{n:e}"), out)); + } + rows +} + +fn fnv1a(s: &str) -> u64 { + s.bytes().fold(0xcbf2_9ce4_8422_2325, |h, b| (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3)) +} + +fn check(name: &str, rows: &[(String, String)], golden: &str, path: &str) { + if let Some(dir) = std::env::var_os("NUMFMT_DUMP_DIR") { + let all: String = rows.iter().map(|(_, b)| b.as_str()).collect(); + std::fs::write(std::path::Path::new(&dir).join(format!("{name}.txt")), all).expect("write dump"); + } + let actual: String = rows.iter().map(|(k, b)| format!("{:016x} {} {k}\n", fnv1a(b), b.lines().count())).collect(); + if std::env::var_os("NUMFMT_BLESS").is_some() { + std::fs::write(path, &actual).expect("write golden"); + return; + } + let bad: Vec = rows + .iter() + .zip(actual.lines().zip(golden.lines())) + .filter(|(_, (a, g))| a != g) + .take(3) + .map(|((_, body), (a, g))| { + format!(" new: {a}\n old: {g}\n new output of that row (first 12 lines):\n{}", body.lines().take(12).collect::>().join("\n")) + }) + .collect(); + assert!( + actual.lines().count() == golden.lines().count() && bad.is_empty(), + "{name}: output differs from the recorded golden ({} vs {} rows). If the change is intended, re-record with\n NUMFMT_BLESS=1 cargo test -p gridcraft-numfmt --test render_equality\n{}", + actual.lines().count(), + golden.lines().count(), + bad.join("\n") + ); +} + +#[test] +fn format_value_and_text_function_match_golden() { + check("codes", &dump_codes(), include_str!("golden/codes.txt"), concat!(env!("CARGO_MANIFEST_DIR"), "/tests/golden/codes.txt")); +} + +#[test] +fn general_fit_matches_golden() { + check("general", &dump_general(), include_str!("golden/general.txt"), concat!(env!("CARGO_MANIFEST_DIR"), "/tests/golden/general.txt")); +} diff --git a/crates/pdf/Cargo.toml b/crates/pdf/Cargo.toml index cd63456c2..9a32a8620 100644 --- a/crates/pdf/Cargo.toml +++ b/crates/pdf/Cargo.toml @@ -3,10 +3,17 @@ name = "gridcraft-pdf" version.workspace = true edition.workspace = true license.workspace = true -description = "GridCraft's tiny self-contained PDF 1.7 writer (standard 14 fonts, vector drawing, RGBA images)" +description = "GridCraft PDF 1.7 writer with shaped Arabic text, vector drawing and RGBA images" readme = "README.md" [lints] workspace = true [dependencies] +harfrust = "=0.12.0" +skrifa = { workspace = true } +unicode-bidi = "=0.3.18" + +[build-dependencies] +sha2 = { workspace = true } +skrifa = { workspace = true } diff --git a/crates/pdf/README.md b/crates/pdf/README.md index cdb6308eb..c8ec1c9b3 100644 --- a/crates/pdf/README.md +++ b/crates/pdf/README.md @@ -1,6 +1,6 @@ # gridcraft-pdf -A small, dependency-free PDF 1.7 writer for vector drawing and RGBA images. +A small PDF 1.7 writer for shaped Arabic text, vector drawing and RGBA images. ## API migration diff --git a/crates/pdf/build.rs b/crates/pdf/build.rs new file mode 100644 index 000000000..a5f132434 --- /dev/null +++ b/crates/pdf/build.rs @@ -0,0 +1,88 @@ +use std::fmt::Write as _; + +fn main() -> Result<(), String> { + arabic_fonts() +} + +/// Embed only Arabic fallback faces from the optional, separately licensed font bundle. +fn arabic_fonts() -> Result<(), String> { + use std::path::PathBuf; + println!("cargo::rerun-if-env-changed=CRAFT_FONTS_DIR"); + println!("cargo::rerun-if-env-changed=CRAFT_FONTS_REQUIRED"); + let required = std::env::var_os("CRAFT_FONTS_REQUIRED").is_some(); + let entries = match std::env::var_os("CRAFT_FONTS_DIR").map(PathBuf::from) { + Some(dir) => arabic_font_entries(&dir), + None if required => Err("CRAFT_FONTS_REQUIRED needs CRAFT_FONTS_DIR".into()), + None => Ok(String::new()), + }; + let entries = match entries { + Ok(entries) => entries, + Err(e) if required => return Err(e), + Err(e) => { + println!("cargo::warning=building without bundled Arabic fonts: {e}"); + String::new() + } + }; + let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap_or_default()).join("arabic_fonts.rs"); + std::fs::write(&out, format!("pub(crate) static ARABIC_FONTS: &[(&str, &[u8])] = &[\n{entries}];\n")) + .map_err(|e| format!("writing {}: {e}", out.display())) +} + +fn arabic_font_entries(dir: &std::path::Path) -> Result { + use sha2::{Digest, Sha256}; + use skrifa::MetadataProvider; + let root = dir.canonicalize().map_err(|e| format!("craft-fonts directory: {e}"))?; + let manifest = root.join("fonts/manifest.txt"); + println!("cargo::rerun-if-changed={}", manifest.display()); + let text = String::from_utf8(read_font_input(&manifest, 1 << 20)?).map_err(|e| e.to_string())?; + let mut out = String::new(); + let mut has_regular = false; + for (line_number, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let fields: Vec<_> = line.split(" | ").map(str::trim).collect(); + let [family, style, file, scripts, _licence, licence_file, checksum, _source] = fields.as_slice() else { + return Err(format!("malformed craft-fonts manifest line {}", line_number + 1)); + }; + if !scripts.split(',').any(|script| script.trim() == "Arab") { + continue; + } + let path = root.join(file).canonicalize().map_err(|e| format!("{file}: {e}"))?; + let licence = root.join(licence_file).canonicalize().map_err(|e| format!("{licence_file}: {e}"))?; + if !path.starts_with(&root) || !licence.starts_with(&root) { + return Err("craft-fonts paths must stay inside the checkout".into()); + } + println!("cargo::rerun-if-changed={}", path.display()); + println!("cargo::rerun-if-changed={}", licence.display()); + let bytes = read_font_input(&path, 64 << 20)?; + if Sha256::digest(&bytes).iter().map(|byte| format!("{byte:02x}")).collect::() != *checksum { + return Err(format!("checksum mismatch for {file}")); + } + let font = skrifa::FontRef::from_index(&bytes, 0).map_err(|e| format!("invalid Arabic font {file}: {e}"))?; + if !"العربية١٢٣".chars().all(|ch| font.charmap().map(ch).is_some()) { + return Err(format!("{file} lacks Arabic glyphs")); + } + let _ = writeln!(out, " ({:?}, include_bytes!({:?})),", format!("craft:{family}:{style}"), path); + has_regular |= *style != "Bold"; + } + if out.is_empty() { + return Err("craft-fonts manifest contains no Arabic face; update the checkout".into()); + } + if !has_regular { + return Err("craft-fonts manifest has only bold Arabic faces; regular text needs a regular or variable face".into()); + } + Ok(out) +} + +fn read_font_input(path: &std::path::Path, limit: u64) -> Result, String> { + use std::io::Read as _; + let file = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut bytes = Vec::new(); + file.take(limit + 1).read_to_end(&mut bytes).map_err(|e| format!("{}: {e}", path.display()))?; + if bytes.len() as u64 > limit { + return Err(format!("{} exceeds the font input size limit", path.display())); + } + Ok(bytes) +} diff --git a/crates/pdf/src/arabic.rs b/crates/pdf/src/arabic.rs new file mode 100644 index 000000000..11a531729 --- /dev/null +++ b/crates/pdf/src/arabic.rs @@ -0,0 +1,669 @@ +//! Arabic shaping and subset Type3 fonts. Outlines use the same variation coordinates as the +//! shaper; ActualText preserves the logical source independently of visual glyph order. +use std::{borrow::Cow, collections::BTreeMap, fmt::Write as _, sync::OnceLock}; + +use harfrust::{Direction, ShaperData, ShaperInstance, UnicodeBuffer}; +use skrifa::{ + FontRef, MetadataProvider, + instance::Size, + outline::{DrawSettings, OutlinePen}, +}; +use unicode_bidi::BidiInfo; + +use crate::{CID_FONT, Error, Font, TextDirection, dict_obj, n, needs_cid, pdf_string, pdf_utf16_string, stream_obj}; + +include!(concat!(env!("OUT_DIR"), "/arabic_fonts.rs")); + +const MAX_TEXT_BYTES: usize = 128 * 1024; +const MAX_OUTLINE_BYTES: usize = 256 * 1024; +const MAX_SUBSET_BYTES: usize = 64 * 1024 * 1024; +const MAX_SUBSET_GLYPHS: usize = 16_384; +const SYNTHETIC_BOLD_STROKE: f32 = 30.0; + +/// How `text` set in `font` is drawn. Symbol keeps its own one-run encoding, and Arabic without +/// an installed or bundled face falls back to the standard fonts. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Shaping { + Plain, + Shaped, + NoFont, +} + +pub(crate) fn shaping(font: Font, text: &str) -> Shaping { + if font == Font::Symbol || !needed(text) { + return Shaping::Plain; + } + match face(bold(font)) { + Ok(None) => Shaping::NoFont, + Ok(Some(_)) | Err(_) => Shaping::Shaped, + } +} + +pub(crate) fn shaped(font: Font, text: &str) -> bool { + shaping(font, text) == Shaping::Shaped +} + +pub(crate) fn needed(text: &str) -> bool { + text.chars().any(|ch| matches!(ch as u32, 0x0600..=0x06ff | 0x0750..=0x077f | 0x0870..=0x089f | 0x08a0..=0x08ff | 0xfb50..=0xfdff | 0xfe70..=0xfefc | 0x10e60..=0x10e7f | 0x1ee00..=0x1eeff)) +} + +fn bold(font: Font) -> bool { + matches!( + font, + Font::HelveticaBold | Font::HelveticaBoldOblique | Font::TimesBold | Font::TimesBoldItalic | Font::CourierBold | Font::CourierBoldOblique + ) +} + +fn italic(font: Font) -> bool { + matches!( + font, + Font::HelveticaOblique + | Font::HelveticaBoldOblique + | Font::TimesItalic + | Font::TimesBoldItalic + | Font::CourierOblique + | Font::CourierBoldOblique + ) +} + +struct Face { + bytes: Cow<'static, [u8]>, + data: ShaperData, + instance: ShaperInstance, + synthetic_bold: bool, +} + +impl Face { + fn new(bytes: Cow<'static, [u8]>, bold: bool) -> Result { + let font = FontRef::from_index(&bytes, 0).map_err(|e| Error(format!("Invalid Arabic PDF font: {e}")))?; + if !"العربية١٢٣".chars().all(|ch| font.charmap().map(ch).is_some()) { + return Err(Error("The PDF font lacks Arabic glyphs".into())); + } + let data = ShaperData::new(&font); + let instance = ShaperInstance::from_variations(&font, [("wght", if bold { 700.0 } else { 400.0 })]); + Ok(Self { bytes, data, instance, synthetic_bold: false }) + } + + fn font(&self) -> Result, Error> { + FontRef::from_index(&self.bytes, 0).map_err(|e| Error(format!("Invalid Arabic PDF font: {e}"))) + } +} + +#[cfg(test)] +pub(crate) fn bold_face_is_synthetic() -> bool { + face(true).is_ok_and(|face| face.is_some_and(|face| face.synthetic_bold)) +} + +/// The Arabic face for a weight, or `None` when none is bundled or installed. The search runs +/// once per weight, so measuring text never rescans the font directories. +fn face(is_bold: bool) -> Result, Error> { + static REGULAR: OnceLock, Error>> = OnceLock::new(); + static BOLD: OnceLock, Error>> = OnceLock::new(); + let cache = if is_bold { &BOLD } else { ®ULAR }; + cache.get_or_init(|| find_face(ARABIC_FONTS, &system_font_dirs(), is_bold)).as_ref().map(Option::as_ref).map_err(Error::clone) +} + +fn loaded_face(font: Font) -> Result<&'static Face, Error> { + face(bold(font))?.ok_or_else(|| Error("Arabic PDF export needs Noto Sans Arabic".into())) +} + +fn bundled_match(fonts: &[(&str, &'static [u8])], is_bold: bool) -> Option<&'static [u8]> { + fonts + .iter() + .find(|(name, bytes)| if is_bold { name.ends_with(":Bold") || has_weight_axis(bytes) } else { !name.ends_with(":Bold") }) + .map(|&(_, bytes)| bytes) +} + +#[cfg(not(target_arch = "wasm32"))] +fn system_font_dirs() -> Vec { + use std::path::PathBuf; + let mut dirs: Vec = [ + "/usr/share/fonts/noto", + "/usr/share/fonts/google-noto-vf", + "/usr/share/fonts/truetype/noto", + "/usr/share/fonts/TTF", + "/usr/local/share/fonts", + "/usr/local/share/fonts/noto", + "/usr/local/share/fonts/TTF", + "/Library/Fonts", + ] + .into_iter() + .map(Into::into) + .collect(); + if let Some(home) = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) { + let home = PathBuf::from(home); + dirs.extend([home.join(".local/share/fonts"), home.join(".fonts"), home.join("Library/Fonts")]); + } + if let Some(root) = std::env::var_os("WINDIR") { + dirs.push(PathBuf::from(root).join("Fonts")); + } + if let Some(root) = std::env::var_os("LOCALAPPDATA") { + dirs.push(PathBuf::from(root).join("Microsoft/Windows/Fonts")); + } + dirs +} + +#[cfg(target_arch = "wasm32")] +fn system_font_dirs() -> Vec { + Vec::new() +} + +fn find_face(bundled: &[(&str, &'static [u8])], dirs: &[std::path::PathBuf], is_bold: bool) -> Result, Error> { + use std::io::Read as _; + if let Some(bytes) = bundled_match(bundled, is_bold) { + return Face::new(Cow::Borrowed(bytes), is_bold).map(Some); + } + let filename = if is_bold { "NotoSansArabic-Bold.ttf" } else { "NotoSansArabic-Regular.ttf" }; + for dir in dirs { + for name in ["NotoSansArabic.ttf", "NotoSansArabic[wght].ttf", filename] { + let path = dir.join(name); + let file = match std::fs::File::open(&path) { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => return Err(Error(format!("Cannot read Arabic PDF font {}: {e}", path.display()))), + }; + let mut bytes = Vec::new(); + file.take((64 << 20) + 1).read_to_end(&mut bytes).map_err(|e| Error(format!("Cannot read Arabic PDF font: {e}")))?; + if bytes.len() > 64 << 20 { + return Err(Error("Arabic PDF font exceeds 64 MiB".into())); + } + if is_bold && name != filename && !has_weight_axis(&bytes) { + continue; + } + return Face::new(Cow::Owned(bytes), is_bold).map(Some); + } + } + if let Some((name, bytes)) = bundled_fallback(bundled, is_bold) { + let mut face = Face::new(Cow::Borrowed(bytes), is_bold)?; + face.synthetic_bold = needs_synthetic_bold(name, bytes, is_bold); + return Ok(Some(face)); + } + Ok(None) +} + +/// The bundled face to use when no face matches the requested weight: regular text never borrows a bold face. +fn bundled_fallback<'a>(fonts: &[(&'a str, &'static [u8])], is_bold: bool) -> Option<(&'a str, &'static [u8])> { + fonts.iter().find(|(name, _)| is_bold || !name.ends_with(":Bold")).copied() +} + +/// A bold request served by a static regular face is emboldened by stroking its outlines. +fn needs_synthetic_bold(name: &str, bytes: &[u8], is_bold: bool) -> bool { + is_bold && !name.ends_with(":Bold") && !has_weight_axis(bytes) +} + +fn has_weight_axis(bytes: &[u8]) -> bool { + FontRef::from_index(bytes, 0).is_ok_and(|font| font.axes().iter().any(|axis| axis.tag() == skrifa::raw::types::Tag::new(b"wght"))) +} + +#[derive(Debug)] +pub(crate) enum Item { + Glyph { id: u32, x: f32, y: f32, source: String }, + Latin { text: String, x: f32 }, + Cid { text: String, x: f32 }, +} + +pub(crate) struct Layout { + pub items: Vec, + pub width: f32, +} + +/// Positions are in 1/1000 em, matching the Type3 font matrix. +pub(crate) fn layout(font: Font, text: &str, direction: TextDirection) -> Result { + if text.len() > MAX_TEXT_BYTES { + return Err(Error("Arabic PDF text exceeds the 128 KiB per-line limit".into())); + } + let direction = if direction == TextDirection::Context { direction.resolve(text) } else { direction }; + let face = loaded_face(font)?; + let raw = face.font()?; + let shaper = face.data.shaper(&raw).instance(Some(&face.instance)).build(); + let units = shaper.units_per_em(); + if units <= 0 { + return Err(Error("Arabic PDF font has invalid units per em".into())); + } + let scale = 1000.0 / units as f32; + let single_line: String = text.chars().map(|ch| if is_paragraph_break(ch) { ' ' } else { ch }).collect(); + let text = single_line.as_str(); + let bidi = BidiInfo::new(text, direction.level()); + let mut layout = Layout { items: Vec::new(), width: 0.0 }; + for para in &bidi.paragraphs { + let (levels, runs) = bidi.visual_runs(para, para.range.clone()); + for run in runs { + let rtl = levels.get(run.start).is_some_and(|level| level.is_rtl()); + let run_text = text.get(run.clone()).ok_or_else(|| Error("Invalid PDF text boundary".into()))?; + let mut spans = Vec::new(); + let mut start = 0; + let mut embedded = None; + for (i, ch) in run_text.char_indices() { + let supported = embeddable(raw.charmap().map(ch).is_some(), ch); + if embedded.is_some_and(|v| v != supported) { + spans.push((start..i, embedded.unwrap_or(false))); + start = i; + } + embedded = Some(supported); + } + spans.push((start..run_text.len(), embedded.unwrap_or(false))); + if rtl { + spans.reverse(); + } + for (span, embedded) in spans { + let part = run_text.get(span.clone()).ok_or_else(|| Error("Invalid PDF text boundary".into()))?; + if !embedded { + let part = part.chars().map(|ch| if is_line_break(ch) { ' ' } else { ch }); + let part: Vec = if rtl { part.rev().map(mirror_ansi).collect() } else { part.collect() }; + for run in part.chunk_by(|a, b| needs_cid(*a) == needs_cid(*b)) { + let (text, x) = (run.iter().collect(), layout.width); + layout.width += run.iter().map(|&ch| f32::from(font.char_width(ch))).sum::(); + layout.items.push(if run.first().is_some_and(|&ch| needs_cid(ch)) { Item::Cid { text, x } } else { Item::Latin { text, x } }); + } + continue; + } + let mut buffer = UnicodeBuffer::new(); + buffer.push_str(part); + buffer.set_pre_context(run_text.get(..span.start).unwrap_or_default()); + buffer.set_post_context(run_text.get(span.end..).unwrap_or_default()); + buffer.set_direction(if rtl { Direction::RightToLeft } else { Direction::LeftToRight }); + buffer.set_flags(harfrust::BufferFlags::REMOVE_DEFAULT_IGNORABLES); + if needed(part) { + buffer.set_script(harfrust::script::ARABIC); + } + buffer.guess_segment_properties(); + let shaped = shaper.shape(buffer, harfrust::ShapeOptions::new()); + let mut clusters: Vec<_> = shaped.glyph_infos().iter().map(|g| g.cluster as usize).collect(); + clusters.push(part.len()); + clusters.sort_unstable(); + clusters.dedup(); + let mut previous_cluster = None; + for (info, pos) in shaped.glyph_infos().iter().zip(shaped.glyph_positions()) { + if info.glyph_id == 0 { + return Err(Error("The Arabic PDF font produced a missing glyph".into())); + } + let cluster = info.cluster as usize; + let source = if previous_cluster == Some(cluster) { + String::new() + } else { + let index = clusters.binary_search(&cluster).map_err(|_| Error("Invalid PDF glyph cluster".into()))?; + let end = clusters.get(index + 1).copied().ok_or_else(|| Error("Invalid PDF glyph cluster".into()))?; + part.get(cluster..end).ok_or_else(|| Error("Invalid PDF glyph cluster".into()))?.to_string() + }; + previous_cluster = Some(cluster); + layout.items.push(Item::Glyph { + id: info.glyph_id, + x: layout.width + pos.x_offset as f32 * scale, + y: pos.y_offset as f32 * scale, + source, + }); + layout.width += pos.x_advance as f32 * scale; + } + } + } + } + Ok(layout) +} + +pub(crate) fn vertical_extents(font: Font, text: &str, direction: TextDirection) -> Result<(f32, f32), Error> { + let layout = layout(font, text, direction)?; + let face = loaded_face(font)?; + let raw = face.font()?; + let metrics = raw.glyph_metrics(Size::new(1000.0), skrifa::instance::LocationRef::new(face.instance.coords())); + let (mut ascent, mut descent) = (0.0_f32, 0.0_f32); + for item in layout.items { + match item { + Item::Glyph { id, y, .. } => { + let (bottom, top) = if let Some(bounds) = metrics.bounds(id.into()) { + (bounds.y_min, bounds.y_max) + } else { + let glyph = raw.outline_glyphs().get(id.into()).ok_or_else(|| Error("Missing Arabic PDF glyph outline".into()))?; + let mut pen = Pen::default(); + glyph + .draw(DrawSettings::unhinted(Size::new(1000.0), skrifa::instance::LocationRef::new(face.instance.coords())), &mut pen) + .map_err(|e| Error(format!("Cannot measure Arabic PDF glyph: {e}")))?; + if pen.full { + return Err(Error("Arabic PDF glyph exceeds the outline size limit".into())); + } + (pen.bounds[1], pen.bounds[3]) + }; + ascent = ascent.max(y + top); + descent = descent.max(-y - bottom); + } + Item::Latin { .. } | Item::Cid { .. } => { + ascent = ascent.max(800.0); + descent = descent.max(220.0); + } + } + } + Ok((ascent, descent)) +} + +/// Whether standard-font `text` is drawn in its logical order, unchanged. +pub(crate) fn is_logical_order(text: &str, direction: TextDirection) -> bool { + direction != TextDirection::RightToLeft && !text.chars().any(is_control) +} + +/// Bidi reordering for standard-font text requires no optional Arabic font. +pub(crate) fn latin_visual(text: &str, direction: TextDirection) -> Result, Error> { + if is_logical_order(text, direction) { + return Ok(Cow::Borrowed(text)); + } + if text.len() > MAX_TEXT_BYTES { + return Err(Error("Bidirectional PDF text exceeds the 128 KiB per-line limit".into())); + } + let single_line: String = text.chars().map(|ch| if is_paragraph_break(ch) { ' ' } else { ch }).collect(); + let text = single_line.as_str(); + let bidi = BidiInfo::new(text, direction.level()); + let mut out = String::with_capacity(text.len()); + for para in &bidi.paragraphs { + let (levels, runs) = bidi.visual_runs(para, para.range.clone()); + for run in runs { + let part = text.get(run.clone()).unwrap_or_default(); + if levels.get(run.start).is_some_and(|level| level.is_rtl()) { + out.extend(part.chars().rev().filter(|ch| !is_control(*ch)).map(mirror_ansi)); + } else { + out.extend(part.chars().filter(|ch| !is_control(*ch))); + } + } + } + Ok(Cow::Owned(out)) +} + +fn mirror_ansi(ch: char) -> char { + match ch { + '(' => ')', + ')' => '(', + '[' => ']', + ']' => '[', + '{' => '}', + '}' => '{', + '<' => '>', + '>' => '<', + '«' => '»', + '»' => '«', + '‹' => '›', + '›' => '‹', + other => other, + } +} + +/// Whether a character is drawn from the embedded Arabic face. Line breaks never are, so the +/// fallback path draws them as spaces even when the face maps them. +fn embeddable(mapped: bool, ch: char) -> bool { + (mapped || is_control(ch)) && !is_line_break(ch) +} + +fn is_line_break(ch: char) -> bool { + matches!(ch, '\n' | '\r' | '\u{2028}' | '\u{2029}') +} + +/// Bidi paragraph separators (class B). A drawn line is one paragraph: left in, they would split it +/// into paragraphs laid out left to right, swapping the parts of a right-to-left line. +fn is_paragraph_break(ch: char) -> bool { + matches!(ch, '\n' | '\r' | '\u{1C}'..='\u{1E}' | '\u{85}' | '\u{2029}') +} + +fn is_control(ch: char) -> bool { + matches!(ch, '\u{061c}' | '\u{200c}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}') +} + +#[derive(Default)] +pub(crate) struct Subset { + keys: BTreeMap<(u32, bool, bool, String), usize>, + glyphs: Vec, + bytes: usize, + /// Some text on the page set a run in the CJK CID font, so the page must declare it. + pub(crate) uses_cid: bool, +} + +struct Glyph { + path: String, + width: f32, + source: String, + bounds: [f32; 4], + emboldened: bool, +} + +/// The Type3 glyph procedure: a filled outline, also stroked in the text colour when emboldened. +fn glyph_procedure(glyph: &Glyph) -> String { + if glyph.emboldened { + format!("{} 0 d0\n{} w 1 j\n{}B\n", n(glyph.width), n(SYNTHETIC_BOLD_STROKE), glyph.path) + } else { + format!("{} 0 d0\n{}f\n", n(glyph.width), glyph.path) + } +} + +impl Subset { + fn insert(&mut self, font: Font, id: u32, source: String) -> Result { + let key = (id, bold(font), italic(font), source.clone()); + if let Some(index) = self.keys.get(&key) { + return Ok(*index); + } + if self.glyphs.len() >= MAX_SUBSET_GLYPHS { + return Err(Error("PDF page has too many distinct Arabic glyphs".into())); + } + let face = loaded_face(font)?; + let raw = face.font()?; + let location = skrifa::instance::LocationRef::new(face.instance.coords()); + let glyph = raw.outline_glyphs().get(id.into()).ok_or_else(|| Error("Missing Arabic PDF glyph outline".into()))?; + let width = + raw.glyph_metrics(Size::new(1000.0), location).advance_width(id.into()).ok_or_else(|| Error("Missing Arabic PDF glyph width".into()))?; + let mut pen = Pen { shear: if italic(font) { 0.212_556_56 } else { 0.0 }, ..Default::default() }; + glyph.draw(DrawSettings::unhinted(Size::new(1000.0), location), &mut pen).map_err(|e| Error(format!("Cannot draw Arabic PDF glyph: {e}")))?; + if pen.full || self.bytes.saturating_add(pen.path.len()).saturating_add(source.len().saturating_mul(2)) > MAX_SUBSET_BYTES { + return Err(Error("Arabic PDF outlines exceed the page size limit".into())); + } + let index = self.glyphs.len(); + self.bytes += pen.path.len() + source.len() * 2; + let emboldened = face.synthetic_bold; + let grow = if emboldened { SYNTHETIC_BOLD_STROKE / 2.0 } else { 0.0 }; + let [x0, y0, x1, y1] = pen.bounds; + self.glyphs.push(Glyph { path: pen.path, width, source, bounds: [x0 - grow, y0 - grow, x1 + grow, y1 + grow], emboldened }); + self.keys.insert(key, index); + Ok(index) + } + + pub(crate) fn text(&mut self, font: Font, text: &str, size: f32, direction: TextDirection) -> Result { + let layout = layout(font, text, direction)?; + let mut out = format!("/Span << /ActualText {} >> BDC\nBT\n", unicode_string(text, true)); + for item in layout.items { + match item { + Item::Glyph { id, x, y, source } => { + let index = self.insert(font, id, source)?; + let _ = writeln!( + out, + "/A{} {} Tf 1 0 0 1 {} {} Tm <{:02X}> Tj", + index / 255 + 1, + n(size), + n(x * size / 1000.0), + n(y * size / 1000.0), + index % 255 + 1 + ); + } + Item::Latin { text, x } => { + let _ = writeln!(out, "/F{} {} Tf 1 0 0 1 {} 0 Tm {} Tj", font.index() + 1, n(size), n(x * size / 1000.0), pdf_string(&text)); + } + Item::Cid { text, x } => { + self.uses_cid = true; + let _ = writeln!(out, "/F{} {} Tf 1 0 0 1 {} 0 Tm {} Tj", CID_FONT + 1, n(size), n(x * size / 1000.0), pdf_utf16_string(&text)); + } + } + } + out.push_str("ET\nEMC"); + Ok(out) + } + + /// Writes the subset fonts into `out` and returns their page resource entries. + pub(crate) fn write_objects(&self, next: &mut u32, out: &mut Vec, offsets: &mut Vec) -> String { + let mut resources = String::new(); + for (batch, glyphs) in self.glyphs.chunks(255).enumerate() { + let id = *next; + let cmap_id = id + 1; + *next += 2; + let mut procs = String::new(); + let mut names = String::new(); + let mut widths = String::new(); + let mut mappings = Vec::new(); + let mut bounds = [0.0_f32; 4]; + for (i, glyph) in glyphs.iter().enumerate() { + let code = i + 1; + let _ = write!(procs, "/g{code} {} 0 R ", *next); + let _ = write!(names, "/g{code} "); + let _ = write!(widths, "{} ", n(glyph.width)); + let body = glyph_procedure(glyph); + stream_obj(out, offsets, *next, "", body.as_bytes()); + *next += 1; + bounds[0] = bounds[0].min(glyph.bounds[0]); + bounds[1] = bounds[1].min(glyph.bounds[1]); + bounds[2] = bounds[2].max(glyph.bounds[2]); + bounds[3] = bounds[3].max(glyph.bounds[3]); + if !glyph.source.is_empty() { + mappings.push(format!("<{code:02X}> {}\n", unicode_string(&glyph.source, false))); + } + } + let mut cmap = "/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n/CMapName /GridCraftArabicUnicode def\n/CMapType 2 def\n1 begincodespacerange\n<00> \nendcodespacerange\n".to_string(); + for group in mappings.chunks(100) { + let _ = writeln!(cmap, "{} beginbfchar", group.len()); + for mapping in group { + cmap.push_str(mapping); + } + cmap.push_str("endbfchar\n"); + } + cmap.push_str("endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n"); + stream_obj(out, offsets, cmap_id, "", cmap.as_bytes()); + dict_obj( + out, + offsets, + id, + &format!( + "<< /Type /Font /Subtype /Type3 /Name /GridCraftArabic /FontBBox [{} {} {} {}] /FontMatrix [0.001 0 0 0.001 0 0] /CharProcs << {procs}>> /Encoding << /Type /Encoding /Differences [1 {names}] >> /FirstChar 1 /LastChar {} /Widths [{widths}] /Resources << >> /ToUnicode {cmap_id} 0 R >>", + n(bounds[0]), + n(bounds[1]), + n(bounds[2]), + n(bounds[3]), + glyphs.len() + ), + ); + let _ = write!(resources, "/A{} {id} 0 R ", batch + 1); + } + resources + } +} + +pub(crate) fn unicode_string(text: &str, bom: bool) -> String { + let mut out = if bom { "'); + out +} + +#[derive(Default)] +struct Pen { + path: String, + point: [f32; 2], + bounds: [f32; 4], + shear: f32, + full: bool, +} + +impl Pen { + fn point(&mut self, x: f32, y: f32) -> [f32; 2] { + let x = x + self.shear * y; + self.bounds[0] = self.bounds[0].min(x); + self.bounds[1] = self.bounds[1].min(y); + self.bounds[2] = self.bounds[2].max(x); + self.bounds[3] = self.bounds[3].max(y); + [x, y] + } + fn op(&mut self, value: &str) { + if self.path.len().saturating_add(value.len()) > MAX_OUTLINE_BYTES { + self.full = true; + } + if !self.full { + self.path.push_str(value); + } + } +} + +impl OutlinePen for Pen { + fn move_to(&mut self, x: f32, y: f32) { + let [px, py] = self.point(x, y); + self.op(&format!("{} {} m\n", n(px), n(py))); + self.point = [x, y]; + } + fn line_to(&mut self, x: f32, y: f32) { + let [px, py] = self.point(x, y); + self.op(&format!("{} {} l\n", n(px), n(py))); + self.point = [x, y]; + } + fn quad_to(&mut self, cx: f32, cy: f32, x: f32, y: f32) { + self.curve_to( + self.point[0] + (cx - self.point[0]) * 2.0 / 3.0, + self.point[1] + (cy - self.point[1]) * 2.0 / 3.0, + x + (cx - x) * 2.0 / 3.0, + y + (cy - y) * 2.0 / 3.0, + x, + y, + ); + } + fn curve_to(&mut self, cx: f32, cy: f32, dx: f32, dy: f32, x: f32, y: f32) { + let [cx, cy] = self.point(cx, cy); + let [dx, dy] = self.point(dx, dy); + let [px, py] = self.point(x, y); + self.op(&format!("{} {} {} {} {} {} c\n", n(cx), n(cy), n(dx), n(dy), n(px), n(py))); + self.point = [x, y]; + } + fn close(&mut self) { + self.op("h\n"); + } +} + +#[cfg(test)] +mod tests { + #[test] + fn fallback_mirrors_only_paired_ansi_punctuation() { + let visual: String = "(مرحبا)".chars().rev().map(super::mirror_ansi).collect(); + assert_eq!(visual, "(ابحرم)"); + assert_eq!("[(<«‹abc›»>)]".chars().rev().map(super::mirror_ansi).collect::(), "[(<«‹cba›»>)]"); + } + + #[test] + fn a_bold_request_served_by_a_static_regular_face_is_emboldened() { + assert!(super::needs_synthetic_bold("craft:Noto Sans Arabic:Regular", b"static", true)); + assert!(!super::needs_synthetic_bold("craft:Noto Sans Arabic:Regular", b"static", false)); + assert!(!super::needs_synthetic_bold("craft:Noto Sans Arabic:Bold", b"static", true)); + let glyph = + |emboldened| super::Glyph { path: "0 0 m 10 0 l 10 10 l h\n".into(), width: 500.0, source: String::new(), bounds: [0.0; 4], emboldened }; + assert_eq!(super::glyph_procedure(&glyph(false)), "500 0 d0\n0 0 m 10 0 l 10 10 l h\nf\n"); + assert_eq!(super::glyph_procedure(&glyph(true)), "500 0 d0\n30 w 1 j\n0 0 m 10 0 l 10 10 l h\nB\n"); + } + + #[test] + fn line_breaks_never_use_the_embedded_face() { + for ch in ['\n', '\r', '\u{2028}', '\u{2029}'] { + assert!(!super::embeddable(true, ch), "{ch:?}"); + } + assert!(super::embeddable(true, 'م')); + assert!(super::embeddable(false, '\u{200f}')); + assert!(!super::embeddable(false, 'A')); + } + + #[test] + fn no_bundled_or_installed_face_is_not_an_error() { + for bold in [false, true] { + assert!(matches!(super::find_face(&[], &[], bold), Ok(None))); + assert!(matches!(super::find_face(&[], &[std::env::temp_dir().join("gridcraft-no-fonts-here")], bold), Ok(None))); + } + } + + #[test] + fn bundled_faces_match_the_requested_weight() { + const REGULAR: &[u8] = b"regular"; + const BOLD: &[u8] = b"bold"; + let bold_first = [("Noto Sans Arabic:Bold", BOLD), ("Noto Sans Arabic", REGULAR)]; + assert_eq!(super::bundled_match(&bold_first, false), Some(REGULAR)); + assert_eq!(super::bundled_match(&bold_first, true), Some(BOLD)); + assert_eq!(super::bundled_match(&[("Noto Sans Arabic", REGULAR)], true), None); + assert_eq!(super::bundled_match(&[], false), None); + assert_eq!(super::bundled_fallback(&[("Noto Sans Arabic:Bold", BOLD)], false), None); + assert_eq!(super::bundled_fallback(&[("Noto Sans Arabic:Regular", REGULAR)], true), Some(("Noto Sans Arabic:Regular", REGULAR))); + } +} diff --git a/crates/pdf/src/lib.rs b/crates/pdf/src/lib.rs index 68983828f..db22568c6 100644 --- a/crates/pdf/src/lib.rs +++ b/crates/pdf/src/lib.rs @@ -5,21 +5,68 @@ //! WinAnsiEncoding (CJK and other scripts) are set in a predefined, non-embedded Adobe-GB1 CID //! font with the `UniGB-UTF16-H` CMap instead of being replaced by `?`. Symbol uses its own //! built-in encoding ([`symbol_code`]), for Greek letters and mathematical operators. +//! - Arabic is shaped and set in embedded Type3 subsets, with exact shaped advances. //! - Vector drawing: rectangles, lines (solid or dashed), polygons, circles (Bézier), clipping. //! - RGBA images as RGB image XObjects with a soft mask for alpha (run-length encoded). //! - Coordinates are points with a **top-left origin** (y grows down); text `y` is the baseline. //! -//! Streams are written uncompressed (images use the trivial RunLengthDecode filter) so the crate -//! needs no dependencies. +//! Streams are written uncompressed (images use the RunLengthDecode filter). #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] #![forbid(unsafe_code)] +mod arabic; mod metrics; mod symbol; pub use symbol::{symbol_code, symbol_width}; +/// Paragraph base direction. Context follows spreadsheet reading order: the first +/// non-whitespace character determines direction, including digits and punctuation. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum TextDirection { + #[default] + Auto, + Context, + LeftToRight, + RightToLeft, +} + +impl TextDirection { + /// Resolve once for a paragraph before wrapping it into visual lines. + pub fn resolve(self, text: &str) -> Self { + use unicode_bidi::{BidiClass, BidiInfo, bidi_class}; + let rtl = match self { + Self::Auto => BidiInfo::new(text, None).paragraphs.first().is_some_and(|p| p.level.is_rtl()), + Self::Context => text.chars().find(|ch| !ch.is_whitespace()).is_some_and(|ch| matches!(bidi_class(ch), BidiClass::R | BidiClass::AL)), + explicit => return explicit, + }; + if rtl { Self::RightToLeft } else { Self::LeftToRight } + } + + fn level(self) -> Option { + match self { + Self::Auto => None, + Self::LeftToRight | Self::Context => Some(unicode_bidi::Level::ltr()), + Self::RightToLeft => Some(unicode_bidi::Level::rtl()), + } + } +} + +/// A PDF export could not preserve the requested text. +#[derive(Clone, Debug)] +pub struct Error(String); + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} +impl std::error::Error for Error {} + +use std::collections::HashMap; +use std::collections::hash_map::DefaultHasher; use std::fmt::Write as _; +use std::hash::{Hash, Hasher}; /// RGB colour, 0–255 per channel. pub type Rgb = [u8; 3]; @@ -37,6 +84,13 @@ impl Point { } } +/// A text baseline origin and its clockwise rotation in radians. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Baseline { + pub origin: Point, + pub angle: f32, +} + /// A page-space rectangle described by its top-left corner and extent. #[derive(Clone, Copy, Debug, Default, PartialEq)] pub struct Rect { @@ -199,10 +253,29 @@ impl Font { } } +/// Whether [`text_width_with_direction`] is the sum of `text`'s character widths: not shaped, +/// not reordered right to left and holding no bidi controls. A prefix's width is then a running sum. +pub fn width_is_char_sum(font: Font, text: &str, direction: TextDirection) -> bool { + let direction = if direction == TextDirection::Context { direction.resolve(text) } else { direction }; + !arabic::shaped(font, text) && arabic::is_logical_order(text, direction) +} + /// Width of `text` in points when set in `font` at `size` points. +/// Arabic uses shaped advances. If its font cannot render the text, measurement returns zero +/// and drawing that text causes [`PdfDoc::finish`] to return the rendering error. pub fn text_width(font: Font, size: f32, text: &str) -> f32 { + text_width_with_direction(font, size, text, TextDirection::Auto) +} + +/// Width measured with the same base direction as drawing. +pub fn text_width_with_direction(font: Font, size: f32, text: &str, direction: TextDirection) -> f32 { + let direction = if direction == TextDirection::Context { direction.resolve(text) } else { direction }; + if arabic::shaped(font, text) { + return arabic::layout(font, text, direction).map(|layout| layout.width * size / 1000.0).unwrap_or(0.0); + } // u64: a u32 sum overflows for ~8 million characters. - let units: u64 = text.chars().map(|c| u64::from(font.char_width(c))).sum(); + let Ok(visual) = arabic::latin_visual(text, direction) else { return 0.0 }; + let units: u64 = visual.chars().map(|c| u64::from(font.char_width(c))).sum(); units as f32 * size / 1000.0 } @@ -322,6 +395,31 @@ fn n(v: f32) -> String { } } +/// Appends `v / 255` with at most three decimals, exactly what `n(v as f32 / 255.0)` gives, but +/// without a float round trip or an allocation per channel. +fn push_unit(s: &mut String, v: u8) { + let m = (u32::from(v) * 1000 + 127) / 255; + match m { + 0 => s.push('0'), + 1000 => s.push('1'), + _ => { + let _ = write!(s, "0.{m:03}"); + while s.ends_with('0') { + s.pop(); + } + } + } +} + +/// Appends a colour operator line without the trailing newline: `r g b rg` (fill) or `r g b RG`. +fn push_rgb(s: &mut String, c: Rgb, stroke: bool) { + for v in c { + push_unit(s, v); + s.push(' '); + } + s.push_str(if stroke { "RG" } else { "rg" }); +} + /// True when every value is finite (NaN and infinities are rejected). fn finite(v: &[f32]) -> bool { v.iter().all(|x| x.is_finite()) @@ -348,16 +446,44 @@ fn dash_values(dash: &[f32]) -> String { d.iter().map(|v| n(*v)).collect::>().join(" ") } -fn rgb_op(c: Rgb, stroke: bool) -> String { - let f = |v: u8| n(v as f32 / 255.0); - format!("{} {} {} {}", f(c[0]), f(c[1]), f(c[2]), if stroke { "RG" } else { "rg" }) -} - struct ImageData { width: u32, height: u32, rgb: Vec, + /// `None` when every pixel is fully opaque (no soft mask is written). alpha: Option>, + /// Hash of all of the above, to find duplicates quickly; equality still compares the bytes. + hash: u64, +} + +impl ImageData { + fn new(width: u32, height: u32, rgb: Vec, alpha: Option>) -> ImageData { + let mut h = DefaultHasher::new(); + width.hash(&mut h); + height.hash(&mut h); + rgb.hash(&mut h); + alpha.hash(&mut h); + ImageData { width, height, rgb, alpha, hash: h.finish() } + } + + /// Same pixels (a hash match alone is never trusted). + fn same(&self, o: &ImageData) -> bool { + self.hash == o.hash && self.width == o.width && self.height == o.height && self.rgb == o.rgb && self.alpha == o.alpha + } +} + +/// The part of the graphics state the writer tracks so that a style operator is only emitted +/// when it changes. `None` = not set yet on this page (or since the last restore that undid it). +#[derive(Clone, Debug, Default, PartialEq)] +struct Style { + fill: Option, + stroke: Option, + /// Formatted line width. + width: Option, + /// Formatted dash array contents. + dash: Option, + /// Line join; PDF's initial value is 0 (miter). + join: u8, } enum LinkTarget { @@ -378,8 +504,21 @@ pub struct Page { content: String, fonts: [bool; FONT_SLOTS], images: Vec, + /// Indexes into `images`, by content hash; a hash match is confirmed with [`ImageData::same`]. + image_index: HashMap>, + /// Pixel comparisons made while looking up duplicate pictures (test instrumentation). + #[cfg(test)] + image_comparisons: usize, + /// Style operators already in effect at the end of `content`. + style: Style, + /// One entry per accepted [`Page::save`], restored by [`Page::restore`]. + saved: Vec