diff --git a/.github/workflows/crw.yml b/.github/workflows/crw.yml new file mode 100644 index 000000000..b1eac4f35 --- /dev/null +++ b/.github/workflows/crw.yml @@ -0,0 +1,79 @@ +name: Shared CRW + +on: + push: + branches: [main] + paths: ["crates/crw/**", "scripts/fetch-crw-samples.py", "Cargo.toml", "Cargo.lock", ".github/workflows/crw.yml"] + pull_request: + paths: ["crates/crw/**", "scripts/fetch-crw-samples.py", "Cargo.toml", "Cargo.lock", ".github/workflows/crw.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: shared-crw-${{ github.ref }} + cancel-in-progress: true + +jobs: + native: + timeout-minutes: 20 + name: CRW lint and tests (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + components: rustfmt, clippy + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + key: craft-crw-${{ matrix.os }} + - run: cargo fmt -p craft-crw --check + - run: cargo clippy -p craft-crw --all-targets --locked -- -D warnings + - run: cargo test -p craft-crw --locked + + wasm: + timeout-minutes: 10 + name: CRW WASM + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + key: craft-crw-wasm + - run: cargo check -p craft-crw --locked --target wasm32-unknown-unknown + + samples: + timeout-minutes: 20 + name: CRW sample files decode exactly + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + key: craft-crw-samples + - name: Cache the CC0 sample files + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ runner.temp }}/crw-samples + key: crw-samples-${{ hashFiles('crates/crw/tests/samples.json') }} + - name: Fetch the pinned CC0 sample files (raw.pixls.us) + run: python scripts/fetch-crw-samples.py "$RUNNER_TEMP/crw-samples" + - run: cargo test -p craft-crw --release --locked --test samples -- --nocapture + env: + CRAFT_CRW_SAMPLES: ${{ runner.temp }}/crw-samples + CRAFT_CRW_REQUIRE_SAMPLES: "1" diff --git a/AGENTS.md b/AGENTS.md index e5adca3b6..b8bd8de82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,6 @@ # AGENTS.md -Shared Rust libraries for the Craft Apps. The shared standards live in -[craftrules](https://github.com/storytold/craftrules) (`../../craftrules` on a dev machine): read its -`AGENTS.md`, then `standards/never-crash.md`, `standards/engineering.md` and `standards/licensing.md`. +Shared Rust libraries for the Craft Apps. - **Never crash.** Every crate's `lib.rs` starts with `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]`. diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index c6345e85e..9585c992a 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -22,6 +22,9 @@ author, source and license. Library code and original assets are under the repos | `crates/ui/tests/fonts/NotoSansArabic-screens.ttf` (instance at weight 400 and width 100, subset to the screen test strings) | The Noto Project Authors | [google/fonts@7085eb8 `ofl/notosansarabic`](https://github.com/google/fonts/tree/7085eb89a950e85db5b166b7a58d414544b4140c/ofl/notosansarabic), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansArabic.txt`) | | `crates/ui/tests/fonts/NotoSansCJKsc-screens.otf` (subset to printable ASCII and the screen test strings) | The Noto Project Authors (Adobe, Google) | [notofonts/noto-cjk@f8d1575 `Sans/OTF/SimplifiedChinese`](https://github.com/notofonts/noto-cjk/tree/f8d157532fbfaeda587e826d4cd5b21a49186f7c/Sans/OTF/SimplifiedChinese), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansCJK.txt`) | | `crates/ui/tests/fonts/NotoSansThai-screens.ttf` (instance at weight 400 and width 100, subset to the screen test strings) | The Noto Project Authors | [google/fonts@8b0a1d0 `ofl/notosansthai`](https://github.com/google/fonts/tree/8b0a1d0f5983c89bc2b93f1b5fb55f9e252744b5/ofl/notosansthai), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansThai.txt`) | +| `crates/crw/tests/samples.json` (manifest only; the 11 raw files are not bundled, `scripts/fetch-crw-samples.py` downloads them outside the checkout) | The photographers who contributed each file to raw.pixls.us (Canon EOS D30, D60, 10D ×3, 300D ×2, Digital Rebel, PowerShot G1, G5, G6) | [raw.pixls.us](https://raw.pixls.us/), each file's `getfile.php` URL and SHA-256 in the manifest | CC0-1.0 (as published by raw.pixls.us for each file) | +| `crates/crw/src/tables.rs`, `crates/crw/clean-room/logs/table_sel*.json` | craft-crw contributors (derived in LightCraft, 2026-10, by known-plaintext alignment of the sample files' entropy streams; `crates/crw/clean-room/02-crw-format.md`) | this repository | MIT OR Apache-2.0 | +| `crates/crw/clean-room/logs/private-corpus.tsv` (file names, sizes and SHA-256 of the maintainer's own EOS D60 files; no image content) | craft-crw contributors (LightCraft derivation record) | this repository | MIT OR Apache-2.0 | Original assets are covered by the repository licence. diff --git a/Cargo.lock b/Cargo.lock index 77ba4276b..4fcd223b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -775,6 +775,17 @@ dependencies = [ "rayon", ] +[[package]] +name = "craft-crw" +version = "0.1.0" +dependencies = [ + "proptest", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.21", +] + [[package]] name = "craft-dialogs" version = "0.1.0" @@ -983,6 +994,7 @@ name = "craft-print" version = "0.1.0" dependencies = [ "craft-winprint-gdi", + "log", "miniz_oxide 0.9.1", "roxmltree 0.21.2", "tempfile", @@ -1061,8 +1073,11 @@ dependencies = [ "chrono", "dunce", "egui", + "egui_kittest", + "epaint_default_fonts", "flate2", "getrandom 0.3.4", + "image", "log", "plist", "semver", diff --git a/Cargo.toml b/Cargo.toml index a4ab549b6..a98e231bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ kurbo = "0.13" thiserror = "2" craft-atomic = { path = "crates/atomic" } craft-cms = { path = "crates/cms" } +craft-crw = { path = "crates/crw" } craft-dialogs = { path = "crates/dialogs" } craft-docpath = { path = "crates/docpath" } craft-font-catalog = { path = "crates/font-catalog" } diff --git a/NOTICE b/NOTICE index a46d37541..6fc5f1ea5 100644 --- a/NOTICE +++ b/NOTICE @@ -200,3 +200,16 @@ https://github.com/storytold/craft-launcher, https://github.com/storytold/photoc https://github.com/storytold/filmcraft, https://github.com/storytold/effectcraft. Copyright (c) 2026 ArtCraft Team and the respective application and craft-libs contributors. All adapted source is licensed MIT OR Apache-2.0. + +The Canon CRW decoder in crates/crw (src/lib.rs, src/tables.rs, src/tests.rs, tests/) and its +derivation record (crates/crw/clean-room/) come from: +- LightCraft, crates/raw/src/vendor/crw.rs, crates/raw/src/vendor/crw_tables.rs, + crates/raw/tests/crw_oracle.rs, crates/raw/tests/it/vendor_robust.rs and clean-room/crw/, + https://github.com/storytold/lightcraft/pull/3021, revision 6ca439b5155820d2befeacd81ec0c6a80b2bd3bc + (decoder ported at 70c755e42a65661401142819278d51b0c5977f7f; the record is identical at both). + https://github.com/storytold/lightcraft + Copyright (c) 2026 ArtCraft Team and the LightCraft contributors. +Its white-level plateau estimate (saturation_plateau) is LightCraft's vendor::white_from_data at the +same revision. The CRW compression was derived clean-room from CC0 sample files and the +maintainer's own photographs by black-box comparison with a reference decoder's command-line +output; no raw-decoder source was consulted (crates/crw/clean-room/sources.md). diff --git a/README.md b/README.md index 55adcbfd9..30d08db6e 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " bool`: the CIFF signature (`II`/`MM`, `HEAPCCDR` at offset 6). +- `header(bytes, &Limits) -> Result`: geometry, bit depth, CFA phase, selector, + make and model, capture time, orientation and white balance, without decoding the samples + (fast: it walks the heap and reads a few records). +- `decode(bytes, &Limits) -> Result`: the header plus the samples and the measured + black and white levels. + +### Output + +`Image { header, samples, black, white }`: + +- `samples`: `width × height` `u16`s, row-major, the whole sensor including the masked borders, + `bits` (10 or 12) deep, black not subtracted. Rows are in sensor order; apply + `header.exif_orientation` (1, 3, 6 or 8) when displaying. +- `header.active_area`: the picture (SensorInfo's inclusive crop); the rest is masked. The default + crop of these bodies is the active area itself. +- `header.cfa`: `Cfa::Rggb` or `Cfa::Gbrg`, the phase at sensor (0, 0); `pattern()` gives the + 2×2 cell as 0 = red, 1 = green, 2 = blue. +- `black`: `BlackLevel { values: [f32; 4] }`, one level per CFA site of the 2×2 cell counted + from the active area's origin (`at(active, x, y)`), measured on the darkest strips of the masked + borders. These bodies sit near 128 (the Digital Rebel sample near 261, the G1 near 29). +- `white`: the saturation plateau when the image has one, else the sensor's clip level when at + least 64 samples reach within 3 % of the full range (the D60, 10D and 300D clip near 4000 at + ISO 100–400), else the full range (4095 or 1023). +- `header.wb_multipliers`: `Some([r, 1.0, b])` from the ColorBalance record on the D60, 10D and + 300D, the slot chosen by the ShotInfo white-balance setting; `None` on the D30 and PowerShots, + whose records the derivation did not decipher. Apps fall back to their own estimate. +- `header.capture_unix`: the TimeStamp record, Unix seconds on the camera's local clock, no UTC + offset. Convert with chrono (`DateTime::from_timestamp`), never by hand. +- `header.make` / `header.model`: the MakeModel record, e.g. `Canon` / `Canon EOS D60`. CRW + files carry no Exif block, so this is where the camera name comes from. + +### Errors and limits + +`Error` is `NotCrw` (not a CIFF raw), `Corrupt(String)` (damaged structure or an undecodable +code), `Unsupported(String)` (an unknown decoder table, or the PowerShot G5) or +`LimitExceeded(&str)`. Every offset read from the file is bounds-checked, the heap walk is bounded +by depth (16) and by a per-file record budget (4,096), and `width × height` is checked against +`Limits::max_samples` (default 2²³, about 8.4 million, on every target; real frames hold at most +7.4 million) before anything is allocated. The decoder holds about three sample-sized `u16` +buffers at its peak. `Limits` is `#[non_exhaustive]`: start from `Limits::default()` and set the +fields you change. + +A stream that ends early is not an error: the remaining samples keep their predictor values (the +frozen stream-end rule; real files end 1–3 samples short). + +### Performance + +About 60 ms for a 6.5 MP EOS file and 70 ms for the 7.4 MP G6 in release mode on a 2024 laptop +(`cargo test -p craft-crw --release --test samples -- --nocapture` prints the times). + +## Tests + +- `cargo test -p craft-crw`: synthetic CIFF files round-tripped through a test-only encoder (the + model's inverse), the frozen tables' structure (prefix-free, all 162 symbols), hostile inputs + (every truncation and byte mutation of a valid file, self-referencing heaps), and a proptest + mutation sweep (`tests/robust.rs`). None needs sample files. +- `tests/samples.rs` pins the eleven CC0 sample files of [`tests/samples.json`](tests/samples.json) + (raw.pixls.us): each decodes to exactly the samples the derivation verified against the + reference, recorded as the SHA-256 of the sample buffer, with its levels, white balance and + metadata. The files (5–11 MB each) are not in the repository: + + ```sh + python scripts/fetch-crw-samples.py ~/crw-samples # downloads and verifies by SHA-256 + CRAFT_CRW_SAMPLES=~/crw-samples cargo test -p craft-crw --release --test samples -- --nocapture + ``` + + Unset, the test prints that it skipped; CI sets `CRAFT_CRW_REQUIRE_SAMPLES=1` and runs it on + Linux (`.github/workflows/crw.yml`; lint and the synthetic tests run on Linux, Windows and + macOS, plus a wasm32 check). +- `CRAFT_CRW_ORACLE=` additionally compares against `.crw.tiff` references next to the + files (the reference decoder's `unprocessed_raw -T` output), the check the derivation used. + Those references are not distributable and stay outside the repository. + +## Origin + +The decoder was written in LightCraft as `crates/raw/src/vendor/crw.rs` and `crw_tables.rs` +(LightCraft PR #3021, revision `6ca439b5155820d2befeacd81ec0c6a80b2bd3bc`, +[storytold/lightcraft](https://github.com/storytold/lightcraft); the decoder itself was ported at +`70c755e4`, and the branch's later review commits changed only a date conversion that is not part +of this crate) and moved here with its tests and +its derivation record. The move replaced LightCraft's types with this crate's own (`Error`, +`Limits`, `Cfa`, `Rect`, `BlackLevel`, `Header`, `Image`), exposed the capture time as Unix +seconds instead of converting it in the decoder, and copied LightCraft's shared plateau estimate +(`white_from_data`) as `saturation_plateau`. The algorithm, tables and thresholds are unchanged, +and the sample-file test proves the decode is byte-identical to the verified LightCraft decode. +Colour (camera matrices, profiles, the JPEG-fitted look) stayed in LightCraft. + +## Frozen tables + +`src/tables.rs` contains frozen code tables from the clean-room derivation record retained in +[LightCraft](https://github.com/storytold/lightcraft). Do not edit them by hand. A table change is +a change to the frozen format and must follow the reopening rule of `clean-room/02-crw-format.md`. + +## Boundaries + +- In scope: the CIFF container records a raw developer needs, the entropy coding, sensor levels, + as-shot white balance, orientation and the records a catalog needs (make, model, time). +- Out of scope: colour (no CRW body carries matrices; apps own their profiles), demosaicing, the + embedded JPEG preview (apps find it with their JPEG scanner; the record is `JpgFromRaw`, 0x2007), + and the full CIFF metadata set (ExifTool's prose documents far more records than the decoder + reads). diff --git a/crates/crw/clean-room/02-crw-format.md b/crates/crw/clean-room/02-crw-format.md new file mode 100644 index 000000000..b89332862 --- /dev/null +++ b/crates/crw/clean-room/02-crw-format.md @@ -0,0 +1,115 @@ +# Canon CRW raw compression — derived format (FROZEN) + +Status: **frozen, revision 2** (plan §5 P4). The derivation is not tuned; any correction +must reopen it explicitly with a recorded cause and a new freeze commit. + +**Revision 2 (2026-10-13), cause:** the maintainer supplied 1,516 D60 files, 50 times the +first corpus. 223 of them failed on code-table entries the first corpus never exercised, +and 9 were misread as having no low-bits plane. Two corrections, both recorded in the +derivation log (not part of this record) and re-verified in `03-verification.md`: the low-bits presence rule +(§1) gained the header-position test, and the selector 0 and 1 second tables were completed +(§3). Nothing else in the model changed, and every file of the first corpus still verifies. Everything here was derived from the CC0/private sample files +and black-box probing of LibRaw 0.22.2 (`unprocessed_raw.exe`), per the approved plan; no +decoder source was consulted. The chronological evidence is the derivation log, which is not part +of this record; the initial payload-layout analysis remains in LightCraft PR #3021. + +## 1 Payload layout + +RawData (CIFF record 0x2005), with w×h from SensorInfo (0x1031) and +(selector, 0, entropyOffset = 514, compressedLength) from DecoderTable (0x1835): + +``` +[ low-bits plane, ceil(w*h/4) bytes ] present on some bodies (rule below) +[ 514 zero bytes ] (the DecoderTable's entropyOffset) +[ entropy bitstream, ff00-stuffed ] +[ tail: 00 00 / ff 00 00 (EOS) or ff d9 (+00) (PowerShot) ] +``` + +- **Low-bits plane**: 2 bits per sensor sample, row-major, LSB-first within each byte + (sample 4i in bits 1:0 … sample 4i+3 in bits 7:6). Present ⇔ either + (a) the first ⌈w·h/4⌉ + 514 bytes of the payload contain any 0xff byte NOT followed by + 0x00 (the entropy stream is always stuffed, the low-bits plane is raw), or + (b) the 514 bytes at offset ⌈w·h/4⌉ are all zero (the zero header sits right after the + plane; without a plane that offset is deep in entropy data). + Rule (a) alone, the revision-1 rule, misses planes that hold no 0xff byte: some D60 + files have a lowest bit that is almost always 0 (low values 1 and 3 on about 0.04 % of + samples), so no four neighbouring samples are all 3. Verified: present on + D30/D60/10D/300D/G5/G6, absent on G1. +- On the EOS bodies the DecoderTable's compressedLength excludes the low-bits plane; + on G5/G6 it includes it. The decoder should not rely on compressedLength: entropy + data starts at (low-bits size, if present) + 514 and runs to the tail. + +## 2 Entropy bitstream + +After removing the stuffing (drop the 0x00 after every 0xff), bits are read +**MSB-first**. The stream codes the w×h sensor samples' 10-bit "high" plane +(the full sample is `(high << 2) | low_bits` where a low-bits plane exists, else +`high` itself): + +- Samples are produced in row-major order, grouped in **blocks of 64 samples** laid + flat over the frame (blocks cross row boundaries; w need not divide by 64). +- Two canonical Huffman code tables per DecoderTableNumber (0, 1, 2): the token that + begins each block is read from the **first table**, every other token from the + **second table**. +- A token decodes to a leaf **(r, s)** with r ≤ 15: skip r samples (each emits its + DPCM chain value unchanged, i.e. a zero diff), then read s magnitude bits and emit + one diff (s = 0 emits a zero diff). Magnitudes follow the JPEG rule, MSB-first: + m ≥ 2^(s−1) → d = +m, else d = m − 2^s + 1. Sample diffs fit s ≤ 10 (10-bit high + plane), but first-table leaves reach s = 11+: the carry-coded block DC difference + spans ±2046. +- The **(0,0) leaf read as a non-first token is EOB**: the rest of the block emits + zero diffs. ((0,0) as the block's first token is a plain zero diff.) +- **Carry**: the diff landing on each block's first sample has the previous block's + (combined) first diff added: `d0 += carry; carry = d0`. When the first sample is + covered by a token's skip (or EOB), its diff is exactly `carry` (and carry is + unchanged). +- **DPCM chains**: two running predictors by sample parity, both initialised to 512, + and both **reset to 512 at the start of every sensor row**. +- If the stream ends before the frame is complete, remaining samples emit their chain + values (observed: the final 2–3 samples of each file). + +## 3 Code tables + +Extracted from the samples by known-plaintext alignment and frozen as data: + +- Selector 0: frozen code tables — 12 first-table + 162 second-table codes. +- Selector 1: frozen code tables — 12 + 162. +- Selector 2: frozen code tables — 12 + 162. + +Each second table now holds all 162 symbols: (r, s) for r = 0..15, s = 1..10, plus (0, 0) +and (15, 0). Revision 1 lacked (7..14, 10) in selector 0 and (14, 10) in selector 1: runs +of 7 or more zero differences followed by a difference of 512 or more, which the first +corpus never contained. The nine codes were learned from the wider corpus by the same +known-plaintext method; each landed in one of the unassigned 16-bit slots inside the code +range, of which there were exactly eight and one. The slots still unassigned are the +all-ones tail of each table (5, 20 and 3 codes) and the first tables' longest code. + +Tables are shared across bodies per selector (verified across D60/10D/300D/Rebel for +selector 0; D30, G1 and the private D60s for selector 1; 10D, 300D and a private D60 +for selector 2). The tables have canonical Huffman structure; codes up to 16 bits, the +`11111111xxxxxxxx` region carrying the larger (run, size) pairs. The JSON rows are a +superset of each file's exercised codes: every row was learned from a real stream +position and validated by the decode reaching the end of that frame; rows never +exercised by two files may in principle be mergeable duplicates of canonical slots, +which the Rust implementation may normalise, provided the verification table of +`03-verification.md` still reproduces exactly. + +## 4 Not yet established / out of scope here + +- **G5 anomaly**: its entropy stream decodes end-to-end with the selector-1 tables, + but values diverge beginning at sample 2056 (= 4 × 514), and its low-bits plane fits + no tested layout (top rows mostly match, decaying to 0 %, no constant shift). The + decoder detects the G5 by its make-model and returns `Error::Unsupported` until a + follow-up derivation resolves it; a caller may fall back to the embedded JPEG. +- The exact meaning of each file's tail bytes (handled by the stream-end rule). +- Big-endian (`MM`) CIFF files: no sample exists. +- PowerShot G2/G3/Pro1/Pro70/S30–S70: no CC0 samples (licence-excluded); decodable + speculatively via the same selectors once their geometry is read from SensorInfo, + to be marked unverified. +- Black level, white balance and colour (plan P3) are documented separately. + +## 5 Verification + +See `03-verification.md` (written after the freeze): full-frame exact-match decode +against LibRaw 0.22.2, with zero tuning after the freeze commit: every sample file, public +and private, matches except the PowerShot G5, which the decoder refuses (28 of 29). diff --git a/crates/crw/clean-room/03-verification.md b/crates/crw/clean-room/03-verification.md new file mode 100644 index 000000000..ff37f67d4 --- /dev/null +++ b/crates/crw/clean-room/03-verification.md @@ -0,0 +1,79 @@ +# CRW decode verification (P4) — against LibRaw 0.22.2, zero tuning + +Method: the frozen model of `02-crw-format.md` plus the frozen tables retained in +LightCraft PR #3021 were run by the local Python reference decoder over every +sample file, comparing **every sensor sample** (full values, low bits merged) against +`unprocessed_raw.exe -T` output. "EXACT" means zero mismatching samples over the whole +frame. The table extraction itself consumed these same files (known-plaintext learning), +so this is a consistency proof against the oracle, not a blind holdout; the cross-body +sharing (tables learned on one body, exact on others) is the strongest independent check. + +Private files were the maintainer's own D60 photos; only aggregate verdicts are recorded here, +never filenames or content. + +## Results + +| File | Selector | Frame | Verdict | +|---|---|---|---| +| `crw-canon-eos-10d-a.crw` | 0 | 3152x2068 | **EXACT** | +| `crw-canon-eos-10d-b.crw` | 2 | 3152x2068 | **EXACT** | +| `crw-canon-eos-10d-c.crw` | 2 | 3152x2068 | **EXACT** | +| `crw-canon-eos-300d-a.crw` | 0 | 3152x2068 | **EXACT** | +| `crw-canon-eos-300d-b.crw` | 2 | 3152x2068 | **EXACT** | +| `crw-canon-eos-d30.crw` | 1 | 2224x1456 | **EXACT** | +| `crw-canon-eos-d60.crw` | 0 | 3152x2068 | **EXACT** | +| `crw-canon-eos-digital-rebel.crw` | 0 | 3152x2068 | **EXACT** | +| `crw-canon-powershot-g1.crw` | 1 | 2144x1560 | **EXACT** | +| `crw-canon-powershot-g5.crw` | 1 | 2672x1968 | 2685618 mismatches, first at 0 (row 0, col 0) | +| `crw-canon-powershot-g6.crw` | 0 | 3160x2344 | **EXACT** | +| `private/CRW_0476.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_0478.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_0486.crw` | 0 | 3152x2068 | **EXACT** | +| `private/CRW_0527.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_0683.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_0717.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_0765.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_0822.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_0879.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_1011.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_1153.crw` | 0 | 3152x2068 | **EXACT** | +| `private/CRW_1277.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_1514.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_1646.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_1740.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_1776.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_2035.crw` | 1 | 3152x2068 | **EXACT** | +| `private/CRW_2219.crw` | 2 | 3152x2068 | **EXACT** | + +**28 of 29 files decode EXACT** — all 10 usable public CC0 files and all 18 private +D60 files; the PowerShot G5 is the known exception (`02-crw-format.md` §4). + +## Notes + +- The final 1–3 samples of each frame follow the stream-end rule (chains persist); + every "EXACT" verdict includes them. +- PowerShot G5: see `02-crw-format.md` §4 — detected and excluded; embedded-JPEG + fallback until resolved. +- The G1 (10-bit, no low-bits plane) verifies the low-bits presence rule; the G6 + (12-bit, different geometry, w not divisible by 64) verifies the row-reset and + block rules on a second geometry. + +## Revision 2: the wider D60 sweep (2026-10-13) + +The maintainer supplied 1,516 Canon EOS D60 files (private; local use only, nothing +copied into the repository, verdicts only recorded here). With the revision-1 decoder, +223 failed on unseen code-table entries and 9 were misread as 10-bit (`02-crw-format.md`, +revision 2). After the two corrections every file was compared against LibRaw 0.22.2 +`unprocessed_raw.exe -T`, every sensor sample, with the Rust decoder +(`plan/crw/model/sweep_all.py`, local): + +| Selector | Files | Verdict | +|---|---:|---| +| 0 | 243 | all **EXACT** | +| 1 | 1,193 | all **EXACT** | +| 2 | 80 | all **EXACT** | +| **Total** | **1,516** | **1,516 EXACT** | + +The nine learned codes came from 30 of the failing files; the other 202 failing files and +the 1,284 that already decoded are a blind check of them. The public files were re-run +with the same build: 10 EXACT, the G5 still refused. diff --git a/crates/crw/clean-room/04-color.md b/crates/crw/clean-room/04-color.md new file mode 100644 index 000000000..c5fa4c647 --- /dev/null +++ b/crates/crw/clean-room/04-color.md @@ -0,0 +1,122 @@ +# CRW colour, geometry and metadata records (P3/P5) + +Status: derived 2026-10-13, after the compression freeze. Everything here comes from the +approved sources: the corpus files themselves (public CC0 + the maintainer's private D60s), +the oracle's decoded sensor data, each file's own embedded JPEG, and ExifTool's prose tag +documentation (`TagNames/CanonRaw.html`, `TagNames/Canon.html` — tag names, table indices and +value enumerations only; fetches logged in `sources.md`). No decoder source was consulted. +Implemented in `crates/raw/src/vendor/crw.rs`. + +## 1 CFA phase + +Determined by two independent sample-derived tests on each body's oracle mosaic: + +- **Chroma correlation**: bin the mosaic 2×2 into RGB under each of the four Bayer phase + hypotheses, roughly gain-match to the body's own embedded JPEG (linearised ≈ γ2.2), and + correlate the R−B chroma plane with the JPEG's. The true phase correlates strongly + (+0.49…+0.94), its R/B swap strongly negatively (`cfa_check.py`, local model dir). +- **Green diagonal**: within 16×16 blocks the two green sites have near-equal sums; the + diagonal with the smaller |difference| carries the greens. + +Results: **RGGB at sensor (0, 0)** for EOS D30, D60, 10D, 300D (both tests decisive) and +PowerShot G6; **GBRG for the PowerShot G1** (greens on the main diagonal; chroma +0.77 GBRG +vs −0.67 GRBG). Unsampled PowerShots default to RGGB, unverified. + +## 2 Active area, black level, white level + +- **Active area**: SensorInfo (0x1031) values 5–8 are an inclusive crop (left, top, right, + bottom); valid on every corpus body (G5/G6 included). +- **Black level**: measured from the masked borders. Observed (oracle data, per body class): + the left border is dark up to 8 columns before the active area on the EOS bodies + (≈127–129 DN on D60/10D, ≈128 on D30, ≈124–127 on 300D) but transitions bright earlier on + the G6 (dark only to col 40 of 64), and on the G1 only the top border is dark + (rows 0–7 ≈ 29 DN; its left border reads ≈98–160). The D60's first ~7 sensor rows are + clamped to 0 (not a black reference). Rule adopted: candidate strips = 8-column groups of + the left border and 4-row groups of the top border; black = per-CFA-site mean of the + darkest strip and all strips within 8 DN of it, ignoring strips with mean ≤ 1 (the clamped + rows). This lands on the true dark level on every corpus body. Cross-check: the trailing + quadruple of ColorBalance (0x10a9) is (126–127)×4 on D60/10D and (124–125)×4 on 300D — + matching the measured borders (ExifTool's prose notes this quadruple as "black levels"). +- **Correction (2026-10-13):** the first implementation also averaged in the clamped strips + it meant to ignore (mean ≤ 1 passes "within 8 DN of the darkest"), which put the D60's + level at 117 instead of 128 and lifted every shadow. Fixed and tested; on 20 private D60 + files the left border's mean is within 2 DN of the record's black quadruple, per site + (127–137, higher on the files with the larger calibration-row value). +- **White level**: saturation plateau of the decoded samples (shared `white_from_data`); + without a plateau, the level at least 64 samples reach when that is within 3 % of the + full range (`white_level`); else the full range, 4095 (12-bit) / 1023 (G1). +- **The sensor clips below 4095 at low ISO** (2026-10-13, 1,516 private D60 files by ISO + from ShotInfo value 2, ISO = 100·2^(v/32 − 5)): at ISO 100, 200 and 400 no sample exceeds + 4009 and the maxima cluster at 4000–4008; at ISO 800 and 1000 saturated files reach 4095. + The 10D and 300D samples stop at 4003–4008 as well. Before `white_level`, a photo with a + small blown area had no plateau and got 4095 (373 of 925 files at ISO 100, 283 of 287 at + ISO 400), leaving blown pixels at 0.977 of white, below the clip threshold. +- **Controlled frames from the maintainer** (lens cap and blown-out, ISO 100 and 1000; all + four decode sample-exact). Cap, ISO 100: active area 129.2–130.1 per site, left border + 128.2–129.6, our level 128.9, record 127–129. Cap, ISO 1000: active area 129.5–130.5, + but left border 134.5–135.7, our level 133.8, record 131–133: at high ISO the masked + border and the record read about 4 DN above the image's true dark level. One frame, so + no correction is applied; noise there is 15 DN. Blown-out, ISO 100: the common values + are 3997–4000 and the maximum 4002 (our level 3987). Blown-out, ISO 1000: every active + sample is 4095 (our level 4080). + +## 3 As-shot white balance + +- **0x10a9 ColorBalance** (D60/10D/300D only): a byte-length u16, then (R, G, G, B) level + quadruples per white-balance setting, the last quadruple being black levels. ExifTool's + prose gives the 10D/300D slot order — Auto, Daylight, Shade, Cloudy, Tungsten, Fluorescent, + Flash, Custom, Kelvin, Black — at indices 1, 5, …, 37, which the record sizes confirm + (82 bytes = 1 + 10×4 u16); the D60's 66-byte record holds 7 slots + black at index 29 + (= ExifTool's "D60 black levels" index), i.e. the same order without Shade and Kelvin. +- Multipliers are **level / green** (e.g. D60 slot 1: 1856, 784, 781, 945 → R 2.37, B 1.21). + Confirmed against the cameras' own JPEGs by a white-preserving matrix fit (rows sum to 1, + least squares from WB-corrected binned raw to the linearised JPEG): plausible slots score + far better than implausible ones; among near-daylight slots the fit cannot fully separate + (γ-model error), so the slot is chosen by metadata, not by fit. +- **Slot selection**: ShotInfo (0x102a) index 7 is the WhiteBalance setting (ExifTool prose: + 0 Auto, 1 Daylight, 2 Cloudy, 3 Tungsten, 4 Fluorescent, 5 Flash, 6 Custom, 8 Shade, + 9 Kelvin). Every corpus file outside the preset captures below reads 0 (Auto) → slot 0. +- **D60 preset mapping verified** (2026-10-14): the maintainer shot one overcast scene at + each setting, Auto, Daylight, Cloudy, Tungsten, Fluorescent, Flash and Custom, ISO 100, + within a minute (all seven decode sample-exact). Slots 1–6 hold the same multipliers in + every file; slot 0 (Auto) varies slightly per shot. On the pixels each camera JPEG + renders neutral, the multipliers the camera applied are nearest to slot 1 for Daylight, + 2 for Cloudy, 4 for Fluorescent and 5 for Flash, i.e. setting n uses slot n. Tungsten and + Custom leave too few neutral pixels for that test (5 and 0), so they were checked by + rendering: the overall tint of our render follows the camera JPEG for every setting + (Tungsten JPEG R/G 0.57 B/G 1.41, ours 0.66 / 1.52; Custom 0.62 / 2.93, ours 0.80 / + 3.02), which no other slot would give. Shade and Kelvin (10D/300D only) remain untested. +- **Auto slot confirmed on the private D60 files** (2026-10-13): on the pixels each camera + JPEG renders neutral (channel spread < 6 of 255, mid-tone, raw unclipped), the median raw + G/R and G/B give the multipliers the camera applied. On 17 files with enough neutral + pixels, slot 0 is within 0.005–0.036 (summed log error) of that on every one, flash + shots included, and is the nearest slot on 16 (the 17th prefers Daylight by 0.005). + Numbers only; no image content recorded. +- **0x10aa** (same three bodies): one (R, G, G, B) quadruple with G ≈ 1024 and R/B as + *levels* (multiplier = G/level); consistent with a measured-white-balance record (Canon + maker-note 0xaa MeasuredColor). On some files it fits the camera JPEG better than slot 0, + on others worse; not used, kept as a documented alternative. +- **Not derived yet**: D30 and PowerShot white balance (no 0x10a9/0x10aa; the 0x1030 + WhiteSample record holds an 8×8 packed sample patch — constant filler on G1/G6, varying on + D30 — undeciphered). These bodies render with the neutral fallback. +- **Colour matrices**: none of the CRW bodies is in `spectral_table.rs`. Rendering uses + the engine's per-file starting look fitted to the file's own embedded JPEG + (`camera_preview::file_local_look` lists CRW; `docs/camera-preview-colour.md`) — accepted + on all 10 decodable public corpus files, holdout RMS 0.009–0.043, including the D30 and + the PowerShots, whose missing white balance the fitted matrix absorbs. + **Update 2026-10-25:** the EOS D60 has dual-illuminant matrices of our own, fitted to + ColorChecker Classic shots under a tungsten bulb and direct sun (the fitting is part of + LightCraft's colour work, `crates/raw/src/chart_table.rs`). They give its white balance a Kelvin scale and are its + colour when the JPEG fit fails; the look stays the JPEG fit. The other CRW bodies have none. + +## 4 Orientation and capture time + +- ImageInfo (0x1810) value 3 is rotation in degrees. The two portrait 300D corpus files + carry 270, and their scene is upright after rotating the stored image 90° counter-clockwise + (checked visually against the embedded JPEG) → 270 maps to Exif orientation 8, hence + 90 → 6 and 180 → 3. All other corpus files carry 0. +- TimeStamp (0x180e) value 0 is a Unix time on the camera's local clock (cross-checked + against the files' Exif-era siblings and plausibility); decoded as a civil time with no + UTC offset. +- Make/model: 0x080a, two NUL-terminated strings ("Canon", "Canon EOS D60"); the G5 + refusal and the G1 CFA special-case key on the model string. diff --git a/crates/crw/clean-room/05-private-corpus.md b/crates/crw/clean-room/05-private-corpus.md new file mode 100644 index 000000000..5bb19f06a --- /dev/null +++ b/crates/crw/clean-room/05-private-corpus.md @@ -0,0 +1,9 @@ +# Private corpus verification summary + +The decoder was checked against 1,516 maintainer-owned EOS D60 files and controlled captures. +No private file names, hashes, image content, or color-chart observations are included in this +crate. The retained aggregate verification results are documented in +[`03-verification.md`](03-verification.md). + +The private corpus and detailed derivation records remain in +[LightCraft PR #3021](https://github.com/storytold/lightcraft/pull/3021). diff --git a/crates/crw/clean-room/README.md b/crates/crw/clean-room/README.md new file mode 100644 index 000000000..cdb728aee --- /dev/null +++ b/crates/crw/clean-room/README.md @@ -0,0 +1,60 @@ +# Canon CRW clean-room derivation record + +What `craft-crw` decodes a CRW from, and the evidence that it is right. The full record was +produced in LightCraft (`clean-room/crw/`, branch `clean-room-crw-format-decode`, revision +`6ca439b5155820d2befeacd81ec0c6a80b2bd3bc`; the record is identical at `70c755e4`, where the decoder was ported) between 2026-10-12 and 2026-10-25; the documents below moved here +**verbatim** when the decoder became this shared crate, and the derivation trail stays at that revision. The documents are the evidence: they are +not rewritten after the fact, so paths they mention are LightCraft's paths of that time (the +mapping is below). + +The derivation trail and frozen table-generation materials remain in +[LightCraft PR #3021](https://github.com/storytold/lightcraft/pull/3021). This crate retains the +format specification, verification summary, colour metadata notes, source log and public-facing +decoder documentation. + +## Documents + +| File | What it is | +|---|---| +| [`sources.md`](sources.md) | Every URL opened and search run, with the time, what was asked and what was taken. The rules in force are at its top. | +| [`02-crw-format.md`](02-crw-format.md) | **The frozen format** (revision 2) and its reopening rule. The decoder implements this document. | +| [`03-verification.md`](03-verification.md) | Full-frame exact comparison against the reference, zero tuning: 28 of 29 files, then 1,516 of 1,516 D60 files. | +| [`04-color.md`](04-color.md) | CFA phase, active area, black and white levels, white-balance records, orientation and time: what the decoder reports beyond the samples. §3's colour matrices concern LightCraft (below). | +| [`05-private-corpus.md`](05-private-corpus.md) | A summary of private-corpus decoder verification; no image data or identifying file list is included. | + +## What stayed local + +- The Python reference model and probe harness (`plan/crw/model/` in LightCraft, gitignored), + by the maintainer's decision #4. Its findings are the entropy-coding derivation, which is not + part of this record. +- The reference decoder: LibRaw 0.22.2, official pre-built Windows binaries installed by the + maintainer; only `unprocessed_raw.exe` was ever run, as a black box. The source tree shipped + alongside it was never opened, read or searched. +- Canon's CIFF specification PDF (copyrighted; archive URL and SHA-256 in `sources.md`). +- The reference TIFFs (`.crw.tiff`) the verification compared against; `CRAFT_CRW_ORACLE` + points the test at a local copy. + +## Path mapping (LightCraft at the time → craft-libs) + +| In the documents | Now | +|---|---| +| `clean-room/crw/…` | `crates/crw/clean-room/…` | +| `crates/raw/src/vendor/crw.rs` | `crates/crw/src/lib.rs` | +| `crates/raw/src/vendor/crw_tables.rs` | `crates/crw/src/tables.rs` | +| `crates/raw/tests/crw_oracle.rs` | `crates/crw/tests/samples.rs` (`CRAFT_CRW_ORACLE`, formerly `LIGHTCRAFT_CRW_ORACLE`) | +| `white_from_data` (LightCraft's shared estimate) | `saturation_plateau` in `crates/crw/src/lib.rs` | +| `xtask/src/main.rs` `RAW_SAMPLES` (corpus URLs) | `crates/crw/tests/samples.json` + `scripts/fetch-crw-samples.py` | +| `plan/crw/model/gen_tables.py` | LightCraft PR #3021 | +| `docs/arw6-compression.md` (the house model for a derivation) | LightCraft's document; the method is summarised in `docs/clean-room.md` of this repository | +| `crates/raw/src/chart_table.rs`, `spectral_table.rs`, `camera_preview`, `assets/camera-profiles/` | LightCraft (colour stays in the app) | + +## Rules that were in force + +From `sources.md`: no raw-decoder source may be opened, fetched, searched inside +or requested (dcraw, LibRaw, rawspeed, rawloader/rawler, dnglab, darktable, RawTherapee, +libopenraw, ExifTool's Perl code, including licence headers, comments and mirrors). Allowed: +published format documentation in prose, published standards, the sample files, and a reference +decoder's binary run as a black box with every probe logged. No tuning after the freeze: a +mismatch reopens the derivation explicitly, with its cause and a new freeze commit recorded +(`02-crw-format.md`, revision 2, is the one reopening). The general standard this followed is +[`docs/clean-room.md`](../../../docs/clean-room.md). diff --git a/crates/crw/clean-room/sources.md b/crates/crw/clean-room/sources.md new file mode 100644 index 000000000..05d8e5304 --- /dev/null +++ b/crates/crw/clean-room/sources.md @@ -0,0 +1,136 @@ +# CRW clean-room derivation — source log + +Every URL opened and every web search run for this work, in order, with the time (local, +Europe-style 24 h), what was asked, and what was taken. If a result or page exposed details of an +existing decoder's algorithm that were not asked for, that is recorded too, with what it said. + +Rules in force (from the task): no raw-decoder source may be opened, fetched, searched inside or +requested — dcraw, LibRaw, rawspeed, rawloader/rawler, dnglab, darktable, RawTherapee, +libopenraw, ExifTool's Perl code — including licence headers, comments and mirrors. Allowed: +published format documentation in prose (ExifTool's CIFF/CRW format and tag-name pages), +published standards, and the sample files. The branch `crw-decode` (an earlier attempt) is not +checked out, diffed, logged or read. + +## Local files consulted (non-web) + +- 2026-10-12 ~19:2x — `CLAUDE.md` (project instructions; in session context from the start) and + `docs/arw6-compression.md` (the house model for deriving and documenting a format). Taken: + process shape (assumptions → spec + samples → Python reference model → freeze → black-box + verification), documentation conventions, corpus table format. +- 2026-10-12 ~19:2x — directory listing and SHA-256 of the maintainer's private + `../RawImages/*.crw` (18 files, Canon EOS D60 era, 5.5–8.6 MB). Taken: file names, sizes, + hashes only. No .THM files present. Contents not yet parsed. + +## Web + +- 2026-10-12 19:26 — **Fetch** `https://exiftool.org/TagNames/CanonRaw.html` (ExifTool's + CanonRaw tag-name page; generated documentation, not Perl source). Asked: CIFF structure, tag + IDs/names, compression info, spec links. Taken: tag IDs — 0x2005 RawData, 0x2007 JpgFromRaw, + 0x2008 ThumbnailImage, 0x1835 DecoderTable (subdir: DecoderTableNumber, CompressedDataOffset, + CompressedDataLength), 0x1031 SensorInfo (subdir), 0x1810 ImageInfo (ImageWidth, ImageHeight, + ComponentBitDepth, ColorBitDepth…), 0x1030 WhiteSample (BlackLevels at index 55), 0x10a9 + ColorBalance, 0x10ae ColorTemperature, 0x080a CanonRawMakeModel, 0x102a CanonShotInfo, 0x1834 + CanonModelID, 0x300a ImageProps, 0x2807 CameraObject, 0x300b ExifInformation, 0x0000 + NullRecord, 0x0001 FreeBytes, and others. No compression algorithm described. Links to + `canon_raw.html` for structure. **No decoder algorithm details shown.** +- 2026-10-12 19:27 — **Fetch** `https://exiftool.org/canon_raw.html` (Phil Harvey's prose "Canon + CRW file format" page). Taken: 26-byte header (ByteOrder `II`/`MM` @0, HeaderLength u32 @2 = + 0x1a, signature `HEAPCCDR` @6, CRWVersion 0x00010002 @14, 8 reserved bytes @18); block = + ValueData, DirCount u16, N×10-byte entries (tag u16, size u32, offset u32), optional + OtherData, u32 DirStart at BlockSize−4; offsets relative to the block's ValueData start; + values 2-byte aligned; tag masks DataLocation 0xc000 (0x0000 = ValueData, 0x4000 = in-entry, + 0x8000/0xc000 unknown), DataFormat 0x3800 (0x0000 bytes, 0x0800 ASCII string, 0x1000 u16s, + 0x1800 u32s/floats, 0x2000 structure, 0x2800/0x3000 subdirectory), TagIndex 0x07ff; + subdirectory = a complete block inside ValueData, its BlockSize = entry Size; root block from + HeaderLength to EOF. RawData 0x2005 noted as "the bulk of the CRW file"; JpgFromRaw 0x2007; + no compression description. Links the original Canon CIFF spec at `http://xyrion.org/ciff/` + (dead) and lclevy's CR2/CR3 pages. **No decoder algorithm details shown.** +- 2026-10-12 19:28 — **Search** (WebSearch, standard): `Canon CIFF specification 1.0 rev 4 PDF + "CIFF" camera image file format specification download`. Results included Wikipedia + "Camera Image File Format", a freedesktop libopenraw wiki page "Canon CRW" (NOT opened — it is + a decoder project's page; left for the maintainer to approve), an unpkg mirror of + exiftool's canon_raw.html, PRONOM. Taken: pointer to Wikipedia for an archived spec URL. + **No decoder algorithm details shown in the result snippets.** +- 2026-10-12 19:28 — **Fetch attempt** `http://xyrion.org/ciff/` — failed (site dead). +- 2026-10-12 19:29 — **Fetch** `https://en.wikipedia.org/wiki/Camera_Image_File_Format`. Taken: + spec part names/dates (Image Data File v1.0 r4, 1997-12-24; File/Directory organization v1.0 + r3, 1998-02-09), archive URL for the PDF, camera list (EOS D30/D60/10D/300D, PowerShot Pro1, + G1–G6, S30–S70). +- 2026-10-12 19:29 — **Download** (curl) + `https://web.archive.org/web/20081230095207/http://xyrion.org/ciff/CIFFspecV1R04.pdf` → + scratchpad `ciff-spec/CIFFspecV1R04.pdf`, 99 422 bytes, SHA-256 + `416d4e49ea3f6ad2ba5ccc76b48ca100605f4a6127816c145e6dd5d6c3b51cf4`, 34 pages. Text extracted + with a local stdlib-Python script (`pdftext.py`, kept in scratchpad; extraction output + `extracted.txt`). This is Canon's own published "CIFF Specification on Image Data File" + v1.0 r4 (1997-12-24); local copies (PDF + text extraction) kept in the gitignored + `plan/crw/` (Canon-copyrighted document, not committed; re-fetchable from the archive URL, + hash above). Taken: the heap file header struct (byteOrder, headerLength, type + "HEAP", subType "JPGM"/"CCDR"/"TIFP"/"ARCH", version 0x00010002), heap = heap space + + offsetTbl, offsetTblOffset as last u32 of the heap, DC_RECORD_ENTRY {typeCode u16, length + u32, offset u32}, numRecords u16, typeCode bit fields (bits 15-14 storage: 00 kStg_InHeapSpace, + 01 kStg_InRecordEntry, 10/11 reserved; bits 13-11 dataType: 000 byte, 001 ASCII, 010 word, + 011 dword, 100 structure, 101/110 heap record, 111 reserved; bits 10-0 IDCode), masks + kStgFormatMask 0xc000 / kDataTypeMask 0x3800 / kIDCodeMask 0x07ff, null/free records, 2-byte + alignment rule, ImageSpec/ImageFormat/CapturedTime/ModelName/… structs, appendix type-code + defines. The spec documents the container and common metadata only — **it does not describe + the CRW raw-data compression** (ImageFormat fileFormat value 2 is merely "Reserved (for + CanonCRW format)"). **No decoder algorithm details shown.** +- 2026-10-12 19:31 — **Fetch** (curl) `https://raw.pixls.us/` and + `https://raw.pixls.us/json/getrepository.php?set=all` (the site's own data endpoint found in + its HTML; first tried `json/data.json`, 404). Taken: the full sample table (2016 rows); + filtered locally for `.crw` (51 rows) and classic CIFF-era bodies; per-file licence, URL and + SHA-256 as published by the site. CC0 CIFF-era bodies: EOS D30, D60, 10D (×3), 300D (×2), + Digital Rebel, PowerShot G1, G5, G6. CC-BY-NC-SA only (NOT downloaded, per task rules): + PowerShot G2, G3, Pro1, Pro70, S30, S40, S45, S50, S60, S70, S2 IS and others. Many other + `.CRW` rows are CHDK sensor dumps from much later PowerShots (A-, SX-, ELPH-series), a + different format. **No decoder algorithm details shown.** +- 2026-10-12 19:32 — **Download** (Python urllib, stdlib) the 11 CC0 files above from + `https://raw.pixls.us/getfile.php/...` into the gitignored `corpus/crw/`; every file verified + against the site's published SHA-256 (all OK; the hashes are pinned in + `crates/crw/tests/samples.json`). + +## Local analysis (scripts in this directory) + +- 2026-10-12 19:35–19:45 — `ciff_dump.py` (written here, stdlib only; container layout + implemented from the two documented sources above) run over the 11 public files and the 18 + private files (whose dumps may contain personal metadata and are never committed). Observed + facts are summarised in the LightCraft derivation record. No raw-data decoding was attempted; statistics + only. The script and its logs are not part of this record. +- 2026-10-12 ~20:0x — P0/P1 local analysis (post-approval): the container notes written from the + already-logged sources + corpus observations. `p1_payload.py`, `p1b_markers.py`, + `p1c_lowbits_bounds.py` run over all 29 files. Established statistically (no decoding): the + three payload layouts, the + 514-byte zero block at the compressed section start of every file, JPEG-style ff00 stuffing + of every entropy stream (zero exceptions), the PowerShot `ff d9` terminator, and the exact + low-bits region bounds on G5/G6. Documented in the LightCraft derivation record. No external sources + consulted in this step. + +## Oracle (P2, from 2026-10-12) + +- The maintainer installed LibRaw 0.22.2 (official pre-built Windows binaries) in + a local `LibRaw-0.22.2\bin` folder. That directory tree also contains LibRaw's source + code; **no source file there is ever opened, read or searched** — only + `bin\unprocessed_raw.exe` is executed, as a black box, per the approved plan (§5a/§7). + All oracle runs and probe campaigns are logged in the derivation log, which is not part of this + record. + +- 2026-10-13 — **Fetch** `https://raw.pixls.us/json/getrepository.php?set=all` (and the site + root to find that endpoint): recovered the stable `getfile.php` URLs of the 11 CC0 corpus + files by matching their SHA-256s, and re-confirmed each row's CC0 licence; URLs pinned in + `xtask/src/main.rs` `RAW_SAMPLES`. No decoder content on these pages. +- 2026-10-13 — **Fetch** `https://exiftool.org/TagNames/CanonRaw.html` (again) and + `https://exiftool.org/TagNames/Canon.html` (prose tag tables only): 0x10a9 ColorBalance → + Canon ColorBalance table (Index2; WB_RGGBLevels slot order Auto/Daylight/Shade/Cloudy/ + Tungsten/Fluorescent/Flash/Custom/Kelvin, index-29 "black levels for the D60" note), + 0x1030 WhiteSample field names, CanonShotInfo index 7 = WhiteBalance and the Canon + WhiteBalance value enumeration, CanonModelID values. Prose documentation, no code. + +- 2026-10-25 — **Fetch** `https://www.xrite.com/service-support/new_color_specifications_for_colorchecker_sg_and_classic_charts` + (prose page): ColorChecker Classic charts made after November 2014 use new reference data, + published as CGATS files. The CGATS download itself was refused to scripts (HTTP 403) and + not worked around. +- 2026-10-25 — **Fetch** `https://raw.githubusercontent.com/colour-science/colour/v0.4.2/colour/characterisation/datasets/colour_checkers/chromaticity_coordinates.py` + (colour-science, BSD-3-Clause; also its rendered docs page): the 24 post-November-2014 + ColorChecker Classic L*a*b* values (D50), credited there to X-Rite's release. Reference + data only, the chart's measured colours; no camera or decoder content. Used for the EOS D60 + colour matrices. diff --git a/crates/crw/src/lib.rs b/crates/crw/src/lib.rs new file mode 100644 index 000000000..345f40af0 --- /dev/null +++ b/crates/crw/src/lib.rs @@ -0,0 +1,760 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Clean-room Canon CRW (CIFF heap) raw decoder: the bodies before the CR2 era (EOS D30, D60, +//! 10D, 300D / Digital Rebel, PowerShot G-series). +//! +//! The container rules come from Canon's published CIFF specification and ExifTool's prose tag +//! documentation; the compression was derived from CC0 and maintainer-owned sample files by +//! black-box analysis against a reference decoder's command-line output, frozen in +//! `clean-room/02-crw-format.md` and verified sample-exact on 28 of 29 corpus files plus 1,516 +//! further EOS D60 files (`clean-room/03-verification.md`). The code tables are frozen data +//! (`tables`, frozen from the clean-room derivation record retained in LightCraft). No other raw decoder's +//! source was consulted; the whole derivation record is in the crate's `clean-room/` folder. +//! +//! - [`probe`] recognises the CIFF signature. +//! - [`header`] reads geometry, bit depth, CFA phase, make and model, capture time, orientation +//! and as-shot white balance without touching the entropy-coded data. +//! - [`decode`] also decodes the sensor samples and measures the black level (masked borders) +//! and the white level (saturation plateau or the sensor's clip level). +//! +//! The decoder never panics on input: every offset read from the file is bounds-checked, the heap +//! walk is bounded by depth and by a per-file record budget, and the sample count is capped by +//! [`Limits`]. Malformed files return [`Error::Corrupt`]; unknown decoder tables and the +//! PowerShot G5 (whose entropy coding diverges from the derived model) return +//! [`Error::Unsupported`]. +//! +//! Known gaps (documented in the freeze): the PowerShot G5 is refused (its embedded JPEG is the +//! app's fallback); PowerShot bodies without CC0 samples (G2, G3, Pro1, S-series) decode through +//! the same model unverified; the D30 and the PowerShots carry no white-balance record the +//! derivation deciphered, so `wb_multipliers` is `None` for them. +//! +//! Colour (camera matrices, profiles) is out of scope: the file carries none, and the app owns +//! its colour model. + +mod tables; +#[cfg(test)] +mod tests; + +use std::collections::BTreeMap; + +/// Why a file could not be decoded. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The bytes do not start with a CIFF `HEAPCCDR` header. + #[error("not a Canon CRW (CIFF) file")] + NotCrw, + /// The file is a CRW but its structure or data is damaged. + #[error("CRW: {0}")] + Corrupt(String), + /// A CRW the derived model does not cover (an unknown decoder table, the PowerShot G5). + #[error("unsupported CRW: {0}")] + Unsupported(String), + /// The file declares more than [`Limits`] allow. + #[error("CRW exceeds a limit: {0}")] + LimitExceeded(&'static str), +} + +/// Caps on what a file may ask the decoder to allocate, checked against the header before +/// anything is allocated. The struct is `#[non_exhaustive]`: start from [`Limits::default`] and +/// set the fields you change. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct Limits { + /// Most sensor samples (`width × height`) a file may declare. The decoder holds about three + /// `u16` buffers of this size at its peak. Default: 2²³ (8.4 million) on every target; real + /// CRW frames hold at most 7.4 million samples. + pub max_samples: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { max_samples: 1 << 23 } + } +} + +/// The 2×2 Bayer phase at sensor position (0, 0). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Cfa { + /// Red, green / green, blue: every verified body except the PowerShot G1. + Rggb, + /// Green, blue / red, green: the PowerShot G1. + Gbrg, +} + +impl Cfa { + /// The conventional four-letter name. + pub fn name(self) -> &'static str { + match self { + Cfa::Rggb => "RGGB", + Cfa::Gbrg => "GBRG", + } + } + + /// The colour at each site of the 2×2 cell, row-major: 0 = red, 1 = green, 2 = blue. + pub fn pattern(self) -> [u8; 4] { + match self { + Cfa::Rggb => [0, 1, 1, 2], + Cfa::Gbrg => [1, 2, 0, 1], + } + } +} + +/// A rectangle in sensor pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Rect { + pub x: usize, + pub y: usize, + pub width: usize, + pub height: usize, +} + +impl Rect { + pub const fn new(x: usize, y: usize, width: usize, height: usize) -> Self { + Self { x, y, width, height } + } +} + +/// Black level per 2×2 CFA site, row-major, anchored at the active area's origin. +/// +/// The anchor differs from [`Header::cfa`]'s: `cfa` is the phase at sensor (0, 0), `values` is +/// indexed from the active area's origin. The active area can start at an odd sensor row (the +/// PowerShot G1 starts at y = 15), so `values[i]` does not line up with `cfa.pattern()[i]`: read a +/// level through [`BlackLevel::at`] rather than pairing the two by index. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct BlackLevel { + pub values: [f32; 4], +} + +impl BlackLevel { + /// The same level at every site. + pub const fn uniform(v: f32) -> Self { + Self { values: [v; 4] } + } + + /// The level at sensor position (`x`, `y`), whose CFA site is counted from `active`'s origin. + pub fn at(&self, active: Rect, x: usize, y: usize) -> f32 { + let px = (x as isize - active.x as isize).rem_euclid(2) as usize; + let py = (y as isize - active.y as isize).rem_euclid(2) as usize; + self.values.get(py * 2 + px).copied().unwrap_or(self.values[0]) + } +} + +/// Everything a CRW tells about its image short of the samples: what [`header`] returns, and +/// what [`Image::header`] carries. +#[derive(Debug, Clone, PartialEq)] +pub struct Header { + /// Sensor width in samples (SensorInfo), masked borders included. + pub width: usize, + /// Sensor height in samples. + pub height: usize, + /// Sample depth: 12 when the file carries a low-bits plane, else 10. + pub bits: u32, + /// The sensor area that holds the image (SensorInfo's inclusive crop); the rest is masked. + pub active_area: Rect, + /// Bayer phase at sensor (0, 0). + pub cfa: Cfa, + /// The DecoderTable number (0, 1 or 2): which frozen code tables the stream uses. + pub selector: u32, + /// Camera make from the MakeModel record (`Canon`). + pub make: Option, + /// Camera model from the MakeModel record (`Canon EOS D60`). + pub model: Option, + /// Capture time as Unix seconds on the camera's local clock (the TimeStamp record); the + /// file carries no UTC offset. Convert with a calendar library (chrono), not by hand. + pub capture_unix: Option, + /// Exif orientation (1, 3, 6 or 8) from the ImageInfo rotation. + pub exif_orientation: u8, + /// As-shot white balance as (R, G, B) multipliers with G = 1, from the ColorBalance record + /// slot the ShotInfo white-balance setting selects; `None` on bodies without the record + /// (D30, PowerShots). + pub wb_multipliers: Option<[f32; 3]>, +} + +/// A decoded CRW. +#[derive(Debug, Clone, PartialEq)] +pub struct Image { + pub header: Header, + /// `width × height` sensor samples, row-major, `bits` deep (10 or 12), black not subtracted. + pub samples: Vec, + /// Black level measured from the masked borders. + pub black: BlackLevel, + /// White level: the saturation plateau, the sensor's clip level, or the full range. + pub white: f32, +} + +/// Result alias for this crate. +pub type Result = std::result::Result; + +fn corrupt(why: &str) -> Error { + Error::Corrupt(why.to_string()) +} + +/// Heap recursion bound: the deepest directory observed in real files is 4. +const MAX_DEPTH: usize = 16; + +/// Records visited per file, across every heap level: real files hold a few dozen. Depth alone +/// doesn't bound the walk — a sub-heap record may point back at its own parent block, and N such +/// records per block multiply into N^MAX_DEPTH visits — so this shared budget does. +const MAX_RECORDS: usize = 4096; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Order { + Le, + Be, +} + +fn u16_at(b: &[u8], at: usize, o: Order) -> Option { + let s: [u8; 2] = b.get(at..at.checked_add(2)?)?.try_into().ok()?; + Some(match o { + Order::Le => u16::from_le_bytes(s), + Order::Be => u16::from_be_bytes(s), + }) +} + +fn u32_at(b: &[u8], at: usize, o: Order) -> Option { + let s: [u8; 4] = b.get(at..at.checked_add(4)?)?.try_into().ok()?; + Some(match o { + Order::Le => u32::from_le_bytes(s), + Order::Be => u32::from_be_bytes(s), + }) +} + +/// Walk a heap block (`start..start + len` of `bytes`) depth-first in record-table order, +/// recording each in-heap record's tag (typeCode & 0x3fff), absolute offset and length — first +/// occurrence wins. In-record values and reserved storage classes are skipped; dataType 5/6 +/// records are sub-heaps. `budget` is the number of records the whole file may still visit; +/// the walk stops wherever it runs out. +fn walk(bytes: &[u8], start: usize, len: usize, o: Order, depth: usize, budget: &mut usize, out: &mut BTreeMap) { + if depth > MAX_DEPTH || len < 6 { + return; + } + let Some(tbl) = start.checked_add(len).and_then(|e| e.checked_sub(4)).and_then(|at| u32_at(bytes, at, o)) else { return }; + let tbl = tbl as usize; + // the offset table must fit between the block start and its own trailing pointer + if tbl.checked_add(6).is_none_or(|e| e > len) { + return; + } + let Some(abs_tbl) = start.checked_add(tbl) else { return }; + let Some(n) = u16_at(bytes, abs_tbl, o) else { return }; + let Some(records_end) = (n as usize).checked_mul(10).and_then(|r| tbl.checked_add(2)?.checked_add(r)) else { return }; + if records_end > len { + return; + } + for k in 0..n as usize { + let Some(left) = budget.checked_sub(1) else { return }; + *budget = left; + let at = abs_tbl + 2 + k * 10; + let (Some(tc), Some(rl), Some(ro)) = (u16_at(bytes, at, o), u32_at(bytes, at + 2, o), u32_at(bytes, at + 6, o)) else { return }; + if tc & 0xc000 != 0 { + continue; // in-record value or reserved storage: nothing we need + } + let (rl, ro) = (rl as usize, ro as usize); + if ro.checked_add(rl).is_none_or(|e| e > len) { + continue; + } + let Some(abs) = start.checked_add(ro) else { continue }; + if matches!((tc >> 11) & 7, 5 | 6) { + walk(bytes, abs, rl, o, depth + 1, budget, out); + } else { + out.entry(tc & 0x3fff).or_insert((abs, rl)); + } + } +} + +/// CIFF records the decoder consumes. +const RAW_DATA: u16 = 0x2005; +const SENSOR_INFO: u16 = 0x1031; +const DECODER_TABLE: u16 = 0x1835; +const MAKE_MODEL: u16 = 0x080a; +const IMAGE_INFO: u16 = 0x1810; +const TIME_STAMP: u16 = 0x180e; +const COLOR_BALANCE: u16 = 0x10a9; +const SHOT_INFO: u16 = 0x102a; + +/// Whether `bytes` start with a CIFF raw header: `II` or `MM`, then `HEAPCCDR` at offset 6. +pub fn probe(bytes: &[u8]) -> bool { + matches!(bytes.get(0..2), Some(b"II") | Some(b"MM")) && bytes.get(6..14) == Some(b"HEAPCCDR") +} + +/// Read the header: geometry, bit depth, CFA phase, metadata and white balance, without +/// decoding the samples. Fails the same way [`decode`] would on the records it needs. +pub fn header(bytes: &[u8], limits: &Limits) -> Result
{ + Ok(parse(bytes, limits, false)?.header) +} + +/// Decode a CRW: the header plus every sensor sample, the black level and the white level. +pub fn decode(bytes: &[u8], limits: &Limits) -> Result { + parse(bytes, limits, true) +} + +fn parse(bytes: &[u8], limits: &Limits, full: bool) -> Result { + let o = match bytes.get(0..2) { + Some(b"II") => Order::Le, + Some(b"MM") => Order::Be, + _ => return Err(Error::NotCrw), + }; + if bytes.get(6..14) != Some(b"HEAPCCDR") { + return Err(Error::NotCrw); + } + let hl = u32_at(bytes, 2, o).ok_or_else(|| corrupt("truncated header"))? as usize; + let heap_len = bytes.len().checked_sub(hl).filter(|&l| l >= 6).ok_or_else(|| corrupt("header length beyond the file"))?; + let mut rec = BTreeMap::new(); + let mut budget = MAX_RECORDS; + walk(bytes, hl, heap_len, o, 0, &mut budget, &mut rec); + + let &(so, sl) = rec.get(&SENSOR_INFO).ok_or_else(|| corrupt("no SensorInfo record"))?; + let sensor: Vec = (0..sl / 2).map_while(|i| u16_at(bytes, so + i * 2, o)).collect(); + let (w, h) = match (sensor.get(1), sensor.get(2)) { + (Some(&w), Some(&h)) => (w as usize, h as usize), + _ => return Err(corrupt("SensorInfo too short")), + }; + if w == 0 || h == 0 { + return Err(corrupt("zero sensor dimensions")); + } + let n = w.checked_mul(h).filter(|&n| n <= limits.max_samples).ok_or(Error::LimitExceeded("image too large"))?; + + let &(dto, dtl) = rec.get(&DECODER_TABLE).ok_or_else(|| corrupt("no DecoderTable record"))?; + if dtl < 16 { + return Err(corrupt("DecoderTable too short")); + } + let selector = u32_at(bytes, dto, o).ok_or_else(|| corrupt("truncated DecoderTable"))?; + let ent_off = u32_at(bytes, dto + 8, o).ok_or_else(|| corrupt("truncated DecoderTable"))? as usize; + let tables = tables::tables(selector).ok_or_else(|| Error::Unsupported(format!("decoder table {selector}")))?; + if ent_off > bytes.len() { + return Err(corrupt("entropy offset beyond the file")); + } + + // "Canon\0\0" + let (make, model) = rec + .get(&MAKE_MODEL) + .and_then(|&(mo, ml)| bytes.get(mo..mo.checked_add(ml)?)) + .map(|v| { + let mut parts = v.split(|&b| b == 0).map(|s| String::from_utf8_lossy(s).trim().to_string()); + (parts.next().unwrap_or_default(), parts.next().unwrap_or_default()) + }) + .unwrap_or_default(); + if model.contains("PowerShot G5") { + return Err(Error::Unsupported( + "Canon PowerShot G5 entropy coding (a documented anomaly of the derivation, clean-room/02-crw-format.md §4); use the embedded JPEG" + .into(), + )); + } + + // active area from the SensorInfo crop (inclusive left, top, right, bottom) + let active = match (sensor.get(5), sensor.get(6), sensor.get(7), sensor.get(8)) { + (Some(&l), Some(&t), Some(&r), Some(&b)) + if (l as usize) < (r as usize) && (r as usize) < w && (t as usize) < (b as usize) && (b as usize) < h => + { + Rect::new(l as usize, t as usize, r as usize - l as usize + 1, b as usize - t as usize + 1) + } + _ => Rect::new(0, 0, w, h), + }; + + let &(ro, rl) = rec.get(&RAW_DATA).ok_or_else(|| corrupt("no raw data record"))?; + let payload = + bytes.get(ro..ro.checked_add(rl).ok_or_else(|| corrupt("raw record size overflow"))?).ok_or_else(|| corrupt("raw data outside the file"))?; + + // Low-bits plane discriminator, two signs of a plane: (a) its raw bytes may hold an 0xff not + // followed by 0x00, which the stuffed entropy stream never does; (b) the zero header of + // `ent_off` bytes sits right after it, where a file without a plane is deep in entropy data. + // (b) catches the D60 files whose lowest bit is almost always 0, so no plane byte is 0xff. + let q = n.div_ceil(4); + let span = q.saturating_add(ent_off); + let head = payload.get(..span.min(payload.len())).unwrap_or(payload); + // Both signs need a payload long enough to hold a plane and the header behind it. A shorter + // payload ends in the PowerShot `ff d9 00` trailer, and `ff d9` is a 0xff not followed by + // 0x00: without this gate a flat no-plane G1 frame read as 12-bit and was then refused. + let plane_fits = payload.len() > span; + let header_after_plane = plane_fits && ent_off >= 64 && payload.get(q..span).is_some_and(|z| z.iter().all(|&b| b == 0)); + let has_low = header_after_plane || (plane_fits && head.windows(2).any(|p| p[0] == 0xff && p[1] != 0x00)); + let bits = if has_low { 12 } else { 10 }; + let low_len = if has_low { q } else { 0 }; + let samples = if full { + let entropy = payload.get(low_len.saturating_add(ent_off)..).ok_or_else(|| corrupt("payload shorter than its header"))?; + let low = has_low.then(|| payload.get(..q)).flatten(); + if has_low && low.is_none() { + return Err(corrupt("payload shorter than its low-bits plane")); + } + let hi = decode_entropy(entropy, w, h, &tables)?; + let mut samples = vec![0u16; n]; + // `samples` is documented as `bits` deep, so a damaged stream whose chain value runs past + // the range is clamped here: a consumer indexing a `1 << bits` LUT must not see more. + let top = ((1u32 << bits) - 1) as u16; + for (i, (dst, &hi)) in samples.iter_mut().zip(hi.iter()).enumerate() { + *dst = match low { + Some(low) => { + let lb = low.get(i >> 2).map_or(0, |&b| (b >> ((i & 3) * 2)) & 3); + let v = (((hi as u32) << 2) | lb as u32) & 0xffff; + v.min(u32::from(top)) as u16 + } + None => hi.min(top), + }; + } + samples + } else { + Vec::new() + }; + + // CFA phase at sensor (0, 0): RGGB on every verified body except the PowerShot G1 (GBRG), + // established by chroma correlation against each body's embedded JPEG (derivation log). + let cfa = if model.contains("PowerShot G1") { Cfa::Gbrg } else { Cfa::Rggb }; + + let black = if full { black_from_borders(&samples, w, h, active) } else { BlackLevel::uniform(0.0) }; + let white = if full { white_level(&samples, bits) } else { ((1u32 << bits) - 1) as f32 }; + + // ImageInfo rotation: degrees; 270 observed on portrait files whose scene is upright after a + // 90° counter-clockwise turn (Exif 8), so 90 is its mirror (Exif 6) + // The record holds a signed degree count, so a file may write −90 for 270: take it modulo + // 360 before mapping, and leave anything off-axis (0, ±360) at the identity. + let rotation_deg = + rec.get(&IMAGE_INFO).and_then(|&(io, il)| (il >= 16).then(|| u32_at(bytes, io + 12, o)).flatten()).map(|r| (r as i32).rem_euclid(360)); + let exif_orientation = match rotation_deg { + Some(90) => 6, + Some(180) => 3, + Some(270) => 8, + _ => 1, + }; + + let capture_unix = rec.get(&TIME_STAMP).and_then(|&(to, tl)| (tl >= 4).then(|| u32_at(bytes, to, o)).flatten()).map(i64::from); + + let header = Header { + width: w, + height: h, + bits, + active_area: active, + cfa, + selector, + make: (!make.is_empty()).then_some(make), + model: (!model.is_empty()).then_some(model), + capture_unix, + exif_orientation, + wb_multipliers: wb_from_color_balance(bytes, &rec, o), + }; + Ok(Image { header, samples, black, white }) +} + +/// As-shot white-balance multipliers from the ColorBalance record (0x10a9, D60/10D/300D): a +/// byte-length word, then (R, G, G, B) level quadruples per white-balance setting, the last +/// quadruple being black levels (ExifTool's prose Canon/CanonRaw tag documentation names the +/// slots; the 10D/300D order is Auto, Daylight, Shade, Cloudy, Tungsten, Fluorescent, Flash, +/// Custom, Kelvin, and the D60's seven slots lack Shade and Kelvin). The slot is chosen by the +/// ShotInfo (0x102a) WhiteBalance value; multipliers are `level / green`, confirmed against the +/// cameras' own JPEG renderings (derivation log). Bodies without the record (D30, PowerShots) +/// return `None`. +fn wb_from_color_balance(bytes: &[u8], rec: &BTreeMap, o: Order) -> Option<[f32; 3]> { + let &(bo, bl) = rec.get(&COLOR_BALANCE)?; + let vals: Vec = (0..bl / 2).map_while(|i| u16_at(bytes, bo + i * 2, o)).collect(); + // quadruples after the length word; the last one is black levels + let slots = vals.len().checked_sub(1)? / 4; + let usable = slots.checked_sub(1)?; + if usable == 0 { + return None; + } + let setting = rec.get(&SHOT_INFO).and_then(|&(so, sl)| (sl / 2 > 7).then(|| u16_at(bytes, so + 14, o)).flatten()).unwrap_or(0); + // Canon WhiteBalance values: 0 Auto, 1 Daylight, 2 Cloudy, 3 Tungsten, 4 Fluorescent, + // 5 Flash, 6 Custom, 8 Shade, 9 Kelvin + let slot = if usable >= 9 { + match setting { + 1 => 1, + 8 => 2, + 2 => 3, + 3 => 4, + 4 => 5, + 5 => 6, + 6 => 7, + 9 => 8, + _ => 0, + } + } else { + match setting { + s @ 1..=6 if (s as usize) < usable => s as usize, + _ => 0, + } + }; + let q = vals.get(1 + slot.min(usable - 1) * 4..5 + slot.min(usable - 1) * 4)?; + let g = (q.get(1).copied()? as f32 + q.get(2).copied()? as f32) / 2.0; + let (r, b) = (q.first().copied()? as f32, q.get(3).copied()? as f32); + if g <= 0.0 || r <= 0.0 || b <= 0.0 { + return None; + } + let (mr, mb) = (r / g, b / g); + ((0.2..=8.0).contains(&mr) && (0.2..=8.0).contains(&mb)).then_some([mr, 1.0, mb]) +} + +/// Fewest samples at the top of the data that count as the sensor's clip level (hot pixels are +/// fewer). +const CLIP_SAMPLES: usize = 64; + +/// White level: the saturation plateau when [`saturation_plateau`] finds one, else the top of +/// the data when it sits just under the full range. +/// +/// These bodies clip below the full range at low ISO: on 1,409 D60 files at ISO 100–400 no +/// sample exceeds 4009 (most stop at 4000–4008), while ISO 800–1000 files reach 4095; the 10D +/// and 300D samples stop at 4003–4008 too. A photo with only a small blown area has no plateau +/// for [`saturation_plateau`], which then answered 4095 (373 of 925 ISO 100 files, 283 of 287 at +/// ISO 400): the blown pixels sat at 0.977 of white, under the clip threshold, and kept a +/// colour after white balance. So when at least [`CLIP_SAMPLES`] samples reach within 3 % of the +/// full range, the level they reach is the white level, lowered by the same 1/256 band the +/// plateau estimate uses. +fn white_level(data: &[u16], bits: u32) -> f32 { + let full = (1u32 << bits.clamp(1, 16)) - 1; + let plateau = saturation_plateau(data, bits); + if plateau < full as f32 { + return plateau; + } + let mut hist = vec![0usize; full as usize + 1]; + for &v in data { + if let Some(h) = hist.get_mut(v as usize) { + *h += 1; + } + } + let mut above = 0usize; + let top = hist.iter().enumerate().rev().find_map(|(v, &n)| { + above += n; + (above >= CLIP_SAMPLES).then_some(v as u32) + }); + match top { + Some(top) if top < full && top as f32 >= 0.97 * full as f32 => (top - (top / 256).max(1)) as f32, + _ => full as f32, + } +} + +/// White level estimate shared by the Craft raw decoders (LightCraft's `white_from_data`): the +/// saturation plateau if a noticeable number of samples sit at the maximum value, otherwise the +/// full `bits` range. A few stray samples above the plateau (hot pixels) don't hide it. +fn saturation_plateau(data: &[u16], bits: u32) -> f32 { + let full_level = (1u32 << bits.clamp(1, 16)) - 1; + let full = full_level as f32; + let step = (data.len() / 2_000_000).max(1); + // samples above the nominal range are padding / invalid, not saturation + let mut hist = vec![0usize; full_level as usize + 1]; + for &v in data.iter().step_by(step) { + if let Some(h) = hist.get_mut(v as usize) { + *h += 1; + } + } + let min_plateau = (data.len() / 20_000).max(16); + // the top of the data, skipping up to half a plateau's worth of strays so a real plateau can't be skipped + let strays = min_plateau / 2; + let mut above = 0usize; + let Some(mx) = hist.iter().enumerate().rev().find_map(|(v, &n)| { + above += n * step; + (above > strays).then_some(v as u32) + }) else { + return full; + }; + let band = (mx / 256).max(1); + let near = mx.saturating_sub(band); + let below = near.saturating_sub(band); + let count = |from: u32, to: u32| hist.get(from as usize..to as usize).map_or(0, |h| h.iter().sum::()); + // the strays count as saturated too + let top = count(near, full_level + 1); + let under = count(below, near); + // a saturation plateau is a spike: many more samples in the top band than in the band just below it + if mx as f32 > full * 0.5 && top * step >= min_plateau && top >= 4 * under + 4 { + // plateau: use its lower edge so everything at saturation maps to ≥ 1.0 + near as f32 + } else { + full + } +} + +/// Black level from the masked sensor borders: the darkest flat strip among 8-column groups of +/// the left border and 4-row groups of the top border, with the strips within 8 DN of it averaged +/// in. A strip whose mean is ≤ 1 is clamped padding (observed on the D60's first rows), not a +/// black reference, and bright transition strips next to the active area only raise means, so the +/// minimum stays on true dark pixels. +fn black_from_borders(data: &[u16], w: usize, h: usize, active: Rect) -> BlackLevel { + let mean_of = |xs: std::ops::Range, ys: std::ops::Range| -> Option { + let (mut sum, mut cnt) = (0f64, 0u64); + for y in ys { + let row = data.get(y.checked_mul(w)?..y.checked_mul(w)?.checked_add(w)?)?; + for &v in row.get(xs.clone())? { + sum += v as f64; + cnt += 1; + } + } + (cnt > 0).then(|| sum / cnt as f64) + }; + let mut strips: Vec<(f64, std::ops::Range, std::ops::Range)> = Vec::new(); + let rows = active.y..active.y.saturating_add(active.height).min(h); + let mut x = 2usize; + while x + 8 <= active.x { + if let Some(m) = mean_of(x..x + 8, rows.clone()) { + strips.push((m, x..x + 8, rows.clone())); + } + x += 8; + } + let cols = active.x..active.x.saturating_add(active.width).min(w); + let mut y = 2usize; + while y + 4 <= active.y { + if let Some(m) = mean_of(cols.clone(), y..y + 4) { + strips.push((m, cols.clone(), y..y + 4)); + } + y += 4; + } + let Some(darkest) = strips.iter().map(|s| s.0).filter(|&m| m > 1.0).reduce(f64::min) else { + return BlackLevel::uniform(0.0); + }; + // per-CFA-site means over every strip near the darkest + let (mut sum, mut cnt) = ([0f64; 4], [0u64; 4]); + for (m, xs, ys) in &strips { + // clamped padding (mean ≤ 1) is no black reference: counting it dragged the D60's level + // from 128 down to 117 and lifted every shadow + if *m <= 1.0 || *m > darkest + 8.0 { + continue; + } + for y in ys.clone() { + let py = (y as isize - active.y as isize).rem_euclid(2) as usize; + for x in xs.clone() { + let px = (x as isize - active.x as isize).rem_euclid(2) as usize; + if let Some(&v) = data.get(y * w + x) { + sum[py * 2 + px] += v as f64; + cnt[py * 2 + px] += 1; + } + } + } + } + if cnt.contains(&0) { + return BlackLevel::uniform(darkest as f32); + } + let mut values = [0f32; 4]; + for (v, (s, c)) in values.iter_mut().zip(sum.iter().zip(&cnt)) { + *v = (s / *c as f64) as f32; + } + BlackLevel { values } +} + +/// A code-table lookup built for 16-bit MSB-first peeks: entry = `len << 16 | run << 8 | size`, +/// zero where no code matches. +fn build_lut(codes: &[(u16, u8, u8, u8)]) -> Vec { + let mut lut = vec![0u32; 1 << 16]; + for &(code, len, r, s) in codes { + if len == 0 || len > 16 { + continue; + } + let span = 1usize << (16 - len); + let base = (code as usize) << (16 - len); + for e in lut.iter_mut().skip(base).take(span) { + *e = (len as u32) << 16 | (r as u32) << 8 | s as u32; + } + } + lut +} + +/// Decode the ff00-stuffed entropy stream into the `w × h` high-plane samples, per the frozen +/// model (`clean-room/02-crw-format.md` §2): 64-sample blocks, first/main code tables, +/// (run, size) leaves with JPEG-rule magnitudes, EOB, block-DC carry, and two parity DPCM chains +/// initialised to 512 and reset at every row start. A stream that ends early leaves the remaining +/// samples at their chain values (the frozen stream-end rule). +fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables) -> Result> { + let d = unstuff(entropy); + let nbits = d.len().saturating_mul(8); + let peek16 = |pos: usize| -> u16 { + let (byte, sh) = (pos >> 3, pos & 7); + let b = |k: usize| d.get(byte + k).copied().unwrap_or(0) as u32; + let w24 = b(0) << 16 | b(1) << 8 | b(2); + ((w24 >> (8 - sh)) & 0xffff) as u16 + }; + + let luts = [build_lut(tables.first), build_lut(tables.main)]; + let n = w.checked_mul(h).ok_or(Error::LimitExceeded("image too large"))?; + let mut out = Chains { dec: vec![0u16; n], w, qi: 0, ce: 512, co: 512 }; + let mut carry: i32 = 0; + let mut pos = 0usize; + while out.qi < n { + let block_end = n.min((out.qi / 64 + 1) * 64); + let mut tok = 0u32; + while out.qi < block_end { + // stream exhausted: every remaining sample keeps its chain value + if pos >= nbits.saturating_sub(1) { + out.fill(n); + break; + } + let e = luts[usize::from(tok > 0)].get(peek16(pos) as usize).copied().unwrap_or(0); + if e == 0 { + return Err(corrupt("undecodable entropy code")); + } + let (len, mut rr, s) = ((e >> 16) as usize, ((e >> 8) & 0xff) as usize, e & 0xff); + if pos.saturating_add(len).saturating_add(s as usize) > nbits { + out.fill(n); + break; + } + pos += len; + tok += 1; + if s == 0 && rr == 0 && tok > 1 { + // end of block + out.fill(block_end); + continue; + } + let diff: i32 = if s > 0 { + let m = (peek16(pos) >> (16 - s)) as i32; + pos += s as usize; + // JPEG magnitude rule + if m >= 1 << (s - 1) { m } else { m - (1 << s) + 1 } + } else { + 0 + }; + let mut first = out.qi.is_multiple_of(64); + if first && rr > 0 { + // the block's first sample covered by the skip: its diff is exactly the carry + out.emit(carry); + rr -= 1; + first = false; + } + out.fill(block_end.min(out.qi + rr)); + if out.qi >= block_end { + continue; + } + if first { + carry = carry.wrapping_add(diff); + out.emit(carry); + } else { + out.emit(diff); + } + } + } + Ok(out.dec) +} + +/// Drop the stuffing byte: an 0x00 after an 0xff. +fn unstuff(entropy: &[u8]) -> Vec { + let mut d = Vec::with_capacity(entropy.len()); + let mut i = 0; + while let Some(&b) = entropy.get(i) { + d.push(b); + i += if b == 0xff && entropy.get(i + 1) == Some(&0x00) { 2 } else { 1 }; + } + d +} + +/// The decoded samples and the two parity DPCM chains that produce them. +struct Chains { + dec: Vec, + w: usize, + qi: usize, + ce: i32, + co: i32, +} + +impl Chains { + /// Emit one diff: row-start chain reset, parity chain update, 16-bit masked store. + fn emit(&mut self, d: i32) { + if self.qi.is_multiple_of(self.w) { + self.ce = 512; + self.co = 512; + } + let c = if self.qi.is_multiple_of(2) { &mut self.ce } else { &mut self.co }; + *c = c.wrapping_add(d); + if let Some(p) = self.dec.get_mut(self.qi) { + *p = (*c & 0xffff) as u16; + } + self.qi += 1; + } + + /// Emit zero diffs up to sample `end`. + fn fill(&mut self, end: usize) { + while self.qi < end { + self.emit(0); + } + } +} diff --git a/crates/crw/src/tables.rs b/crates/crw/src/tables.rs new file mode 100644 index 000000000..b66d71af7 --- /dev/null +++ b/crates/crw/src/tables.rs @@ -0,0 +1,127 @@ +//! Canon CRW entropy code tables, frozen clean-room data. +//! +//! Frozen code tables extracted by known-plaintext alignment in the clean-room derivation record +//! retained at https://github.com/storytold/lightcraft (see ATTRIBUTION.md). Do not edit by hand. +//! Each entry is `(code bits, code length, run, size)`; codes are MSB-first. + +/// One selector's pair of canonical Huffman tables: the code starting each 64-sample +/// block reads from `first`, every other code from `main`. +pub(crate) struct SelTables { + pub first: &'static [(u16, u8, u8, u8)], + pub main: &'static [(u16, u8, u8, u8)], +} + +#[rustfmt::skip] +const FIRST_0: &[(u16, u8, u8, u8)] = &[ + (0b00, 2, 0, 4), (0b010, 3, 0, 3), (0b011, 3, 0, 5), (0b100, 3, 0, 6), (0b101, 3, 0, 2), (0b1100, 4, 0, 7), + (0b1101, 4, 0, 1), (0b11100, 5, 0, 8), (0b11101, 5, 0, 9), (0b11110, 5, 0, 0), (0b111110, 6, 0, 10), (0b1111110, 7, 0, 11), +]; +#[rustfmt::skip] +const MAIN_0: &[(u16, u8, u8, u8)] = &[ + (0b00, 2, 0, 3), (0b01, 2, 0, 4), (0b100, 3, 0, 2), (0b101, 3, 0, 5), (0b1100, 4, 0, 1), (0b1101, 4, 0, 6), + (0b11100, 5, 0, 7), (0b111010, 6, 0, 8), (0b111011, 6, 1, 2), (0b111100, 6, 1, 3), (0b111101, 6, 1, 1), (0b1111100, 7, 1, 4), + (0b1111101, 7, 0, 9), (0b11111100, 8, 1, 5), (0b111111010, 9, 2, 2), (0b111111011, 9, 0, 0), (0b1111111000, 10, 2, 1), (0b1111111001, 10, 1, 6), + (0b1111111010, 10, 0, 10), (0b1111111011, 10, 15, 0), (0b1111111100, 10, 2, 3), (0b11111111010, 11, 1, 7), (0b111111110110, 12, 2, 4), (0b1111111101110000, 16, 3, 1), + (0b1111111101110001, 16, 3, 2), (0b1111111101110010, 16, 1, 8), (0b1111111101110011, 16, 1, 9), (0b1111111101110100, 16, 3, 3), (0b1111111101110101, 16, 2, 5), (0b1111111101110110, 16, 4, 1), + (0b1111111101110111, 16, 3, 4), (0b1111111101111000, 16, 4, 2), (0b1111111101111001, 16, 3, 5), (0b1111111101111010, 16, 5, 1), (0b1111111101111011, 16, 3, 6), (0b1111111101111100, 16, 3, 7), + (0b1111111101111101, 16, 3, 8), (0b1111111101111110, 16, 2, 9), (0b1111111101111111, 16, 7, 9), (0b1111111110000000, 16, 2, 6), (0b1111111110000001, 16, 1, 10), (0b1111111110000010, 16, 3, 9), + (0b1111111110000011, 16, 5, 6), (0b1111111110000100, 16, 5, 7), (0b1111111110000101, 16, 2, 8), (0b1111111110000110, 16, 2, 7), (0b1111111110000111, 16, 5, 2), (0b1111111110001000, 16, 5, 5), + (0b1111111110001001, 16, 5, 8), (0b1111111110001010, 16, 4, 3), (0b1111111110001011, 16, 7, 6), (0b1111111110001100, 16, 5, 9), (0b1111111110001101, 16, 7, 7), (0b1111111110001110, 16, 5, 4), + (0b1111111110001111, 16, 6, 1), (0b1111111110010000, 16, 15, 9), (0b1111111110010001, 16, 7, 1), (0b1111111110010010, 16, 7, 8), (0b1111111110010011, 16, 7, 5), (0b1111111110010100, 16, 9, 6), + (0b1111111110010101, 16, 9, 7), (0b1111111110010110, 16, 4, 9), (0b1111111110010111, 16, 11, 7), (0b1111111110011000, 16, 5, 3), (0b1111111110011001, 16, 13, 7), (0b1111111110011010, 16, 7, 4), + (0b1111111110011011, 16, 11, 6), (0b1111111110011100, 16, 9, 8), (0b1111111110011101, 16, 4, 7), (0b1111111110011110, 16, 4, 8), (0b1111111110011111, 16, 9, 5), (0b1111111110100000, 16, 6, 9), + (0b1111111110100001, 16, 9, 9), (0b1111111110100010, 16, 9, 1), (0b1111111110100011, 16, 15, 10), (0b1111111110100100, 16, 11, 8), (0b1111111110100101, 16, 6, 8), (0b1111111110100110, 16, 11, 5), + (0b1111111110100111, 16, 11, 9), (0b1111111110101000, 16, 13, 6), (0b1111111110101001, 16, 15, 7), (0b1111111110101010, 16, 13, 8), (0b1111111110101011, 16, 6, 7), (0b1111111110101100, 16, 4, 6), + (0b1111111110101101, 16, 4, 5), (0b1111111110101110, 16, 9, 4), (0b1111111110101111, 16, 8, 9), (0b1111111110110000, 16, 15, 8), (0b1111111110110001, 16, 8, 1), (0b1111111110110010, 16, 13, 5), + (0b1111111110110011, 16, 15, 6), (0b1111111110110100, 16, 11, 4), (0b1111111110110101, 16, 8, 8), (0b1111111110110110, 16, 11, 1), (0b1111111110110111, 16, 2, 10), (0b1111111110111000, 16, 4, 4), + (0b1111111110111001, 16, 7, 2), (0b1111111110111010, 16, 13, 9), (0b1111111110111011, 16, 8, 7), (0b1111111110111100, 16, 6, 6), (0b1111111110111101, 16, 13, 4), (0b1111111110111110, 16, 15, 5), + (0b1111111110111111, 16, 3, 10), (0b1111111111000000, 16, 10, 7), (0b1111111111000001, 16, 7, 3), (0b1111111111000010, 16, 10, 9), (0b1111111111000011, 16, 10, 8), (0b1111111111000100, 16, 8, 6), + (0b1111111111000101, 16, 6, 2), (0b1111111111000110, 16, 12, 7), (0b1111111111000111, 16, 6, 5), (0b1111111111001000, 16, 12, 8), (0b1111111111001001, 16, 12, 9), (0b1111111111001010, 16, 10, 1), + (0b1111111111001011, 16, 15, 4), (0b1111111111001100, 16, 13, 1), (0b1111111111001101, 16, 14, 9), (0b1111111111001110, 16, 5, 10), (0b1111111111001111, 16, 9, 2), (0b1111111111010000, 16, 8, 5), + (0b1111111111010001, 16, 10, 6), (0b1111111111010010, 16, 14, 7), (0b1111111111010011, 16, 9, 3), (0b1111111111010100, 16, 14, 8), (0b1111111111010101, 16, 12, 1), (0b1111111111010110, 16, 12, 6), + (0b1111111111010111, 16, 7, 10), (0b1111111111011000, 16, 6, 4), (0b1111111111011001, 16, 14, 1), (0b1111111111011010, 16, 4, 10), (0b1111111111011011, 16, 6, 10), (0b1111111111011100, 16, 14, 6), + (0b1111111111011101, 16, 11, 3), (0b1111111111011110, 16, 15, 1), (0b1111111111011111, 16, 13, 3), (0b1111111111100000, 16, 10, 5), (0b1111111111100001, 16, 8, 10), (0b1111111111100010, 16, 11, 2), + (0b1111111111100011, 16, 9, 10), (0b1111111111100100, 16, 11, 10), (0b1111111111100101, 16, 8, 4), (0b1111111111100110, 16, 10, 4), (0b1111111111100111, 16, 6, 3), (0b1111111111101000, 16, 14, 5), + (0b1111111111101001, 16, 12, 5), (0b1111111111101010, 16, 15, 3), (0b1111111111101011, 16, 13, 2), (0b1111111111101100, 16, 12, 4), (0b1111111111101101, 16, 8, 2), (0b1111111111101110, 16, 10, 10), + (0b1111111111101111, 16, 13, 10), (0b1111111111110000, 16, 14, 4), (0b1111111111110001, 16, 15, 2), (0b1111111111110010, 16, 12, 10), (0b1111111111110011, 16, 8, 3), (0b1111111111110100, 16, 10, 3), + (0b1111111111110101, 16, 10, 2), (0b1111111111110110, 16, 12, 3), (0b1111111111110111, 16, 14, 10), (0b1111111111111000, 16, 12, 2), (0b1111111111111001, 16, 14, 2), (0b1111111111111010, 16, 14, 3), +]; +#[rustfmt::skip] +const FIRST_1: &[(u16, u8, u8, u8)] = &[ + (0b00, 2, 0, 3), (0b01, 2, 0, 2), (0b100, 3, 0, 4), (0b101, 3, 0, 1), (0b1100, 4, 0, 5), (0b1101, 4, 0, 0), + (0b1110, 4, 0, 6), (0b11110, 5, 0, 7), (0b111110, 6, 0, 9), (0b1111110, 7, 0, 8), (0b11111110, 8, 0, 10), (0b111111110, 9, 0, 11), +]; +#[rustfmt::skip] +const MAIN_1: &[(u16, u8, u8, u8)] = &[ + (0b00, 2, 0, 2), (0b01, 2, 0, 3), (0b100, 3, 0, 1), (0b101, 3, 0, 4), (0b1100, 4, 0, 5), (0b11010, 5, 1, 2), + (0b11011, 5, 1, 1), (0b11100, 5, 0, 6), (0b11101, 5, 1, 3), (0b111100, 6, 0, 7), (0b1111010, 7, 0, 8), (0b1111011, 7, 1, 4), + (0b1111100, 7, 2, 2), (0b1111101, 7, 0, 9), (0b11111100, 8, 2, 1), (0b111111010, 9, 0, 0), (0b111111011, 9, 2, 3), (0b111111100, 9, 1, 5), + (0b1111111010, 10, 3, 1), (0b1111111011, 10, 3, 2), (0b1111111100, 10, 0, 10), (0b11111111010, 11, 1, 6), (0b1111111101100000, 16, 15, 0), (0b1111111101100001, 16, 2, 4), + (0b1111111101100010, 16, 3, 3), (0b1111111101100011, 16, 4, 1), (0b1111111101100100, 16, 4, 2), (0b1111111101100101, 16, 1, 9), (0b1111111101100110, 16, 1, 7), (0b1111111101100111, 16, 2, 5), + (0b1111111101101000, 16, 1, 8), (0b1111111101101001, 16, 5, 1), (0b1111111101101010, 16, 3, 4), (0b1111111101101011, 16, 4, 3), (0b1111111101101100, 16, 5, 2), (0b1111111101101101, 16, 2, 9), + (0b1111111101101110, 16, 3, 5), (0b1111111101101111, 16, 6, 1), (0b1111111101110000, 16, 3, 9), (0b1111111101110001, 16, 7, 1), (0b1111111101110010, 16, 6, 2), (0b1111111101110011, 16, 3, 6), + (0b1111111101110100, 16, 5, 3), (0b1111111101110101, 16, 2, 6), (0b1111111101110110, 16, 3, 8), (0b1111111101110111, 16, 1, 10), (0b1111111101111000, 16, 3, 7), (0b1111111101111001, 16, 8, 1), + (0b1111111101111010, 16, 2, 7), (0b1111111101111011, 16, 9, 1), (0b1111111101111100, 16, 7, 9), (0b1111111101111101, 16, 5, 5), (0b1111111101111110, 16, 4, 5), (0b1111111101111111, 16, 2, 8), + (0b1111111110000000, 16, 7, 2), (0b1111111110000001, 16, 5, 9), (0b1111111110000010, 16, 10, 1), (0b1111111110000011, 16, 11, 1), (0b1111111110000100, 16, 4, 4), (0b1111111110000101, 16, 6, 9), + (0b1111111110000110, 16, 5, 4), (0b1111111110000111, 16, 5, 8), (0b1111111110001000, 16, 13, 1), (0b1111111110001001, 16, 15, 10), (0b1111111110001010, 16, 5, 7), (0b1111111110001011, 16, 14, 1), + (0b1111111110001100, 16, 15, 1), (0b1111111110001101, 16, 11, 9), (0b1111111110001110, 16, 4, 9), (0b1111111110001111, 16, 4, 7), (0b1111111110010000, 16, 6, 3), (0b1111111110010001, 16, 6, 10), + (0b1111111110010010, 16, 15, 9), (0b1111111110010011, 16, 5, 6), (0b1111111110010100, 16, 4, 6), (0b1111111110010101, 16, 10, 8), (0b1111111110010110, 16, 2, 10), (0b1111111110010111, 16, 4, 10), + (0b1111111110011000, 16, 7, 8), (0b1111111110011001, 16, 9, 9), (0b1111111110011010, 16, 3, 10), (0b1111111110011011, 16, 7, 5), (0b1111111110011100, 16, 7, 4), (0b1111111110011101, 16, 8, 6), + (0b1111111110011110, 16, 6, 5), (0b1111111110011111, 16, 12, 1), (0b1111111110100000, 16, 7, 6), (0b1111111110100001, 16, 11, 6), (0b1111111110100010, 16, 9, 6), (0b1111111110100011, 16, 13, 6), + (0b1111111110100100, 16, 8, 9), (0b1111111110100101, 16, 8, 5), (0b1111111110100110, 16, 12, 9), (0b1111111110100111, 16, 15, 5), (0b1111111110101000, 16, 9, 5), (0b1111111110101001, 16, 11, 4), + (0b1111111110101010, 16, 12, 7), (0b1111111110101011, 16, 15, 7), (0b1111111110101100, 16, 8, 10), (0b1111111110101101, 16, 9, 7), (0b1111111110101110, 16, 11, 8), (0b1111111110101111, 16, 7, 3), + (0b1111111110110000, 16, 11, 7), (0b1111111110110001, 16, 13, 8), (0b1111111110110010, 16, 13, 9), (0b1111111110110011, 16, 8, 7), (0b1111111110110100, 16, 10, 7), (0b1111111110110101, 16, 7, 10), + (0b1111111110110110, 16, 4, 8), (0b1111111110110111, 16, 8, 2), (0b1111111110111000, 16, 8, 4), (0b1111111110111001, 16, 14, 10), (0b1111111110111010, 16, 15, 4), (0b1111111110111011, 16, 10, 6), + (0b1111111110111100, 16, 12, 5), (0b1111111110111101, 16, 5, 10), (0b1111111110111110, 16, 9, 4), (0b1111111110111111, 16, 10, 4), (0b1111111111000000, 16, 12, 6), (0b1111111111000001, 16, 9, 2), + (0b1111111111000010, 16, 12, 3), (0b1111111111000011, 16, 6, 8), (0b1111111111000100, 16, 11, 5), (0b1111111111000101, 16, 12, 8), (0b1111111111000110, 16, 14, 4), (0b1111111111000111, 16, 14, 5), + (0b1111111111001000, 16, 14, 6), (0b1111111111001001, 16, 14, 9), (0b1111111111001010, 16, 10, 2), (0b1111111111001011, 16, 10, 3), (0b1111111111001100, 16, 14, 3), (0b1111111111001101, 16, 12, 2), + (0b1111111111001110, 16, 6, 6), (0b1111111111001111, 16, 6, 7), (0b1111111111010000, 16, 9, 3), (0b1111111111010001, 16, 10, 10), (0b1111111111010010, 16, 13, 4), (0b1111111111010011, 16, 13, 5), + (0b1111111111010100, 16, 14, 7), (0b1111111111010101, 16, 15, 8), (0b1111111111010110, 16, 8, 8), (0b1111111111010111, 16, 9, 10), (0b1111111111011000, 16, 13, 7), (0b1111111111011001, 16, 7, 7), + (0b1111111111011010, 16, 12, 4), (0b1111111111011011, 16, 6, 4), (0b1111111111011100, 16, 14, 2), (0b1111111111011101, 16, 9, 8), (0b1111111111011110, 16, 10, 5), (0b1111111111011111, 16, 12, 10), + (0b1111111111100000, 16, 13, 10), (0b1111111111100001, 16, 14, 8), (0b1111111111100010, 16, 15, 3), (0b1111111111100011, 16, 15, 6), (0b1111111111100100, 16, 10, 9), (0b1111111111100101, 16, 11, 2), + (0b1111111111100110, 16, 11, 3), (0b1111111111100111, 16, 15, 2), (0b1111111111101000, 16, 13, 2), (0b1111111111101001, 16, 8, 3), (0b1111111111101010, 16, 11, 10), (0b1111111111101011, 16, 13, 3), +]; +#[rustfmt::skip] +const FIRST_2: &[(u16, u8, u8, u8)] = &[ + (0b000, 3, 0, 6), (0b001, 3, 0, 5), (0b010, 3, 0, 7), (0b011, 3, 0, 4), (0b100, 3, 0, 8), (0b101, 3, 0, 3), + (0b1100, 4, 0, 9), (0b1101, 4, 0, 2), (0b1110, 4, 0, 0), (0b11110, 5, 0, 10), (0b111110, 6, 0, 1), (0b1111110, 7, 0, 11), +]; +#[rustfmt::skip] +const MAIN_2: &[(u16, u8, u8, u8)] = &[ + (0b000, 3, 0, 4), (0b001, 3, 0, 5), (0b010, 3, 0, 3), (0b011, 3, 0, 6), (0b100, 3, 0, 2), (0b101, 3, 0, 7), + (0b1100, 4, 0, 1), (0b1101, 4, 0, 8), (0b11100, 5, 0, 9), (0b111010, 6, 1, 2), (0b111011, 6, 1, 3), (0b111100, 6, 1, 4), + (0b1111010, 7, 1, 1), (0b1111011, 7, 1, 5), (0b1111100, 7, 0, 10), (0b11111010, 8, 1, 6), (0b11111011, 8, 1, 7), (0b111111000, 9, 15, 0), + (0b111111001, 9, 0, 0), (0b111111010, 9, 2, 2), (0b111111011, 9, 2, 1), (0b111111100, 9, 1, 8), (0b1111111010, 10, 2, 3), (0b11111110110, 11, 1, 9), + (0b11111110111, 11, 2, 4), (0b111111110000, 12, 3, 2), (0b111111110001, 12, 3, 1), (0b1111111100100, 13, 2, 5), (0b1111111100101, 13, 3, 3), (0b1111111100110, 13, 3, 8), + (0b1111111100111, 13, 3, 7), (0b1111111101000, 13, 3, 4), (0b1111111101001, 13, 3, 5), (0b1111111101010, 13, 3, 6), (0b1111111101011, 13, 3, 9), (0b11111111011000, 14, 7, 9), + (0b11111111011001, 14, 5, 7), (0b11111111011010, 14, 5, 8), (0b11111111011011, 14, 5, 9), (0b11111111011100, 14, 2, 8), (0b11111111011101, 14, 5, 6), (0b11111111011110, 14, 7, 8), + (0b11111111011111, 14, 2, 7), (0b11111111100000, 14, 4, 1), (0b11111111100001, 14, 2, 9), (0b1111111110001000, 16, 7, 7), (0b1111111110001001, 16, 2, 6), (0b1111111110001010, 16, 4, 2), + (0b1111111110001011, 16, 7, 6), (0b1111111110001100, 16, 9, 9), (0b1111111110001101, 16, 1, 10), (0b1111111110001110, 16, 5, 5), (0b1111111110001111, 16, 9, 8), (0b1111111110010000, 16, 9, 7), + (0b1111111110010001, 16, 15, 9), (0b1111111110010010, 16, 4, 8), (0b1111111110010011, 16, 5, 4), (0b1111111110010100, 16, 9, 6), (0b1111111110010101, 16, 8, 9), (0b1111111110010110, 16, 4, 7), + (0b1111111110010111, 16, 11, 7), (0b1111111110011000, 16, 4, 9), (0b1111111110011001, 16, 15, 10), (0b1111111110011010, 16, 7, 5), (0b1111111110011011, 16, 6, 8), (0b1111111110011100, 16, 11, 6), + (0b1111111110011101, 16, 6, 7), (0b1111111110011110, 16, 6, 9), (0b1111111110011111, 16, 11, 9), (0b1111111110100000, 16, 11, 8), (0b1111111110100001, 16, 13, 8), (0b1111111110100010, 16, 5, 2), + (0b1111111110100011, 16, 13, 7), (0b1111111110100100, 16, 8, 8), (0b1111111110100101, 16, 11, 5), (0b1111111110100110, 16, 7, 4), (0b1111111110100111, 16, 5, 1), (0b1111111110101000, 16, 4, 6), + (0b1111111110101001, 16, 13, 9), (0b1111111110101010, 16, 15, 8), (0b1111111110101011, 16, 3, 10), (0b1111111110101100, 16, 13, 6), (0b1111111110101101, 16, 8, 7), (0b1111111110101110, 16, 4, 5), + (0b1111111110101111, 16, 7, 10), (0b1111111110110000, 16, 9, 5), (0b1111111110110001, 16, 13, 5), (0b1111111110110010, 16, 15, 6), (0b1111111110110011, 16, 8, 6), (0b1111111110110100, 16, 11, 4), + (0b1111111110110101, 16, 10, 9), (0b1111111110110110, 16, 9, 4), (0b1111111110110111, 16, 5, 3), (0b1111111110111000, 16, 2, 10), (0b1111111110111001, 16, 10, 8), (0b1111111110111010, 16, 4, 3), + (0b1111111110111011, 16, 15, 5), (0b1111111110111100, 16, 15, 7), (0b1111111110111101, 16, 13, 4), (0b1111111110111110, 16, 6, 6), (0b1111111110111111, 16, 10, 7), (0b1111111111000000, 16, 5, 10), + (0b1111111111000001, 16, 4, 4), (0b1111111111000010, 16, 8, 10), (0b1111111111000011, 16, 12, 9), (0b1111111111000100, 16, 14, 8), (0b1111111111000101, 16, 12, 8), (0b1111111111000110, 16, 14, 7), + (0b1111111111000111, 16, 9, 10), (0b1111111111001000, 16, 6, 10), (0b1111111111001001, 16, 7, 3), (0b1111111111001010, 16, 4, 10), (0b1111111111001011, 16, 6, 1), (0b1111111111001100, 16, 12, 7), + (0b1111111111001101, 16, 15, 4), (0b1111111111001110, 16, 12, 6), (0b1111111111001111, 16, 6, 5), (0b1111111111010000, 16, 14, 9), (0b1111111111010001, 16, 7, 2), (0b1111111111010010, 16, 14, 6), + (0b1111111111010011, 16, 7, 1), (0b1111111111010100, 16, 9, 1), (0b1111111111010101, 16, 9, 3), (0b1111111111010110, 16, 10, 6), (0b1111111111010111, 16, 13, 10), (0b1111111111011000, 16, 9, 2), + (0b1111111111011001, 16, 8, 5), (0b1111111111011010, 16, 6, 2), (0b1111111111011011, 16, 15, 3), (0b1111111111011100, 16, 12, 5), (0b1111111111011101, 16, 11, 2), (0b1111111111011110, 16, 10, 4), + (0b1111111111011111, 16, 8, 4), (0b1111111111100000, 16, 11, 10), (0b1111111111100001, 16, 6, 4), (0b1111111111100010, 16, 10, 5), (0b1111111111100011, 16, 11, 3), (0b1111111111100100, 16, 13, 2), + (0b1111111111100101, 16, 8, 1), (0b1111111111100110, 16, 14, 5), (0b1111111111100111, 16, 13, 3), (0b1111111111101000, 16, 10, 10), (0b1111111111101001, 16, 12, 4), (0b1111111111101010, 16, 12, 10), + (0b1111111111101011, 16, 15, 2), (0b1111111111101100, 16, 11, 1), (0b1111111111101101, 16, 14, 4), (0b1111111111101110, 16, 13, 1), (0b1111111111101111, 16, 8, 3), (0b1111111111110000, 16, 6, 3), + (0b1111111111110001, 16, 14, 10), (0b1111111111110010, 16, 12, 3), (0b1111111111110011, 16, 14, 2), (0b1111111111110100, 16, 8, 2), (0b1111111111110101, 16, 15, 1), (0b1111111111110110, 16, 10, 3), + (0b1111111111110111, 16, 12, 2), (0b1111111111111000, 16, 10, 1), (0b1111111111111001, 16, 12, 1), (0b1111111111111010, 16, 14, 3), (0b1111111111111011, 16, 10, 2), (0b1111111111111100, 16, 14, 1), +]; +/// The tables for DecoderTable selector 0, 1 or 2. +pub(crate) fn tables(selector: u32) -> Option { + match selector { + 0 => Some(SelTables { first: FIRST_0, main: MAIN_0 }), + 1 => Some(SelTables { first: FIRST_1, main: MAIN_1 }), + 2 => Some(SelTables { first: FIRST_2, main: MAIN_2 }), + _ => None, + } +} diff --git a/crates/crw/src/tests.rs b/crates/crw/src/tests.rs new file mode 100644 index 000000000..a858cef9d --- /dev/null +++ b/crates/crw/src/tests.rs @@ -0,0 +1,668 @@ +//! Unit tests: synthetic CIFF files built with a test-only encoder that is the inverse of the +//! frozen model, plus the frozen tables' structural properties. The decoder's behaviour on real +//! files is pinned in `tests/samples.rs`. + +use super::*; + +fn put16(v: u16, o: Order) -> [u8; 2] { + match o { + Order::Le => v.to_le_bytes(), + Order::Be => v.to_be_bytes(), + } +} +fn put32(v: u32, o: Order) -> [u8; 4] { + match o { + Order::Le => v.to_le_bytes(), + Order::Be => v.to_be_bytes(), + } +} + +/// A heap block: values, record table, trailing offset-table pointer. +fn heap(records: &[(u16, Vec)], o: Order) -> Vec { + let mut b = Vec::new(); + let mut entries = Vec::new(); + for (tc, v) in records { + let off = b.len(); + b.extend_from_slice(v); + if b.len() % 2 == 1 { + b.push(0); + } + entries.push((*tc, v.len() as u32, off as u32)); + } + let tbl = b.len() as u32; + b.extend_from_slice(&put16(entries.len() as u16, o)); + for (tc, l, off) in entries { + b.extend_from_slice(&put16(tc, o)); + b.extend_from_slice(&put32(l, o)); + b.extend_from_slice(&put32(off, o)); + } + b.extend_from_slice(&put32(tbl, o)); + b +} + +/// A complete CIFF file: 26-byte header + the root heap. +fn ciff(records: &[(u16, Vec)], o: Order) -> Vec { + let mut f = Vec::new(); + f.extend_from_slice(match o { + Order::Le => b"II", + Order::Be => b"MM", + }); + f.extend_from_slice(&put32(26, o)); + f.extend_from_slice(b"HEAPCCDR"); + f.extend_from_slice(&put32(0x0001_0002, o)); + f.extend_from_slice(&[0; 8]); + f.extend_from_slice(&heap(records, o)); + f +} + +fn sensor_info(w: u16, h: u16, crop: Option<(u16, u16, u16, u16)>, o: Order) -> Vec { + let (l, t, r, b) = crop.unwrap_or((0, 0, 0, 0)); + [34, w, h, 0, 0, l, t, r, b, 0, 0, 0, 0, 0, 0, 0, 0].iter().flat_map(|&v| put16(v, o)).collect() +} + +fn decoder_table(sel: u32, o: Order) -> Vec { + [sel, 0, 514, 0].iter().flat_map(|&v| put32(v, o)).collect() +} + +/// Entropy-encode a 10-bit high plane with the model's inverse: per-block first/main tables, +/// (0, s) tokens only (the caller keeps mid-block diffs nonzero), JPEG magnitudes, carry, +/// parity chains reset at row starts; MSB-first packing, zero padding, ff00 stuffing. +fn encode_hi(hi: &[u16], w: usize, t: &tables::SelTables) -> Vec { + let mut bits: Vec = Vec::new(); + let push_code = |bits: &mut Vec, tab: &[(u16, u8, u8, u8)], s: u32| { + let &(c, l, ..) = tab.iter().find(|e| e.2 == 0 && e.3 as u32 == s).expect("(0, s) code exists"); + bits.extend((0..l).rev().map(|i| (c >> i) & 1 == 1)); + }; + let size_of = |d: i32| if d == 0 { 0 } else { 32 - d.unsigned_abs().leading_zeros() }; + let (mut ce, mut co) = (512i32, 512i32); + let mut carry = 0i32; + for (qi, &v) in hi.iter().enumerate() { + if qi % w == 0 { + ce = 512; + co = 512; + } + let c = if qi % 2 == 0 { &mut ce } else { &mut co }; + let dd = v as i32 - *c; + *c = v as i32; + let (tab, d) = if qi % 64 == 0 { + let d = dd - carry; + carry = dd; + (t.first, d) + } else { + assert_ne!(dd, 0, "test images avoid zero diffs mid-block"); + (t.main, dd) + }; + let s = size_of(d); + push_code(&mut bits, tab, s); + if s > 0 { + let m = if d > 0 { d } else { d + (1 << s) - 1 } as u32; + bits.extend((0..s).rev().map(|i| (m >> i) & 1 == 1)); + } + } + pack_entropy(&bits) +} + +/// MSB-first packing into bytes, zero padding in the last byte, then ff00 stuffing. +fn pack_entropy(bits: &[bool]) -> Vec { + let mut stuffed = Vec::new(); + for chunk in bits.chunks(8) { + let mut b = 0u8; + for (i, &bit) in chunk.iter().enumerate() { + b |= (bit as u8) << (7 - i); + } + stuffed.push(b); + if b == 0xff { + stuffed.push(0x00); + } + } + stuffed +} + +/// One end-of-block token pair per 64-sample block: the block's first sample takes the carry +/// (zero here), the other 63 keep their chain values. A flat frame like this compresses to a +/// small fraction of its low-bits plane, the shape a lens-cap or blown-out frame has. +fn encode_eob(n: usize, t: &tables::SelTables) -> Vec { + let code = |tab: &[(u16, u8, u8, u8)]| { + let &(c, l, ..) = tab.iter().find(|e| e.2 == 0 && e.3 == 0).expect("an end-of-block code"); + (c, l) + }; + let (fc, fl) = code(t.first); + let (mc, ml) = code(t.main); + let mut bits: Vec = Vec::new(); + for _ in 0..n.div_ceil(64) { + for (c, l) in [(fc, fl), (mc, ml)] { + bits.extend((0..l).rev().map(|i| (c >> i) & 1 == 1)); + } + } + pack_entropy(&bits) +} + +/// A high plane whose diffs are all nonzero, bounded around 512: values follow +/// `(3 qi) mod 5 + 1`, so same-parity neighbours differ by 1 or −4 (never 0) and +/// row starts sit 1–5 above the reset value. +fn test_hi(n: usize) -> Vec { + (0..n).map(|qi| 512 + (qi as u16 * 3) % 5 + 1).collect() +} + +fn payload(low: Option<&[u8]>, entropy: &[u8]) -> Vec { + let mut p = low.map(|l| l.to_vec()).unwrap_or_default(); + p.extend_from_slice(&[0; 514]); + p.extend_from_slice(entropy); + p.extend_from_slice(&[0, 0]); + p +} + +/// The real PowerShot record shape: the payload closes with `ff d9` and a pad byte, not `00 00`. +fn payload_with_trailer(low: Option<&[u8]>, entropy: &[u8]) -> Vec { + let mut p = low.map(|l| l.to_vec()).unwrap_or_default(); + p.extend_from_slice(&[0; 514]); + p.extend_from_slice(entropy); + p.extend_from_slice(&[0xff, 0xd9, 0x00]); + p +} + +/// A file whose raw record is `payload` verbatim, with `extra` records beside the usual three. +fn file_with_payload(w: u16, h: u16, model: &str, payload: &[u8], extra: &[(u16, Vec)], o: Order) -> Vec { + let mut props = vec![(SENSOR_INFO, sensor_info(w, h, None, o)), (DECODER_TABLE, decoder_table(0, o)), (MAKE_MODEL, make_model(model))]; + props.extend_from_slice(extra); + ciff(&[(RAW_DATA, payload.to_vec()), (0x300a, heap(&props, o))], o) +} + +fn make_model(model: &str) -> Vec { + let mut v = b"Canon\0".to_vec(); + v.extend_from_slice(model.as_bytes()); + v.push(0); + v +} + +fn roundtrip_file(w: u16, h: u16, model: &str, low: Option<&[u8]>, hi: &[u16], o: Order) -> Vec { + let tables = tables::tables(0).expect("selector 0"); + let entropy = encode_hi(hi, w as usize, &tables); + // the realistic shape: raw data in the root, properties in a nested heap + let props = heap(&[(SENSOR_INFO, sensor_info(w, h, None, o)), (DECODER_TABLE, decoder_table(0, o)), (MAKE_MODEL, make_model(model))], o); + ciff(&[(RAW_DATA, payload(low, &entropy)), (0x300a, props)], o) +} + +fn full(bytes: &[u8]) -> Result { + decode(bytes, &Limits::default()) +} + +#[test] +fn probe_needs_the_signature() { + assert!(probe(b"II\x1a\0\0\0HEAPCCDR\0\0")); + assert!(probe(b"MM\0\0\0\x1aHEAPCCDR")); + assert!(!probe(b"II\x1a\0\0\0HEAP")); + assert!(!probe(b"II*\0\x08\0\0\0"), "a TIFF"); + assert!(!probe(b"")); + assert_eq!(full(b"II*\0\x08\0\0\0"), Err(Error::NotCrw)); + // a signature with no heap behind it is corrupt, not a panic + assert!(matches!(full(b"II HEAPCCDR "), Err(Error::Corrupt(_)))); +} + +#[test] +fn roundtrips_a_10_bit_frame() { + let (w, h) = (16u16, 8u16); + let hi = test_hi(128); + let f = roundtrip_file(w, h, "Canon EOS D60", None, &hi, Order::Le); + let img = full(&f).expect("decodes"); + assert_eq!((img.header.width, img.header.height, img.header.bits), (16, 8, 10)); + assert_eq!(img.samples, hi); + assert_eq!(img.header.cfa, Cfa::Rggb); + assert_eq!(img.header.selector, 0); + assert_eq!(img.header.make.as_deref(), Some("Canon")); + assert_eq!(img.header.model.as_deref(), Some("Canon EOS D60")); + assert_eq!(img.header.exif_orientation, 1); + assert_eq!(img.header.capture_unix, None); + assert_eq!(img.header.wb_multipliers, None); +} + +#[test] +fn roundtrips_a_12_bit_frame_with_a_low_bits_plane() { + let (w, h) = (16u16, 8u16); + let hi = test_hi(128); + // low bits all 3: the plane is all 0xff bytes, which trips the presence rule + let low = vec![0xffu8; 32]; + let f = roundtrip_file(w, h, "Canon EOS D60", Some(&low), &hi, Order::Le); + let img = full(&f).expect("decodes"); + assert_eq!(img.header.bits, 12); + let expected: Vec = hi.iter().map(|&v| (v << 2) | 3).collect(); + assert_eq!(img.samples, expected); +} + +/// A low-bits plane without a single 0xff byte (real D60 files whose lowest bit is nearly +/// always 0) used to be taken for "no plane": the plane was read as entropy data and the +/// file came out as 10-bit garbage. The zero header after the plane gives it away. +#[test] +fn a_low_bits_plane_without_ff_bytes_is_still_a_plane() { + let hi = test_hi(128); + for byte in [0x22u8, 0x00] { + let low = vec![byte; 32]; + let f = roundtrip_file(16, 8, "Canon EOS D60", Some(&low), &hi, Order::Le); + let img = full(&f).expect("decodes"); + assert_eq!(img.header.bits, 12); + let expected: Vec = hi.iter().enumerate().map(|(i, &v)| (v << 2) | u16::from((byte >> ((i & 3) * 2)) & 3)).collect(); + assert_eq!(img.samples, expected); + } +} + +#[test] +fn big_endian_ciff_decodes_too() { + let hi = test_hi(128); + let f = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, Order::Be); + let img = full(&f).expect("decodes"); + assert_eq!(img.samples, hi); +} + +#[test] +fn header_reads_geometry_without_samples() { + let hi = test_hi(128); + let f = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, Order::Le); + let hdr = header(&f, &Limits::default()).expect("header decodes"); + assert_eq!((hdr.width, hdr.height, hdr.bits), (16, 8, 10)); + assert_eq!(hdr, full(&f).expect("decodes").header, "the header is the same either way"); +} + +#[test] +fn limits_cap_the_sample_count() { + let hi = test_hi(128); + let f = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, Order::Le); + let mut limits = Limits { max_samples: 127 }; + assert_eq!(decode(&f, &limits), Err(Error::LimitExceeded("image too large"))); + assert!(header(&f, &limits).is_err()); + limits.max_samples = 128; + assert!(decode(&f, &limits).is_ok()); +} + +/// A header declaring 2²⁴ samples over a 128-sample stream is refused by the default limits +/// before any sample-sized buffer exists: the cap must stay above every real frame (7.4 million) +/// and below anything that would let a crafted file demand gigabytes. +#[test] +fn default_limits_refuse_a_crafted_giant_frame() { + let hi = test_hi(128); + let f = roundtrip_file(4096, 4096, "Canon EOS D60", None, &hi, Order::Le); + assert_eq!(decode(&f, &Limits::default()), Err(Error::LimitExceeded("image too large"))); + assert_eq!(header(&f, &Limits::default()), Err(Error::LimitExceeded("image too large"))); + assert!(Limits::default().max_samples >= 7_407_040, "the cap must admit the largest real frame"); +} + +#[test] +fn g1_is_gbrg() { + let hi = test_hi(128); + let f = roundtrip_file(16, 8, "Canon PowerShot G1", None, &hi, Order::Le); + let img = full(&f).expect("decodes"); + assert_eq!(img.header.cfa, Cfa::Gbrg); + assert_eq!(img.header.cfa.pattern(), [1, 2, 0, 1]); + assert_eq!(Cfa::Rggb.pattern(), [0, 1, 1, 2]); +} + +#[test] +fn g5_is_refused_with_its_anomaly_note() { + let hi = test_hi(128); + let f = roundtrip_file(16, 8, "Canon PowerShot G5", None, &hi, Order::Le); + let Err(Error::Unsupported(why)) = full(&f) else { panic!("expected Unsupported") }; + assert!(why.contains("G5"), "{why}"); + assert!(matches!(header(&f, &Limits::default()), Err(Error::Unsupported(_)))); +} + +/// A D60 at ISO 100 clips near 4000, and a photo with a small blown area has no plateau: +/// the white level must still come down to where the data stops, not stay at 4095. A few +/// hot pixels, or a photo that never gets near the top, leave the full range. +#[test] +fn white_level_follows_a_small_clipped_area() { + let mut data: Vec = (0..200_000).map(|i| 300 + (i % 3000) as u16).collect(); + assert_eq!(white_level(&data, 12), 4095.0, "nothing near the top"); + for v in data.iter_mut().take(100) { + *v = 4002; + } + assert_eq!(white_level(&data, 12), (4002 - 15) as f32, "100 samples at the clip level"); + let mut hot: Vec = (0..200_000).map(|i| 300 + (i % 3000) as u16).collect(); + for v in hot.iter_mut().take(10) { + *v = 4002; + } + assert_eq!(white_level(&hot, 12), 4095.0, "ten hot pixels are not a clip level"); +} + +#[test] +fn saturation_plateau_is_found_with_strays_above_it() { + let mut d: Vec = (0..100_000).map(|i| (i % 10000) as u16).collect(); + assert_eq!(saturation_plateau(&d, 14), 16383.0); + d.extend(std::iter::repeat_n(15000u16, 1000)); + let w = saturation_plateau(&d, 14); + assert!((14900.0..=15000.0).contains(&w), "{w}"); + assert_eq!(saturation_plateau(&[], 12), 4095.0); + let mut d: Vec = (0..100_000).map(|i| (2048 + i % 10000) as u16).collect(); + d.extend(std::iter::repeat_n(13583u16, 5000)); + d.extend([13700, 13900, 14100, 14426]); + let w = saturation_plateau(&d, 14); + assert!((13500.0..=13583.0).contains(&w), "{w}"); +} + +/// The D60 layout: sensor rows clamped to 0 above the image, dark columns left of it. The +/// clamped rows must not count towards the black level. +#[test] +fn black_level_ignores_clamped_rows() { + let (w, h) = (64usize, 40usize); + let active = Rect::new(24, 16, 40, 24); + let mut data = vec![900u16; w * h]; + for y in 0..h { + for x in 0..w { + data[y * w + x] = if y < 7 { + 0 + } else if x < 24 { + 128 + } else { + 900 + }; + } + } + let black = black_from_borders(&data, w, h, active); + assert!(black.values.iter().all(|v| (v - 128.0).abs() < 0.5), "{:?}", black.values); + assert_eq!(black.at(active, 24, 16), black.values[0]); + assert_eq!(black.at(active, 25, 16), black.values[1]); + assert_eq!(black.at(active, 24, 17), black.values[2]); + assert_eq!(black.at(active, 23, 15), black.values[3], "sites are counted from the active origin, modulo 2"); + assert_eq!(black_from_borders(&data, w, h, Rect::new(0, 0, w, h)), BlackLevel::uniform(0.0), "no border, no level"); +} + +/// The frozen stream-end rule: a truncated entropy stream leaves the remaining samples at +/// their chain values instead of failing the file. +#[test] +fn truncated_stream_keeps_chain_values() { + let (w, h) = (16u16, 8u16); + let hi = test_hi(128); + let tables = tables::tables(0).expect("selector 0"); + let entropy = encode_hi(&hi, 16, &tables); + let cut = entropy.len() / 2; + let props = heap( + &[(SENSOR_INFO, sensor_info(w, h, None, Order::Le)), (DECODER_TABLE, decoder_table(0, Order::Le)), (MAKE_MODEL, make_model("Canon EOS D60"))], + Order::Le, + ); + let f = ciff(&[(RAW_DATA, payload(None, &entropy[..cut])), (0x300a, props)], Order::Le); + let img = full(&f).expect("truncated stream still decodes"); + assert_eq!(img.samples.len(), 128); + assert_eq!(&img.samples[..32], &hi[..32], "the leading samples decode exactly"); +} + +/// Every record the decoder needs missing, bogus selectors, dimensions and offsets: errors, +/// never panics; and every truncation of a valid file survives. +#[test] +fn hostile_inputs_error_cleanly() { + let o = Order::Le; + let hi = test_hi(128); + let good = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, o); + for n in 0..good.len() { + let _ = full(&good[..n]); + let _ = header(&good[..n], &Limits::default()); + } + // single-byte mutations over the whole file + for i in 0..good.len() { + let mut bad = good.clone(); + bad[i] ^= 0xa5; + let _ = full(&bad); + } + let props_no_sensor = heap(&[(DECODER_TABLE, decoder_table(0, o))], o); + let f = ciff(&[(RAW_DATA, vec![0; 600]), (0x300a, props_no_sensor)], o); + assert!(matches!(full(&f), Err(Error::Corrupt(_)))); + let props_bad_sel = heap(&[(SENSOR_INFO, sensor_info(16, 8, None, o)), (DECODER_TABLE, decoder_table(7, o))], o); + let f = ciff(&[(RAW_DATA, vec![0; 600]), (0x300a, props_bad_sel)], o); + assert!(matches!(full(&f), Err(Error::Unsupported(_)))); + let props_zero = heap(&[(SENSOR_INFO, sensor_info(0, 0, None, o)), (DECODER_TABLE, decoder_table(0, o))], o); + let f = ciff(&[(RAW_DATA, vec![0; 600]), (0x300a, props_zero)], o); + assert!(full(&f).is_err()); + // a heap that nests into itself must hit the depth bound, not recurse forever + let mut self_heap = heap(&[(0x300a, vec![0; 10])], o); + let len = self_heap.len() as u32; + // point the nested record at the whole block + let tbl = u32::from_le_bytes(self_heap[len as usize - 4..].try_into().unwrap()) as usize; + self_heap[tbl + 4..tbl + 8].copy_from_slice(&len.to_le_bytes()); + self_heap[tbl + 8..tbl + 12].copy_from_slice(&0u32.to_le_bytes()); + let mut f = ciff(&[], o); + f.truncate(26); + f.extend_from_slice(&self_heap); + assert!(full(&f).is_err()); +} + +/// The frozen tables are prefix-free within each table, so the 16-bit lookup is unambiguous. +#[test] +fn frozen_tables_are_prefix_free() { + for sel in 0..3 { + let t = tables::tables(sel).expect("selector"); + for (name, tab) in [("first", t.first), ("main", t.main)] { + let mut lut = vec![false; 1 << 16]; + for &(code, len, ..) in tab { + assert!((1..=16).contains(&len), "sel {sel} {name}: code length {len}"); + let base = (code as usize) << (16 - len); + for cell in lut.iter_mut().skip(base).take(1 << (16 - len)) { + assert!(!*cell, "sel {sel} {name}: overlapping codes"); + *cell = true; + } + } + } + } + assert!(tables::tables(3).is_none()); +} + +/// Every second table codes all 162 symbols: (r, s) for r 0..=15 and s 1..=10, end of block +/// and (15, 0). Selectors 0 and 1 lacked (7..=14, 10) and (14, 10), learned from too few +/// files, and 223 of 1,516 real D60 files failed with "undecodable entropy code". +#[test] +fn second_tables_code_every_symbol() { + for sel in 0..3 { + let t = tables::tables(sel).expect("selector"); + let have: std::collections::BTreeSet<(u8, u8)> = t.main.iter().map(|e| (e.2, e.3)).collect(); + let mut want: std::collections::BTreeSet<(u8, u8)> = (0..16).flat_map(|r| (1..=10).map(move |s| (r, s))).collect(); + want.extend([(0, 0), (15, 0)]); + assert_eq!(have, want, "selector {sel}"); + assert_eq!(t.main.len(), 162, "selector {sel}: one code per symbol"); + } +} + +#[test] +fn white_balance_comes_from_the_shot_info_slot() { + let o = Order::Le; + let hi = test_hi(128); + // D60-shaped table: 7 slots + black levels; Daylight slot (1) = 1800, 800, 800, 960 + let mut cb: Vec = vec![66]; + cb.extend([2000u16, 760, 760, 830]); + cb.extend([1800u16, 800, 800, 960]); + for _ in 0..5 { + cb.extend([1500u16, 900, 900, 1500]); + } + cb.extend([126u16, 127, 126, 126]); + let cb: Vec = cb.iter().flat_map(|&v| put16(v, o)).collect(); + let mut shot: Vec = vec![54; 27]; + shot[7] = 1; // Daylight + let shot: Vec = shot.iter().flat_map(|&v| put16(v, o)).collect(); + let tables = tables::tables(0).expect("selector 0"); + let props = heap( + &[ + (SENSOR_INFO, sensor_info(16, 8, None, o)), + (DECODER_TABLE, decoder_table(0, o)), + (MAKE_MODEL, make_model("Canon EOS D60")), + (COLOR_BALANCE, cb), + (SHOT_INFO, shot), + ], + o, + ); + let f = ciff(&[(RAW_DATA, payload(None, &encode_hi(&hi, 16, &tables))), (0x300a, props)], o); + let img = full(&f).expect("decodes"); + let wb = img.header.wb_multipliers.expect("wb"); + assert!((wb[0] - 2.25).abs() < 1e-3 && wb[1] == 1.0 && (wb[2] - 1.2).abs() < 1e-3, "{wb:?}"); +} + +#[test] +fn rotation_maps_to_exif_orientation_and_the_time_stamp_is_unix_seconds() { + let o = Order::Le; + let hi = test_hi(128); + let tables = tables::tables(0).expect("selector 0"); + // The record holds a signed degree count, so 270 may also be written as −90. + for (deg, want) in [(0u32, 1u8), (90, 6), (180, 3), (270, 8), (-90i32 as u32, 8), (-270i32 as u32, 6)] { + let image_info: Vec = [3072u32, 2048, 0, deg, 12, 12, 0].iter().flat_map(|&v| put32(v, o)).collect(); + let stamp: Vec = [1083410638u32, 0, 0].iter().flat_map(|&v| put32(v, o)).collect(); + let extra = [(IMAGE_INFO, image_info), (TIME_STAMP, stamp)]; + let f = file_with_payload(16, 8, "Canon EOS 300D DIGITAL", &payload(None, &encode_hi(&hi, 16, &tables)), &extra, o); + let img = full(&f).expect("decodes"); + assert_eq!(img.header.exif_orientation, want, "rotation {deg}"); + assert_eq!(img.header.capture_unix, Some(1083410638)); + } +} + +#[test] +fn sensor_info_crop_is_the_active_area() { + let o = Order::Le; + let hi = test_hi(128); + let tables = tables::tables(0).expect("selector 0"); + let with = |crop| { + let props = + heap(&[(SENSOR_INFO, sensor_info(16, 8, crop, o)), (DECODER_TABLE, decoder_table(0, o)), (MAKE_MODEL, make_model("Canon EOS D60"))], o); + let f = ciff(&[(RAW_DATA, payload(None, &encode_hi(&hi, 16, &tables))), (0x300a, props)], o); + full(&f).expect("decodes").header.active_area + }; + assert_eq!(with(Some((4, 1, 15, 7))), Rect::new(4, 1, 12, 7), "inclusive left, top, right, bottom"); + assert_eq!(with(None), Rect::new(0, 0, 16, 8), "no crop: the whole sensor"); + assert_eq!(with(Some((4, 1, 16, 7))), Rect::new(0, 0, 16, 8), "a crop beyond the sensor is ignored"); + assert_eq!(with(Some((5, 1, 4, 7))), Rect::new(0, 0, 16, 8), "an inverted crop is ignored"); +} + +/// A block whose every record is a sub-heap pointing back at the block itself: depth alone +/// would allow N^MAX_DEPTH visits (here 64^17), the record budget ends the walk promptly. +#[test] +fn self_referencing_sub_heaps_terminate() { + const N: usize = 64; + for o in [Order::Le, Order::Be] { + let len = (2 + N * 10 + 4) as u32; + let mut f = Vec::new(); + f.extend_from_slice(match o { + Order::Le => b"II", + Order::Be => b"MM", + }); + f.extend_from_slice(&put32(26, o)); + f.extend_from_slice(b"HEAPCCDR"); + f.extend_from_slice(&put32(0x0001_0002, o)); + f.extend_from_slice(&[0; 8]); + f.extend_from_slice(&put16(N as u16, o)); // record table at block offset 0 + for _ in 0..N { + f.extend_from_slice(&put16(5 << 11 | 1, o)); // storage class 5: sub-heap + f.extend_from_slice(&put32(len, o)); // ... covering the whole parent block + f.extend_from_slice(&put32(0, o)); + } + f.extend_from_slice(&put32(0, o)); // trailing offset-table pointer + let started = std::time::Instant::now(); + assert!(matches!(header(&f, &Limits::default()), Err(Error::Corrupt(_)))); + assert!(matches!(full(&f), Err(Error::Corrupt(_)))); + assert!(started.elapsed() < std::time::Duration::from_secs(5), "walk did not stay within budget"); + let mut rec = BTreeMap::new(); + let mut budget = MAX_RECORDS; + walk(&f, 26, len as usize, o, 0, &mut budget, &mut rec); + assert_eq!(budget, 0); + assert!(rec.is_empty()); + } +} + +/// A no-plane G1 frame that compresses well closes with the record's real `ff d9 00` trailer and +/// is still shorter than plane + header — the shape of a lens-cap or blown-out frame. Rule (a) +/// read that `ff d9` as a plane byte, reported 12 bits and then refused the file. +#[test] +fn a_short_powershot_payload_closing_with_the_trailer_has_no_plane() { + let o = Order::Le; + let (w, h) = (64u16, 64u16); + let n = w as usize * h as usize; + let tables = tables::tables(0).expect("selector 0"); + let entropy = encode_eob(n, &tables); + let p = payload_with_trailer(None, &entropy); + let span = n.div_ceil(4) + 514; + assert!(p.len() < span, "the payload must be shorter than plane + header: {} vs {span}", p.len()); + assert!(p.windows(2).any(|q| q[0] == 0xff && q[1] != 0x00), "the trailer is the unstuffed ff pair"); + + let f = file_with_payload(w, h, "Canon PowerShot G1", &p, &[], o); + let hdr = header(&f, &Limits::default()).expect("header"); + assert_eq!(hdr.bits, 10, "the trailer is not a low-bits plane"); + // Every block is one end-of-block token, so the chain values stand (512 at each row start): + // the frame decodes instead of being refused for a plane it does not have. + let img = full(&f).expect("decodes"); + assert_eq!((img.header.bits, img.samples.iter().filter(|&&v| v == 512).count()), (10, n)); + + // The same frame without the trailer was already read as 10-bit. + let f = file_with_payload(w, h, "Canon PowerShot G1", &payload(None, &entropy), &[], o); + assert_eq!(header(&f, &Limits::default()).expect("header").bits, 10); +} + +/// An end-of-block token at the head of a block emits the carry and leaves the other 63 samples +/// at their chain values: the runs the sample hashes cover, on a file the unit tests can read. +#[test] +fn end_of_block_keeps_the_chain_values() { + let o = Order::Le; + let (w, h) = (64u16, 64u16); + let n = w as usize * h as usize; + let tables = tables::tables(0).expect("selector 0"); + let f = file_with_payload(w, h, "Canon EOS D30", &payload(None, &encode_eob(n, &tables)), &[], o); + let img = full(&f).expect("decodes"); + assert_eq!(img.samples.len(), n); + assert!(img.samples.iter().all(|&v| v == 512), "every sample keeps its row's chain value"); +} + +/// Zero sensor dimensions are damage, not a request for too much memory. +#[test] +fn zero_sensor_dimensions_are_corrupt_not_a_limit() { + let f = roundtrip_file(0, 0, "Canon EOS D60", None, &[], Order::Le); + assert!(matches!(full(&f), Err(Error::Corrupt(_)))); + assert!(matches!(header(&f, &Limits::default()), Err(Error::Corrupt(_)))); +} + +/// A hostile stream can run the chain past the bit depth (here a "10-bit" body writing 1535). +/// `samples` is documented as `bits` deep, so the surplus never reaches a `1 << bits` LUT. +#[test] +fn samples_are_clamped_to_the_declared_depth() { + let f = roundtrip_file(2, 1, "Canon PowerShot G1", None, &[512, 1535], Order::Le); + let img = full(&f).expect("decodes"); + assert_eq!((img.header.bits, img.samples.as_slice()), (10, &[512u16, 1023][..])); + // the 12-bit merge of the same chain cannot leave 0..=4095 either + let low = vec![0xffu8; 1]; + let f = roundtrip_file(2, 1, "Canon EOS D60", Some(&low), &[512, 1535], Order::Le); + let img = full(&f).expect("decodes"); + assert_eq!(img.header.bits, 12); + assert!(img.samples.iter().all(|&v| v <= 4095), "{:?}", img.samples); +} + +/// The two ColorBalance layouts and every Canon setting that picks a slot. Every corpus file is +/// set to Auto, so a wrong arm would still pass on the samples alone. +#[test] +fn white_balance_slot_arms_follow_the_camera_setting() { + let o = Order::Le; + // Quadruple k is (100+k, 200, 200, 300+k): the multipliers name the slot chosen, and the last + // quadruple is black levels, which is never a slot. + let wb = |usable: u16, setting: u16| { + let mut vals = vec![(usable + 1) * 8]; + for k in 0..=usable { + vals.extend([100 + k, 200, 200, 300 + k]); + } + let mut bytes: Vec = vals.iter().flat_map(|&v| put16(v, o)).collect(); + let cb_len = bytes.len(); + let si_off = bytes.len(); + let mut shot = [0u16; 8]; + shot[7] = setting; + bytes.extend(shot.iter().flat_map(|&v| put16(v, o))); + let mut rec = BTreeMap::new(); + rec.insert(COLOR_BALANCE, (0usize, cb_len)); + rec.insert(SHOT_INFO, (si_off, 16usize)); + wb_from_color_balance(&bytes, &rec, o) + }; + let expect = |k: u16| Some([(100 + k) as f32 / 200.0, 1.0, (300 + k) as f32 / 200.0]); + // 10D/300D: Auto, Daylight, Shade, Cloudy, Tungsten, Fluorescent, Flash, Custom, Kelvin + for (setting, slot) in [(0u16, 0u16), (1, 1), (8, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (9, 8), (7, 0)] { + assert_eq!(wb(9, setting), expect(slot), "nine-slot setting {setting}"); + } + // D60: Auto, Daylight, Cloudy, Tungsten, Fluorescent, Flash, Custom. Shade and Kelvin have no + // slot of their own and fall back to Auto. + for (setting, slot) in [(0u16, 0u16), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (8, 0), (9, 0)] { + assert_eq!(wb(7, setting), expect(slot), "seven-slot setting {setting}"); + } +} + +#[test] +fn errors_display_their_cause() { + assert_eq!(Error::NotCrw.to_string(), "not a Canon CRW (CIFF) file"); + assert_eq!(corrupt("no raw data record").to_string(), "CRW: no raw data record"); + assert_eq!(Error::Unsupported("decoder table 7".into()).to_string(), "unsupported CRW: decoder table 7"); + assert_eq!(Error::LimitExceeded("image too large").to_string(), "CRW exceeds a limit: image too large"); +} diff --git a/crates/crw/tests/robust.rs b/crates/crw/tests/robust.rs new file mode 100644 index 000000000..92b5355a4 --- /dev/null +++ b/crates/crw/tests/robust.rs @@ -0,0 +1,108 @@ +//! Malformed CRW files must decode to an error, never panic: a CIFF with the records the decoder +//! reads and a pseudo-random payload, mutated by bit flips and truncation. + +use craft_crw::{Limits, decode, header, probe}; +use proptest::prelude::*; + +const W: usize = 48; +const H: usize = 20; + +/// A Canon CRW: CIFF header, a raw-data payload of pseudo-random bytes, and a nested properties +/// heap with the records the decoder reads. +fn crw() -> Vec { + let le16 = |v: u16| v.to_le_bytes(); + let le32 = |v: u32| v.to_le_bytes(); + let heap = |records: &[(u16, Vec)]| -> Vec { + let mut b = Vec::new(); + let mut entries = Vec::new(); + for (tc, v) in records { + entries.push((*tc, v.len() as u32, b.len() as u32)); + b.extend_from_slice(v); + if b.len() % 2 == 1 { + b.push(0); + } + } + let tbl = b.len() as u32; + b.extend_from_slice(&le16(entries.len() as u16)); + for (tc, l, off) in entries { + b.extend_from_slice(&le16(tc)); + b.extend_from_slice(&le32(l)); + b.extend_from_slice(&le32(off)); + } + b.extend_from_slice(&le32(tbl)); + b + }; + let sensor: Vec = [34u16, W as u16, H as u16, 0, 0, 2, 2, W as u16 - 3, H as u16 - 3].iter().flat_map(|&v| le16(v)).collect(); + let dtable: Vec = [0u32, 0, 514, 0].iter().flat_map(|&v| le32(v)).collect(); + let image_info: Vec = [W as u32, H as u32, 0, 270, 12, 12, 0].iter().flat_map(|&v| le32(v)).collect(); + let mut color_balance: Vec = vec![66]; + color_balance.extend((0..8).flat_map(|i| [1800 + i * 10, 800, 800, 960 + i * 5])); + let color_balance: Vec = color_balance.iter().flat_map(|&v| le16(v)).collect(); + let shot: Vec = [54u16; 27].iter().flat_map(|&v| le16(v)).collect(); + let props = heap(&[ + (0x1031, sensor), + (0x1835, dtable), + (0x080a, b"Canon\0Canon EOS D60\0".to_vec()), + (0x1810, image_info), + (0x180e, [1083410638u32, 0, 0].iter().flat_map(|&v| le32(v)).collect()), + (0x10a9, color_balance), + (0x102a, shot), + ]); + let payload: Vec = (0..514 + 256).map(|i| (i * 41 % 253) as u8).collect(); + let root = heap(&[(0x2005, payload), (0x300a, props)]); + let mut f = b"II".to_vec(); + f.extend_from_slice(&le32(26)); + f.extend_from_slice(b"HEAPCCDR"); + f.extend_from_slice(&le32(0x0001_0002)); + f.extend_from_slice(&[0; 8]); + f.extend_from_slice(&root); + f +} + +#[test] +fn the_synthetic_file_is_recognised() { + let f = crw(); + assert!(probe(&f)); + let hdr = header(&f, &Limits::default()).expect("header"); + assert_eq!((hdr.width, hdr.height), (W, H)); + assert_eq!(hdr.exif_orientation, 8); + assert_eq!(hdr.capture_unix, Some(1083410638)); + assert!(hdr.wb_multipliers.is_some()); + // pseudo-random entropy data decodes or is refused, never panics + let _ = decode(&f, &Limits::default()); +} + +#[test] +fn every_truncation_and_single_byte_mutation_survives() { + let good = crw(); + for n in 0..good.len() { + let _ = header(&good[..n], &Limits::default()); + let _ = decode(&good[..n], &Limits::default()); + } + for i in 0..good.len() { + let mut bad = good.clone(); + bad[i] ^= 0xa5; + let _ = decode(&bad, &Limits::default()); + bad[i] = 0xff; + let _ = decode(&bad, &Limits::default()); + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 2000, .. ProptestConfig::default() })] + + #[test] + fn mutated_files_never_panic(flips in proptest::collection::vec((any::(), any::()), 1..16), cut in any::(), max_samples in 1usize..4096) { + let mut data = crw(); + let n = data.len(); + for (i, v) in flips { + data[i % n] ^= v; + } + data.truncate(cut % (n + 1)); + let mut limits = Limits::default(); + limits.max_samples = max_samples; + let _ = header(&data, &limits); + let _ = decode(&data, &limits); + let _ = decode(&data, &Limits::default()); + } +} diff --git a/crates/crw/tests/samples.json b/crates/crw/tests/samples.json new file mode 100644 index 000000000..df90b371b --- /dev/null +++ b/crates/crw/tests/samples.json @@ -0,0 +1,140 @@ +[ + { + "name": "crw-canon-eos-d30.crw", + "body": "Canon EOS D30", + "source": "https://raw.pixls.us/getfile.php/1307/nice/Canon%20-%20EOS%20D30%20-%20RAW%20(3:2).CRW", + "sha256": "4ee482f2b6f9f1a8d370703ea04e9f2abaa49356973a7f56b229af93a78e8a2e", + "license": "CC0-1.0", + "expect": { + "width": 2224, "height": 1456, "bits": 12, "selector": 1, "cfa": "RGGB", + "active": [56, 12, 2160, 1440], "model": "Canon EOS D30", "exif_orientation": 1, "capture_unix": 1482512730, + "wb": null, "black": [127.9338, 127.8187, 128.2763, 128.4221], "white": 4095.0, + "samples_sha256": "8ac62f45a4ee04efebed7900eb46dd263dc4a03a96c4fe711dc41b6bec6078fe" + } + }, + { + "name": "crw-canon-eos-d60.crw", + "body": "Canon EOS D60", + "source": "https://raw.pixls.us/getfile.php/2010/nice/Canon%20-%20EOS%20D60%20-%20RAW%20(3:2).CRW", + "sha256": "2b98659ecff170af1dfe44f89b493139e75e995b2979686f6ff0847cfb56b97f", + "license": "CC0-1.0", + "expect": { + "width": 3152, "height": 2068, "bits": 12, "selector": 0, "cfa": "RGGB", + "active": [72, 16, 3072, 2048], "model": "Canon EOS D60", "exif_orientation": 1, "capture_unix": 1467566074, + "wb": [2.627477, 1.0, 1.0990753], "black": [127.4332, 127.0843, 127.3606, 126.9793], "white": 3984.0, + "samples_sha256": "e01a6bc5cb8f67cd066f68863a04050f183c41e45395e8d4bc3b5ba4393561bb" + } + }, + { + "name": "crw-canon-eos-10d-a.crw", + "body": "Canon EOS 10D", + "source": "https://raw.pixls.us/getfile.php/2027/nice/Canon%20-%20EOS%2010D%20-%20RAW%20(3:2).CRW", + "sha256": "b0730eb3a67cc2281a32e0b2ee104199246928d954a68b7e5b72366514c73d02", + "license": "CC0-1.0", + "expect": { + "width": 3152, "height": 2068, "bits": 12, "selector": 0, "cfa": "RGGB", + "active": [72, 16, 3072, 2048], "model": "Canon EOS 10D", "exif_orientation": 1, "capture_unix": 1228574022, + "wb": [2.09375, 1.0, 1.4915866], "black": [127.8010, 126.4857, 127.8322, 126.5070], "white": 3983.0, + "samples_sha256": "3aecb01756492569734885710dafd6ffc26421ac0b5a7fa1c151973d48cd389a" + } + }, + { + "name": "crw-canon-eos-10d-b.crw", + "body": "Canon EOS 10D", + "source": "https://raw.pixls.us/getfile.php/3164/nice/Canon%20-%20EOS%2010D%20-%20RAW%20(3:2).crw", + "sha256": "e879c94afdc64dabc88b01ce2d02628aa225259303ec71dd61f7fb0322bfea2d", + "license": "CC0-1.0", + "expect": { + "width": 3152, "height": 2068, "bits": 12, "selector": 2, "cfa": "RGGB", + "active": [72, 16, 3072, 2048], "model": "Canon EOS 10D", "exif_orientation": 1, "capture_unix": 1105175848, + "wb": [1.8184005, 1.0, 1.3529766], "black": [124.6726, 125.9032, 124.6995, 125.9359], "white": 3990.0, + "samples_sha256": "25d8c540bace7ae9b92ab631fae3c4effee9625e388af600cdf60894764d3f92" + } + }, + { + "name": "crw-canon-eos-10d-c.crw", + "body": "Canon EOS 10D", + "source": "https://raw.pixls.us/getfile.php/3165/nice/Canon%20-%20EOS%2010D%20-%20RAW%20(3:2).crw", + "sha256": "58144118854657df5cd2185df40fecf6b665fc6754fef0416c2f222b68b025c6", + "license": "CC0-1.0", + "expect": { + "width": 3152, "height": 2068, "bits": 12, "selector": 2, "cfa": "RGGB", + "active": [72, 16, 3072, 2048], "model": "Canon EOS 10D", "exif_orientation": 1, "capture_unix": 1109761444, + "wb": [1.9759471, 1.0, 1.1220685], "black": [127.7952, 128.2460, 127.8664, 128.3089], "white": 3988.0, + "samples_sha256": "c1bc8c958de399ecb5fbaeb3bd473baabbb534c5874508f9b51c3984a1319c7f" + } + }, + { + "name": "crw-canon-eos-300d-a.crw", + "body": "Canon EOS 300D", + "source": "https://raw.pixls.us/getfile.php/1983/nice/Canon%20-%20EOS%20300D%20-%20RAW%20(3:2).crw", + "sha256": "47da16e1668af5500eea63cf42cf2992d9cb3d32180b8d786f17f1e2a57f39f4", + "license": "CC0-1.0", + "expect": { + "width": 3152, "height": 2068, "bits": 12, "selector": 0, "cfa": "RGGB", + "active": [72, 16, 3072, 2048], "model": "Canon EOS 300D DIGITAL", "exif_orientation": 8, "capture_unix": 1083408238, + "wb": [1.9579327, 1.0, 1.2632211], "black": [125.1676, 124.4162, 125.1797, 124.3886], "white": 3987.0, + "samples_sha256": "87848353618212be07370d659ceaa2ab349ba79b29c73702368233be2371de65" + } + }, + { + "name": "crw-canon-eos-300d-b.crw", + "body": "Canon EOS 300D", + "source": "https://raw.pixls.us/getfile.php/3162/nice/Canon%20-%20EOS%20300D%20-%20RAW%20(3:2).CRW", + "sha256": "9e7f62239fe728046af28d0d4a8a722d35d40146a79794a5bda1775306af4383", + "license": "CC0-1.0", + "expect": { + "width": 3152, "height": 2068, "bits": 12, "selector": 2, "cfa": "RGGB", + "active": [72, 16, 3072, 2048], "model": "Canon EOS 300D DIGITAL", "exif_orientation": 8, "capture_unix": 1182188238, + "wb": [2.230908, 1.0, 1.1737823], "black": [124.8050, 124.1321, 125.1695, 124.2802], "white": 3992.0, + "samples_sha256": "b1b61526c67948b600a03a89df0a301bc220f303bf8b70c93ea657356875a715" + } + }, + { + "name": "crw-canon-eos-digital-rebel.crw", + "body": "Canon EOS Digital Rebel", + "source": "https://raw.pixls.us/getfile.php/1308/nice/Canon%20-%20EOS%20Digital%20Rebel%20-%20RAW%20(3:2).CRW", + "sha256": "d6a3131bfac390fde01b96882ec378211f04570c3252708b58f0d05b33a9fdfc", + "license": "CC0-1.0", + "expect": { + "width": 3152, "height": 2068, "bits": 12, "selector": 0, "cfa": "RGGB", + "active": [72, 16, 3072, 2048], "model": "Canon EOS DIGITAL REBEL", "exif_orientation": 1, "capture_unix": 1480372394, + "wb": [1.6094965, 1.0, 1.676627], "black": [262.0760, 260.0167, 261.7442, 260.0508], "white": 4095.0, + "samples_sha256": "ec487fc432a31a648fc8238452976fd96115ff058bc6989352bdebdf997d99aa" + } + }, + { + "name": "crw-canon-powershot-g1.crw", + "body": "Canon PowerShot G1", + "source": "https://raw.pixls.us/getfile.php/2073/nice/Canon%20-%20PowerShot%20G1%20-%20RAW%20(4:3).crw", + "sha256": "f39be5acf14057ce04527fd8d0574e57ec8ed5fed082d7bfaff763075a44ee8a", + "license": "CC0-1.0", + "expect": { + "width": 2144, "height": 1560, "bits": 10, "selector": 1, "cfa": "GBRG", + "active": [24, 15, 2048, 1536], "model": "Canon PowerShot G1", "exif_orientation": 1, "capture_unix": 1065594172, + "wb": null, "black": [28.7314, 28.7637, 28.7544, 28.6963], "white": 1023.0, + "samples_sha256": "2316b72fd8d84cff2abbac1e56d9a6ea76997ee9a4182a040d9e261735393ba2" + } + }, + { + "name": "crw-canon-powershot-g5.crw", + "body": "Canon PowerShot G5", + "source": "https://raw.pixls.us/getfile.php/1746/nice/Canon%20-%20PowerShot%20G5%20-%20RAW%20(4:3).CRW", + "sha256": "bb084223d714871b99de4f549eaf00513e30a7080feeead700252c2008060196", + "license": "CC0-1.0", + "expect": "unsupported" + }, + { + "name": "crw-canon-powershot-g6.crw", + "body": "Canon PowerShot G6", + "source": "https://raw.pixls.us/getfile.php/4226/nice/Canon%20-%20PowerShot%20G6%20-%20RAW%20(4:3).CRW", + "sha256": "244378e1f5660b6cc6cefc93a75ed4928d0b28aa6304ff327b61c748538f4fef", + "license": "CC0-1.0", + "expect": { + "width": 3160, "height": 2344, "bits": 12, "selector": 0, "cfa": "RGGB", + "active": [64, 24, 3072, 2304], "model": "Canon PowerShot G6", "exif_orientation": 1, "capture_unix": 1607955982, + "wb": null, "black": [128.9415, 128.9516, 129.0142, 129.0901], "white": 4080.0, + "samples_sha256": "17804ed2ff64873e7d24ca06c58a4c8d095aad1907affb938ebcd12b618fa7ac" + } + } +] diff --git a/crates/crw/tests/samples.rs b/crates/crw/tests/samples.rs new file mode 100644 index 000000000..1f729f02f --- /dev/null +++ b/crates/crw/tests/samples.rs @@ -0,0 +1,231 @@ +//! The CC0 sample files (raw.pixls.us) pinned in `samples.json`: each decodes to exactly the +//! sensor samples the derivation verified against the black-box reference (`clean-room/ +//! 03-verification.md`), recorded as the SHA-256 of the little-endian sample buffer, with the +//! levels, white balance and metadata the decoder read then. +//! +//! The files are not in the repository (5–11 MB each): `python scripts/fetch-crw-samples.py +//! ` downloads and verifies them, and `CRAFT_CRW_SAMPLES=` points the test at them. +//! Without the variable the test prints that it skipped and passes; `CRAFT_CRW_REQUIRE_SAMPLES=1` +//! (CI) turns the skip into a failure. +//! +//! `CRAFT_CRW_ORACLE=` additionally compares every sample against `.crw.tiff` +//! references next to the files (16-bit grey TIFFs of the whole sensor, as written by the +//! reference decoder's `unprocessed_raw -T`), the check the derivation used. Those references +//! are not distributable and live outside the repository. + +use craft_crw::{Error, Limits, decode, probe}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; + +#[derive(Deserialize)] +struct Sample { + name: String, + body: String, + sha256: String, + expect: Expect, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum Expect { + Unsupported(String), + Decoded(Decoded), +} + +#[derive(Deserialize)] +struct Decoded { + width: usize, + height: usize, + bits: u32, + selector: u32, + cfa: String, + active: [usize; 4], + model: String, + exif_orientation: u8, + capture_unix: i64, + wb: Option<[f32; 3]>, + black: [f32; 4], + white: f32, + samples_sha256: String, +} + +fn manifest() -> Vec { + serde_json::from_str(include_str!("samples.json")).expect("samples.json parses") +} + +fn hex(digest: impl AsRef<[u8]>) -> String { + digest.as_ref().iter().map(|b| format!("{b:02x}")).collect() +} + +fn samples_dir() -> Option { + match std::env::var_os("CRAFT_CRW_SAMPLES") { + Some(dir) => Some(PathBuf::from(dir)), + None if std::env::var_os("CRAFT_CRW_REQUIRE_SAMPLES").is_some_and(|v| v == "1") => { + panic!("CRAFT_CRW_REQUIRE_SAMPLES=1 but CRAFT_CRW_SAMPLES is unset") + } + None => { + eprintln!("CRAFT_CRW_SAMPLES not set; skipping the sample-file tests (scripts/fetch-crw-samples.py)"); + None + } + } +} + +#[test] +fn manifest_is_well_formed() { + let m = manifest(); + assert_eq!(m.len(), 11); + for s in &m { + assert!(s.name.ends_with(".crw") && !s.name.contains('/') && !s.name.contains('\\'), "{}", s.name); + assert_eq!(s.sha256.len(), 64, "{}", s.name); + if let Expect::Decoded(d) = &s.expect { + assert_eq!(d.samples_sha256.len(), 64, "{}", s.name); + assert!(matches!(d.bits, 10 | 12), "{}", s.name); + } else { + assert!(s.body.contains("G5"), "only the PowerShot G5 is refused: {}", s.name); + } + } +} + +#[test] +fn sample_files_decode_to_the_verified_samples() { + let Some(dir) = samples_dir() else { return }; + let mut checked = 0; + for s in manifest() { + let path = dir.join(&s.name); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + assert_eq!(hex(Sha256::digest(&bytes)), s.sha256, "{}: not the pinned CC0 file", s.name); + assert!(probe(&bytes), "{}: not recognised", s.name); + let started = std::time::Instant::now(); + let result = decode(&bytes, &Limits::default()); + let took = started.elapsed(); + match (&s.expect, result) { + (Expect::Unsupported(kind), Err(Error::Unsupported(why))) => { + assert_eq!(kind, "unsupported", "{}", s.name); + assert!(why.contains("G5"), "{}: {why}", s.name); + eprintln!("{}: refused as documented ({why})", s.name); + } + (Expect::Unsupported(_), other) => panic!("{}: expected Unsupported, got {other:?}", s.name.clone()), + (Expect::Decoded(d), Ok(img)) => { + let h = &img.header; + assert_eq!((h.width, h.height, h.bits, h.selector), (d.width, d.height, d.bits, d.selector), "{}", s.name); + assert_eq!(h.cfa.name(), d.cfa, "{}", s.name); + assert_eq!([h.active_area.x, h.active_area.y, h.active_area.width, h.active_area.height], d.active, "{}", s.name); + assert_eq!(h.model.as_deref(), Some(d.model.as_str()), "{}", s.name); + assert_eq!(h.exif_orientation, d.exif_orientation, "{}", s.name); + assert_eq!(h.capture_unix, Some(d.capture_unix), "{}", s.name); + match (h.wb_multipliers, d.wb) { + (None, None) => {} + (Some(got), Some(want)) => { + assert!(got.iter().zip(&want).all(|(a, b)| (a - b).abs() < 1e-4), "{}: wb {got:?} vs {want:?}", s.name) + } + (got, want) => panic!("{}: wb {got:?} vs {want:?}", s.name), + } + assert!( + img.black.values.iter().zip(&d.black).all(|(a, b)| (a - b).abs() < 0.01), + "{}: black {:?} vs {:?}", + s.name, + img.black.values, + d.black + ); + assert_eq!(img.white, d.white, "{}", s.name); + assert_eq!(img.samples.len(), d.width * d.height, "{}", s.name); + let mut hasher = Sha256::new(); + for v in &img.samples { + hasher.update(v.to_le_bytes()); + } + assert_eq!(hex(hasher.finalize()), d.samples_sha256, "{}: the decoded samples differ from the verified decode", s.name); + eprintln!("{}: {}x{} {}-bit selector {} EXACT in {took:.0?}", s.name, h.width, h.height, h.bits, h.selector); + } + (Expect::Decoded(_), Err(e)) => panic!("{}: {e}", s.name), + } + checked += 1; + } + assert_eq!(checked, 11); +} + +/// Minimal reader for the reference TIFFs: one 16-bit grey image, stripped, either byte order. +fn oracle_samples(bytes: &[u8]) -> (usize, usize, Vec) { + let be = match &bytes[..4] { + b"II*\0" => false, + b"MM\0*" => true, + other => panic!("not a TIFF: {other:?}"), + }; + let u16_at = |at: usize| -> u16 { + let b: [u8; 2] = bytes[at..at + 2].try_into().unwrap(); + if be { u16::from_be_bytes(b) } else { u16::from_le_bytes(b) } + }; + let u32_at = |at: usize| -> u32 { + let b: [u8; 4] = bytes[at..at + 4].try_into().unwrap(); + if be { u32::from_be_bytes(b) } else { u32::from_le_bytes(b) } + }; + let ifd = u32_at(4) as usize; + let n = u16_at(ifd) as usize; + let (mut w, mut h, mut bits, mut spp) = (0usize, 0usize, 0u16, 1u16); + let (mut offsets, mut counts) = (Vec::new(), Vec::new()); + for k in 0..n { + let e = ifd + 2 + k * 12; + let (tag, typ, cnt) = (u16_at(e), u16_at(e + 2), u32_at(e + 4) as usize); + let size = match typ { + 3 => 2, + 4 => 4, + _ => 1, + }; + let values_at = if cnt * size <= 4 { e + 8 } else { u32_at(e + 8) as usize }; + let value = |i: usize| -> usize { if typ == 3 { u16_at(values_at + i * 2) as usize } else { u32_at(values_at + i * 4) as usize } }; + match tag { + 256 => w = value(0), + 257 => h = value(0), + 258 => bits = value(0) as u16, + 277 => spp = value(0) as u16, + 273 => offsets = (0..cnt).map(value).collect(), + 279 => counts = (0..cnt).map(value).collect(), + _ => {} + } + } + assert_eq!((bits, spp), (16, 1), "oracle TIFF is 16-bit grey"); + let mut data = Vec::with_capacity(w * h); + for (&o, &c) in offsets.iter().zip(&counts) { + let strip = &bytes[o..o + c]; + data.extend(strip.as_chunks::<2>().0.iter().map(|p| if be { u16::from_be_bytes(*p) } else { u16::from_le_bytes(*p) })); + } + data.truncate(w * h); + (w, h, data) +} + +#[test] +fn oracle_tiffs_match_every_sample() { + let Some(dir) = std::env::var_os("CRAFT_CRW_ORACLE").map(PathBuf::from) else { + eprintln!("CRAFT_CRW_ORACLE not set; skipping the reference-TIFF comparison"); + return; + }; + let mut entries: Vec = std::fs::read_dir(&dir).expect("oracle directory").filter_map(|e| e.ok()).map(|e| e.path()).collect(); + entries.sort(); + let mut checked = 0; + for path in entries.iter().filter(|p| p.extension().and_then(|e| e.to_str()).is_some_and(|e| e.eq_ignore_ascii_case("crw"))) { + let name = path.file_name().and_then(|n| n.to_str()).unwrap_or_default(); + let reference: &Path = &path.with_extension("crw.tiff"); + if !reference.is_file() { + eprintln!("{name}: no reference TIFF, skipped"); + continue; + } + let bytes = std::fs::read(path).expect("read crw"); + let img = match decode(&bytes, &Limits::default()) { + Ok(img) => img, + Err(Error::Unsupported(why)) if why.contains("G5") => { + eprintln!("{name}: G5 refused as documented"); + continue; + } + Err(e) => panic!("{name}: {e}"), + }; + let (w, h, oracle) = oracle_samples(&std::fs::read(reference).expect("read oracle tiff")); + assert_eq!((img.header.width, img.header.height), (w, h), "{name}: dimensions"); + assert_eq!(oracle.len(), w * h, "{name}: oracle sample count"); + assert_eq!(img.samples.len(), w * h, "{name}: decoded sample count"); + let mismatches = img.samples.iter().zip(&oracle).filter(|(a, b)| a != b).count(); + assert_eq!(mismatches, 0, "{name}: {mismatches} mismatching samples"); + eprintln!("{name}: EXACT ({w}x{h})"); + checked += 1; + } + assert!(checked > 0, "no .crw files with references found in {}", dir.display()); +} diff --git a/crates/print/Cargo.toml b/crates/print/Cargo.toml index 1e6a63507..cd3dd0c41 100644 --- a/crates/print/Cargo.toml +++ b/crates/print/Cargo.toml @@ -14,6 +14,10 @@ craft-winprint-gdi = { git = "https://github.com/storytold/winprint.rs", rev = " [lints] workspace = true +[target.'cfg(unix)'.dependencies] +# Says why the printer list is empty when lpstat can't run (e.g. a Flatpak without CUPS tools). +log = "0.4" + [target.'cfg(windows)'.dependencies] # Printing without starting anything (storytold/pdfcraft#1107): the printer list (EnumPrinters) # and the job (Windows.Data.Pdf drawn with Direct2D into the XPS print API, options as a diff --git a/crates/print/README.md b/crates/print/README.md index 11f7184ab..547af90f8 100644 --- a/crates/print/README.md +++ b/crates/print/README.md @@ -15,6 +15,25 @@ opens Windows printer preferences and submits jobs with copies, collation, duple - **macOS/Linux:** CUPS `lpstat`, `lpoptions` and `lp`. Jobs stream through stdin. Exact-size sheets use `fit-to-page=false`; shrink-to-fit uses IPP `print-scaling=auto-fit`. +- **Flatpak:** the Freedesktop runtime has libcups but not `lpstat`/`lp`/`lpoptions`, so the app + bundles them (the apps' manifests build them from CUPS as a `cups-client` module). What each + piece is for: + - `lpstat` and `--socket=cups`: the printer list. Without either it is empty, and a missing + `lpstat` is logged. + - `--system-talk-name=org.freedesktop.Avahi`: driverless network printers (`lpstat -e`). + Without it the list keeps the spooler's queues and loses only those. + - `lp`: submitting jobs. + - Driver options (`printer_options`) come from the queue's PPD in `/etc/cups/ppd`; `lpoptions` + only marks which choices are current. Flatpak reserves `/etc`, so the host's PPDs, and with + them these options, are out of reach in a Flatpak whether or not `lpoptions` is there. + - `--filesystem=~/.cups:ro`: the user's default printer and saved options, and a server named + in their `client.conf`. A network server also needs `--share=network`. A server named only + in the host's `/etc/cups/client.conf` is invisible in the sandbox, as is all of `/etc`; + `flatpak override --user --env=CUPS_SERVER=host:port ` names it for that one app. + - A socket path in `client.conf`: Flatpak mounts the socket it chooses at + `/var/run/cups/cups.sock`, so the manifests' wrapper around the tools points CUPS there when + CUPS' socket is the one Flatpak chose, and fails when it isn't rather than reach another + server. Server names and `CUPS_SERVER` are left alone. - **Windows:** in this process, with no PowerShell, script or other program started, so printing works where App Control for Business or AppLocker enforce script rules. Printers come from `EnumPrinters` (local queues and `\\server\queue` connections) and the default from diff --git a/crates/print/src/lib.rs b/crates/print/src/lib.rs index ecf975083..accf83921 100644 --- a/crates/print/src/lib.rs +++ b/crates/print/src/lib.rs @@ -394,8 +394,19 @@ fn lpstat_e_command() -> std::process::Command { pub fn printers() -> Vec { #[cfg(all(unix, not(target_arch = "wasm32")))] { - let Some(queues) = lpstat_command().output().ok().map(|o| String::from_utf8_lossy(&o.stdout).into_owned()) else { - return Vec::new(); + let queues = match lpstat_command().output() { + Ok(o) => { + // Exits 1 with only driverless printers ("No destinations added"); `-e` still finds them. + if !o.status.success() { + log::info!("lpstat -p -d exited with {}: {}", o.status, String::from_utf8_lossy(&o.stderr).trim()); + } + String::from_utf8_lossy(&o.stdout).into_owned() + } + Err(e) => { + // A Flatpak needs lpstat/lp bundled plus the CUPS socket and Avahi (see README). + log::warn!("no printers: lpstat could not run ({e}); install the CUPS client tools"); + return Vec::new(); + } }; // A spooler without `-e`, or one whose discovery times out, only loses the driverless names. let available = lpstat_e_command().output().ok().map(|o| String::from_utf8_lossy(&o.stdout).into_owned()); diff --git a/crates/update/Cargo.toml b/crates/update/Cargo.toml index 399347480..4e074060a 100644 --- a/crates/update/Cargo.toml +++ b/crates/update/Cargo.toml @@ -12,11 +12,15 @@ workspace = true [features] default = [] -egui = ["dep:egui"] +egui = ["dep:egui", "dep:image"] local-mirror = [] +# tests/dialog_screenshots.rs: renders the dialogs to PNG with egui_kittest's wgpu renderer. +snapshot = ["egui", "egui_kittest/wgpu"] [dependencies] egui = { workspace = true, optional = true } +# Decodes the app icon the startup prompt shows (`App::icon_png`); PNG only. +image = { version = "0.25", default-features = false, features = ["png"], optional = true } serde = { workspace = true } serde_json = { workspace = true } log = "0.4" @@ -29,6 +33,9 @@ semver = { version = "1", features = ["serde"] } [dev-dependencies] flate2 = "1" +egui_kittest = "0.37" +# egui's own Hack and Ubuntu Light for the screenshots (egui is built without default fonts). +epaint_default_fonts = "0.37" [target.'cfg(unix)'.dependencies] flate2 = "1" diff --git a/crates/update/src/app.rs b/crates/update/src/app.rs index c5e0ce25c..59146cf41 100644 --- a/crates/update/src/app.rs +++ b/crates/update/src/app.rs @@ -17,12 +17,21 @@ pub struct App { pub former_names: &'static [&'static str], /// Its 128 px PNG icon, for the menu entry written beside an AppImage. `None`: no icon. pub icon_png: Option<&'static [u8]>, + /// Its accent colour (sRGB), for the update prompt's buttons and highlights: the app's brand + /// colour, as ArtCraft Launcher shows it. `None`: the default blue. + pub accent: Option<[u8; 3]>, + /// Who makes it, shown as "by " in the update prompt. [`DEVELOPER`] unless the app + /// says otherwise. + pub developer: &'static str, } +/// The developer of every app in the storytold organisation. +pub const DEVELOPER: &str = "ArtCraft"; + impl App { /// An app with no former names and no icon. pub const fn new(id: &'static str, name: &'static str, tagline: &'static str, repo: &'static str) -> Self { - Self { id, name, tagline, repo, former_ids: &[], former_names: &[], icon_png: None } + Self { id, name, tagline, repo, former_ids: &[], former_names: &[], icon_png: None, accent: None, developer: DEVELOPER } } /// `ai.storyteller.`: macOS bundle id, Flatpak id and Linux desktop-file id. diff --git a/crates/update/src/dialog.rs b/crates/update/src/dialog.rs index d6bf1dd4b..66dd956cb 100644 --- a/crates/update/src/dialog.rs +++ b/crates/update/src/dialog.rs @@ -4,6 +4,7 @@ use egui::{Align, Align2, Color32, Layout, RichText, Vec2}; use crate::app::App; use crate::release::Release; +use crate::updater::Status; /// What the user chose in the startup update prompt. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -45,23 +46,14 @@ impl StartupPrompt { let mut response = StartupPromptResponse::Pending; - // Modal dimmed background overlay; it senses clicks so they don't reach widgets underneath. - let screen_rect = ctx.content_rect(); - egui::Area::new(egui::Id::unique("craft_update_backdrop")).order(egui::Order::Middle).fixed_pos(screen_rect.min).show(ctx, |ui| { - let (rect, _) = ui.allocate_exact_size(screen_rect.size(), egui::Sense::click_and_drag()); - ui.painter().rect_filled(rect, 0.0, Color32::from_black_alpha(150)); - }); + backdrop(ctx); if ctx.input_mut(|i| i.consume_key(egui::Modifiers::NONE, egui::Key::Escape)) { response = StartupPromptResponse::Later; } - let is_dark = ctx.theme() == egui::Theme::Dark; - let card_bg = if is_dark { Color32::from_rgb(26, 26, 28) } else { Color32::from_rgb(255, 255, 255) }; - let border_color = if is_dark { Color32::from_rgb(52, 52, 56) } else { Color32::from_rgb(215, 215, 220) }; - let text_primary = if is_dark { Color32::from_rgb(245, 245, 248) } else { Color32::from_rgb(20, 20, 25) }; - let text_muted = if is_dark { Color32::from_rgb(150, 150, 160) } else { Color32::from_rgb(105, 105, 115) }; - let accent_color = Color32::from_rgb(52, 120, 246); + let pal = Palette::new(ctx, self.app); + let Palette { is_dark, card_bg, border_color, text_primary, text_muted, fill, on_fill } = pal; egui::Area::new(egui::Id::unique("craft_update.startup_prompt")) .anchor(Align2::CENTER_CENTER, Vec2::ZERO) @@ -74,43 +66,11 @@ impl StartupPrompt { .inner_margin(egui::Margin::symmetric(20, 18)) .show(ui, |ui| { ui.set_width(400.0); - ui.spacing_mut().item_spacing = Vec2::new(8.0, 6.0); - - // Header row: App badge + title + close - ui.horizontal(|ui| { - let (badge_rect, _) = ui.allocate_exact_size(Vec2::new(34.0, 34.0), egui::Sense::hover()); - ui.painter().rect_filled(badge_rect, egui::CornerRadius::ZERO, accent_color); - let initial = self.app.name.chars().next().map(|c| c.to_string()).unwrap_or_else(|| "A".to_string()); - ui.painter().text(badge_rect.center(), Align2::CENTER_CENTER, initial, egui::FontId::monospace(18.0), Color32::WHITE); - - ui.add_space(4.0); - - ui.vertical(|ui| { - ui.spacing_mut().item_spacing.y = 2.0; - ui.horizontal(|ui| { - ui.label(RichText::new(self.app.name).size(15.0).strong().color(text_primary)); - ui.label(RichText::new("UPDATE").size(10.0).monospace().color(accent_color)); - }); - // ASCII only, like the rest of the card: an app whose fonts lack a glyph may panic - // instead of drawing a box, and this prompt shows up unasked. - let version_pill = format!("v{} to {}", self.running_version, self.release.version_label()); - ui.label(RichText::new(version_pill).size(11.0).monospace().color(text_muted)); - }); - - ui.with_layout(Layout::right_to_left(Align::Min), |ui| { - // Two strokes, not a "✕" glyph: none of the bundled faces has U+2715. - let (rect, close) = ui.allocate_exact_size(Vec2::splat(18.0), egui::Sense::click()); - close.widget_info(|| egui::WidgetInfo::labeled(egui::Role::Button, true, "Close")); - let color = if close.hovered() { text_primary } else { text_muted }; - let (center, reach) = (rect.center(), 4.5); - let stroke = egui::Stroke::new(1.5, color); - ui.painter().line_segment([center + Vec2::new(-reach, -reach), center + Vec2::new(reach, reach)], stroke); - ui.painter().line_segment([center + Vec2::new(-reach, reach), center + Vec2::new(reach, -reach)], stroke); - if close.on_hover_cursor(egui::CursorIcon::PointingHand).clicked() { - response = StartupPromptResponse::Later; - } - }); - }); + style_card(ui, &pal); + let version_line = format!("v{} to {}", self.running_version, self.release.version_label()); + if card_header(ui, self.app, &pal, true, &version_line) { + response = StartupPromptResponse::Later; + } ui.add_space(8.0); @@ -145,18 +105,29 @@ impl StartupPrompt { ui.painter().hline(hairline_rect.x_range(), hairline_rect.top(), egui::Stroke::new(1.0, border_color)); ui.add_space(8.0); - // Footer actions - ui.horizontal(|ui| { - let what_new = egui::Hyperlink::from_label_and_url( - RichText::new("What's new").size(11.5).color(accent_color), - &self.release.html_url, - ); - ui.add(what_new); + // Footer actions: a row as tall as the buttons, everything centred in it, so the link + // lines up with them (a plain horizontal row puts it at the top). + ui.with_layout(Layout::left_to_right(Align::Center), |ui| { + ui.set_min_height(26.0); + // Body text, so it reads on any card; the underline carries the accent. + let link = + ui.add(egui::Label::new(RichText::new("What's new").size(11.5).color(text_primary)).sense(egui::Sense::click())); + link.widget_info(|| egui::WidgetInfo::labeled(egui::Role::Link, true, "What's new")); + let y = link.rect.bottom() - 0.5; + ui.painter() + .line_segment([egui::pos2(link.rect.left(), y), egui::pos2(link.rect.right(), y)], egui::Stroke::new(1.0, fill)); + // Keyboard focus (Tab, then Enter or Space opens it): the buttons' 2 px ring. + if link.has_focus() { + let ring = egui::Stroke::new(2.0, text_primary); + ui.painter().rect_stroke(link.rect.expand(2.0), egui::CornerRadius::ZERO, ring, egui::StrokeKind::Outside); + } + if link.on_hover_cursor(egui::CursorIcon::PointingHand).clicked() { + ui.ctx().open_url(egui::OpenUrl::new_tab(&self.release.html_url)); + } ui.with_layout(Layout::right_to_left(Align::Center), |ui| { if self.is_installing { let dismiss_btn = egui::Button::new(RichText::new("DISMISS").size(10.5).monospace().color(text_muted)) - .stroke(egui::Stroke::new(1.0, border_color)) .corner_radius(egui::CornerRadius::ZERO) .min_size(Vec2::new(72.0, 26.0)); if ui.add(dismiss_btn).clicked() { @@ -164,28 +135,25 @@ impl StartupPrompt { } } else { if self.is_self_updating { - let update_btn = egui::Button::new( - RichText::new("UPDATE AND RESTART").size(10.5).strong().monospace().color(Color32::WHITE), - ) - .fill(accent_color) - .corner_radius(egui::CornerRadius::ZERO) - .min_size(Vec2::new(140.0, 26.0)); + let update_btn = + egui::Button::new(RichText::new("UPDATE AND RESTART").size(10.5).strong().monospace().color(on_fill)) + .fill(fill) + .corner_radius(egui::CornerRadius::ZERO) + .min_size(Vec2::new(140.0, 26.0)); if ui.add(update_btn).clicked() { response = StartupPromptResponse::UpdateAndRestart; } } else { - let dl_btn = - egui::Button::new(RichText::new("DOWNLOAD").size(10.5).strong().monospace().color(Color32::WHITE)) - .fill(accent_color) - .corner_radius(egui::CornerRadius::ZERO) - .min_size(Vec2::new(90.0, 26.0)); + let dl_btn = egui::Button::new(RichText::new("DOWNLOAD").size(10.5).strong().monospace().color(on_fill)) + .fill(fill) + .corner_radius(egui::CornerRadius::ZERO) + .min_size(Vec2::new(90.0, 26.0)); if ui.add(dl_btn).clicked() { response = StartupPromptResponse::Download; } } let later_btn = egui::Button::new(RichText::new("LATER").size(10.5).monospace().color(text_primary)) - .stroke(egui::Stroke::new(1.0, border_color)) .corner_radius(egui::CornerRadius::ZERO) .min_size(Vec2::new(65.0, 26.0)); if ui.add(later_btn).clicked() { @@ -205,6 +173,301 @@ impl StartupPrompt { } } +/// The card's colours for the current theme and app. +#[derive(Clone, Copy)] +struct Palette { + is_dark: bool, + card_bg: Color32, + border_color: Color32, + text_primary: Color32, + text_muted: Color32, + /// The app's accent. + fill: Color32, + /// Text on `fill`. + on_fill: Color32, +} + +impl Palette { + fn new(ctx: &egui::Context, app: &App) -> Self { + let is_dark = ctx.theme() == egui::Theme::Dark; + let fill = app.accent.map_or(DEFAULT_ACCENT, |[r, g, b]| Color32::from_rgb(r, g, b)); + Self { + is_dark, + card_bg: if is_dark { Color32::from_rgb(26, 26, 28) } else { Color32::from_rgb(255, 255, 255) }, + border_color: if is_dark { Color32::from_rgb(52, 52, 56) } else { Color32::from_rgb(215, 215, 220) }, + text_primary: if is_dark { Color32::from_rgb(245, 245, 248) } else { Color32::from_rgb(20, 20, 25) }, + text_muted: if is_dark { Color32::from_rgb(150, 150, 160) } else { Color32::from_rgb(105, 105, 115) }, + fill, + on_fill: ink_on(fill), + } + } +} + +/// The dimmed modal backdrop; it senses clicks so they don't reach widgets underneath. True when +/// it was clicked (a click outside the card). +fn backdrop(ctx: &egui::Context) -> bool { + let screen_rect = ctx.content_rect(); + egui::Area::new(egui::Id::unique("craft_update_backdrop")) + .order(egui::Order::Middle) + .fixed_pos(screen_rect.min) + .show(ctx, |ui| { + let (rect, response) = ui.allocate_exact_size(screen_rect.size(), egui::Sense::click_and_drag()); + ui.painter().rect_filled(rect, 0.0, Color32::from_black_alpha(150)); + response.clicked() + }) + .inner +} + +/// The host app's widget style isn't ours to rely on (a light theme's hover can be dark under +/// this card's dark text), so the card's buttons get their own states. +fn style_card(ui: &mut egui::Ui, pal: &Palette) { + let widgets = &mut ui.visuals_mut().widgets; + let (hover_bg, press_bg) = button_states(pal.is_dark); + for (state, bg) in [(&mut widgets.inactive, pal.card_bg), (&mut widgets.hovered, hover_bg), (&mut widgets.active, press_bg)] { + state.weak_bg_fill = bg; + state.bg_fill = bg; + state.bg_stroke = egui::Stroke::new(1.0, pal.border_color); + } + // egui draws a focused button (and a pressed one) in the active state: its border is + // the focus ring, in the body text colour so it shows on the card and around a fill. + widgets.active.bg_stroke = egui::Stroke::new(2.0, pal.text_primary); + ui.spacing_mut().item_spacing = Vec2::new(8.0, 6.0); +} + +/// Header row: app icon (or its initial in a badge), name, developer, `second_line` and a close +/// button. `update_pill` adds the "UPDATE" chip. True when the close button was clicked. +fn card_header(ui: &mut egui::Ui, app: &App, pal: &Palette, update_pill: bool, second_line: &str) -> bool { + let Palette { text_primary, text_muted, fill, on_fill, .. } = *pal; + let mut closed = false; + ui.horizontal(|ui| { + let (badge_rect, _) = ui.allocate_exact_size(Vec2::new(34.0, 34.0), egui::Sense::hover()); + if let Some(icon) = icon_texture(ui.ctx(), app) { + let uv = egui::Rect::from_min_max(egui::pos2(0.0, 0.0), egui::pos2(1.0, 1.0)); + ui.painter().image(icon.id(), badge_rect, uv, Color32::WHITE); + } else { + ui.painter().rect_filled(badge_rect, egui::CornerRadius::ZERO, fill); + let initial = app.name.chars().next().map(|c| c.to_string()).unwrap_or_else(|| "A".to_string()); + ui.painter().text(badge_rect.center(), Align2::CENTER_CENTER, initial, egui::FontId::monospace(18.0), on_fill); + } + + ui.add_space(4.0); + + ui.vertical(|ui| { + ui.spacing_mut().item_spacing.y = 2.0; + ui.horizontal(|ui| { + ui.label(RichText::new(app.name).size(15.0).strong().color(text_primary)); + ui.label(RichText::new(format!("by {}", app.developer)).size(11.5).color(text_muted)); + if update_pill { + // A chip, not accent-coloured text: small accent text misses WCAG's 4.5:1 for + // several apps (PdfCraft's red on the dark card, amber on the light one). + egui::Frame::new().fill(fill).inner_margin(egui::Margin::symmetric(4, 1)).show(ui, |ui| { + ui.label(RichText::new("UPDATE").size(10.0).monospace().color(on_fill)); + }); + } + }); + // ASCII only, like the rest of the card: an app whose fonts lack a glyph may panic + // instead of drawing a box, and the startup prompt shows up unasked. + ui.label(RichText::new(second_line).size(11.0).monospace().color(text_muted)); + }); + + ui.with_layout(Layout::right_to_left(Align::Min), |ui| { + // Two strokes, not a "✕" glyph: none of the bundled faces has U+2715. + let (rect, close) = ui.allocate_exact_size(Vec2::splat(18.0), egui::Sense::click()); + close.widget_info(|| egui::WidgetInfo::labeled(egui::Role::Button, true, "Close")); + let color = if close.hovered() { text_primary } else { text_muted }; + let (center, reach) = (rect.center(), 4.5); + let stroke = egui::Stroke::new(1.5, color); + ui.painter().line_segment([center + Vec2::new(-reach, -reach), center + Vec2::new(reach, reach)], stroke); + ui.painter().line_segment([center + Vec2::new(-reach, reach), center + Vec2::new(reach, -reach)], stroke); + if close.on_hover_cursor(egui::CursorIcon::PointingHand).clicked() { + closed = true; + } + }); + }); + closed +} + +/// What the user chose in the result dialog of a manual update check. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CheckResultResponse { + /// Dialog is still open and waiting for user input. + None, + /// OK, the close button, Esc or a click outside: the app stops showing the dialog. + Close, + /// Failed state only: the app starts another requested check. + CheckAgain, +} + +/// The result of a manual "Check for Updates", in the same card as [`StartupPrompt`]: a spinner +/// while checking, "newest version" when up to date, the error and a retry when it failed. +/// +/// An app shows it while a requested check is in view: open it when the user starts a check +/// (`Updater::check_now`), call [`show`](Self::show) every frame with `updater.status()` until it +/// returns [`CheckResultResponse::Close`], then stop calling it. On +/// [`CheckResultResponse::CheckAgain`] start another requested check and keep showing it. An +/// available update goes to [`StartupPrompt`] instead; the app closes this dialog when the prompt opens. +#[derive(Clone, Copy, Debug)] +pub struct CheckResultDialog { + app: &'static App, + running_version: &'static str, +} + +impl CheckResultDialog { + /// A dialog for `app`, which is running `running_version`. + pub fn new(app: &'static App, running_version: &'static str) -> Self { + Self { app, running_version } + } + + /// Draws the modal card for `status` and returns what the user chose. [`Status::Idle`] draws + /// nothing and returns [`CheckResultResponse::None`]. + pub fn show(&self, ctx: &egui::Context, status: &Status) -> CheckResultResponse { + if matches!(status, Status::Idle) { + return CheckResultResponse::None; + } + let mut response = CheckResultResponse::None; + if backdrop(ctx) { + response = CheckResultResponse::Close; + } + if ctx.input_mut(|i| i.consume_key(egui::Modifiers::NONE, egui::Key::Escape)) { + response = CheckResultResponse::Close; + } + + let pal = Palette::new(ctx, self.app); + let Palette { card_bg, border_color, text_primary, text_muted, fill, on_fill, .. } = pal; + egui::Area::new(egui::Id::unique("craft_update.check_result")).anchor(Align2::CENTER_CENTER, Vec2::ZERO).order(egui::Order::Foreground).show( + ctx, + |ui| { + egui::Frame::new() + .fill(card_bg) + .stroke(egui::Stroke::new(1.0, border_color)) + .corner_radius(egui::CornerRadius::ZERO) + .inner_margin(egui::Margin::symmetric(20, 18)) + .show(ui, |ui| { + ui.set_width(400.0); + style_card(ui, &pal); + if card_header(ui, self.app, &pal, false, &format!("v{}", self.running_version)) { + response = CheckResultResponse::Close; + } + ui.add_space(8.0); + + match status { + Status::Checking => { + ui.horizontal(|ui| { + ui.spinner(); + ui.label(RichText::new("Checking for updates…").size(12.5).color(text_primary)); + }); + } + Status::UpToDate => { + let text = format!("{} {} is the newest version available.", self.app.name, self.running_version); + ui.label(RichText::new(text).size(12.5).color(text_muted)); + } + Status::Failed(error) => { + let alert_color = Color32::from_rgb(225, 95, 30); + ui.label(RichText::new("Could not check for updates.").size(12.5).strong().color(alert_color)); + ui.add(egui::Label::new(RichText::new(error).size(12.0).color(text_muted)).selectable(true)); + ui.label( + RichText::new(format!("All releases are listed at {}", self.app.releases_url())).size(11.5).color(text_muted), + ); + } + Status::Idle => {} + } + + if !matches!(status, Status::Checking) { + ui.add_space(10.0); + let (hairline_rect, _) = ui.allocate_exact_size(Vec2::new(ui.available_width(), 1.0), egui::Sense::hover()); + ui.painter().hline(hairline_rect.x_range(), hairline_rect.top(), egui::Stroke::new(1.0, border_color)); + ui.add_space(8.0); + ui.with_layout(Layout::right_to_left(Align::Center), |ui| { + ui.set_min_height(26.0); + let ok = egui::Button::new(RichText::new("OK").size(10.5).strong().monospace().color(on_fill)) + .fill(fill) + .corner_radius(egui::CornerRadius::ZERO) + .min_size(Vec2::new(65.0, 26.0)); + if ui.add(ok).clicked() { + response = CheckResultResponse::Close; + } + if matches!(status, Status::Failed(_)) { + let again = egui::Button::new(RichText::new("CHECK AGAIN").size(10.5).monospace().color(text_primary)) + .corner_radius(egui::CornerRadius::ZERO) + .min_size(Vec2::new(100.0, 26.0)); + if ui.add(again).clicked() { + response = CheckResultResponse::CheckAgain; + } + } + }); + } + }); + }, + ); + response + } +} + +/// Backgrounds of an unfilled button (LATER, DISMISS) when hovered and when pressed: a step from +/// the card, so its text keeps the contrast it has at rest. +fn button_states(is_dark: bool) -> (Color32, Color32) { + if is_dark { + (Color32::from_rgb(44, 44, 48), Color32::from_rgb(58, 58, 64)) + } else { + (Color32::from_rgb(236, 236, 241), Color32::from_rgb(222, 222, 230)) + } +} + +/// The prompt's blue for an app that names no accent ([`App::accent`]). +const DEFAULT_ACCENT: Color32 = Color32::from_rgb(52, 120, 246); + +/// WCAG relative luminance of `c`: 0 for black to 1 for white. +fn luminance(c: Color32) -> f32 { + let channel = |v: u8| { + let v = f32::from(v) / 255.0; + if v <= 0.04045 { v / 12.92 } else { ((v + 0.055) / 1.055).powf(2.4) } + }; + 0.2126 * channel(c.r()) + 0.7152 * channel(c.g()) + 0.0722 * channel(c.b()) +} + +/// WCAG contrast ratio of two colours: 1 (none) to 21 (black on white). +fn contrast(a: Color32, b: Color32) -> f32 { + let (a, b) = (luminance(a), luminance(b)); + (a.max(b) + 0.05) / (a.min(b) + 0.05) +} + +/// Text for a label on `fill`: white or black, whichever contrasts more. Every Craft app accent +/// then reaches WCAG AA's 4.5:1 for the prompt's small labels (5:1 at the least). +fn ink_on(fill: Color32) -> Color32 { + if contrast(fill, Color32::WHITE) >= contrast(fill, Color32::BLACK) { Color32::WHITE } else { Color32::BLACK } +} + +/// The largest icon the prompt decodes, a side; app icons are 128 px. +const ICON_MAX_SIDE: u32 = 1024; + +/// `app`'s icon as a texture, decoded once per context and app; `None` when the app has none or +/// it doesn't decode, and the prompt shows the app's initial instead. +fn icon_texture(ctx: &egui::Context, app: &App) -> Option { + let png = app.icon_png?; + let id = egui::Id::unique(("craft_update.icon", app.id)); + if let Some(cached) = ctx.data(|d| d.get_temp::>(id)) { + return cached; + } + // The renderer's own limit too: load_texture panics (debug) on a texture larger than it takes. + let max_side = u32::try_from(ctx.input(|i| i.max_texture_side)).unwrap_or(u32::MAX).min(ICON_MAX_SIDE); + let texture = + decode_icon(png, max_side).map(|image| ctx.load_texture(format!("craft_update.icon.{}", app.id), image, egui::TextureOptions::LINEAR)); + ctx.data_mut(|d| d.insert_temp(id, texture.clone())); + texture +} + +/// A PNG as an RGBA image, or `None` when it doesn't decode or is over `max_side` a side. +fn decode_icon(png: &[u8], max_side: u32) -> Option { + let mut reader = image::ImageReader::with_format(std::io::Cursor::new(png), image::ImageFormat::Png); + let mut limits = image::Limits::default(); + limits.max_image_width = Some(max_side); + limits.max_image_height = Some(max_side); + reader.limits(limits); + let rgba = reader.decode().ok()?.to_rgba8(); + let size = [usize::try_from(rgba.width()).ok()?, usize::try_from(rgba.height()).ok()?]; + Some(egui::ColorImage::from_rgba_unmultiplied(size, rgba.as_raw())) +} + #[cfg(test)] mod tests { use super::*; @@ -290,6 +553,243 @@ mod tests { fills } + /// A `width` x `height` PNG of one opaque color. + fn png(width: u32, height: u32, color: [u8; 4]) -> Vec { + let mut out = std::io::Cursor::new(Vec::new()); + image::RgbaImage::from_pixel(width, height, image::Rgba(color)).write_to(&mut out, image::ImageFormat::Png).unwrap(); + out.into_inner() + } + + /// The textured meshes painted this frame (the app icon; text is drawn as `Shape::Text`). + fn textured_meshes(output: &FullOutput) -> Vec { + fn collect(shape: &egui::Shape, out: &mut Vec) { + match shape { + egui::Shape::Vec(shapes) => shapes.iter().for_each(|s| collect(s, out)), + egui::Shape::Mesh(mesh) if mesh.texture_id != egui::TextureId::default() => out.push(mesh.texture_id), + _ => {} + } + } + let mut ids = Vec::new(); + for clipped in &output.shapes { + collect(&clipped.shape, &mut ids); + } + ids + } + + #[test] + fn icons_decode_and_bad_ones_are_refused() { + let image = decode_icon(&png(3, 2, [10, 20, 30, 255]), ICON_MAX_SIDE).unwrap(); + assert_eq!(image.size, [3, 2]); + assert_eq!(image.pixels[0], Color32::from_rgb(10, 20, 30)); + assert!(decode_icon(b"\x89PNG\r\n\x1a\nnot really", ICON_MAX_SIDE).is_none(), "a corrupt PNG must not decode"); + assert!(decode_icon(&png(ICON_MAX_SIDE + 1, 1, [0, 0, 0, 255]), ICON_MAX_SIDE).is_none(), "an oversized icon must not decode"); + // A renderer that takes smaller textures than the icon: no texture, so the initial is shown. + assert!(decode_icon(&png(3, 2, [0, 0, 0, 255]), 2).is_none(), "an icon over the renderer's limit must not decode"); + } + + #[test] + fn prompt_shows_the_app_icon_or_else_its_initial() { + static ICON_APP: std::sync::OnceLock = std::sync::OnceLock::new(); + let icon_png: &'static [u8] = Box::leak(png(128, 128, [200, 40, 40, 255]).into_boxed_slice()); + let with_icon: &'static App = ICON_APP.get_or_init(|| App { icon_png: Some(icon_png), ..PHOTOCRAFT }); + let ctx = dark_context(); + let mut prompt = StartupPrompt { app: with_icon, ..prompt_for(false) }; + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + assert_eq!(textured_meshes(&output).len(), 1, "the app icon is drawn"); + let badge = rect_fills(&output).iter().any(|(rect, fill)| *fill == DEFAULT_ACCENT && rect.width() == 34.0); + assert!(!badge, "no letter badge beside the icon"); + assert!(!painted_text(&output).lines().any(|l| l == "P"), "no initial beside the icon"); + + // No icon (or one that won't decode): the initial in the accent badge, as before. + let ctx = dark_context(); + let mut prompt = prompt_for(false); + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + assert!(textured_meshes(&output).is_empty()); + assert!(painted_text(&output).lines().any(|l| l == "P"), "the initial stands in for a missing icon"); + } + + #[test] + fn an_apps_accent_fills_the_buttons_with_readable_text() { + // An amber accent: its buttons take it, with black text (white is under 3:1 on it). + static AMBER: std::sync::OnceLock = std::sync::OnceLock::new(); + let amber: &'static App = AMBER.get_or_init(|| App { accent: Some([0xf2, 0xa5, 0x16]), ..PHOTOCRAFT }); + let ctx = dark_context(); + let mut prompt = StartupPrompt { app: amber, ..prompt_for(false) }; + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + let fill = Color32::from_rgb(0xf2, 0xa5, 0x16); + assert!(rect_fills(&output).iter().any(|(_, f)| *f == fill), "the accent fills the button and badge"); + assert!(!rect_fills(&output).iter().any(|(_, f)| *f == DEFAULT_ACCENT), "no default blue beside an accent"); + assert_eq!(ink_on(fill), Color32::BLACK); + // Whichever of white and black contrasts more: black on the default blue, white on + // WordCraft's deeper blue and PdfCraft's red. + assert_eq!(ink_on(DEFAULT_ACCENT), Color32::BLACK); + assert_eq!(ink_on(Color32::from_rgb(0x3f, 0x5f, 0xe0)), Color32::WHITE); + assert_eq!(ink_on(Color32::from_rgb(0xb7, 0x02, 0x28)), Color32::WHITE); + // The accent underlines "What's new" (whose text is the body colour). + let underlined = line_strokes(&output).contains(&fill); + assert!(underlined, "the link's underline takes the accent"); + assert!((contrast(Color32::BLACK, Color32::WHITE) - 21.0).abs() < 0.01); + } + + #[test] + fn hovering_later_uses_the_prompts_own_background_not_the_hosts() { + // A host whose light theme hovers in black: LATER's dark text would vanish on it. + let ctx = Context::default(); + ctx.set_theme(egui::ThemePreference::Light); + ctx.global_style_mut(|style| { + style.animation_time = 0.0; + style.visuals.widgets.hovered.weak_bg_fill = Color32::BLACK; + style.visuals.widgets.hovered.bg_fill = Color32::BLACK; + }); + let mut prompt = prompt_for(false); + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + let later = text_pos(&output, "LATER").expect("LATER is painted"); + frame(&ctx, &mut prompt, vec![Event::PointerMoved(later)]); + let (_, hovered) = frame(&ctx, &mut prompt, vec![Event::PointerMoved(later)]); + let fills = rect_fills(&hovered); + assert!(fills.iter().any(|(r, f)| *f == button_states(false).0 && r.contains(later)), "LATER hovers in the prompt's grey: {fills:?}"); + assert!(!fills.iter().any(|(r, f)| *f == Color32::BLACK && r.contains(later)), "not in the host's black"); + } + + /// The strokes of the rectangles painted this frame, with each rectangle. + fn rect_strokes(output: &FullOutput) -> Vec<(Rect, egui::Stroke)> { + fn collect(shape: &egui::Shape, out: &mut Vec<(Rect, egui::Stroke)>) { + match shape { + egui::Shape::Vec(shapes) => shapes.iter().for_each(|s| collect(s, out)), + egui::Shape::Rect(rect) => out.push((rect.rect, rect.stroke)), + _ => {} + } + } + let mut strokes = Vec::new(); + for clipped in &output.shapes { + collect(&clipped.shape, &mut strokes); + } + strokes + } + + #[test] + fn a_pressed_or_focused_button_shows_a_ring() { + // Focus and a press both draw the active state; its 2 px ring must show on LATER (whose + // border used to be fixed) and on the filled DOWNLOAD. + for label in ["LATER", "DOWNLOAD"] { + let ctx = dark_context(); + let mut prompt = prompt_for(false); + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + let at = text_pos(&output, label).expect("the button is painted"); + let down = Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: true, modifiers: Modifiers::NONE }; + frame(&ctx, &mut prompt, vec![Event::PointerMoved(at)]); + let (_, pressed) = frame(&ctx, &mut prompt, vec![down]); + let ring = rect_strokes(&pressed).iter().any(|(r, s)| r.contains(at) && s.width == 2.0 && s.color == Color32::from_rgb(245, 245, 248)); + assert!(ring, "{label} shows the focus ring while pressed"); + } + } + + #[test] + fn a_focused_whats_new_link_shows_a_ring() { + let ctx = dark_context(); + let mut prompt = prompt_for(false); + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + let link = text_pos(&output, "What's new").expect("the link is painted"); + let tab = Event::Key { key: Key::Tab, physical_key: None, pressed: true, repeat: false, modifiers: Modifiers::NONE }; + let ringed = (0..8).any(|_| { + let (_, out) = frame(&ctx, &mut prompt, vec![tab.clone()]); + rect_strokes(&out).iter().any(|(r, s)| r.contains(link) && s.width == 2.0 && s.color == Color32::from_rgb(245, 245, 248)) + }); + assert!(ringed, "tabbing to the link shows the focus ring"); + } + + #[test] + fn whats_new_lines_up_with_the_buttons() { + let ctx = dark_context(); + let mut prompt = prompt_for(false); + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + let link = text_pos(&output, "What's new").expect("the link is painted"); + let button = text_pos(&output, "DOWNLOAD").expect("the button is painted"); + assert!((link.y - button.y).abs() <= 1.0, "link centred at {} but the button at {}", link.y, button.y); + } + + #[test] + fn clicking_download_answers_download() { + let ctx = dark_context(); + let mut prompt = prompt_for(false); + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + let at = text_pos(&output, "DOWNLOAD").expect("DOWNLOAD is painted"); + let button = |pressed| Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed, modifiers: Modifiers::NONE }; + frame(&ctx, &mut prompt, vec![Event::PointerMoved(at)]); + frame(&ctx, &mut prompt, vec![button(true)]); + let (response, _) = frame(&ctx, &mut prompt, vec![button(false)]); + assert_eq!(response, StartupPromptResponse::Download); + } + + /// The colours of the line segments painted this frame. + fn line_strokes(output: &FullOutput) -> Vec { + fn collect(shape: &egui::Shape, out: &mut Vec) { + match shape { + egui::Shape::Vec(shapes) => shapes.iter().for_each(|s| collect(s, out)), + egui::Shape::LineSegment { stroke, .. } => out.push(stroke.color), + _ => {} + } + } + let mut colors = Vec::new(); + for clipped in &output.shapes { + collect(&clipped.shape, &mut colors); + } + colors + } + + #[test] + fn every_craft_accent_reaches_aa_for_its_labels() { + // The 13 apps' accents (as ArtCraft Launcher shows them, PdfCraft's new red) and the default. + let accents = [ + 0x2f7bf5, 0xe8573f, 0x8b5cf6, 0xf2a516, 0xb70228, 0xe0368f, 0x7bb51c, 0x02da96, 0x3f5fe0, 0x1fa5c9, 0xf07a1e, 0x3ea6e8, 0x1f9d55, + 0x3478f6, + ]; + for hex in accents { + let fill = Color32::from_rgb((hex >> 16) as u8, (hex >> 8) as u8, hex as u8); + let ratio = contrast(fill, ink_on(fill)); + assert!(ratio >= 4.5, "#{hex:06x}: {ratio:.2}:1"); + } + } + + /// Where the frame painted `text` (inside its galley, so a pointer there is over it). + fn text_pos(output: &FullOutput, text: &str) -> Option { + fn find(shape: &egui::Shape, text: &str) -> Option { + match shape { + egui::Shape::Vec(shapes) => shapes.iter().find_map(|s| find(s, text)), + egui::Shape::Text(t) if t.galley.text() == text => Some(t.pos + t.galley.rect.center().to_vec2()), + _ => None, + } + } + output.shapes.iter().find_map(|c| find(&c.shape, text)) + } + + #[test] + fn the_prompt_names_the_developer() { + let ctx = dark_context(); + let mut prompt = prompt_for(false); + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + assert!(painted_text(&output).lines().any(|l| l == "by ArtCraft"), "every suite app is ArtCraft's by default"); + + // An app can name its own developer. + static OTHER: std::sync::OnceLock = std::sync::OnceLock::new(); + let other: &'static App = OTHER.get_or_init(|| App { developer: "Someone Else", ..PHOTOCRAFT }); + let ctx = dark_context(); + let mut prompt = StartupPrompt { app: other, ..prompt_for(false) }; + frame(&ctx, &mut prompt, Vec::new()); + let (_, output) = frame(&ctx, &mut prompt, Vec::new()); + let painted = painted_text(&output); + assert!(painted.lines().any(|l| l == "by Someone Else") && !painted.contains("by ArtCraft"), "{painted}"); + } + #[test] fn creates_startup_prompt_dialog() { let release = Release { version: crate::version::parse("0.2.0"), tag: "v0.2.0".to_string(), ..Release::default() }; @@ -402,4 +902,40 @@ mod tests { assert!(output.shapes.is_empty(), "closed prompt painted {} shapes", output.shapes.len()); assert!(painted_text(&output).is_empty()); } + + #[test] + fn check_result_paints_each_state_and_escape_closes() { + let dialog = CheckResultDialog::new(&PHOTOCRAFT, "0.1.0"); + let run = |ctx: &Context, status: &Status, events: Vec| { + // The first frame of an area only sizes it; the second paints. + let mut frames = [Vec::new(), events].into_iter().map(|events| { + let mut response = CheckResultResponse::None; + let mut output = ctx.run_ui(RawInput { screen_rect: Some(screen()), events, ..RawInput::default() }, |ui| { + response = dialog.show(ui.ctx(), status); + }); + output.textures_delta.clear(); + (response, output) + }); + frames.next(); + frames.next().unwrap() + }; + let ctx = dark_context(); + let (_, checking) = run(&ctx, &Status::Checking, Vec::new()); + assert!(painted_text(&checking).contains("Checking for updates…")); + assert!(!painted_text(&checking).contains("UPDATE"), "no update pill on a result"); + let (_, up_to_date) = run(&ctx, &Status::UpToDate, Vec::new()); + let text = painted_text(&up_to_date); + assert!(text.contains("PhotoCraft 0.1.0 is the newest version available.") && text.contains("OK"), "{text}"); + let (response, failed) = run(&ctx, &Status::Failed("no network".into()), Vec::new()); + let text = painted_text(&failed); + for want in ["Could not check for updates.", "no network", "All releases are listed at https://github.com/", "CHECK AGAIN", "OK"] { + assert!(text.contains(want), "{want} missing in {text}"); + } + assert_eq!(response, CheckResultResponse::None); + let (_, idle) = run(&ctx, &Status::Idle, Vec::new()); + assert!(painted_text(&idle).is_empty()); + + let escape = Event::Key { key: Key::Escape, physical_key: None, pressed: true, repeat: false, modifiers: Modifiers::NONE }; + assert_eq!(run(&dark_context(), &Status::UpToDate, vec![escape]).0, CheckResultResponse::Close); + } } diff --git a/crates/update/src/folders.rs b/crates/update/src/folders.rs index a144a3ad1..0fc6da8c5 100644 --- a/crates/update/src/folders.rs +++ b/crates/update/src/folders.rs @@ -40,6 +40,20 @@ pub fn data_local_dir() -> Option { } } +/// The host's per-user data folder, also from inside a Flatpak, where `XDG_DATA_HOME` points into +/// the sandbox (`~/.var/app//data`). Files other programs on the host must find go here. +pub fn host_data_local_dir() -> Option { + pick_host_data_dir(std::path::Path::new("/.flatpak-info").exists(), absolute_env("HOST_XDG_DATA_HOME"), home_dir(), data_local_dir()) +} + +/// [`host_data_local_dir`] with its inputs passed in, so every CI OS can test the Flatpak case. +fn pick_host_data_dir(in_flatpak: bool, host_xdg: Option, home: Option, local: Option) -> Option { + if in_flatpak { + return host_xdg.or_else(|| home.map(|h| h.join(".local").join("share"))); + } + local +} + /// Where per-user caches live on this machine. pub fn cache_dir() -> Option { if cfg!(target_os = "windows") { @@ -62,4 +76,15 @@ mod tests { assert!(cache_dir().is_some_and(|p| p.is_absolute())); } } + + #[test] + fn inside_a_flatpak_the_hosts_data_folder_is_used() { + let home = Some(PathBuf::from("/home/u")); + let sandboxed = Some(PathBuf::from("/home/u/.var/app/ai.storyteller.artcraft-launcher/data")); + let host = Some(PathBuf::from("/srv/u/data")); + assert_eq!(pick_host_data_dir(true, host.clone(), home.clone(), sandboxed.clone()), host); + assert_eq!(pick_host_data_dir(true, None, home.clone(), sandboxed.clone()), Some(PathBuf::from("/home/u/.local/share"))); + assert_eq!(pick_host_data_dir(false, host, home, sandboxed.clone()), sandboxed); + assert_eq!(pick_host_data_dir(true, None, None, None), None); + } } diff --git a/crates/update/src/http.rs b/crates/update/src/http.rs index 963eed1e5..6a1543483 100644 --- a/crates/update/src/http.rs +++ b/crates/update/src/http.rs @@ -11,6 +11,8 @@ use sha2::{Digest, Sha256}; /// Largest JSON or text answer we read. const MAX_SMALL: u64 = 4 << 20; +/// Largest GitHub API answer we read: 100 releases of an app with 49 assets are about 10 MB. +const MAX_API: u64 = 16 << 20; /// What a GET of a small resource produced. #[derive(Debug, PartialEq, Eq)] @@ -154,7 +156,7 @@ impl Http { match status { 200 => { let etag = header("etag"); - let text = read_text_capped(&mut resp, MAX_SMALL).map_err(|e| format!("unreadable answer from GitHub ({e})"))?; + let text = read_text_capped(&mut resp, MAX_API).map_err(|e| format!("unreadable answer from GitHub ({e})"))?; Ok(Fetched::Body { text, etag }) } 304 => Ok(Fetched::NotModified), @@ -996,6 +998,19 @@ pub(crate) mod tests { assert_eq!(http.get_feed(&url).unwrap(), "{\"ok\":true}"); } + #[test] + fn a_long_release_list_is_read_up_to_the_api_cap() { + // SolidCraft's list of 100 releases was 6.5 MB, past MAX_SMALL (craft-launcher#1021). + let long = vec![b' '; (MAX_SMALL as usize) + 1024]; + let (url, _) = serve(long.clone(), vec![Reply::Full]); + let http = Http::new(crate::TEST_AGENT).unwrap(); + assert!(matches!(http.get_api(&url, None), Ok(Fetched::Body { text, .. }) if text.len() == long.len())); + // Past MAX_API it is refused outright, not truncated and parsed. + let (url, _) = serve(vec![b' '; (MAX_API as usize) + 1024], vec![Reply::Full]); + let err = http.get_api(&url, None).unwrap_err(); + assert!(err.contains("larger than"), "{err}"); + } + #[test] fn a_gzip_bomb_answer_is_refused_before_parsing() { // A few hundred bytes on the wire that decode to more than MAX_SMALL. ureq's own limit diff --git a/crates/update/src/install.rs b/crates/update/src/install.rs index ece71d49e..f8547fe44 100644 --- a/crates/update/src/install.rs +++ b/crates/update/src/install.rs @@ -933,6 +933,86 @@ pub(crate) mod linux { kinds } + /// Flags for `flatpak install` of one bundle. A per-user install stays noninteractive. + /// A system-wide install is also noninteractive: it runs under `pkexec`, which asks for the + /// password. Flatpak's own install-bundle action does not. Its polkit message needs `$(path)`, + /// and current polkit denies the action when that detail is missing, with no dialog. + pub(crate) fn flatpak_bundle_flags(system: bool) -> &'static [&'static str] { + if system { + &["install", "--system", "-y", "--noninteractive", "--reinstall", "--bundle"] + } else { + &["install", "--user", "-y", "--noninteractive", "--reinstall", "--bundle"] + } + } + + /// Program prefix for a system-wide bundle reinstall. Inside the sandbox the host runs it, + /// because this process cannot see the host's `pkexec` or write `/var/lib/flatpak`. + pub(crate) fn system_bundle_prefix(in_sandbox: bool) -> &'static [&'static str] { + if in_sandbox { &["flatpak-spawn", "--host", "pkexec", "/usr/bin/flatpak"] } else { &["pkexec", "/usr/bin/flatpak"] } + } + + fn system_bundle_command(file: &Path) -> Result { + if !process::in_sandbox() && pkexec().is_none() { + return Err("installing system-wide needs polkit (pkexec), which isn't installed".into()); + } + let prefix = system_bundle_prefix(process::in_sandbox()); + let Some((program, rest)) = prefix.split_first() else { + return Err("couldn't start Flatpak".into()); + }; + let mut cmd = process::command(*program); + cmd.args(rest); + cmd.args(flatpak_bundle_flags(true)); + cmd.arg(file); + Ok(cmd) + } + + pub(crate) fn flatpak_bundle_error(output: &std::process::Output) -> Result<(), String> { + if output.status.success() { + return Ok(()); + } + // pkexec: 126 = the password prompt was dismissed, 127 = not authorized. + if matches!(output.status.code(), Some(126 | 127)) { + let detail = String::from_utf8_lossy(&output.stderr); + log::warn!("flatpak bundle install ended with {:?}: {}", output.status.code(), detail.trim()); + return Err(CANCELLED.into()); + } + let stderr = String::from_utf8_lossy(&output.stderr); + let err = stderr.trim(); + let err = if err.is_empty() { String::from_utf8_lossy(&output.stdout).trim().to_string() } else { err.to_string() }; + if err.is_empty() { + Err("Flatpak couldn't install the bundle (it needs the Flathub remote for its runtime)".into()) + } else { + let clean_err = err.strip_prefix("error: ").or_else(|| err.strip_prefix("Error: ")).unwrap_or(&err); + Err(format!("Flatpak couldn't install the bundle: {clean_err}")) + } + } + + /// Reinstalls a downloaded `.flatpak` bundle. `system` replaces the system-wide copy through + /// `pkexec`. Per-user stays a normal `flatpak install --user`, which needs no password. + pub(crate) fn install_flatpak_bundle(file: &Path, system: bool, stop: &Stop, phase: &mut dyn FnMut(JobPhase)) -> Result<(), String> { + install_flatpak_bundle_via(&SYSTEM, file, system, stop, phase) + } + + fn install_flatpak_bundle_via(root: &Root<'_>, file: &Path, system: bool, stop: &Stop, phase: &mut dyn FnMut(JobPhase)) -> Result<(), String> { + let install_one = || { + log::info!("executing flatpak install --reinstall --bundle for {}", file.display()); + let mut cmd = if system { + system_bundle_command(file)? + } else { + let mut cmd = process::flatpak_command(); + cmd.args(flatpak_bundle_flags(false)); + cmd.arg(file); + cmd + }; + let output = cmd.output().map_err(|e| format!("couldn't start Flatpak ({e})"))?; + if output.status.success() { + log::info!("flatpak install succeeded for {}", file.display()); + } + flatpak_bundle_error(&output) + }; + root.queue.alone(&cancelled(stop), &mut turn(phase, JobPhase::Installing), install_one).unwrap_or(Err(CANCELLED.into())) + } + /// Installs (or upgrades) a downloaded package with the system's own tools. pub fn install_package(kind: PackageKind, file: &Path, stop: &Stop, phase: &mut dyn FnMut(JobPhase)) -> Result<(), String> { if kind == PackageKind::Deb && system_tool("apt-get").is_none() { @@ -950,31 +1030,7 @@ pub(crate) mod linux { return as_root(root, command, f, stop, phase, JobPhase::Installing); } match kind { - PackageKind::Flatpak => { - let install_one = || { - log::info!("executing flatpak install --reinstall --bundle for {}", file.display()); - let output = process::flatpak_command() - .args(["install", "--user", "-y", "--noninteractive", "--reinstall", "--bundle"]) - .arg(f) - .output() - .map_err(|e| format!("couldn't start Flatpak ({e})"))?; - if output.status.success() { - log::info!("flatpak install succeeded for {}", file.display()); - Ok(()) - } else { - let stderr = String::from_utf8_lossy(&output.stderr); - let err = stderr.trim(); - let err = if err.is_empty() { String::from_utf8_lossy(&output.stdout).trim().to_string() } else { err.to_string() }; - if err.is_empty() { - Err("Flatpak couldn't install the bundle (it needs the Flathub remote for its runtime)".into()) - } else { - let clean_err = err.strip_prefix("error: ").or_else(|| err.strip_prefix("Error: ")).unwrap_or(&err); - Err(format!("Flatpak couldn't install the bundle: {clean_err}")) - } - } - }; - root.queue.alone(&cancelled(stop), &mut turn(phase, JobPhase::Installing), install_one).unwrap_or(Err(CANCELLED.into())) - } + PackageKind::Flatpak => install_flatpak_bundle_via(root, file, false, stop, phase), other => Err(format!("{} isn't a system package", other.label())), } } @@ -1138,13 +1194,7 @@ pub(crate) mod linux { /// Where menu entries and icons go: `$XDG_DATA_HOME`, else `~/.local/share`. Inside the /// launcher's own Flatpak that variable points into the sandbox, so the host's is used. pub fn host_data_dir() -> Option { - if Path::new("/.flatpak-info").exists() { - if let Some(dir) = std::env::var_os("HOST_XDG_DATA_HOME").map(PathBuf::from).filter(|p| p.is_absolute()) { - return Some(dir); - } - return crate::folders::home_dir().map(|h| h.join(".local").join("share")); - } - crate::folders::data_local_dir() + crate::folders::host_data_local_dir() } /// The app's 128 px icon, as bundled in `assets/app-icons`. @@ -1681,7 +1731,7 @@ fn reinstall_command(program: &Path, msi: &Path, context: &[&str]) -> Result PathBuf { +pub(crate) fn msi_log(msi: &Path) -> PathBuf { msi.with_extension("log") } @@ -1738,9 +1788,15 @@ pub mod windows { /// Runs an app's own new MSI for a self-update, with the same progress window and no restart /// as [`run_msi`], and waits for it. `context` keeps the installed copy's Windows Installer - /// context. A failure names the installer's log, which is the only place 1603 is explained. - pub fn reinstall(msi: &Path, context: &[&str]) -> Result<(), String> { - let status = reinstall_command(&msiexec(), msi, context)?.status().map_err(|e| format!("couldn't start the Windows Installer ({e})"))?; + /// context; `folder`, when given, is where it goes. A failure names the installer's log, + /// which is the only place 1603 is explained. + pub fn reinstall(msi: &Path, context: &[&str], folder: Option<&Path>) -> Result<(), String> { + use std::os::windows::process::CommandExt; + let mut command = reinstall_command(&msiexec(), msi, context)?; + if let Some(dir) = folder { + command.raw_arg(msi_folder_argument(dir)?); + } + let status = command.status().map_err(|e| format!("couldn't start the Windows Installer ({e})"))?; explain(status.code()).map_err(|e| match status.code() { Some(1602) | None => e, _ => format!("{e}. Its log is at {}", msi_log(msi).display()), @@ -1753,6 +1809,20 @@ pub mod windows { INSTALLER.alone(&cancelled(stop), &mut turn(phase, working), work).unwrap_or_else(|| Err(CANCELLED.into())) } + /// Removes one installed product by its code, with a progress window and no restart, and waits. + /// A per-user product asks for no permission. `log` receives the installer's verbose log. + pub fn remove_product(product_code: &str, log: &Path) -> Result<(), String> { + if !crate::msi_context::valid_product_code(product_code) { + return Err(format!("{product_code} isn't a Windows Installer product code")); + } + let status = process::command(msiexec()) + .args(["/x", product_code, "/passive", "/norestart", "/l*v"]) + .arg(log) + .status() + .map_err(|e| format!("couldn't start the Windows Installer ({e})"))?; + explain(status.code()).map_err(|e| format!("{e}. Its log is at {}", log.display())) + } + pub fn uninstall_msi(app: &App, stop: &Stop, phase: &mut dyn FnMut(JobPhase)) -> Result<(), String> { let installs = crate::detect::windows::msi_installs(app); if installs.is_empty() { @@ -1775,6 +1845,31 @@ pub mod windows { mod tests { use super::*; + #[cfg(target_os = "linux")] + #[test] + fn a_system_flatpak_bundle_is_reinstalled_with_pkexec_on_that_installation() { + assert_eq!(super::linux::flatpak_bundle_flags(false), ["install", "--user", "-y", "--noninteractive", "--reinstall", "--bundle"]); + let system = super::linux::flatpak_bundle_flags(true); + assert_eq!(system, ["install", "--system", "-y", "--noninteractive", "--reinstall", "--bundle"]); + assert!(!system.contains(&"--user")); + assert_eq!(super::linux::system_bundle_prefix(true), ["flatpak-spawn", "--host", "pkexec", "/usr/bin/flatpak"]); + assert_eq!(super::linux::system_bundle_prefix(false), ["pkexec", "/usr/bin/flatpak"]); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_dismissed_system_bundle_password_cancels_and_a_flatpak_error_is_kept() { + use std::os::unix::process::ExitStatusExt; + use std::process::Output; + let dismissed = Output { status: std::process::ExitStatus::from_raw(126 << 8), stdout: Vec::new(), stderr: Vec::new() }; + assert_eq!(super::linux::flatpak_bundle_error(&dismissed).unwrap_err(), "cancelled"); + let failed = + Output { status: std::process::ExitStatus::from_raw(1 << 8), stdout: Vec::new(), stderr: b"error: something went wrong\n".to_vec() }; + assert_eq!(super::linux::flatpak_bundle_error(&failed).unwrap_err(), "Flatpak couldn't install the bundle: something went wrong"); + let ok = Output { status: std::process::ExitStatus::from_raw(0), stdout: Vec::new(), stderr: Vec::new() }; + assert!(super::linux::flatpak_bundle_error(&ok).is_ok()); + } + #[cfg(unix)] #[test] fn msi_install_uses_the_package_target_instead_of_a_symlinked_cache() { diff --git a/crates/update/src/lib.rs b/crates/update/src/lib.rs index 0da16b040..4f14c0b5a 100644 --- a/crates/update/src/lib.rs +++ b/crates/update/src/lib.rs @@ -12,8 +12,10 @@ pub mod install; pub mod manifest; pub mod msi_context; mod msi_copy; +mod msi_migrate; pub mod packages; pub mod paths; +pub mod pins; pub mod platform; #[cfg(windows)] pub mod portable; @@ -46,10 +48,12 @@ pub use time::parse_rfc3339; pub use version::{Version, is_newer, parse as parse_version}; #[cfg(feature = "egui")] -pub use dialog::{StartupPrompt, StartupPromptResponse}; +pub use dialog::{CheckResultDialog, CheckResultResponse, StartupPrompt, StartupPromptResponse}; /// Checks for an update against GitHub releases for the given app. pub fn check_for_update(app: &App, http: &Http, current_version: &str) -> Result, String> { + let version = self_update::current_version(app, current_version).map(|v| v.to_string()); + let current_version = version.as_deref().unwrap_or(current_version); let url = format!("https://api.github.com/repos/{}/releases?per_page=30", app.repo); match http.get_api(&url, None)? { crate::http::Fetched::Body { text, .. } => offered_release(&text, app.repo, current_version), diff --git a/crates/update/src/msi_copy.rs b/crates/update/src/msi_copy.rs index 05e1ca3ff..a78ad435d 100644 --- a/crates/update/src/msi_copy.rs +++ b/crates/update/src/msi_copy.rs @@ -42,20 +42,32 @@ const ANOTHER_COPY: &str = "Windows lists this app for a different account setti const NOT_ONE_PRODUCT: &str = "Windows lists this app's installation more than once or not at all, so it can't tell which copy to update. Use the app's installer instead."; -/// The `msiexec` properties for `copy`'s update. `product_codes` are the app's Windows Installer +/// The `msiexec` properties for `copy`'s update (as [`crate::msi_migrate`] uses them). `product_codes` are the app's Windows Installer /// registrations; `probe` asks Windows Installer for one product's context, and `record_scopes` /// lists the context of every uninstall record of that product, read only when the lookup cannot /// run. Exactly one product must be registered: with two, either could be the copy that runs. The /// records must agree on one context. Whichever answers, it must be the context of the hive that /// named this exe, or the registration is another copy's. +#[cfg(test)] pub(crate) fn update_arguments( copy: MsiCopy, product_codes: &[String], probe: impl FnOnce(&str) -> Result, record_scopes: impl FnOnce(&str) -> Vec, ) -> Result<&'static [&'static str], String> { + Ok(resolve(copy, product_codes, probe, record_scopes)?.map_or(&[][..], |(scope, _)| crate::msi_context::scope_arguments(scope))) +} + +/// The confirmed context and product code of `copy`, by the rules of [`update_arguments`]. `None` +/// for the launcher's own package, which keeps its package default and is never moved. +pub(crate) fn resolve( + copy: MsiCopy, + product_codes: &[String], + probe: impl FnOnce(&str) -> Result, + record_scopes: impl FnOnce(&str) -> Vec, +) -> Result, String> { let hive = match copy { - MsiCopy::Legacy => return Ok(&[]), + MsiCopy::Legacy => return Ok(None), MsiCopy::Conflicting => return Err(CONFLICTING.into()), MsiCopy::Registered { hive } => hive, }; @@ -65,7 +77,7 @@ pub(crate) fn update_arguments( if scope != hive { return Err(ANOTHER_COPY.into()); } - Ok(crate::msi_context::scope_arguments(scope)) + Ok(Some((scope, code.clone()))) } /// How a copy was installed, from what is known about it. The launcher's own per-user package in @@ -156,15 +168,16 @@ pub(crate) mod windows { super::classify(is_launcher, is_legacy(app, &exe), || match_registration(&exe, ®istered_exes(app))) } - /// The `msiexec` properties for `exe`'s update, asking Windows Installer for its context. - pub(crate) fn update_arguments(app: &App, exe: &Path) -> Result<&'static [&'static str], String> { + /// The confirmed context and product code of `exe`'s copy, asking Windows Installer; `None` + /// for the launcher's own package. See [`super::resolve`]. + pub(crate) fn resolve(app: &App, exe: &Path) -> Result, String> { let copy = find(app, exe).ok_or_else(|| format!("{} isn't registered with the Windows Installer", app.name))?; let codes: Vec = crate::detect::windows::msi_installs(app) .into_iter() .map(|m| m.product_code) .filter(|code| crate::msi_context::valid_product_code(code)) .collect(); - super::update_arguments(copy, &codes, crate::msi_context::probe, record_scopes) + super::resolve(copy, &codes, crate::msi_context::probe, record_scopes) } /// The context of every uninstall record of `code`: the machine's (both registry views) are diff --git a/crates/update/src/msi_migrate.rs b/crates/update/src/msi_migrate.rs new file mode 100644 index 000000000..5ceb90020 --- /dev/null +++ b/crates/update/src/msi_migrate.rs @@ -0,0 +1,301 @@ +//! TEMPORARY: moves an app installed for one account to an install for everyone during its own +//! update. Remove this module two releases after the app packages become per-machine only +//! (craft-libs#422, step 4); from then on there are no per-user copies left to move. +//! +//! The released packages refuse a per-machine install while a per-user copy exists, so the move +//! removes first and installs second: +//! +//! 1. Remove the per-user copy (no permission prompt). If that fails nothing has changed. +//! 2. Install the new version for everyone (Windows asks for permission). +//! 3. If that fails or permission is refused, install the new version for the account again, in +//! the folder it had. The person always keeps a working app; settings and documents live +//! outside the program folder and are never touched. +//! +//! Which copies move: one in a per-user default folder goes to `Program Files\ArtCraft\`; one +//! in a folder the person chose outside their profile moves in place; one in a chosen folder +//! inside the profile keeps updating for the account, since program files for everyone must not +//! live in one person's profile. The launcher's own package never moves. + +#![cfg_attr(not(windows), allow(dead_code))] + +use std::path::{Path, PathBuf}; + +use crate::msi_context::{MsiScope, scope_arguments}; + +/// What an MSI self-update does. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Step { + /// Update in place with these `msiexec` properties. + Update(&'static [&'static str]), + /// Remove the per-user product, install for everyone into `folder`, and fall back to the + /// account in `old_folder` if that fails. + Move { product_code: String, folder: PathBuf, old_folder: PathBuf }, +} + +/// Where this account's files live. `roots` are `USERPROFILE`, `LOCALAPPDATA` and `APPDATA`, which +/// may be redirected away from one another; `defaults` are the app's per-user default folders +/// ([`default_folders`]). All resolved the way the exe's folder is, so links and `subst` drives +/// compare as the same place. +#[derive(Clone, Debug, Default)] +pub(crate) struct Profile { + pub roots: Vec, + pub defaults: Vec, +} + +/// The per-user default folders of `app_name` under `local_app_data`: the suite folder +/// `Programs\ArtCraft\`, then the older `Programs\`. +pub(crate) fn default_folders(local_app_data: &Path, app_name: &str) -> [PathBuf; 2] { + let programs = local_app_data.join("Programs"); + [programs.join("ArtCraft").join(app_name), programs.join(app_name)] +} + +/// The step for a copy whose exe is in `dir`. `resolved` is its confirmed context and product +/// code (`None` for the launcher's own package); `program_files` is the root a new install for +/// everyone goes under, for this package's architecture. +pub(crate) fn plan(resolved: Option<(MsiScope, String)>, dir: &Path, app_name: &str, profile: &Profile, program_files: Option<&Path>) -> Step { + let Some((scope, product_code)) = resolved else { return Step::Update(&[]) }; + if scope == MsiScope::PerMachine { + return Step::Update(scope_arguments(scope)); + } + let stay = Step::Update(scope_arguments(MsiScope::PerUser)); + let folder = if profile.defaults.iter().any(|default| same_path(default, dir)) { + match program_files { + Some(root) => root.join("ArtCraft").join(app_name), + None => return stay, + } + } else if profile.roots.iter().any(|root| is_within(dir, root)) { + return stay; + } else { + dir.to_path_buf() + }; + Step::Move { product_code, folder, old_folder: dir.to_path_buf() } +} + +/// Windows paths compare without regard to case or a trailing separator. +fn normal(path: &Path) -> String { + path.as_os_str().to_string_lossy().replace('/', "\\").trim_end_matches('\\').to_ascii_lowercase() +} + +fn same_path(a: &Path, b: &Path) -> bool { + normal(a) == normal(b) +} + +/// Whether `path` is `root` or inside it, by whole folder names. +fn is_within(path: &Path, root: &Path) -> bool { + let (path, root) = (normal(path), normal(root)); + !root.is_empty() && (path == root || path.strip_prefix(&root).is_some_and(|rest| rest.starts_with('\\'))) +} + +/// Moves a copy in the only safe order: `remove` the per-user copy (stop if that fails, nothing has +/// changed), install `to_everyone`, and only if that fails `keep_log` of it and install +/// `back_to_account`. The error says where the app is now. +pub(crate) fn run_move( + remove: impl FnOnce() -> Result<(), String>, + to_everyone: impl FnOnce() -> Result<(), String>, + keep_log: impl FnOnce(), + back_to_account: impl FnOnce() -> Result<(), String>, +) -> Result<(), String> { + remove()?; + let Err(installed) = to_everyone() else { return Ok(()) }; + keep_log(); + Err(move_failure(&installed, back_to_account())) +} + +/// The error a failed move reports, saying where the app now is. +pub(crate) fn move_failure(installed: &str, rollback: Result<(), String>) -> String { + match rollback { + Ok(()) => format!("couldn't move the app to an install for everyone ({installed}); it was updated for your account instead"), + Err(again) => format!( + "couldn't move the app to an install for everyone ({installed}), and reinstalling it for your account failed too ({again}). Run its installer to get it back; your settings and documents are untouched" + ), + } +} + +#[cfg(windows)] +pub(crate) mod windows { + use std::path::{Path, PathBuf}; + + use super::{Profile, Step, plan, run_move}; + use crate::app::App; + use crate::install::windows::{reinstall, remove_product}; + use crate::msi_context::{MsiScope, scope_arguments}; + + fn env_path(name: &str) -> Option { + std::env::var_os(name).map(PathBuf::from).filter(|p| p.is_absolute()) + } + + /// The step for `exe`'s update, asking Windows Installer for its context. + pub(crate) fn step(app: &App, exe: &Path) -> Result { + let resolved = crate::msi_copy::windows::resolve(app, exe)?; + let dir = exe.parent().ok_or_else(|| format!("{} isn't in a folder", app.name))?; + let dir = dunce::canonicalize(dir).unwrap_or_else(|_| dir.to_path_buf()); + // Resolved like `dir`, whole path at a time, so a profile or a default folder reached + // through a junction or a `subst` drive still matches. + let canonical = |p: PathBuf| dunce::canonicalize(&p).unwrap_or(p); + let profile = Profile { + roots: ["USERPROFILE", "LOCALAPPDATA", "APPDATA"].into_iter().filter_map(env_path).map(canonical).collect(), + defaults: crate::folders::data_local_dir() + .map(|local| super::default_folders(&local, app.name).into_iter().map(canonical).collect()) + .unwrap_or_default(), + }; + // This process has the package's architecture (the helper is a copy of the app), so its + // own `ProgramFiles` is the right root: `Program Files (x86)` for a 32-bit app. + Ok(plan(resolved, &dir, app.name, &profile, env_path("ProgramFiles").as_deref())) + } + + /// Runs `step` with the downloaded package `msi`. + pub(crate) fn run(msi: &Path, step: &Step) -> Result<(), String> { + match step { + Step::Update(context) => reinstall(msi, context, None), + Step::Move { product_code, folder, old_folder } => run_move( + || remove_product(product_code, &msi.with_extension("remove.log")), + || reinstall(msi, scope_arguments(MsiScope::PerMachine), Some(folder)), + // The fallback writes a fresh log; keep the failed install's beside it. + || { + let _ = std::fs::rename(crate::install::msi_log(msi), msi.with_extension("everyone.log")); + }, + || reinstall(msi, scope_arguments(MsiScope::PerUser), Some(old_folder)), + ), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const CODE: &str = "{500F5254-26F4-4AAA-B90D-6F3FD9947D14}"; + + fn p(s: &str) -> PathBuf { + PathBuf::from(s) + } + + fn profile() -> Profile { + Profile { + roots: vec![p(r"C:\Users\Ann"), p(r"C:\Users\Ann\AppData\Local"), p(r"C:\Users\Ann\AppData\Roaming")], + defaults: [r"C:\Users\Ann\AppData\Local\Programs\ArtCraft\GridCraft", r"C:\Users\Ann\AppData\Local\Programs\GridCraft"].map(p).to_vec(), + } + } + + #[test] + fn the_default_folders_are_the_suite_folder_then_the_older_one() { + let local = p(r"C:\Users\Ann\AppData\Local"); + let expected = [local.join("Programs").join("ArtCraft").join("GridCraft"), local.join("Programs").join("GridCraft")]; + assert_eq!(default_folders(&local, "GridCraft"), expected); + } + + #[test] + fn a_default_folder_reached_through_a_link_still_moves_to_the_suite_folder() { + // The Windows layer resolves each default folder whole, so a linked `Programs` arrives + // here as the real path the exe's folder also resolves to. + let old = r"D:\Redirected\Programs\ArtCraft\GridCraft"; + let linked = Profile { defaults: vec![p(old)], ..profile() }; + let suite = pf().map(|root| root.join("ArtCraft").join("GridCraft")).unwrap(); + let expected = Step::Move { product_code: CODE.to_string(), folder: suite, old_folder: p(old) }; + assert_eq!(plan(user(), &p(old), "GridCraft", &linked, pf()), expected); + } + + fn user() -> Option<(MsiScope, String)> { + Some((MsiScope::PerUser, CODE.to_string())) + } + + fn moved(folder: &str, old: &str) -> Step { + Step::Move { product_code: CODE.to_string(), folder: p(folder), old_folder: p(old) } + } + + fn pf() -> Option<&'static Path> { + Some(Path::new(r"C:\Program Files")) + } + + #[test] + fn a_copy_for_everyone_and_the_launcher_update_in_place() { + let machine = Some((MsiScope::PerMachine, CODE.to_string())); + assert_eq!(plan(machine, &p(r"C:\Program Files\GridCraft"), "GridCraft", &profile(), pf()), Step::Update(&["ALLUSERS=1"])); + assert_eq!( + plan(None, &p(r"C:\Users\Ann\AppData\Local\Programs\ArtCraft\Launcher"), "ArtCraft Launcher", &profile(), pf()), + Step::Update(&[]) + ); + } + + #[test] + fn a_per_user_copy_in_a_default_folder_moves_to_the_suite_folder() { + for old in [r"C:\Users\Ann\AppData\Local\Programs\ArtCraft\GridCraft", r"c:\users\ann\appdata\local\programs\gridcraft\"] { + // Joined like the plan joins it: `\` on Windows, `/` where these tests also run. + let suite = pf().map(|root| root.join("ArtCraft").join("GridCraft")).unwrap(); + let expected = Step::Move { product_code: CODE.to_string(), folder: suite, old_folder: p(old) }; + assert_eq!(plan(user(), &p(old), "GridCraft", &profile(), pf()), expected); + } + } + + #[test] + fn a_chosen_folder_outside_the_profile_moves_in_place() { + let old = r"D:\Apps\GridCraft"; + assert_eq!(plan(user(), &p(old), "GridCraft", &profile(), pf()), moved(old, old)); + } + + #[test] + fn a_chosen_folder_inside_the_profile_keeps_updating_for_the_account() { + for dir in [r"C:\Users\Ann\Tools\GridCraft", r"C:\Users\Ann\AppData\Local\Other\GridCraft", r"C:\Users\Ann\AppData\Roaming\GridCraft"] { + assert_eq!(plan(user(), &p(dir), "GridCraft", &profile(), pf()), Step::Update(&["ALLUSERS=2", "MSIINSTALLPERUSER=1"]), "{dir}"); + } + } + + #[test] + fn a_folder_that_only_shares_a_prefix_with_the_profile_is_outside_it() { + let old = r"C:\Users\Anna\GridCraft"; + assert_eq!(plan(user(), &p(old), "GridCraft", &profile(), pf()), moved(old, old)); + } + + #[test] + fn without_a_program_files_folder_a_default_copy_stays_for_the_account() { + let dir = p(r"C:\Users\Ann\AppData\Local\Programs\ArtCraft\GridCraft"); + assert_eq!(plan(user(), &dir, "GridCraft", &profile(), None), Step::Update(&["ALLUSERS=2", "MSIINSTALLPERUSER=1"])); + } + + fn steps(results: [Result<(), String>; 3]) -> (Result<(), String>, Vec<&'static str>) { + let done = std::cell::RefCell::new(Vec::new()); + let [remove, everyone, account] = results; + let result = run_move( + || { + done.borrow_mut().push("remove"); + remove + }, + || { + done.borrow_mut().push("everyone"); + everyone + }, + || done.borrow_mut().push("keep log"), + || { + done.borrow_mut().push("account"); + account + }, + ); + (result, done.into_inner()) + } + + #[test] + fn a_move_removes_first_and_installs_for_everyone() { + assert_eq!(steps([Ok(()), Ok(()), Ok(())]), (Ok(()), vec!["remove", "everyone"])); + } + + #[test] + fn a_failed_removal_changes_nothing_else() { + let (result, done) = steps([Err("error 1603".into()), Ok(()), Ok(())]); + assert_eq!(result, Err("error 1603".into())); + assert_eq!(done, ["remove"]); + } + + #[test] + fn a_refused_install_for_everyone_puts_the_app_back_for_the_account() { + let (result, done) = steps([Ok(()), Err("cancelled".into()), Ok(())]); + assert_eq!(done, ["remove", "everyone", "keep log", "account"]); + assert!(result.is_err_and(|e| e.contains("updated for your account instead"))); + } + + #[test] + fn a_failed_move_says_where_the_app_is() { + assert!(move_failure("error 1602", Ok(())).contains("updated for your account instead")); + let both = move_failure("error 1602", Err("error 1603".into())); + assert!(both.contains("error 1603") && both.contains("Run its installer"), "{both}"); + } +} diff --git a/crates/update/src/pins.rs b/crates/update/src/pins.rs new file mode 100644 index 000000000..c0921d0f3 --- /dev/null +++ b/crates/update/src/pins.rs @@ -0,0 +1,237 @@ +//! The version each app should follow, shared by the launcher and the apps. +//! +//! The launcher writes one small file when someone chooses a version in "Choose version"; an app's +//! own update check reads it, so an app kept on an older release, or on nightlies, is not offered +//! the latest stable release every time it starts. Only the launcher writes the file; apps read. +//! +//! The file sits in a folder of its own, `ArtCraft/versions` in the host's per-user data folder +//! (`%LOCALAPPDATA%`, `~/Library/Application Support`, `$XDG_DATA_HOME` or `~/.local/share`), +//! never in the launcher's own folder: a Flatpak launcher's own folder is inside its sandbox, and a +//! Flatpak app can be given read access to this one folder alone +//! (`--filesystem=xdg-data/ArtCraft/versions:ro`) without seeing the launcher's settings. +//! Inside a Flatpak, the host's folder is used (see [`crate::folders::host_data_local_dir`]). +//! +//! Invariants: +//! - A missing, unreadable or damaged file means every app follows stable releases, as before. +//! - An entry this version cannot read (written by a newer launcher) counts as managed by the +//! launcher, never as stable, and [`Pins::save`] writes it back unchanged. +//! - Choosing stable removes the app's entry, so the file only lists apps kept on something else. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// Names a file to use in place of the default (absolute paths only). The launcher sets it for an +/// app whose home it redirects (tarball installs on Linux), so the app still finds the real file. +pub const PATH_ENV: &str = "CRAFT_VERSIONS_FILE"; + +/// The line a Flatpak app's manifest adds to its `finish-args` so it can read the file. It names +/// the folder [`Pins::default_path`] uses; the two change together or Flatpak apps lose the file. +pub const FLATPAK_GRANT: &str = "--filesystem=xdg-data/ArtCraft/versions:ro"; + +/// Which published build an app follows. A pinned version never changes on its own. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum VersionTrack { + #[default] + Stable, + Nightly, + Pinned(String), +} + +#[derive(Default, Serialize, Deserialize)] +struct File { + /// App id to its track, kept as raw values so entries from a newer launcher survive a save. + #[serde(default)] + apps: BTreeMap, +} + +/// The saved version choices. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Pins { + apps: BTreeMap, +} + +/// What an app's own update check should do. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Choice { + /// Follow stable releases: check and offer as usual. + Stable, + /// The launcher keeps this app on another track; the app offers nothing itself. + Managed(VersionTrack), + /// An entry this version cannot read; also left to the launcher. + Unknown, +} + +impl Pins { + /// The shared file (see the module notes), or the one [`PATH_ENV`] names. + pub fn default_path() -> Option { + if let Some(path) = std::env::var_os(PATH_ENV).map(PathBuf::from).filter(|p| p.is_absolute()) { + return Some(path); + } + crate::folders::host_data_local_dir().map(|d| d.join("ArtCraft").join("versions").join("versions.json")) + } + + /// Reads `path`. A missing or damaged file reads as empty: every app follows stable releases. + pub fn load(path: &Path) -> Self { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Self::default(), + Err(e) => { + log::warn!("couldn't read {} ({e}); every app follows stable releases", path.display()); + return Self::default(); + } + }; + match serde_json::from_str::(&text) { + Ok(file) => Self { apps: file.apps }, + Err(e) => { + log::warn!("{} is damaged ({e}); every app follows stable releases", path.display()); + Self::default() + } + } + } + + /// Reads the default file (see [`Self::default_path`]). + pub fn load_default() -> Self { + Self::default_path().map(|p| Self::load(&p)).unwrap_or_default() + } + + /// Writes `path` in one step, so a reader never sees half a file. + pub fn save(&self, path: &Path) -> Result<(), String> { + let file = File { apps: self.apps.clone() }; + let text = serde_json::to_string_pretty(&file).map_err(|e| format!("couldn't encode the version choices ({e})"))?; + crate::paths::write_atomic(path, text.as_bytes()) + } + + /// What `app`'s own update check should do. + pub fn choice(&self, app: &str) -> Choice { + match self.apps.get(app) { + None => Choice::Stable, + Some(raw) => match serde_json::from_value::(raw.clone()) { + Ok(VersionTrack::Stable) => Choice::Stable, + Ok(track) => Choice::Managed(track), + Err(_) => Choice::Unknown, + }, + } + } + + /// The track `app` follows, when this version can read it. + pub fn track(&self, app: &str) -> Option { + match self.choice(app) { + Choice::Stable => Some(VersionTrack::Stable), + Choice::Managed(track) => Some(track), + Choice::Unknown => None, + } + } + + /// Sets `app`'s track. Stable removes the entry. + pub fn set(&mut self, app: &str, track: &VersionTrack) { + match track { + VersionTrack::Stable => { + self.apps.remove(app); + } + other => { + if let Ok(raw) = serde_json::to_value(other) { + self.apps.insert(app.to_string(), raw); + } + } + } + } + + /// Every app with a track this version can read, stable ones excluded. + pub fn tracks(&self) -> impl Iterator { + self.apps + .iter() + .filter_map(|(id, raw)| serde_json::from_value(raw.clone()).ok().map(|t| (id.as_str(), t))) + .filter(|(_, track)| *track != VersionTrack::Stable) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pinned(tag: &str) -> VersionTrack { + VersionTrack::Pinned(tag.into()) + } + + #[test] + fn a_missing_or_damaged_file_means_every_app_follows_stable() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("versions.json"); + assert_eq!(Pins::load(&path).choice("pdfcraft"), Choice::Stable); + std::fs::write(&path, "{ not json").unwrap(); + assert_eq!(Pins::load(&path).choice("pdfcraft"), Choice::Stable); + std::fs::write(&path, r#"{"apps":"wrong shape"}"#).unwrap(); + assert_eq!(Pins::load(&path).choice("pdfcraft"), Choice::Stable); + } + + #[test] + fn choices_survive_a_save_and_only_the_named_app_is_held() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("nested").join("versions.json"); + let mut pins = Pins::default(); + pins.set("pdfcraft", &pinned("v0.13.0")); + pins.set("photocraft", &VersionTrack::Nightly); + pins.save(&path).unwrap(); + let read = Pins::load(&path); + assert_eq!(read, pins); + assert_eq!(read.choice("pdfcraft"), Choice::Managed(pinned("v0.13.0"))); + assert_eq!(read.choice("photocraft"), Choice::Managed(VersionTrack::Nightly)); + assert_eq!(read.choice("wordcraft"), Choice::Stable); + assert_eq!(read.tracks().count(), 2); + } + + #[test] + fn choosing_stable_removes_the_entry() { + let mut pins = Pins::default(); + pins.set("pdfcraft", &pinned("v0.13.0")); + pins.set("pdfcraft", &VersionTrack::Stable); + assert_eq!(pins, Pins::default()); + assert_eq!(pins.track("pdfcraft"), Some(VersionTrack::Stable)); + } + + #[test] + fn the_file_uses_the_launchers_own_shape() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("versions.json"); + std::fs::write(&path, r#"{"apps":{"pdfcraft":{"Pinned":"v0.13.0"},"filmcraft":"Nightly","wordcraft":"Stable"}}"#).unwrap(); + let pins = Pins::load(&path); + assert_eq!(pins.track("pdfcraft"), Some(pinned("v0.13.0"))); + assert_eq!(pins.track("filmcraft"), Some(VersionTrack::Nightly)); + assert_eq!(pins.choice("wordcraft"), Choice::Stable); + let listed: Vec<&str> = pins.tracks().map(|(id, _)| id).collect(); + assert_eq!(listed, ["filmcraft", "pdfcraft"], "an explicit Stable entry is not listed"); + } + + #[test] + fn an_entry_from_a_newer_launcher_is_left_to_the_launcher_and_kept_on_save() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("versions.json"); + std::fs::write(&path, r#"{"apps":{"pdfcraft":{"Beta":{"since":"v0.20.0"}}}}"#).unwrap(); + let mut pins = Pins::load(&path); + assert_eq!(pins.choice("pdfcraft"), Choice::Unknown); + assert_eq!(pins.track("pdfcraft"), None); + pins.set("wordcraft", &pinned("v1.0.0")); + pins.save(&path).unwrap(); + let text = std::fs::read_to_string(&path).unwrap(); + assert!(text.contains("Beta") && text.contains("v0.20.0"), "kept: {text}"); + assert_eq!(Pins::load(&path).choice("pdfcraft"), Choice::Unknown); + } + + #[test] + fn the_default_file_sits_in_its_own_folder_outside_the_launchers() { + // The environment override is read by the same function; it is not set in tests. + if std::env::var_os(PATH_ENV).is_some() { + return; + } + if let Some(path) = Pins::default_path() { + assert!(path.is_absolute()); + assert_eq!(path.file_name().and_then(|n| n.to_str()), Some("versions.json")); + // The folder is the one the Flatpak grant names, under the host's data folder. + let granted = FLATPAK_GRANT.strip_prefix("--filesystem=xdg-data/").and_then(|g| g.strip_suffix(":ro")).unwrap_or_default(); + let expected = granted.split('/').fold(crate::folders::host_data_local_dir().unwrap_or_default(), |p, part| p.join(part)); + assert_eq!(path.parent(), Some(expected.as_path()), "{FLATPAK_GRANT}"); + } + } +} diff --git a/crates/update/src/self_update.rs b/crates/update/src/self_update.rs index 7b92429d4..f294ff9af 100644 --- a/crates/update/src/self_update.rs +++ b/crates/update/src/self_update.rs @@ -13,7 +13,7 @@ //! | Windows portable zip | after exit, unzip over this folder (settings in `ArtCraftLauncherData` stay) | //! | Linux AppImage | rename the new AppImage over this one (fine while running), start it | //! | Linux `.deb` / `.rpm` | the system package manager, with the password prompt; then restart | -//! | Linux Flatpak | reinstall the bundle with `--reinstall` via the host portal; then restart | +//! | Linux Flatpak | reinstall the bundle on the same installation (per-user, or system-wide through `pkexec`); then restart | //! //! Anything else (FreeBSD, a development build) gets the download link instead. @@ -85,7 +85,7 @@ pub const DEV_VAR: &str = "CRAFT_UPDATE_DEV"; /// The version this app counts as (see [`PRETEND_VAR`]). pub fn current_version(app: &App, fallback_pkg_version: &str) -> Option { - let app_pretend = format!("{}_PRETEND_VERSION", app.id.replace('-', "_").to_uppercase()); + let app_pretend = pretend_var(app); std::env::var(&app_pretend) .or_else(|_| std::env::var(PRETEND_VAR)) .ok() @@ -93,6 +93,15 @@ pub fn current_version(app: &App, fallback_pkg_version: &str) -> Option .or_else(|| version::parse(fallback_pkg_version)) } +fn pretend_var(app: &App) -> String { + format!("{}_PRETEND_VERSION", app.id.replace('-', "_").to_uppercase()) +} + +/// A pretend version lasts one run, including when an update schedules a restart. +fn clear_pretend_version(cmd: &mut std::process::Command, app: &App) { + cmd.env_remove(PRETEND_VAR).env_remove(pretend_var(app)); +} + /// How long a just-installed version stays exempt from being offered again. const JUST_UPDATED_FOR: i64 = 24 * 3600; @@ -201,11 +210,18 @@ fn detect(app: &App, exe: &Path) -> Option { None } -/// The `msiexec` properties that keep `exe`'s Windows Installer context, or why its update stops. -/// Asked while the app still runs (so it can show the error) and again by the helper. +/// What `exe`'s MSI update does (update in place, or move a per-user copy to everyone), or why it +/// stops. Asked while the app still runs (so it can show the error) and again by the helper. #[cfg(windows)] -fn msi_arguments(app: &App, exe: &Path) -> Result<&'static [&'static str], String> { - crate::msi_copy::windows::update_arguments(app, exe) +fn msi_step(app: &App, exe: &Path) -> Result { + crate::msi_migrate::windows::step(app, exe) +} + +/// Whether the helper starts the app after the update step: always after a successful one, and +/// after a failed one only while its exe is still there. +#[cfg_attr(not(windows), allow(dead_code))] +fn restarts(step_succeeded: bool, exe_exists: bool) -> bool { + step_succeeded || exe_exists } /// The exe to start after an MSI update of a registered copy: the one its registrations name now @@ -220,16 +236,19 @@ fn exe_after_msi(app: &App, registered: bool, exe: &Path) -> PathBuf { } /// How the running copy at `exe` was installed on Linux, or `None` when it isn't this app's to -/// update: a Flatpak bundle installed per user, an AppImage, the deb/rpm/Arch package at -/// `/usr/bin/`, or a standalone copy outside the package managers' folders ([`linux_location`]). +/// update: a Flatpak release bundle (per user or system-wide), an AppImage, the deb/rpm/Arch +/// package at `/usr/bin/`, or a standalone copy outside the package managers' folders +/// ([`linux_location`]). #[cfg(target_os = "linux")] fn detect(app: &App, exe: &Path) -> Option { if process::in_sandbox() { - // Only the release bundle installed per user updates itself, with `flatpak install --user - // --reinstall --bundle`. A system-wide copy would gain a second, per-user one beside it; - // a copy from a repository is that repository's to update (and an unsigned bundle can't - // go into a signed one). - return (is_user_install() && from_bundle(app)).then_some(Install::Flatpak); + // A release bundle updates itself in place. Per-user uses `flatpak install --user`. + // System-wide uses `pkexec flatpak install --system`: `--user` would leave a second copy + // beside the system one, and Flatpak's own install-bundle polkit action is denied on + // current polkit when its message cannot fill in `$(path)`. A copy from a repository is + // that repository's to update, and an unsigned bundle can't go into a signed one. + let level = flatpak_level()?; + return from_bundle(app, level).then_some(Install::Flatpak); } if let Some(appimage) = std::env::var_os("APPIMAGE") { return Some(Install::AppImage(PathBuf::from(appimage))); @@ -283,14 +302,22 @@ fn linux_location(app: &App, exe: &Path) -> LinuxLocation { } } -/// Whether this sandbox runs the per-user installation's copy, from `app-path` in `/.flatpak-info`. +/// Where this sandbox's Flatpak copy is installed, from `app-path` in `/.flatpak-info`. #[cfg(target_os = "linux")] -fn is_user_install() -> bool { - std::fs::read_to_string("/.flatpak-info").is_ok_and(|info| parse_is_user_install(&info)) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FlatpakLevel { + User, + System, } #[cfg(target_os = "linux")] -fn parse_is_user_install(info: &str) -> bool { +fn flatpak_level() -> Option { + let info = std::fs::read_to_string("/.flatpak-info").ok()?; + parse_flatpak_level(&info) +} + +#[cfg(target_os = "linux")] +fn parse_flatpak_level(info: &str) -> Option { let mut in_instance = false; for line in info.lines() { let line = line.trim(); @@ -299,18 +326,28 @@ fn parse_is_user_install(info: &str) -> bool { } else if in_instance && let Some(path) = line.strip_prefix("app-path=") { // The host's path. The per-user installation is ~/.local/share/flatpak there; it can't // be looked up from in here, where XDG_DATA_HOME points into ~/.var/app instead. - return path.trim().contains("/.local/share/flatpak/"); + let path = path.trim(); + if path.contains("/.local/share/flatpak/") { + return Some(FlatpakLevel::User); + } + if path.contains("/var/lib/flatpak/") { + return Some(FlatpakLevel::System); + } + return None; } } - false + None } -/// Whether the per-user copy was installed from a `.flatpak` bundle. `/.flatpak-info` doesn't say -/// where it came from, so the host's Flatpak is asked. +/// Whether this copy was installed from a `.flatpak` bundle. `/.flatpak-info` doesn't say where +/// it came from, so the host's Flatpak is asked on the same installation the copy runs from. #[cfg(target_os = "linux")] -fn from_bundle(app: &App) -> bool { - let id = app.bundle_id(); - process::capture_flatpak(&["info", "--user", "--show-origin", &id], std::time::Duration::from_secs(3)) +fn from_bundle(app: &App, level: FlatpakLevel) -> bool { + let scope = match level { + FlatpakLevel::User => "--user", + FlatpakLevel::System => "--system", + }; + process::capture_flatpak(&["info", scope, "--show-origin", &app.bundle_id()], std::time::Duration::from_secs(3)) .is_some_and(|origin| is_bundle_origin(app, origin.trim())) } @@ -346,7 +383,8 @@ pub fn update( if let (Some(rel_v), Some(cur_v)) = (release.version.as_ref(), cur_v.as_ref()) && !version::offers_update(rel_v, cur_v) { - // A build from source is only replaced by a strictly higher release (see `offers_update`). + // Compare the effective version: an explicit pretend version also exercises installation. + // Without that opt-in, a build from source requires a strictly higher release. return Err(format!("{} {} is not newer than currently running version {}", app.name, release.version_label(), cur_v)); } let (asset, _) = pick_asset(release, &[app.id], platform, &[install.kind()]) @@ -373,24 +411,31 @@ pub fn update( let placed = install::macos::install_dmg(app, &file, Some(&existing), None, false, paths, &install::macos::Expect::release(release)); let _ = std::fs::remove_file(&file); let placed = placed?; - relaunch_after_exit(pid, &[Path::new("/usr/bin/open").as_os_str(), placed.as_os_str()]) + relaunch_after_exit(app, pid, &[Path::new("/usr/bin/open").as_os_str(), placed.as_os_str()]) } #[cfg(target_os = "linux")] Install::AppImage(path) => { replace_appimage(&file, path)?; let _ = std::fs::remove_file(&file); - relaunch_after_exit(pid, &[path.as_os_str()]) + relaunch_after_exit(app, pid, &[path.as_os_str()]) } #[cfg(target_os = "linux")] Install::Package(kind, exe) => { let result = install::linux::install_package(*kind, &file, stop, phase); let _ = std::fs::remove_file(&file); result?; - relaunch_after_exit(pid, &[exe.as_os_str()]) + relaunch_after_exit(app, pid, &[exe.as_os_str()]) } #[cfg(target_os = "linux")] Install::Flatpak => { - let result = install::linux::install_package(PackageKind::Flatpak, &file, stop, phase); + let system = match flatpak_level() { + Some(FlatpakLevel::System) => true, + Some(FlatpakLevel::User) => false, + // Re-read rather than trusting a stale guess: `--user` on a system copy + // would install a second one beside it. + None => return Err(format!("this copy of {} can't update itself; download the new version instead", app.name)), + }; + let result = install::linux::install_flatpak_bundle(&file, system, stop, phase); let _ = std::fs::remove_file(&file); result?; relaunch_flatpak_after_exit(app) @@ -412,7 +457,7 @@ pub fn update( let result = replace_standalone(app, &file, exe); let _ = std::fs::remove_file(&file); result?; - relaunch_after_exit(pid, &[exe.as_os_str()]) + relaunch_after_exit(app, pid, &[exe.as_os_str()]) } #[allow(unreachable_patterns)] _ => { @@ -496,20 +541,19 @@ fn replace_standalone(app: &App, tarball: &Path, exe: &Path) -> Result<(), Strin /// Starts `sh`, which waits for `pid` to exit and then runs `command`. Detached, so it outlives us. #[cfg(unix)] -fn relaunch_after_exit(pid: u32, command: &[&std::ffi::OsStr]) -> Result<(), String> { - process::spawn_detached(helper_command(pid, command)).map_err(|e| format!("couldn't schedule the restart ({e})")) +fn relaunch_after_exit(app: &App, pid: u32, command: &[&std::ffi::OsStr]) -> Result<(), String> { + process::spawn_detached(helper_command(app, pid, command)).map_err(|e| format!("couldn't schedule the restart ({e})")) } #[cfg(unix)] -fn helper_command(pid: u32, command: &[&std::ffi::OsStr]) -> std::process::Command { +fn helper_command(app: &App, pid: u32, command: &[&std::ffi::OsStr]) -> std::process::Command { let mut cmd = process::command("/bin/sh"); cmd.arg("-c") .arg(r#"pid=$1; shift; i=0; while kill -0 "$pid" 2>/dev/null && [ $i -lt 600 ]; do sleep 0.1; i=$((i+1)); done; exec "$@""#) .arg("sh") .arg(pid.to_string()) - .args(command) - // The new launcher reports its real version; a pretend one was for this run only. - .env_remove(PRETEND_VAR); + .args(command); + clear_pretend_version(&mut cmd, app); cmd } @@ -573,10 +617,19 @@ fn relaunch_flatpak_after_exit(app: &App) -> Result<(), String> { (None, None) => return Err(format!("{} is updated; quit and reopen it to start the new version", app.name)), }; let script = format!("({wait_clause}) &"); + process::spawn_detached(flatpak_restart_command(app, &script, &target_arg)).map_err(|e| format!("couldn't schedule the Flatpak restart ({e})")) +} + +#[cfg(target_os = "linux")] +fn flatpak_restart_command(app: &App, script: &str, target: &str) -> std::process::Command { let app_id = app.bundle_id(); + // flatpak run reapplies persistent overrides; removing inherited variables alone is insufficient. + let global_unset = format!("--unset-env={PRETEND_VAR}"); + let app_unset = format!("--unset-env={}", pretend_var(app)); let mut cmd = process::command("flatpak-spawn"); - cmd.args(["--host", "sh", "-c", &script, "sh", &target_arg, "flatpak", "run", &app_id]).env_remove(PRETEND_VAR); - process::spawn_detached(cmd).map_err(|e| format!("couldn't schedule the Flatpak restart ({e})")) + cmd.args(["--host", "sh", "-c", script, "sh", target, "flatpak", "run", &global_unset, &app_unset, &app_id]); + clear_pretend_version(&mut cmd, app); + cmd } /// The first argument of the Windows update helper: a copy of this program that finishes an @@ -637,7 +690,7 @@ const PORTABLE_STAGING: &str = "launcher-update"; /// `pid` to exit, does `finish` and starts `exe`. Run from the cache, the helper doesn't hold the /// installed exe that the step replaces. #[cfg(windows)] -fn windows_helper(_app: &App, paths: &Paths, pid: u32, exe: &Path, finish: &Finish) -> Result<(), String> { +fn windows_helper(app: &App, paths: &Paths, pid: u32, exe: &Path, finish: &Finish) -> Result<(), String> { let program = std::env::current_exe().map_err(|e| format!("couldn't find this program ({e})"))?; let dir = paths.cache.join(HELPER_DIR); // The previous update's copy, when it's still there. @@ -646,7 +699,8 @@ fn windows_helper(_app: &App, paths: &Paths, pid: u32, exe: &Path, finish: &Fini let helper = dir.join(program.file_name().ok_or("this program has no file name")?); std::fs::copy(&program, &helper).map_err(|e| format!("couldn't schedule the update ({e})"))?; let mut cmd = process::command(&helper); - cmd.args(helper_args(pid, exe, finish)).env_remove(PRETEND_VAR); + cmd.args(helper_args(pid, exe, finish)); + clear_pretend_version(&mut cmd, app); process::spawn_detached(cmd).map_err(|e| format!("couldn't schedule the update ({e})")) } @@ -678,6 +732,11 @@ pub fn finish(app: &App, args: &[OsString]) -> Option> { } // The new app, or the old one when the step failed. let exe = if done.is_ok() { exe_after_msi(app, registered, &exe) } else { exe }; + // A failed move can leave no program to start; its recorded error already says how to get + // the app back, and a restart error must not replace it. + if !restarts(done.is_ok(), exe.is_file()) { + return Some(done); + } let started = signed_like(app, &me, &exe) .and_then(|()| process::spawn_detached(std::process::Command::new(&exe)).map_err(|e| format!("couldn't start {} ({e})", exe.display()))); if let Err(e) = &started { @@ -733,7 +792,7 @@ fn windows_precheck(app: &App, install: &Install, paths: &Paths, download: &Path let me = std::env::current_exe().map_err(|e| format!("couldn't find this program ({e})"))?; match install { Install::Msi(exe) => signed_like(app, &me, exe) - .and_then(|()| msi_arguments(app, exe).map(drop)) + .and_then(|()| msi_step(app, exe).map(drop)) .and_then(|()| signed_like(app, &me, download)) .and_then(|()| own_executable(app, exe)) .and_then(|()| same_cache_as_helper(app, paths, exe)), @@ -870,9 +929,9 @@ fn run_step(app: &App, reference: &Path, pid: u32, exe: &Path, step: &Finish, wa wait_watch::returned(pid, waited); let result = if waited { match step { - Finish::Msi(msi) => signed_like(app, reference, msi) - .and_then(|()| msi_arguments(app, exe)) - .and_then(|arguments| install::windows::reinstall(msi, arguments)), + Finish::Msi(msi) => { + signed_like(app, reference, msi).and_then(|()| msi_step(app, exe)).and_then(|step| crate::msi_migrate::windows::run(msi, &step)) + } Finish::Portable { zip, tmp } => match (exe.parent(), exe.file_name()) { (Some(dir), Some(executable)) => crate::portable::unpack_over(zip, tmp, dir, executable, &|file| { // The expected executable must be signed even when its contents aren't a PE file. @@ -1489,10 +1548,38 @@ mod tests { #[cfg(unix)] #[test] fn the_restarted_launcher_does_not_inherit_a_pretend_version() { - let cmd = helper_command(1, &[std::ffi::OsStr::new("/usr/bin/true")]); + let cmd = helper_command(&crate::suite::LAUNCHER, 1, &[std::ffi::OsStr::new("/usr/bin/true")]); assert!(cmd.get_envs().any(|(k, v)| k == PRETEND_VAR && v.is_none()), "the helper must remove {PRETEND_VAR}"); } + #[test] + fn restarted_apps_do_not_inherit_their_pretend_versions() { + for app in crate::suite::SUITE.iter().chain(std::iter::once(&crate::suite::LAUNCHER)) { + let mut cmd = std::process::Command::new("unused"); + clear_pretend_version(&mut cmd, app); + for key in [PRETEND_VAR.to_string(), pretend_var(app)] { + assert!(cmd.get_envs().any(|(k, v)| k == key.as_str() && v.is_none()), "{} retained {key}", app.id); + } + } + } + + #[cfg(target_os = "linux")] + #[test] + fn flatpak_restart_unsets_persistent_pretend_version_overrides() { + for app in &crate::suite::SUITE { + let cmd = flatpak_restart_command(app, "exec \"$@\"", "123"); + let args: Vec<_> = cmd.get_args().map(|v| v.to_string_lossy().into_owned()).collect(); + let expected = [ + "flatpak".to_string(), + "run".to_string(), + format!("--unset-env={PRETEND_VAR}"), + format!("--unset-env={}", pretend_var(app)), + app.bundle_id(), + ]; + assert!(args.windows(expected.len()).any(|w| w == expected), "{}: {args:?}", app.id); + } + } + /// The helper waits for the old launcher to exit before it starts the new one. #[cfg(unix)] #[test] @@ -1500,7 +1587,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let marker = dir.path().join("started"); let mut old = std::process::Command::new("sleep").arg("1").spawn().unwrap(); - relaunch_after_exit(old.id(), &[std::ffi::OsStr::new("touch"), marker.as_os_str()]).unwrap(); + relaunch_after_exit(&crate::suite::LAUNCHER, old.id(), &[std::ffi::OsStr::new("touch"), marker.as_os_str()]).unwrap(); std::thread::sleep(std::time::Duration::from_millis(400)); assert!(!marker.exists(), "started while the old launcher was still running"); old.wait().unwrap(); @@ -1759,6 +1846,14 @@ mod tests { assert_eq!(pick(Os::Linux, Install::Package(PackageKind::ArchPkg, PathBuf::new()).kind()).as_deref(), Some(names[6])); } + #[test] + fn a_failed_step_that_removed_the_app_starts_nothing_and_keeps_its_message() { + assert!(restarts(true, true)); + assert!(restarts(true, false)); + assert!(restarts(false, true)); + assert!(!restarts(false, false)); + } + #[test] fn update_refuses_to_downgrade_when_release_is_older() { let dir = tempfile::tempdir().unwrap(); @@ -1892,19 +1987,26 @@ mod tests { assert_eq!(parse_flatpak_host_pid(ps, "999999999"), None); } - /// Only the per-user installation's copy updates itself; a system-wide one is left alone. + /// A per-user bundle and a system-wide bundle are told apart. Anything else is left alone, + /// including `flatpak run --app-path=`, which mounts an empty /app. #[cfg(target_os = "linux")] #[test] - fn only_a_per_user_install_updates_itself() { + fn a_flatpak_copy_is_updated_on_the_installation_it_runs_from() { let info = |path: &str| format!("[Application]\nname=ai.storyteller.artcraft-launcher\n\n[Instance]\ninstance-id=1\napp-path={path}\n"); - assert!(parse_is_user_install(&info("/home/alice/.local/share/flatpak/app/ai.storyteller.artcraft-launcher/x86_64/stable/0a3de1c/files"))); - assert!(!parse_is_user_install(&info("/var/lib/flatpak/app/ai.storyteller.artcraft-launcher/x86_64/stable/0a3de1c/files"))); - // `flatpak run --app-path=` mounts an empty /app. - assert!(!parse_is_user_install(&info(""))); + assert_eq!( + parse_flatpak_level(&info("/home/alice/.local/share/flatpak/app/ai.storyteller.artcraft-launcher/x86_64/stable/0a3de1c/files")), + Some(FlatpakLevel::User) + ); + assert_eq!( + parse_flatpak_level(&info("/var/lib/flatpak/app/ai.storyteller.artcraft-launcher/x86_64/stable/0a3de1c/files")), + Some(FlatpakLevel::System) + ); + assert_eq!(parse_flatpak_level(&info("")), None); + assert_eq!(parse_flatpak_level(&info("/opt/flatpak/app/ai.storyteller.artcraft-launcher/files")), None); // The key belongs to [Instance] only. - assert!(!parse_is_user_install("[Application]\napp-path=/home/alice/.local/share/flatpak/app/x/files\n")); - assert!(!parse_is_user_install("[Instance]\ninstance-id=123\n")); - assert!(!parse_is_user_install("")); + assert_eq!(parse_flatpak_level("[Application]\napp-path=/home/alice/.local/share/flatpak/app/x/files\n"), None); + assert_eq!(parse_flatpak_level("[Instance]\ninstance-id=123\n"), None); + assert_eq!(parse_flatpak_level(""), None); } /// A copy from a repository (Flathub or any other) isn't the bundle's to replace. diff --git a/crates/update/src/source.rs b/crates/update/src/source.rs index e71e8dcbd..75a46fd26 100644 --- a/crates/update/src/source.rs +++ b/crates/update/src/source.rs @@ -112,7 +112,7 @@ impl GitHubSource { Ok(Some(found(None))) } Fetched::RateLimited { reset } => { - *self.limited_until.lock().unwrap_or_else(PoisonError::into_inner) = Some(reset.unwrap_or_else(|| crate::now() + 15 * 60)); + *self.limited_until.lock().unwrap_or_else(PoisonError::into_inner) = Some(retry_at(reset, crate::now())); Ok(None) } } @@ -138,6 +138,14 @@ impl GitHubSource { } } +/// GitHub's quota refills within the hour, so a later reset is a bad answer. +const MAX_LIMIT_WAIT: i64 = 60 * 60; + +/// When to ask the API again after a refusal. +fn retry_at(reset: Option, now: i64) -> i64 { + reset.unwrap_or(now + 15 * 60).min(now + MAX_LIMIT_WAIT) +} + impl ReleaseSource for GitHubSource { fn latest(&self, repo: &str, prereleases: bool) -> Answer { let api_error = if self.limited() { @@ -196,3 +204,20 @@ impl ReleaseSource for GitHubSource { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_refusal_waits_for_the_reset_github_names() { + assert_eq!(retry_at(Some(1_000 + 600), 1_000), 1_600); + assert_eq!(retry_at(None, 1_000), 1_000 + 15 * 60); + } + + #[test] + fn a_reset_past_the_quota_window_cannot_stop_the_api_for_good() { + assert_eq!(retry_at(Some(i64::MAX), 1_000), 1_000 + 60 * 60); + assert_eq!(retry_at(Some(99_999_999_999), 1_000), 1_000 + 60 * 60); + } +} diff --git a/crates/update/src/updater.rs b/crates/update/src/updater.rs index 836bf441e..a20f14053 100644 --- a/crates/update/src/updater.rs +++ b/crates/update/src/updater.rs @@ -2,12 +2,18 @@ //! and the self-update worker. An app creates one [`Updater`], calls [`Updater::tick`] and //! [`Updater::show`] each frame, and acts on the [`Action`] it returns, so every app gets the same //! handling of failures, dropped workers, repaints and quitting. +//! +//! An app the launcher keeps on another version (see [`crate::pins`]) offers nothing itself: the +//! check reads the shared choice first, and a requested check reports [`Status::UpToDate`] while +//! [`Updater::managed`] says why. +use std::path::PathBuf; use std::sync::Arc; use std::sync::mpsc::{Receiver, TryRecvError, channel}; use std::time::Duration; use crate::dialog::{StartupPrompt, StartupPromptResponse}; +use crate::pins::{Choice, Pins}; use crate::release::Release; use crate::{App, Http, Paths, Platform, Stop, check_for_update, self_update}; @@ -55,13 +61,19 @@ pub struct Updater { restart: bool, /// An update is installed and waits for the app to exit; nothing starts a second one. installed: bool, + /// The version-choice file to read in place of [`Pins::default_path`] (tests). + pins_path: Option, + /// The launcher's choice for this app when it is not stable, as of the last check or offer. + managed: Option, } impl Updater { pub fn new(app: &'static App, version: impl Into) -> Self { + let version = version.into(); + let version = self_update::current_version(app, &version).map_or(version, |v| v.to_string()); Self { app, - version: version.into(), + version, source: None, status: Status::Idle, requested: false, @@ -71,6 +83,8 @@ impl Updater { prompt: None, restart: false, installed: false, + pins_path: None, + managed: None, } } @@ -80,6 +94,37 @@ impl Updater { self } + /// Reads the version choices from `path` instead of the launcher's file (tests). + pub fn with_pins_path(mut self, path: impl Into) -> Self { + self.pins_path = Some(path.into()); + self + } + + /// The launcher's choice for this app when it keeps it off stable releases (a pinned version, + /// nightlies, or a choice this version cannot read), as of the last check. A window answering + /// Help ▸ Check for Updates can say so instead of "newest version". + pub fn managed(&self) -> Option<&Choice> { + self.managed.as_ref() + } + + /// Reads the launcher's choice for this app; remembers it when it is not stable. + fn read_choice(&mut self) -> bool { + let pins = match &self.pins_path { + Some(path) => Pins::load(path), + None => Pins::load_default(), + }; + self.managed = Some(pins.choice(self.app.id)).filter(|c| *c != Choice::Stable); + if let Some(choice) = &self.managed { + log::info!("{} follows the launcher's choice ({choice:?}); no update offered here", self.app.name); + // A prompt opened before the pin would still install the stable release. One that is + // installing or finished stays: `tick` reports through it. + if self.install.is_none() && !self.installed { + self.prompt = None; + } + } + self.managed.is_some() + } + /// The check at start, once. Later calls do nothing. pub fn check_at_start(&mut self, ctx: &egui::Context) { if !std::mem::replace(&mut self.started, true) { @@ -96,7 +141,7 @@ impl Updater { /// Opens the startup prompt for a release the app found itself. pub fn offer(&mut self, release: Release) { - if self.install.is_none() && !self.installed { + if self.install.is_none() && !self.installed && !self.read_choice() { self.prompt = Some(StartupPrompt::new(self.app, release, self.version.clone())); } } @@ -128,6 +173,11 @@ impl Updater { if self.check.is_some() { return; } + // The file is a few bytes, read once per check; the launcher may have changed it since. + if self.read_choice() { + self.finish_check(Ok(None)); + return; + } let (tx, rx) = channel(); let (source, app, version, ctx) = (self.source.clone(), self.app, self.version.clone(), ctx.clone()); let spawned = std::thread::Builder::new().name("update-check".into()).spawn(move || { @@ -148,6 +198,15 @@ impl Updater { fn finish_check(&mut self, answer: Checked) { let requested = std::mem::take(&mut self.requested); + // The launcher may have pinned or unpinned the app while the check ran: whatever the + // answer, an app it manages offers nothing and reports nothing to do. + if self.read_choice() { + if let Err(e) = &answer { + log::info!("{} update check failed ({e}), but the launcher manages its version", self.app.name); + } + self.status = if requested { Status::UpToDate } else { Status::Idle }; + return; + } match answer { Ok(Some(release)) => { self.status = Status::Idle; @@ -262,6 +321,10 @@ impl Updater { } fn start_install(&mut self, ctx: &egui::Context) { + // The launcher may have pinned the app while the prompt was open; that closes it. + if self.install.is_none() && !self.installed && self.read_choice() { + return; + } let Some(prompt) = &mut self.prompt else { return }; if self.installed { prompt.status_message = Some(format!("The update is installed. Close {} to finish; it restarts by itself.", self.app.name)); @@ -311,12 +374,17 @@ mod tests { use super::*; use crate::suite::PHOTOCRAFT; + /// An updater that never reads the real version-choice file. fn updater(answer: Checked) -> Updater { - Updater::new(&PHOTOCRAFT, "0.1.0").with_source(Arc::new(move || answer.clone())) + Updater::new(&PHOTOCRAFT, "0.1.0").with_source(Arc::new(move || answer.clone())).with_pins_path(no_pins()) + } + + pub(super) fn no_pins() -> PathBuf { + std::env::temp_dir().join("craft-update-tests-no-such-dir").join("versions.json") } /// Ticks until the running check has reported. - fn settle(updater: &mut Updater, ctx: &egui::Context) { + pub(super) fn settle(updater: &mut Updater, ctx: &egui::Context) { let deadline = std::time::Instant::now() + Duration::from_secs(10); while updater.check.is_some() && std::time::Instant::now() < deadline { updater.tick(ctx); @@ -324,10 +392,58 @@ mod tests { } } - fn newer() -> Release { + pub(super) fn newer() -> Release { Release { tag: "v9.0.0".into(), version: crate::version::parse("9.0.0"), ..Release::default() } } + // Child processes isolate environment overrides from parallel tests without unsafe set_var. + #[test] + fn pretend_versions_trigger_startup_prompts_for_every_app() { + const CHILD: &str = "CRAFT_UPDATE_PRETEND_TEST_APP"; + if let Ok(id) = std::env::var(CHILD) { + let app = crate::suite::by_id(&id).unwrap(); + let mut u = Updater::new(app, "99.0.0").with_pins_path(no_pins()); + assert_eq!(u.version, "0.0.1", "{} ignored its pretend version", app.id); + let version = u.version.clone(); + let repo = app.repo; + u = u.with_source(Arc::new(move || { + crate::offered_release(r#"[{"tag_name":"v1.0.0","name":"Test release","draft":false,"prerelease":false}]"#, repo, &version) + })); + let ctx = egui::Context::default(); + u.check_at_start(&ctx); + settle(&mut u, &ctx); + assert!(u.prompt_open(), "{} did not offer the newer release", app.id); + // The explicit test override also applies to the pre-existing installation guard. + // No assets means the installer stops before any network request or file replacement. + let dir = tempfile::tempdir().unwrap(); + let error = self_update::update( + app, + &self_update::Install::AppImage(dir.path().join("test.AppImage")), + &newer(), + Platform { os: crate::Os::Linux, arch: crate::Arch::X64 }, + &Http::new(crate::TEST_AGENT).unwrap(), + &Paths::under(dir.path()), + &Stop::default(), + &mut |_| {}, + &|| Ok(()), + Some("99.0.0"), + ) + .unwrap_err(); + assert!(error.contains("has no"), "{}: {error}", app.id); + return; + } + for app in &crate::suite::SUITE { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "updater::tests::pretend_versions_trigger_startup_prompts_for_every_app", "--nocapture"]) + .env(CHILD, app.id) + .env(format!("{}_PRETEND_VERSION", app.id.to_uppercase()), "0.0.1") + .env_remove(self_update::PRETEND_VAR) + .status() + .unwrap(); + assert!(status.success(), "{} pretend-version regression failed", app.id); + } + } + #[test] fn a_requested_check_reports_an_up_to_date_app() { let (ctx, mut u) = (egui::Context::default(), updater(Ok(None))); @@ -394,7 +510,7 @@ mod tests { #[test] fn a_panicking_check_is_an_error_not_a_hang() { let ctx = egui::Context::default(); - let mut u = Updater::new(&PHOTOCRAFT, "0.1.0").with_source(Arc::new(|| std::panic::panic_any("boom"))); + let mut u = Updater::new(&PHOTOCRAFT, "0.1.0").with_source(Arc::new(|| std::panic::panic_any("boom"))).with_pins_path(no_pins()); u.check_now(&ctx); settle(&mut u, &ctx); assert!(matches!(u.status(), Status::Failed(_))); @@ -484,3 +600,7 @@ mod tests { assert_eq!(u.show(&ctx), Action::None); } } + +#[cfg(test)] +#[path = "updater_pin_tests.rs"] +mod pin_tests; diff --git a/crates/update/src/updater_pin_tests.rs b/crates/update/src/updater_pin_tests.rs new file mode 100644 index 000000000..2a4f58f12 --- /dev/null +++ b/crates/update/src/updater_pin_tests.rs @@ -0,0 +1,166 @@ +//! The startup prompt and requested checks follow the launcher's shared version choice. + +use std::sync::Arc; + +use super::tests::{newer, settle}; +use super::*; +use crate::suite::PHOTOCRAFT; + +/// A version-choice file holding `track` for PhotoCraft. +fn pins_with(track: crate::pins::VersionTrack) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + let mut pins = Pins::default(); + pins.set(PHOTOCRAFT.id, &track); + pins.save(&dir.path().join("versions.json")).unwrap(); + dir +} + +#[test] +fn a_pinned_app_is_not_prompted_and_the_network_is_not_asked() { + use crate::pins::VersionTrack; + let ctx = egui::Context::default(); + for track in [VersionTrack::Pinned("v0.13.0".into()), VersionTrack::Nightly] { + let dir = pins_with(track.clone()); + let mut u = Updater::new(&PHOTOCRAFT, "0.13.0") + .with_source(Arc::new(|| std::panic::panic_any("the network must not be asked"))) + .with_pins_path(dir.path().join("versions.json")); + u.check_at_start(&ctx); + settle(&mut u, &ctx); + assert!(!u.prompt_open(), "{track:?}"); + assert_eq!(u.status(), &Status::Idle); + u.check_now(&ctx); + settle(&mut u, &ctx); + assert_eq!(u.status(), &Status::UpToDate, "{track:?}"); + assert_eq!(u.managed(), Some(&Choice::Managed(track.clone()))); + u.offer(newer()); + assert!(!u.prompt_open(), "an offer is held too: {track:?}"); + } +} + +#[test] +fn unpinning_in_the_launcher_brings_updates_back_on_the_next_check() { + let ctx = egui::Context::default(); + let dir = pins_with(crate::pins::VersionTrack::Pinned("v0.13.0".into())); + let path = dir.path().join("versions.json"); + let mut u = Updater::new(&PHOTOCRAFT, "0.13.0").with_source(Arc::new(|| Ok(Some(newer())))).with_pins_path(&path); + u.check_now(&ctx); + settle(&mut u, &ctx); + assert!(!u.prompt_open()); + Pins::default().save(&path).unwrap(); + u.check_now(&ctx); + settle(&mut u, &ctx); + assert!(u.prompt_open()); + assert_eq!(u.managed(), None); +} + +#[test] +fn an_unreadable_choice_from_a_newer_launcher_also_holds_updates() { + let ctx = egui::Context::default(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("versions.json"); + std::fs::write(&path, r#"{"apps":{"photocraft":{"Beta":1}}}"#).unwrap(); + let mut u = Updater::new(&PHOTOCRAFT, "0.13.0").with_source(Arc::new(|| Ok(Some(newer())))).with_pins_path(&path); + u.check_at_start(&ctx); + settle(&mut u, &ctx); + assert!(!u.prompt_open()); + assert_eq!(u.managed(), Some(&Choice::Unknown)); +} + +#[test] +fn another_apps_pin_does_not_hold_this_one() { + let ctx = egui::Context::default(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("versions.json"); + let mut pins = Pins::default(); + pins.set("pdfcraft", &crate::pins::VersionTrack::Pinned("v0.2.0".into())); + pins.save(&path).unwrap(); + let mut u = Updater::new(&PHOTOCRAFT, "0.13.0").with_source(Arc::new(|| Ok(Some(newer())))).with_pins_path(&path); + u.check_at_start(&ctx); + settle(&mut u, &ctx); + assert!(u.prompt_open()); +} + +#[test] +fn a_prompt_opened_before_the_pin_cannot_install_the_stable_release() { + let ctx = egui::Context::default(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("versions.json"); + let mut u = Updater::new(&PHOTOCRAFT, "0.13.0").with_source(Arc::new(|| Ok(Some(newer())))).with_pins_path(&path); + u.offer(newer()); + assert!(u.prompt_open()); + let mut pins = Pins::default(); + pins.set(PHOTOCRAFT.id, &crate::pins::VersionTrack::Pinned("v0.13.0".into())); + pins.save(&path).unwrap(); + u.request_install(&ctx, true); + assert!(u.install.is_none(), "nothing installs once the launcher pinned the app"); + assert!(!u.prompt_open()); +} + +#[test] +fn a_check_after_the_pin_closes_a_prompt_opened_before_it() { + let ctx = egui::Context::default(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("versions.json"); + let mut u = Updater::new(&PHOTOCRAFT, "0.13.0").with_source(Arc::new(|| Ok(Some(newer())))).with_pins_path(&path); + u.offer(newer()); + let mut pins = Pins::default(); + pins.set(PHOTOCRAFT.id, &crate::pins::VersionTrack::Nightly); + pins.save(&path).unwrap(); + u.check_now(&ctx); + settle(&mut u, &ctx); + assert!(!u.prompt_open()); + assert_eq!(u.status(), &Status::UpToDate); +} + +#[test] +fn a_running_install_keeps_its_prompt_when_the_app_is_pinned() { + let dir = pins_with(crate::pins::VersionTrack::Pinned("v0.13.0".into())); + let mut u = Updater::new(&PHOTOCRAFT, "0.13.0").with_pins_path(dir.path().join("versions.json")); + u.prompt = Some(StartupPrompt::new(&PHOTOCRAFT, newer(), "0.13.0")); + let (_tx, rx) = channel(); + u.install = Some((rx, newer())); + assert!(u.read_choice()); + assert!(u.prompt_open(), "tick reports the running install through the prompt"); +} + +#[test] +fn a_pin_made_while_the_check_runs_still_holds_its_answer() { + let ctx = egui::Context::default(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("versions.json"); + let during = path.clone(); + // The check itself pins the app before answering, as the launcher could mid-check. + let source: CheckSource = Arc::new(move || { + let mut pins = Pins::default(); + pins.set(PHOTOCRAFT.id, &crate::pins::VersionTrack::Pinned("v0.13.0".into())); + pins.save(&during)?; + Ok(Some(newer())) + }); + let mut u = Updater::new(&PHOTOCRAFT, "0.13.0").with_source(source).with_pins_path(&path); + u.check_now(&ctx); + settle(&mut u, &ctx); + assert!(!u.prompt_open()); + assert_eq!(u.status(), &Status::UpToDate); +} + +#[test] +fn every_answer_after_a_pin_made_mid_check_reports_the_launchers_choice() { + let ctx = egui::Context::default(); + let answers: [Checked; 2] = [Ok(None), Err("offline".into())]; + for answer in answers { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("versions.json"); + let during = path.clone(); + let source: CheckSource = Arc::new(move || { + let mut pins = Pins::default(); + pins.set(PHOTOCRAFT.id, &crate::pins::VersionTrack::Nightly); + pins.save(&during)?; + answer.clone() + }); + let mut u = Updater::new(&PHOTOCRAFT, "0.13.0").with_source(source).with_pins_path(&path); + u.check_now(&ctx); + settle(&mut u, &ctx); + assert_eq!(u.status(), &Status::UpToDate); + assert_eq!(u.managed(), Some(&Choice::Managed(crate::pins::VersionTrack::Nightly))); + } +} diff --git a/crates/update/tests/dialog_screenshots.rs b/crates/update/tests/dialog_screenshots.rs new file mode 100644 index 000000000..19123b82f --- /dev/null +++ b/crates/update/tests/dialog_screenshots.rs @@ -0,0 +1,78 @@ +#![cfg(feature = "snapshot")] +//! Renders the update dialogs to PNG, light and dark at 1.5x: +//! `CRAFT_UPDATE_SNAPSHOT_DIR=~/.cache/cl-update-shots cargo test -p craft-update --features snapshot --test dialog_screenshots` +//! Without the directory variable the test does nothing; these are not golden comparisons. + +use std::path::PathBuf; + +use craft_update::dialog::{CheckResultDialog, StartupPrompt}; +use craft_update::release::Release; +use craft_update::suite::PHOTOCRAFT; +use craft_update::updater::Status; +use egui::Theme; +use egui_kittest::Harness; + +fn fonts(ctx: &egui::Context) { + let mut defs = egui::FontDefinitions::default(); + for (name, bytes, family) in [ + ("Hack", epaint_default_fonts::HACK_REGULAR, egui::FontFamily::Monospace), + ("Ubuntu", epaint_default_fonts::UBUNTU_LIGHT, egui::FontFamily::Proportional), + ] { + defs.font_data.insert(name.into(), std::sync::Arc::new(egui::FontData::from_static(bytes))); + defs.families.entry(family).or_default().push(name.into()); + } + // Both faces serve both families: the card uses monospace labels and proportional text. + for family in defs.families.values_mut() { + for name in ["Ubuntu", "Hack"] { + if !family.iter().any(|f| f == name) { + family.push(name.into()); + } + } + } + ctx.set_fonts(defs); +} + +static APP: craft_update::App = PHOTOCRAFT; + +#[test] +fn dialog_screenshots() { + let Some(dir) = std::env::var_os("CRAFT_UPDATE_SNAPSHOT_DIR").map(PathBuf::from) else { return }; + std::fs::create_dir_all(&dir).unwrap_or_else(|e| panic!("{}: {e}", dir.display())); + let long_error = "couldn't reach https://api.github.com/repos/storytold/photocraft/releases/latest: \ + connection timed out after 30 seconds (os error 110); check the network or proxy settings"; + let states: [(&str, Option); 4] = [ + ("checking", Some(Status::Checking)), + ("up-to-date", Some(Status::UpToDate)), + ("failed", Some(Status::Failed(long_error.into()))), + ("startup-prompt", None), + ]; + for (name, status) in states { + for (theme_name, theme) in [("light", Theme::Light), ("dark", Theme::Dark)] { + let dialog = CheckResultDialog::new(&APP, "0.1.0"); + let mut prompt = StartupPrompt::new( + &APP, + Release { version: Some(semver::Version::new(0, 2, 0)), tag: "v0.2.0".into(), ..Release::default() }, + "0.1.0", + ); + prompt.is_self_updating = false; + let mut h = Harness::builder() + .with_size(egui::vec2(560.0, 360.0)) + .with_pixels_per_point(1.5) + .with_theme(theme) + .with_context_setup(fonts) + .wgpu() + .build_ui(|ui| match &status { + Some(status) => { + dialog.show(ui.ctx(), status); + } + None => { + prompt.show(ui.ctx()); + } + }); + h.run_steps(4); + let image = h.render().unwrap_or_else(|e| panic!("render {name}: {e}")); + let path = dir.join(format!("update-{name}-{theme_name}.png")); + image.save(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + } + } +} diff --git a/docs/clean-room.md b/docs/clean-room.md new file mode 100644 index 000000000..e4140e104 --- /dev/null +++ b/docs/clean-room.md @@ -0,0 +1,87 @@ +# Clean-room format work in craft-libs + +Several crates here read or write formats that other open-source projects also implement, often +under licences the Craft apps cannot take code from (GPL, LGPL, AGPL, CDDL) or under terms that +forbid it (Adobe's). This document is the standard a format crate follows so that every line of +it is ours, and so that a reviewer can check that claim from the repository alone. It is the +written form of what `craft-jpeg`, `craft-media` and `craft-crw` did. + +## What "clean-room" means here + +A clean-room crate is written only from: + +- **published specifications and standards** (ITU-T, ISO/IEC, IETF RFCs, a vendor's own published + format document, a registry's pages), read as documents; +- **prose documentation** of a format or its tags (ExifTool's tag-name and format pages, a wiki + page that describes a container), never a project's code; +- **sample files** we may use: CC0, or owned by a contributor who consents, or produced by us. + Not NC-licensed files, not files whose licence is unclear, never anything from inside a + proprietary application's bundle; +- **a reference implementation run as a black box**: an official pre-built binary of another + decoder, fed inputs and observed for outputs, with every run logged. Its source is never + opened, read, searched, diffed or quoted, including comments, licence headers, mirrors and + excerpts in forum posts or AI answers. + +Never consulted, in any form: GPL/LGPL/AGPL/CDDL decoder and encoder sources (dcraw, LibRaw, +rawspeed, rawloader/rawler, dnglab, darktable, RawTherapee, ART, libopenraw, lensfun, ExifTool's +Perl, FFmpeg/libav, x264/x265 and the like), Adobe's SDKs, profiles (DCP, LCP), presets, fonts, +icons or binaries, and any earlier attempt of ours that was itself not clean. + +## The process (what the record must show) + +Modelled on LightCraft's ARW compression derivation and repeated for CRW +(`crates/crw/clean-room/`). A crate that derives an undocumented format commits a `clean-room/` +folder with: + +1. **Starting assumptions**, written before any source is consulted, every item marked + unverified, committed once and never edited. A reviewer diffs them against the result to see + what was assumed and what was established. +2. **A plan** naming the sources that will be used, the sample corpus (public files with URL, + licence and SHA-256; private files by name, size and hash only, never content), the phases, + the freeze point, and the questions that are the maintainer's to decide. Work past the plan + waits for the maintainer's recorded approval. +3. **A source log**: every URL opened and every search run, in order, with the time, what was + asked and what was taken. If a page exposed implementation details that were not asked for, + the log says so and what it said. +4. **Derivation documents** in which every fact cites the experiment (script and log) or the + document that established it, and corrections are stated, not silently applied. Statistics + before decoding; hypotheses tested against internal consistency first; black-box probes + logged one by one. +5. **A freeze**: one document holding the complete derived format, with its tables as data. Its + commit is the freeze point. Nothing in it is tuned afterwards. A mismatch found later reopens + the derivation *explicitly*: the cause, the change and a new freeze revision are recorded in + that document. +6. **Verification** after the freeze, with zero tuning: every sample compared exactly against the + reference, every file's verdict recorded, failures recorded as failures. Cross-body checks + (tables learned on one body, exact on another) are the strongest independent evidence. +7. **The implementation** from the frozen document, with: never-crash rules (`get()` for every + file-derived offset, checked arithmetic, bounded recursion and allocation, no `unwrap`/`panic`), + synthetic tests that need no sample files (a test-only encoder that inverts the model is the + usual tool), hostile-input tests, and a sample-file test pinning the verified decode (a digest + of the output per file) so the crate can prove it still matches without redistributing the + references. +8. **Table data** lives in the record as JSON and is rendered into Rust by a committed script; the + generated file says so and is never edited by hand. + +When the derivation was done in another repository, the crate's `clean-room/` may hold only items +3, 5, 6 and 8, provided it pins that repository, path and revision (in its README and in `NOTICE`) +for the starting assumptions, the plan and the derivation documents, as `craft-crw` does. + +Scripts in the record use the Python standard library only and run under `python -I`. Local-only +material (reference models, oracle binaries, copyrighted specification PDFs, dumps that may carry +personal metadata) is named in the record with how to obtain it, and is gitignored. + +## Assets and attribution + +Sample files are never committed (sizes aside, they are other people's photographs): tests +download them outside the checkout by pinned URL and SHA-256, licence checked (`scripts/ +fetch-*.py`), and skip cleanly without them. Each manifest, each table derived from samples and +each log set has a row in `ATTRIBUTION.md`; code and records that came from an app are listed in +`NOTICE` under *Source provenance* with the repository, paths and revision. + +## When something is not clean + +Stop, record what was seen and where (the source log), and tell the maintainer. Code written +afterwards by the same person for the same part of the format is suspect; the maintainer decides +whether the part is re-derived by someone else, documented as derived from that source under its +licence terms, or dropped. Never quietly continue. diff --git a/docs/update-testing.md b/docs/update-testing.md new file mode 100644 index 000000000..1dee183bd --- /dev/null +++ b/docs/update-testing.md @@ -0,0 +1,33 @@ +# Testing update prompts + +Every native Craft App accepts an environment variable derived from its app id: +uppercase, hyphens replaced with underscores, followed by `_PRETEND_VERSION`. +Run each command from the corresponding app checkout, with a `craft-update` pin that +includes this fix. These app packages are not part of the `craft-libs` workspace. +For example: + +```sh +LIGHTCRAFT_PRETEND_VERSION=0.0.1 cargo run -p lightcraft +WORDCRAFT_PRETEND_VERSION=0.0.1 cargo run -p wordcraft +PDFCRAFT_PRETEND_VERSION=0.0.1 cargo run -p pdfcraft +``` + +The shared updater uses this version for startup checks, manual checks, the prompt's +running-version label, and installation checks. It still fetches real release metadata; +a network error or an app with no published newer release cannot produce an update prompt. +Use Help → Check for Updates to see errors that the startup check only logs. + +An app-specific setting takes priority over the legacy shared +`ARTCRAFT_LAUNCHER_PRETEND_VERSION` setting. An invalid version falls back to the +compiled package version. Leave the variables unset for normal version detection. +The restart helper clears both settings for the app so a newly installed copy reports +its real version. + +`CRAFT_UPDATE_DEV=1` separately lets a development executable be treated as an installed +copy for testing installation. It is unnecessary to preview a prompt. Download/install +buttons perform a real update; a pretend version does not simulate installation. +The explicit override also applies to the installation version check, allowing tests +to reinstall the current release or replace a newer development build. Leave it unset +to retain the normal protection against installing an older or equal version. + +Web builds have no updater. These settings apply to native builds only. diff --git a/scripts/fetch-crw-samples.py b/scripts/fetch-crw-samples.py new file mode 100644 index 000000000..281d95097 --- /dev/null +++ b/scripts/fetch-crw-samples.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Fetch the pinned CC0 Canon CRW sample files for craft-crw's tests, outside the checkout. + +The manifest is crates/crw/tests/samples.json (name, raw.pixls.us URL, SHA-256, licence). Files +already present with the right hash are kept. Then run the tests with +`CRAFT_CRW_SAMPLES= cargo test -p craft-crw --release --test samples`. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + +MAX_BYTES = 16 * 1024 * 1024 + + +class _HttpsOnlyRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if urllib.parse.urlsplit(newurl).scheme != "https": + raise ValueError("refusing to follow a non-HTTPS redirect") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +urllib.request.install_opener(urllib.request.build_opener(_HttpsOnlyRedirectHandler)) + + +def fetch(url): + if urllib.parse.urlsplit(url).scheme != "https": + raise ValueError("refusing to fetch a non-HTTPS URL") + for attempt in range(3): + try: + with urllib.request.urlopen(url, timeout=120) as response: + return response.read(MAX_BYTES + 1) + except urllib.error.HTTPError as error: + if error.code not in (408, 429) and not 500 <= error.code <= 599: + raise + if attempt == 2: + raise + error.close() + except (urllib.error.URLError, TimeoutError, ConnectionError): + if attempt == 2: + raise + time.sleep(attempt + 1) + raise RuntimeError("sample download retries exhausted") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("destination", type=Path) + args = parser.parse_args() + root = Path(__file__).resolve().parent.parent + destination = args.destination.resolve() + if destination == root or root in destination.parents: + parser.error("sample files must be stored outside the checkout (they are never committed)") + samples = json.loads((root / "crates/crw/tests/samples.json").read_text(encoding="utf-8")) + destination.mkdir(parents=True, exist_ok=True) + for sample in samples: + name, url, digest, licence = sample["name"], sample["source"], sample["sha256"], sample["license"] + if Path(name).name != name or name in ("", ".", "..") or not url.startswith("https://"): + raise ValueError("invalid sample name or source") + if licence != "CC0-1.0": + raise ValueError(f"{name}: only CC0 samples are fetched (manifest says {licence})") + path = (destination / name).resolve() + if path.parent != destination: + raise ValueError(f"sample path escapes destination: {name}") + if path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == digest: + print(f"Verified {name}") + continue + data = fetch(url) + if len(data) > MAX_BYTES or hashlib.sha256(data).hexdigest() != digest: + raise ValueError(f"sample size or SHA-256 mismatch: {name}") + fd, tmp = tempfile.mkstemp(dir=destination, prefix=f".{name}.", suffix=".tmp") + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + os.replace(tmp, destination / name) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + print(f"Fetched and verified {name}") + + +if __name__ == "__main__": + main()