mirror of
https://github.com/storytold/craft-libs.git
synced 2026-10-31 07:39:23 +00:00
Merge pull request #365 from rwoldberg/clean-room-crw-format-decode
Clean room crw format decode
This commit is contained in:
30 files changed
+7499
No files matched your search
@@ -0,0 +1,79 @@
|
||||
name: Shared CRW
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ["crates/crw/**", "scripts/fetch-crw-samples.py", "Cargo.toml", "Cargo.lock", ".github/workflows/crw.yml"]
|
||||
pull_request:
|
||||
paths: ["crates/crw/**", "scripts/fetch-crw-samples.py", "Cargo.toml", "Cargo.lock", ".github/workflows/crw.yml"]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: shared-crw-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
native:
|
||||
timeout-minutes: 20
|
||||
name: CRW lint and tests (${{ matrix.os }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
|
||||
with:
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
||||
with:
|
||||
key: craft-crw-${{ matrix.os }}
|
||||
- run: cargo fmt -p craft-crw --check
|
||||
- run: cargo clippy -p craft-crw --all-targets --locked -- -D warnings
|
||||
- run: cargo test -p craft-crw --locked
|
||||
|
||||
wasm:
|
||||
timeout-minutes: 10
|
||||
name: CRW WASM
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
|
||||
with:
|
||||
targets: wasm32-unknown-unknown
|
||||
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
||||
with:
|
||||
key: craft-crw-wasm
|
||||
- run: cargo check -p craft-crw --locked --target wasm32-unknown-unknown
|
||||
|
||||
samples:
|
||||
timeout-minutes: 20
|
||||
name: CRW sample files decode exactly
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
|
||||
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
||||
with:
|
||||
key: craft-crw-samples
|
||||
- name: Cache the CC0 sample files
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: ${{ runner.temp }}/crw-samples
|
||||
key: crw-samples-${{ hashFiles('crates/crw/tests/samples.json') }}
|
||||
- name: Fetch the pinned CC0 sample files (raw.pixls.us)
|
||||
run: python scripts/fetch-crw-samples.py "$RUNNER_TEMP/crw-samples"
|
||||
- run: cargo test -p craft-crw --release --locked --test samples -- --nocapture
|
||||
env:
|
||||
CRAFT_CRW_SAMPLES: ${{ runner.temp }}/crw-samples
|
||||
CRAFT_CRW_REQUIRE_SAMPLES: "1"
|
||||
@@ -22,6 +22,9 @@ author, source and license. Library code and original assets are under the repos
|
||||
| `crates/ui/tests/fonts/NotoSansArabic-screens.ttf` (instance at weight 400 and width 100, subset to the screen test strings) | The Noto Project Authors | [google/fonts@7085eb8 `ofl/notosansarabic`](https://github.com/google/fonts/tree/7085eb89a950e85db5b166b7a58d414544b4140c/ofl/notosansarabic), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansArabic.txt`) |
|
||||
| `crates/ui/tests/fonts/NotoSansCJKsc-screens.otf` (subset to printable ASCII and the screen test strings) | The Noto Project Authors (Adobe, Google) | [notofonts/noto-cjk@f8d1575 `Sans/OTF/SimplifiedChinese`](https://github.com/notofonts/noto-cjk/tree/f8d157532fbfaeda587e826d4cd5b21a49186f7c/Sans/OTF/SimplifiedChinese), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansCJK.txt`) |
|
||||
| `crates/ui/tests/fonts/NotoSansThai-screens.ttf` (instance at weight 400 and width 100, subset to the screen test strings) | The Noto Project Authors | [google/fonts@8b0a1d0 `ofl/notosansthai`](https://github.com/google/fonts/tree/8b0a1d0f5983c89bc2b93f1b5fb55f9e252744b5/ofl/notosansthai), via craft-fonts | OFL-1.1 (`licenses/ui-test-fonts/OFL-NotoSansThai.txt`) |
|
||||
| `crates/crw/tests/samples.json` (manifest only; the 11 raw files are not bundled, `scripts/fetch-crw-samples.py` downloads them outside the checkout) | The photographers who contributed each file to raw.pixls.us (Canon EOS D30, D60, 10D ×3, 300D ×2, Digital Rebel, PowerShot G1, G5, G6) | [raw.pixls.us](https://raw.pixls.us/), each file's `getfile.php` URL and SHA-256 in the manifest | CC0-1.0 (as published by raw.pixls.us for each file) |
|
||||
| `crates/crw/src/tables.rs`, `crates/crw/clean-room/logs/table_sel*.json` | craft-crw contributors (derived in LightCraft, 2026-10, by known-plaintext alignment of the sample files' entropy streams; `crates/crw/clean-room/02-crw-format.md`) | this repository | MIT OR Apache-2.0 |
|
||||
| `crates/crw/clean-room/logs/private-corpus.tsv` (file names, sizes and SHA-256 of the maintainer's own EOS D60 files; no image content) | craft-crw contributors (LightCraft derivation record) | this repository | MIT OR Apache-2.0 |
|
||||
|
||||
Original assets are covered by the repository licence.
|
||||
|
||||
|
||||
Generated
+11
@@ -775,6 +775,17 @@ dependencies = [
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-crw"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"proptest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-dialogs"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -25,6 +25,7 @@ kurbo = "0.13"
|
||||
thiserror = "2"
|
||||
craft-atomic = { path = "crates/atomic" }
|
||||
craft-cms = { path = "crates/cms" }
|
||||
craft-crw = { path = "crates/crw" }
|
||||
craft-dialogs = { path = "crates/dialogs" }
|
||||
craft-docpath = { path = "crates/docpath" }
|
||||
craft-font-catalog = { path = "crates/font-catalog" }
|
||||
|
||||
@@ -200,3 +200,16 @@ https://github.com/storytold/craft-launcher, https://github.com/storytold/photoc
|
||||
https://github.com/storytold/filmcraft, https://github.com/storytold/effectcraft.
|
||||
Copyright (c) 2026 ArtCraft Team and the respective application and craft-libs contributors.
|
||||
All adapted source is licensed MIT OR Apache-2.0.
|
||||
|
||||
The Canon CRW decoder in crates/crw (src/lib.rs, src/tables.rs, src/tests.rs, tests/) and its
|
||||
derivation record (crates/crw/clean-room/) come from:
|
||||
- LightCraft, crates/raw/src/vendor/crw.rs, crates/raw/src/vendor/crw_tables.rs,
|
||||
crates/raw/tests/crw_oracle.rs, crates/raw/tests/it/vendor_robust.rs and clean-room/crw/,
|
||||
branch clean-room-crw-format-decode, revision 6ca439b5155820d2befeacd81ec0c6a80b2bd3bc
|
||||
(decoder ported at 70c755e42a65661401142819278d51b0c5977f7f; the record is identical at both).
|
||||
https://github.com/storytold/lightcraft
|
||||
Copyright (c) 2026 ArtCraft Team and the LightCraft contributors.
|
||||
Its white-level plateau estimate (saturation_plateau) is LightCraft's vendor::white_from_data at the
|
||||
same revision. The CRW compression was derived clean-room from CC0 sample files and the
|
||||
maintainer's own photographs by black-box comparison with a reference decoder's command-line
|
||||
output; no raw-decoder source was consulted (crates/crw/clean-room/sources.md).
|
||||
@@ -34,6 +34,10 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
|
||||
- [`craft-cms`](crates/cms/README.md): pure-Rust ICC v2/v4 colour management: profile parsing
|
||||
and writing, transforms, rendering intents, black point compensation, soft proofing, gamut
|
||||
checks, 3D LUTs and built-in profiles (sRGB, Display P3, synthetic coated CMYK and others).
|
||||
- [`craft-crw`](crates/crw/README.md): clean-room Canon CRW (CIFF) raw decoder, from LightCraft:
|
||||
the heap container, the entropy coding derived by black-box analysis (frozen code tables), sensor
|
||||
black and white levels and as-shot white balance. The complete derivation record (plan, sources
|
||||
log, scripts, verification) is in `crates/crw/clean-room/`. Colour stays in each app.
|
||||
- [`craft-dialogs`](crates/dialogs/README.md): native file and message dialogs that never block
|
||||
the event loop: queued requests shown at the end of the frame and answered on a later frame,
|
||||
free-standing Open and Folder panels and Save sheets on macOS, case-insensitive filters. What
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "craft-crw"
|
||||
description = "Clean-room Canon CRW (CIFF) raw decoder: the heap container, the derived entropy coding, sensor levels and as-shot white balance."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
thiserror = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
# Pinned digests of the decoded CC0 sample files (tests/samples.rs); not a product dependency.
|
||||
sha2 = { version = "0.10", default-features = false, features = ["std"] }
|
||||
@@ -0,0 +1,153 @@
|
||||
# craft-crw
|
||||
|
||||
A clean-room Canon CRW (CIFF) raw decoder for the Craft apps: the raw format of Canon's bodies
|
||||
before the CR2 era (EOS D30, D60, 10D, 300D / Digital Rebel; PowerShot G1 and G6). No `unsafe`,
|
||||
no panics on malformed input, and it builds for `wasm32-unknown-unknown`. The only dependency is
|
||||
`thiserror`.
|
||||
|
||||
```toml
|
||||
craft-crw = { git = "https://github.com/storytold/craft-libs", rev = "<commit>" }
|
||||
```
|
||||
|
||||
The container (a CIFF heap file) follows Canon's published CIFF specification and ExifTool's prose
|
||||
tag documentation. The compression was never documented by Canon: it was **derived** from CC0 and
|
||||
maintainer-owned sample files, frozen, and verified sample-exact against a reference decoder's
|
||||
command-line output, treated as a black box. No raw-decoder source was consulted, at any point.
|
||||
The whole record is in [`clean-room/`](clean-room/README.md); the frozen format is
|
||||
[`clean-room/02-crw-format.md`](clean-room/02-crw-format.md).
|
||||
|
||||
## What it decodes
|
||||
|
||||
| Body | Selector(s) seen | Verified | Notes |
|
||||
|---|---|---|---|
|
||||
| EOS D30 | 1 | exact (1 CC0 file) | no white-balance record: `wb_multipliers` is `None` |
|
||||
| EOS D60 | 0, 1, 2 | exact (1 CC0 + 1,561 maintainer files) | sensor clips near 4000 at low ISO (`white`) |
|
||||
| EOS 10D | 0, 2 | exact (3 CC0 files) | |
|
||||
| EOS 300D / Digital Rebel | 0, 2 | exact (3 CC0 files) | portrait files carry the rotation (`exif_orientation` 8) |
|
||||
| PowerShot G1 | 1 | exact (1 CC0 file) | 10-bit (no low-bits plane), `Cfa::Gbrg`, no white balance |
|
||||
| PowerShot G6 | 0 | exact (1 CC0 file) | no white balance |
|
||||
| PowerShot G5 | 1 | **refused** | its entropy stream diverges from the model from sample 2056 on (`Error::Unsupported`); apps show the embedded JPEG |
|
||||
| PowerShot G2, G3, Pro1, Pro70, S30–S70 | | unverified | no CC0 samples exist; they decode through the same model if their records match, with no guarantee |
|
||||
|
||||
"Exact" means every sensor sample equals the reference decode (`clean-room/03-verification.md`).
|
||||
Big-endian (`MM`) files are handled from the specification and tested synthetically; none has
|
||||
been seen in the wild. CHDK sensor dumps named `.CRW` are a different format and are not CIFF files
|
||||
(`probe` says no).
|
||||
|
||||
## Use
|
||||
|
||||
```rust
|
||||
use craft_crw::{decode, header, probe, Error, Limits};
|
||||
|
||||
if craft_crw::probe(&bytes) {
|
||||
let img = decode(&bytes, &Limits::default())?;
|
||||
let h = &img.header;
|
||||
// h.width × h.height samples, h.bits deep, Bayer phase h.cfa at sensor (0, 0);
|
||||
// the picture is h.active_area; img.black (per CFA site) and img.white are sensor levels;
|
||||
// h.wb_multipliers is the as-shot white balance when the body records one.
|
||||
}
|
||||
```
|
||||
|
||||
- `probe(bytes) -> bool`: the CIFF signature (`II`/`MM`, `HEAPCCDR` at offset 6).
|
||||
- `header(bytes, &Limits) -> Result<Header, Error>`: geometry, bit depth, CFA phase, selector,
|
||||
make and model, capture time, orientation and white balance, without decoding the samples
|
||||
(fast: it walks the heap and reads a few records).
|
||||
- `decode(bytes, &Limits) -> Result<Image, Error>`: the header plus the samples and the measured
|
||||
black and white levels.
|
||||
|
||||
### Output
|
||||
|
||||
`Image { header, samples, black, white }`:
|
||||
|
||||
- `samples`: `width × height` `u16`s, row-major, the whole sensor including the masked borders,
|
||||
`bits` (10 or 12) deep, black not subtracted. Rows are in sensor order; apply
|
||||
`header.exif_orientation` (1, 3, 6 or 8) when displaying.
|
||||
- `header.active_area`: the picture (SensorInfo's inclusive crop); the rest is masked. The default
|
||||
crop of these bodies is the active area itself.
|
||||
- `header.cfa`: `Cfa::Rggb` or `Cfa::Gbrg`, the phase at sensor (0, 0); `pattern()` gives the
|
||||
2×2 cell as 0 = red, 1 = green, 2 = blue.
|
||||
- `black`: `BlackLevel { values: [f32; 4] }`, one level per CFA site of the 2×2 cell counted
|
||||
from the active area's origin (`at(active, x, y)`), measured on the darkest strips of the masked
|
||||
borders. These bodies sit near 128 (the Digital Rebel sample near 261, the G1 near 29).
|
||||
- `white`: the saturation plateau when the image has one, else the sensor's clip level when at
|
||||
least 64 samples reach within 3 % of the full range (the D60, 10D and 300D clip near 4000 at
|
||||
ISO 100–400), else the full range (4095 or 1023).
|
||||
- `header.wb_multipliers`: `Some([r, 1.0, b])` from the ColorBalance record on the D60, 10D and
|
||||
300D, the slot chosen by the ShotInfo white-balance setting; `None` on the D30 and PowerShots,
|
||||
whose records the derivation did not decipher. Apps fall back to their own estimate.
|
||||
- `header.capture_unix`: the TimeStamp record, Unix seconds on the camera's local clock, no UTC
|
||||
offset. Convert with chrono (`DateTime::from_timestamp`), never by hand.
|
||||
- `header.make` / `header.model`: the MakeModel record, e.g. `Canon` / `Canon EOS D60`. CRW
|
||||
files carry no Exif block, so this is where the camera name comes from.
|
||||
|
||||
### Errors and limits
|
||||
|
||||
`Error` is `NotCrw` (not a CIFF raw), `Corrupt(String)` (damaged structure or an undecodable
|
||||
code), `Unsupported(String)` (an unknown decoder table, or the PowerShot G5) or
|
||||
`LimitExceeded(&str)`. Every offset read from the file is bounds-checked, the heap walk is bounded
|
||||
by depth (16) and by a per-file record budget (4,096), and `width × height` is checked against
|
||||
`Limits::max_samples` (default 2²³, about 8.4 million, on every target; real frames hold at most
|
||||
7.4 million) before anything is allocated. The decoder holds about three sample-sized `u16`
|
||||
buffers at its peak. `Limits` is `#[non_exhaustive]`: start from `Limits::default()` and set the
|
||||
fields you change.
|
||||
|
||||
A stream that ends early is not an error: the remaining samples keep their predictor values (the
|
||||
frozen stream-end rule; real files end 1–3 samples short).
|
||||
|
||||
### Performance
|
||||
|
||||
About 60 ms for a 6.5 MP EOS file and 70 ms for the 7.4 MP G6 in release mode on a 2024 laptop
|
||||
(`cargo test -p craft-crw --release --test samples -- --nocapture` prints the times).
|
||||
|
||||
## Tests
|
||||
|
||||
- `cargo test -p craft-crw`: synthetic CIFF files round-tripped through a test-only encoder (the
|
||||
model's inverse), the frozen tables' structure (prefix-free, all 162 symbols), hostile inputs
|
||||
(every truncation and byte mutation of a valid file, self-referencing heaps), and a proptest
|
||||
mutation sweep (`tests/robust.rs`). None needs sample files.
|
||||
- `tests/samples.rs` pins the eleven CC0 sample files of [`tests/samples.json`](tests/samples.json)
|
||||
(raw.pixls.us): each decodes to exactly the samples the derivation verified against the
|
||||
reference, recorded as the SHA-256 of the sample buffer, with its levels, white balance and
|
||||
metadata. The files (5–11 MB each) are not in the repository:
|
||||
|
||||
```sh
|
||||
python scripts/fetch-crw-samples.py ~/crw-samples # downloads and verifies by SHA-256
|
||||
CRAFT_CRW_SAMPLES=~/crw-samples cargo test -p craft-crw --release --test samples -- --nocapture
|
||||
```
|
||||
|
||||
Unset, the test prints that it skipped; CI sets `CRAFT_CRW_REQUIRE_SAMPLES=1` and runs it on
|
||||
Linux (`.github/workflows/crw.yml`; lint and the synthetic tests run on Linux, Windows and
|
||||
macOS, plus a wasm32 check).
|
||||
- `CRAFT_CRW_ORACLE=<dir>` additionally compares against `<name>.crw.tiff` references next to the
|
||||
files (the reference decoder's `unprocessed_raw -T` output), the check the derivation used.
|
||||
Those references are not distributable and stay outside the repository.
|
||||
|
||||
## Origin
|
||||
|
||||
The decoder was written in LightCraft as `crates/raw/src/vendor/crw.rs` and `crw_tables.rs`
|
||||
(branch `clean-room-crw-format-decode`, revision `6ca439b5155820d2befeacd81ec0c6a80b2bd3bc`,
|
||||
[storytold/lightcraft](https://github.com/storytold/lightcraft); the decoder itself was ported at
|
||||
`70c755e4`, and the branch's later review commits changed only a date conversion that is not part
|
||||
of this crate) and moved here with its tests and
|
||||
its derivation record. The move replaced LightCraft's types with this crate's own (`Error`,
|
||||
`Limits`, `Cfa`, `Rect`, `BlackLevel`, `Header`, `Image`), exposed the capture time as Unix
|
||||
seconds instead of converting it in the decoder, and copied LightCraft's shared plateau estimate
|
||||
(`white_from_data`) as `saturation_plateau`. The algorithm, tables and thresholds are unchanged,
|
||||
and the sample-file test proves the decode is byte-identical to the verified LightCraft decode.
|
||||
Colour (camera matrices, profiles, the JPEG-fitted look) stayed in LightCraft.
|
||||
|
||||
## Regenerating the tables
|
||||
|
||||
`src/tables.rs` is generated data. Edit nothing by hand: the source of truth is
|
||||
`clean-room/logs/table_sel{0,1,2}.json`, and `python crates/crw/clean-room/tools/gen_tables.py`
|
||||
renders it (byte-identical to the committed file). A change to the tables is a change to the frozen
|
||||
format and must follow the reopening rule of `clean-room/02-crw-format.md`.
|
||||
|
||||
## Boundaries
|
||||
|
||||
- In scope: the CIFF container records a raw developer needs, the entropy coding, sensor levels,
|
||||
as-shot white balance, orientation and the records a catalog needs (make, model, time).
|
||||
- Out of scope: colour (no CRW body carries matrices; apps own their profiles), demosaicing, the
|
||||
embedded JPEG preview (apps find it with their JPEG scanner; the record is `JpgFromRaw`, 0x2007),
|
||||
and the full CIFF metadata set (ExifTool's prose documents far more records than the decoder
|
||||
reads).
|
||||
@@ -0,0 +1,105 @@
|
||||
# CRW RawData payload layout (P1) — established by statistics, no decoding
|
||||
|
||||
Status: derived 2026-10-12 from the 29-file corpus by statistics alone. The scripts that produced
|
||||
it (`p1_payload.py`, `p1b_markers.py`, `p1c_lowbits_bounds.py`) and their logs are not part of
|
||||
this record. **The entropy-coded bitstream itself has not been
|
||||
decoded;** everything here comes from sizes, offsets, byte histograms and ff-digram statistics.
|
||||
Corrections are stated against the starting assumptions, by their labels (B2, B3, …).
|
||||
|
||||
## 1 The three layouts
|
||||
|
||||
With S = RawData record size, (N, 0, 514, L) = the DecoderTable u32s, w×h = SensorInfo raw
|
||||
dimensions, Q = ⌈w·h/4⌉ (one byte per four pixels — the "low-bits" region candidate):
|
||||
|
||||
| Class (bodies) | RawData layout | Evidence |
|
||||
|---|---|---|
|
||||
| EOS: D30, D60, 10D, 300D (23 files) | `[low bits, Q][514 zero bytes][stuffed entropy, L−514−t][tail t]` | S−L = Q+2 or Q+3 on all 23; the section after Q starts with exactly 514 zero bytes in all 23; the entropy part contains **zero** non-stuffed ff (every 0xff is followed by 0x00) in all 23 |
|
||||
| PowerShot 12-bit: G5, G6 | `[514 zero bytes][low bits, Q][stuffed entropy][ff d9 (+00)]` | S−L = 2; first 514 bytes zero; **all** non-stuffed ff (5002 of 5002 on G5, 7928 of 7928 on G6) fall inside [514, 514+Q); the rest of the stream is perfectly stuffed; payload ends `ff d9` |
|
||||
| PowerShot 10-bit: G1 | `[514 zero bytes][stuffed entropy][ff d9 + 00]` | S−L = 3; first 514 bytes zero; zero non-stuffed ff anywhere; payload ends `ff d9 00` |
|
||||
|
||||
Private corpus: all 18 D60 files match the EOS row exactly.
|
||||
|
||||
## 2 DecoderTable (0x1835) semantics
|
||||
|
||||
`(N, 0, 514, L)` where:
|
||||
|
||||
- **N = decoder table selector**, 0–2, varies per shot on the same body (B5 supported; all
|
||||
three values exercised by the corpus). Presumably chosen by the camera per image; the
|
||||
mapping (ISO? content?) is not yet established and is not needed for decoding.
|
||||
- Second u32: 0 in all 29 files.
|
||||
- **514 = offset to the entropy data within the compressed section** (ExifTool names the field
|
||||
*CompressedDataOffset*). Behaviourally: the compressed section begins with exactly 514 zero
|
||||
bytes in all 29 files, EOS and PowerShot alike — a reserved/padding block the decoder skips.
|
||||
(On G5/G6 the low-bits region sits between this zero block and the entropy data.)
|
||||
- **L = CompressedDataLength**: the byte length of the compressed section — from the end of
|
||||
the low-bits prefix on EOS, from 0 on PowerShots — excluding the small tail.
|
||||
|
||||
## 3 Tails
|
||||
|
||||
- EOS: 2 bytes `00 00` (D30, and files where S−L−Q = 2) or 3 bytes `ff 00 00` (D60/10D/Rebel
|
||||
files with S−L−Q = 3). The `ff 00` there may be a final stuffed 0xff of the entropy stream
|
||||
sitting just outside L — to be settled in P2 when the stream end is decodable.
|
||||
- PowerShots: `ff d9` (the JPEG EOI marker used as a terminator) plus one `00` pad when needed
|
||||
to keep the record length even (G1: S−L = 3 = `ff d9 00`; G5/G6: S−L = 2 = `ff d9`).
|
||||
|
||||
## 4 The low-bits regions (still a hypothesis as to *content*, region *extent* established)
|
||||
|
||||
- Extent Q = w·h/4 exactly, in all 25 files that have the region (B6's arithmetic holds).
|
||||
- D60-class prefix: reshaped at w/4 = 788 bytes/row, the first **7 rows are exactly zero**
|
||||
(5516 = 7×788 bytes) in the public D60 file, and the leading zero run is row-aligned —
|
||||
strong evidence the region is row-major at 2 bits/pixel.
|
||||
Below the crop-top band the bytes are near-uniform (entropy ≈ 7.99), as LSB noise should be.
|
||||
- D30's region is high-entropy from byte 0 and ends in a run of `ff` bytes (saturated LSBs?);
|
||||
no zero rows — the D30's masked rows evidently carry noise. Placement for D30 is supported
|
||||
by the size arithmetic and by its entropy part being perfectly stuffed (0 non-stuffed ff).
|
||||
- **Not yet established:** the bit packing order within each byte, and that the bits really
|
||||
are the two LSBs of the final samples (vs. e.g. MSBs). Decisive test planned (P2, with the
|
||||
oracle): the oracle's output codes' 2 LSBs must equal this region bit-for-bit under one
|
||||
fixed packing rule, for every file.
|
||||
|
||||
## 5 Corrections to the starting assumptions
|
||||
|
||||
- **B3 (bare bitstream, "no JPEG wrapper")** — half right: there are no JPEG markers or
|
||||
segment structure, but the entropy data **is byte-stuffed exactly like JPEG entropy data**
|
||||
(0xff is always followed by 0x00; 2 576–28 275 stuffed ff bytes per file, zero exceptions
|
||||
across 29 files), and the PowerShot payloads even end with a JPEG EOI. **B8's "no
|
||||
byte-stuffing" is refuted.** The decoder must unstuff `ff 00 → ff`.
|
||||
- **B6 (low bits)** — the region exists, but the recollection had its distribution backwards:
|
||||
it is present on all EOS bodies (even the 10-bit-era D30 — see open question below) and on
|
||||
the later 12-bit PowerShots (G5/G6), absent on the 10-bit G1; and its position differs by
|
||||
class (prefix on EOS, after the 514-byte zero block on PowerShots).
|
||||
- **B2 (bit depth)** — if the low-bits hypothesis holds, G1 is the only "no extra bits" body
|
||||
here; whether EOS bodies are 10+2 or the region means something else is a P2 question. (The
|
||||
D30 having the region argues the split is not simply "12-bit bodies".)
|
||||
- **A9 DecoderTable** — layout now pinned: (selector, 0, entropy-offset 514, compressed length).
|
||||
|
||||
## 6 Per-file accounting (public corpus)
|
||||
|
||||
| File | w×h | S | N | L | S−L | Q | S−L−Q | zero run @comp |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| d30 | 2224×1456 | 2 824 648 | 1 | 2 015 110 | 809 538 | 809 536 | 2 | 514 |
|
||||
| d60 | 3152×2068 | 6 694 146 | 0 | 5 064 559 | 1 629 587 | 1 629 584 | 3 | 514 |
|
||||
| 10d-a | 3152×2068 | 7 134 454 | 0 | 5 504 868 | 1 629 586 | 1 629 584 | 2 | 514 |
|
||||
| 10d-b | 3152×2068 | 7 244 858 | 2 | 5 615 272 | 1 629 586 | 1 629 584 | 2 | 514 |
|
||||
| 10d-c | 3152×2068 | 7 455 524 | 2 | 5 825 938 | 1 629 586 | 1 629 584 | 2 | 514 |
|
||||
| 300d-a | 3152×2068 | 6 929 638 | 0 | 5 300 052 | 1 629 586 | 1 629 584 | 2 | 514 |
|
||||
| 300d-b | 3152×2068 | 7 206 250 | 2 | 5 576 664 | 1 629 586 | 1 629 584 | 2 | 514 |
|
||||
| rebel | 3152×2068 | 6 406 164 | 0 | 4 776 577 | 1 629 587 | 1 629 584 | 3 | 514 |
|
||||
| g1 | 2144×1560 | 2 257 354 | 1 | 2 257 351 | 3 | — | — | 514 |
|
||||
| g5 | 2672×1968 | 4 861 806 | 1 | 4 861 804 | 2 | 1 314 624 (inside L) | — | 514 |
|
||||
| g6 | 3160×2344 | 7 313 156 | 0 | 7 313 154 | 2 | 1 851 760 (inside L) | — | 514 |
|
||||
|
||||
(S here is the RawData record size, not the file size.)
|
||||
|
||||
## 7 Open for P2 (entropy derivation)
|
||||
|
||||
1. The entropy code itself (B4): alphabet, tables per selector N, block structure, predictors
|
||||
(B7), bit order within bytes (rest of B8), bit depth per body (B2).
|
||||
2. Low-bits packing order and recombination (B6) — decisive oracle test in §4.
|
||||
3. Why 514 (= 512 + 2?) — cosmetic; decoder just skips to the offset given by the record.
|
||||
4. The D60-class "7 zero rows" and D30's trailing `ff` run in the low-bits region — expect an
|
||||
explanation to fall out of the geometry work (P3).
|
||||
5. EOS tail `ff 00 00` vs `00 00` — settle when the stream end is decodable.
|
||||
|
||||
P2 requires the approved oracle (LibRaw pre-built binary, run black-box, per the plan the
|
||||
maintainer approved) for probing and for the low-bits test. **Requested from the maintainer before P2 starts.**
|
||||
@@ -0,0 +1,115 @@
|
||||
# Canon CRW raw compression — derived format (FROZEN)
|
||||
|
||||
Status: **frozen, revision 2** (plan §5 P4). The derivation is not tuned; any correction
|
||||
must reopen it explicitly with a recorded cause and a new freeze commit.
|
||||
|
||||
**Revision 2 (2026-10-13), cause:** the maintainer supplied 1,516 D60 files, 50 times the
|
||||
first corpus. 223 of them failed on code-table entries the first corpus never exercised,
|
||||
and 9 were misread as having no low-bits plane. Two corrections, both recorded in the
|
||||
derivation log (not part of this record) and re-verified in `03-verification.md`: the low-bits presence rule
|
||||
(§1) gained the header-position test, and the selector 0 and 1 second tables were completed
|
||||
(§3). Nothing else in the model changed, and every file of the first corpus still verifies. Everything here was derived from the CC0/private sample files
|
||||
and black-box probing of LibRaw 0.22.2 (`unprocessed_raw.exe`), per the approved plan; no
|
||||
decoder source was consulted. The chronological evidence is the derivation log, which is not part
|
||||
of this record; the payload groundwork is `01-payload-layout.md`.
|
||||
|
||||
## 1 Payload layout
|
||||
|
||||
RawData (CIFF record 0x2005), with w×h from SensorInfo (0x1031) and
|
||||
(selector, 0, entropyOffset = 514, compressedLength) from DecoderTable (0x1835):
|
||||
|
||||
```
|
||||
[ low-bits plane, ceil(w*h/4) bytes ] present on some bodies (rule below)
|
||||
[ 514 zero bytes ] (the DecoderTable's entropyOffset)
|
||||
[ entropy bitstream, ff00-stuffed ]
|
||||
[ tail: 00 00 / ff 00 00 (EOS) or ff d9 (+00) (PowerShot) ]
|
||||
```
|
||||
|
||||
- **Low-bits plane**: 2 bits per sensor sample, row-major, LSB-first within each byte
|
||||
(sample 4i in bits 1:0 … sample 4i+3 in bits 7:6). Present ⇔ either
|
||||
(a) the first ⌈w·h/4⌉ + 514 bytes of the payload contain any 0xff byte NOT followed by
|
||||
0x00 (the entropy stream is always stuffed, the low-bits plane is raw), or
|
||||
(b) the 514 bytes at offset ⌈w·h/4⌉ are all zero (the zero header sits right after the
|
||||
plane; without a plane that offset is deep in entropy data).
|
||||
Rule (a) alone, the revision-1 rule, misses planes that hold no 0xff byte: some D60
|
||||
files have a lowest bit that is almost always 0 (low values 1 and 3 on about 0.04 % of
|
||||
samples), so no four neighbouring samples are all 3. Verified: present on
|
||||
D30/D60/10D/300D/G5/G6, absent on G1.
|
||||
- On the EOS bodies the DecoderTable's compressedLength excludes the low-bits plane;
|
||||
on G5/G6 it includes it. The decoder should not rely on compressedLength: entropy
|
||||
data starts at (low-bits size, if present) + 514 and runs to the tail.
|
||||
|
||||
## 2 Entropy bitstream
|
||||
|
||||
After removing the stuffing (drop the 0x00 after every 0xff), bits are read
|
||||
**MSB-first**. The stream codes the w×h sensor samples' 10-bit "high" plane
|
||||
(the full sample is `(high << 2) | low_bits` where a low-bits plane exists, else
|
||||
`high` itself):
|
||||
|
||||
- Samples are produced in row-major order, grouped in **blocks of 64 samples** laid
|
||||
flat over the frame (blocks cross row boundaries; w need not divide by 64).
|
||||
- Two canonical Huffman code tables per DecoderTableNumber (0, 1, 2): the token that
|
||||
begins each block is read from the **first table**, every other token from the
|
||||
**second table**.
|
||||
- A token decodes to a leaf **(r, s)** with r ≤ 15: skip r samples (each emits its
|
||||
DPCM chain value unchanged, i.e. a zero diff), then read s magnitude bits and emit
|
||||
one diff (s = 0 emits a zero diff). Magnitudes follow the JPEG rule, MSB-first:
|
||||
m ≥ 2^(s−1) → d = +m, else d = m − 2^s + 1. Sample diffs fit s ≤ 10 (10-bit high
|
||||
plane), but first-table leaves reach s = 11+: the carry-coded block DC difference
|
||||
spans ±2046.
|
||||
- The **(0,0) leaf read as a non-first token is EOB**: the rest of the block emits
|
||||
zero diffs. ((0,0) as the block's first token is a plain zero diff.)
|
||||
- **Carry**: the diff landing on each block's first sample has the previous block's
|
||||
(combined) first diff added: `d0 += carry; carry = d0`. When the first sample is
|
||||
covered by a token's skip (or EOB), its diff is exactly `carry` (and carry is
|
||||
unchanged).
|
||||
- **DPCM chains**: two running predictors by sample parity, both initialised to 512,
|
||||
and both **reset to 512 at the start of every sensor row**.
|
||||
- If the stream ends before the frame is complete, remaining samples emit their chain
|
||||
values (observed: the final 2–3 samples of each file).
|
||||
|
||||
## 3 Code tables
|
||||
|
||||
Extracted from the samples by known-plaintext alignment and frozen as data:
|
||||
|
||||
- Selector 0: `logs/table_sel0.json` — 12 first-table + 162 second-table codes.
|
||||
- Selector 1: `logs/table_sel1.json` — 12 + 162.
|
||||
- Selector 2: `logs/table_sel2.json` — 12 + 162.
|
||||
|
||||
Each second table now holds all 162 symbols: (r, s) for r = 0..15, s = 1..10, plus (0, 0)
|
||||
and (15, 0). Revision 1 lacked (7..14, 10) in selector 0 and (14, 10) in selector 1: runs
|
||||
of 7 or more zero differences followed by a difference of 512 or more, which the first
|
||||
corpus never contained. The nine codes were learned from the wider corpus by the same
|
||||
known-plaintext method; each landed in one of the unassigned 16-bit slots inside the code
|
||||
range, of which there were exactly eight and one. The slots still unassigned are the
|
||||
all-ones tail of each table (5, 20 and 3 codes) and the first tables' longest code.
|
||||
|
||||
Tables are shared across bodies per selector (verified across D60/10D/300D/Rebel for
|
||||
selector 0; D30, G1 and the private D60s for selector 1; 10D, 300D and a private D60
|
||||
for selector 2). The tables have canonical Huffman structure; codes up to 16 bits, the
|
||||
`11111111xxxxxxxx` region carrying the larger (run, size) pairs. The JSON rows are a
|
||||
superset of each file's exercised codes: every row was learned from a real stream
|
||||
position and validated by the decode reaching the end of that frame; rows never
|
||||
exercised by two files may in principle be mergeable duplicates of canonical slots,
|
||||
which the Rust implementation may normalise, provided the verification table of
|
||||
`03-verification.md` still reproduces exactly.
|
||||
|
||||
## 4 Not yet established / out of scope here
|
||||
|
||||
- **G5 anomaly**: its entropy stream decodes end-to-end with the selector-1 tables,
|
||||
but values diverge beginning at sample 2056 (= 4 × 514), and its low-bits plane fits
|
||||
no tested layout (top rows mostly match, decaying to 0 %, no constant shift). The
|
||||
decoder detects the G5 by its make-model and returns `Error::Unsupported` until a
|
||||
follow-up derivation resolves it; a caller may fall back to the embedded JPEG.
|
||||
- The exact meaning of each file's tail bytes (handled by the stream-end rule).
|
||||
- Big-endian (`MM`) CIFF files: no sample exists.
|
||||
- PowerShot G2/G3/Pro1/Pro70/S30–S70: no CC0 samples (licence-excluded); decodable
|
||||
speculatively via the same selectors once their geometry is read from SensorInfo,
|
||||
to be marked unverified.
|
||||
- Black level, white balance and colour (plan P3) are documented separately.
|
||||
|
||||
## 5 Verification
|
||||
|
||||
See `03-verification.md` (written after the freeze): full-frame exact-match decode
|
||||
against LibRaw 0.22.2, with zero tuning after the freeze commit: every sample file, public
|
||||
and private, matches except the PowerShot G5, which the decoder refuses (28 of 29).
|
||||
@@ -0,0 +1,79 @@
|
||||
# CRW decode verification (P4) — against LibRaw 0.22.2, zero tuning
|
||||
|
||||
Method: the frozen model of `02-crw-format.md` plus the frozen tables
|
||||
(`logs/table_sel{0,1,2}.json`) were run by the local Python reference decoder over every
|
||||
sample file, comparing **every sensor sample** (full values, low bits merged) against
|
||||
`unprocessed_raw.exe -T` output. "EXACT" means zero mismatching samples over the whole
|
||||
frame. The table extraction itself consumed these same files (known-plaintext learning),
|
||||
so this is a consistency proof against the oracle, not a blind holdout; the cross-body
|
||||
sharing (tables learned on one body, exact on others) is the strongest independent check.
|
||||
|
||||
Private files are the maintainer's own D60 photos (hashes in `logs/private-corpus.tsv`);
|
||||
only their verdicts are recorded here, never content.
|
||||
|
||||
## Results
|
||||
|
||||
| File | Selector | Frame | Verdict |
|
||||
|---|---|---|---|
|
||||
| `crw-canon-eos-10d-a.crw` | 0 | 3152x2068 | **EXACT** |
|
||||
| `crw-canon-eos-10d-b.crw` | 2 | 3152x2068 | **EXACT** |
|
||||
| `crw-canon-eos-10d-c.crw` | 2 | 3152x2068 | **EXACT** |
|
||||
| `crw-canon-eos-300d-a.crw` | 0 | 3152x2068 | **EXACT** |
|
||||
| `crw-canon-eos-300d-b.crw` | 2 | 3152x2068 | **EXACT** |
|
||||
| `crw-canon-eos-d30.crw` | 1 | 2224x1456 | **EXACT** |
|
||||
| `crw-canon-eos-d60.crw` | 0 | 3152x2068 | **EXACT** |
|
||||
| `crw-canon-eos-digital-rebel.crw` | 0 | 3152x2068 | **EXACT** |
|
||||
| `crw-canon-powershot-g1.crw` | 1 | 2144x1560 | **EXACT** |
|
||||
| `crw-canon-powershot-g5.crw` | 1 | 2672x1968 | 2685618 mismatches, first at 0 (row 0, col 0) |
|
||||
| `crw-canon-powershot-g6.crw` | 0 | 3160x2344 | **EXACT** |
|
||||
| `private/CRW_0476.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_0478.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_0486.crw` | 0 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_0527.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_0683.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_0717.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_0765.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_0822.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_0879.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_1011.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_1153.crw` | 0 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_1277.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_1514.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_1646.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_1740.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_1776.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_2035.crw` | 1 | 3152x2068 | **EXACT** |
|
||||
| `private/CRW_2219.crw` | 2 | 3152x2068 | **EXACT** |
|
||||
|
||||
**28 of 29 files decode EXACT** — all 10 usable public CC0 files and all 18 private
|
||||
D60 files; the PowerShot G5 is the known exception (`02-crw-format.md` §4).
|
||||
|
||||
## Notes
|
||||
|
||||
- The final 1–3 samples of each frame follow the stream-end rule (chains persist);
|
||||
every "EXACT" verdict includes them.
|
||||
- PowerShot G5: see `02-crw-format.md` §4 — detected and excluded; embedded-JPEG
|
||||
fallback until resolved.
|
||||
- The G1 (10-bit, no low-bits plane) verifies the low-bits presence rule; the G6
|
||||
(12-bit, different geometry, w not divisible by 64) verifies the row-reset and
|
||||
block rules on a second geometry.
|
||||
|
||||
## Revision 2: the wider D60 sweep (2026-10-13)
|
||||
|
||||
The maintainer supplied 1,516 Canon EOS D60 files (private; local use only, nothing
|
||||
copied into the repository, verdicts only recorded here). With the revision-1 decoder,
|
||||
223 failed on unseen code-table entries and 9 were misread as 10-bit (`02-crw-format.md`,
|
||||
revision 2). After the two corrections every file was compared against LibRaw 0.22.2
|
||||
`unprocessed_raw.exe -T`, every sensor sample, with the Rust decoder
|
||||
(`plan/crw/model/sweep_all.py`, local):
|
||||
|
||||
| Selector | Files | Verdict |
|
||||
|---|---:|---|
|
||||
| 0 | 243 | all **EXACT** |
|
||||
| 1 | 1,193 | all **EXACT** |
|
||||
| 2 | 80 | all **EXACT** |
|
||||
| **Total** | **1,516** | **1,516 EXACT** |
|
||||
|
||||
The nine learned codes came from 30 of the failing files; the other 202 failing files and
|
||||
the 1,284 that already decoded are a blind check of them. The public files were re-run
|
||||
with the same build: 10 EXACT, the G5 still refused.
|
||||
@@ -0,0 +1,122 @@
|
||||
# CRW colour, geometry and metadata records (P3/P5)
|
||||
|
||||
Status: derived 2026-10-13, after the compression freeze. Everything here comes from the
|
||||
approved sources: the corpus files themselves (public CC0 + the maintainer's private D60s),
|
||||
the oracle's decoded sensor data, each file's own embedded JPEG, and ExifTool's prose tag
|
||||
documentation (`TagNames/CanonRaw.html`, `TagNames/Canon.html` — tag names, table indices and
|
||||
value enumerations only; fetches logged in `sources.md`). No decoder source was consulted.
|
||||
Implemented in `crates/raw/src/vendor/crw.rs`.
|
||||
|
||||
## 1 CFA phase
|
||||
|
||||
Determined by two independent sample-derived tests on each body's oracle mosaic:
|
||||
|
||||
- **Chroma correlation**: bin the mosaic 2×2 into RGB under each of the four Bayer phase
|
||||
hypotheses, roughly gain-match to the body's own embedded JPEG (linearised ≈ γ2.2), and
|
||||
correlate the R−B chroma plane with the JPEG's. The true phase correlates strongly
|
||||
(+0.49…+0.94), its R/B swap strongly negatively (`cfa_check.py`, local model dir).
|
||||
- **Green diagonal**: within 16×16 blocks the two green sites have near-equal sums; the
|
||||
diagonal with the smaller |difference| carries the greens.
|
||||
|
||||
Results: **RGGB at sensor (0, 0)** for EOS D30, D60, 10D, 300D (both tests decisive) and
|
||||
PowerShot G6; **GBRG for the PowerShot G1** (greens on the main diagonal; chroma +0.77 GBRG
|
||||
vs −0.67 GRBG). Unsampled PowerShots default to RGGB, unverified.
|
||||
|
||||
## 2 Active area, black level, white level
|
||||
|
||||
- **Active area**: SensorInfo (0x1031) values 5–8 are an inclusive crop (left, top, right,
|
||||
bottom); valid on every corpus body (G5/G6 included).
|
||||
- **Black level**: measured from the masked borders. Observed (oracle data, per body class):
|
||||
the left border is dark up to 8 columns before the active area on the EOS bodies
|
||||
(≈127–129 DN on D60/10D, ≈128 on D30, ≈124–127 on 300D) but transitions bright earlier on
|
||||
the G6 (dark only to col 40 of 64), and on the G1 only the top border is dark
|
||||
(rows 0–7 ≈ 29 DN; its left border reads ≈98–160). The D60's first ~7 sensor rows are
|
||||
clamped to 0 (not a black reference). Rule adopted: candidate strips = 8-column groups of
|
||||
the left border and 4-row groups of the top border; black = per-CFA-site mean of the
|
||||
darkest strip and all strips within 8 DN of it, ignoring strips with mean ≤ 1 (the clamped
|
||||
rows). This lands on the true dark level on every corpus body. Cross-check: the trailing
|
||||
quadruple of ColorBalance (0x10a9) is (126–127)×4 on D60/10D and (124–125)×4 on 300D —
|
||||
matching the measured borders (ExifTool's prose notes this quadruple as "black levels").
|
||||
- **Correction (2026-10-13):** the first implementation also averaged in the clamped strips
|
||||
it meant to ignore (mean ≤ 1 passes "within 8 DN of the darkest"), which put the D60's
|
||||
level at 117 instead of 128 and lifted every shadow. Fixed and tested; on 20 private D60
|
||||
files the left border's mean is within 2 DN of the record's black quadruple, per site
|
||||
(127–137, higher on the files with the larger calibration-row value).
|
||||
- **White level**: saturation plateau of the decoded samples (shared `white_from_data`);
|
||||
without a plateau, the level at least 64 samples reach when that is within 3 % of the
|
||||
full range (`white_level`); else the full range, 4095 (12-bit) / 1023 (G1).
|
||||
- **The sensor clips below 4095 at low ISO** (2026-10-13, 1,516 private D60 files by ISO
|
||||
from ShotInfo value 2, ISO = 100·2^(v/32 − 5)): at ISO 100, 200 and 400 no sample exceeds
|
||||
4009 and the maxima cluster at 4000–4008; at ISO 800 and 1000 saturated files reach 4095.
|
||||
The 10D and 300D samples stop at 4003–4008 as well. Before `white_level`, a photo with a
|
||||
small blown area had no plateau and got 4095 (373 of 925 files at ISO 100, 283 of 287 at
|
||||
ISO 400), leaving blown pixels at 0.977 of white, below the clip threshold.
|
||||
- **Controlled frames from the maintainer** (lens cap and blown-out, ISO 100 and 1000; all
|
||||
four decode sample-exact). Cap, ISO 100: active area 129.2–130.1 per site, left border
|
||||
128.2–129.6, our level 128.9, record 127–129. Cap, ISO 1000: active area 129.5–130.5,
|
||||
but left border 134.5–135.7, our level 133.8, record 131–133: at high ISO the masked
|
||||
border and the record read about 4 DN above the image's true dark level. One frame, so
|
||||
no correction is applied; noise there is 15 DN. Blown-out, ISO 100: the common values
|
||||
are 3997–4000 and the maximum 4002 (our level 3987). Blown-out, ISO 1000: every active
|
||||
sample is 4095 (our level 4080).
|
||||
|
||||
## 3 As-shot white balance
|
||||
|
||||
- **0x10a9 ColorBalance** (D60/10D/300D only): a byte-length u16, then (R, G, G, B) level
|
||||
quadruples per white-balance setting, the last quadruple being black levels. ExifTool's
|
||||
prose gives the 10D/300D slot order — Auto, Daylight, Shade, Cloudy, Tungsten, Fluorescent,
|
||||
Flash, Custom, Kelvin, Black — at indices 1, 5, …, 37, which the record sizes confirm
|
||||
(82 bytes = 1 + 10×4 u16); the D60's 66-byte record holds 7 slots + black at index 29
|
||||
(= ExifTool's "D60 black levels" index), i.e. the same order without Shade and Kelvin.
|
||||
- Multipliers are **level / green** (e.g. D60 slot 1: 1856, 784, 781, 945 → R 2.37, B 1.21).
|
||||
Confirmed against the cameras' own JPEGs by a white-preserving matrix fit (rows sum to 1,
|
||||
least squares from WB-corrected binned raw to the linearised JPEG): plausible slots score
|
||||
far better than implausible ones; among near-daylight slots the fit cannot fully separate
|
||||
(γ-model error), so the slot is chosen by metadata, not by fit.
|
||||
- **Slot selection**: ShotInfo (0x102a) index 7 is the WhiteBalance setting (ExifTool prose:
|
||||
0 Auto, 1 Daylight, 2 Cloudy, 3 Tungsten, 4 Fluorescent, 5 Flash, 6 Custom, 8 Shade,
|
||||
9 Kelvin). Every corpus file outside the preset captures below reads 0 (Auto) → slot 0.
|
||||
- **D60 preset mapping verified** (2026-10-14): the maintainer shot one overcast scene at
|
||||
each setting, Auto, Daylight, Cloudy, Tungsten, Fluorescent, Flash and Custom, ISO 100,
|
||||
within a minute (all seven decode sample-exact). Slots 1–6 hold the same multipliers in
|
||||
every file; slot 0 (Auto) varies slightly per shot. On the pixels each camera JPEG
|
||||
renders neutral, the multipliers the camera applied are nearest to slot 1 for Daylight,
|
||||
2 for Cloudy, 4 for Fluorescent and 5 for Flash, i.e. setting n uses slot n. Tungsten and
|
||||
Custom leave too few neutral pixels for that test (5 and 0), so they were checked by
|
||||
rendering: the overall tint of our render follows the camera JPEG for every setting
|
||||
(Tungsten JPEG R/G 0.57 B/G 1.41, ours 0.66 / 1.52; Custom 0.62 / 2.93, ours 0.80 /
|
||||
3.02), which no other slot would give. Shade and Kelvin (10D/300D only) remain untested.
|
||||
- **Auto slot confirmed on the private D60 files** (2026-10-13): on the pixels each camera
|
||||
JPEG renders neutral (channel spread < 6 of 255, mid-tone, raw unclipped), the median raw
|
||||
G/R and G/B give the multipliers the camera applied. On 17 files with enough neutral
|
||||
pixels, slot 0 is within 0.005–0.036 (summed log error) of that on every one, flash
|
||||
shots included, and is the nearest slot on 16 (the 17th prefers Daylight by 0.005).
|
||||
Numbers only; no image content recorded.
|
||||
- **0x10aa** (same three bodies): one (R, G, G, B) quadruple with G ≈ 1024 and R/B as
|
||||
*levels* (multiplier = G/level); consistent with a measured-white-balance record (Canon
|
||||
maker-note 0xaa MeasuredColor). On some files it fits the camera JPEG better than slot 0,
|
||||
on others worse; not used, kept as a documented alternative.
|
||||
- **Not derived yet**: D30 and PowerShot white balance (no 0x10a9/0x10aa; the 0x1030
|
||||
WhiteSample record holds an 8×8 packed sample patch — constant filler on G1/G6, varying on
|
||||
D30 — undeciphered). These bodies render with the neutral fallback.
|
||||
- **Colour matrices**: none of the CRW bodies is in `spectral_table.rs`. Rendering uses
|
||||
the engine's per-file starting look fitted to the file's own embedded JPEG
|
||||
(`camera_preview::file_local_look` lists CRW; `docs/camera-preview-colour.md`) — accepted
|
||||
on all 10 decodable public corpus files, holdout RMS 0.009–0.043, including the D30 and
|
||||
the PowerShots, whose missing white balance the fitted matrix absorbs.
|
||||
**Update 2026-10-25:** the EOS D60 has dual-illuminant matrices of our own, fitted to
|
||||
ColorChecker Classic shots under a tungsten bulb and direct sun (the fitting is part of
|
||||
LightCraft's colour work, `crates/raw/src/chart_table.rs`). They give its white balance a Kelvin scale and are its
|
||||
colour when the JPEG fit fails; the look stays the JPEG fit. The other CRW bodies have none.
|
||||
|
||||
## 4 Orientation and capture time
|
||||
|
||||
- ImageInfo (0x1810) value 3 is rotation in degrees. The two portrait 300D corpus files
|
||||
carry 270, and their scene is upright after rotating the stored image 90° counter-clockwise
|
||||
(checked visually against the embedded JPEG) → 270 maps to Exif orientation 8, hence
|
||||
90 → 6 and 180 → 3. All other corpus files carry 0.
|
||||
- TimeStamp (0x180e) value 0 is a Unix time on the camera's local clock (cross-checked
|
||||
against the files' Exif-era siblings and plausibility); decoded as a civil time with no
|
||||
UTC offset.
|
||||
- Make/model: 0x080a, two NUL-terminated strings ("Canon", "Canon EOS D60"); the G5
|
||||
refusal and the G1 CFA special-case key on the model string.
|
||||
@@ -0,0 +1,58 @@
|
||||
# Private EOS D60 corpus and controlled captures
|
||||
|
||||
The maintainer's own Canon EOS D60 files in `../RawImages/` (local only, never copied into
|
||||
the repository). Names, sizes and SHA-256 of every file are in
|
||||
`logs/private-corpus.tsv`; no image content is recorded anywhere. All 1,561 files decode
|
||||
sample-exact against LibRaw 0.22.2 `unprocessed_raw.exe -T`.
|
||||
|
||||
| Set | Files | What it is | What it established |
|
||||
|---|---:|---|---|
|
||||
| `RawImages/` | 1,516 | Everyday photos, 2005–2014, all Auto white balance; ISO 100 (925), 200 (197), 400 (287), 800 (34), 1000 (73) | Freeze revision 2: two incomplete code tables and the plane-without-0xff case (`02-crw-format.md`); the low-ISO clip level (`04-color.md`); the D60 colour profile (`assets/camera-profiles/Canon_EOS_D60.json`, 1,424 usable photos) |
|
||||
| `Cap&WhiteOut/` | 4 | Lens cap and blown-out frames, ISO 100 and 1000 (2026-10-13) | Black level at ISO 100 matches the image area; at ISO 1000 the border reads about 4 DN high. Clip level about 4000 at ISO 100, 4095 at ISO 1000 (`04-color.md`) |
|
||||
| `WBModes/` | 7 | One overcast scene at Auto, Daylight, Cloudy, Tungsten, Fluorescent, Flash, Custom; ISO 100, within a minute (2026-10-14) | White-balance setting n uses ColorBalance slot n (`04-color.md`) |
|
||||
| `TripodGrayCard/` | 14 | The same card filling most of the frame, tripod, f/8, ISO 100; two brackets (Auto 1/16–1/6 s, Cloudy 1/12–1/5 s), each starting and ending on the same speed; 14 s each (2026-10-14) | Black level confirmed; sensor linear; the spill strip (below) |
|
||||
| `GrayCard/` | 20 | An 18 % grey card (film-era), handheld, overcast; 5 frames at f/4 with exposure compensation, 15 at f/1.8 stepping 1/640–1/1600 s under Cloudy, Daylight and Auto (2026-10-14) | See below |
|
||||
|
||||
## Grey-card results (2026-10-14)
|
||||
|
||||
Measured in the middle 28 % of each frame, which lies on the card in every frame (the card
|
||||
is larger and was handheld; spread across the box about 3 %, sensor noise included).
|
||||
|
||||
- **Tone matches the camera.** Our render's card lightness is within 0.3 L\* of the camera
|
||||
JPEG's in all six frames checked (L\* 37.6–59.0 across exposures), so the profile's
|
||||
per-photo tone fit reproduces Canon's curve at mid-grey.
|
||||
- **Neutrals carry a slight magenta.** Our card is a\* +0.6 to +1.5 above the camera's,
|
||||
b\* 0.4–1.0 below (about 1–1.5 ΔE, under the usual threshold of visibility). It is the
|
||||
same small bias seen on the white-balance series' foliage, measured here on a true
|
||||
neutral. The profile matrix's rows sum to 1.005, 0.998 and 1.008, which accounts for part
|
||||
of it. Left for the chart refit, which can hold neutrals exactly.
|
||||
- **Overcast neutral.** The card needs R × 2.68 and B × 1.13 (G = 1) to render neutral, the
|
||||
same in all four groups to ±0.03. That lies between the camera's Daylight (2.54 / 1.22)
|
||||
and Cloudy (2.80 / 1.13) presets; its Auto chose R 2.54–2.60, B 1.14–1.16, slightly cool,
|
||||
which our As Shot reproduces, as it should.
|
||||
- **Black level:** these handheld brackets could not pin it through exposure (shutter
|
||||
accuracy at 1/640–1/1600 s, and the light dimmed about 10 % over the 90 s); the tripod
|
||||
set below settles it. *Correction:* the "border measurement of 130.4–132.0" first written
|
||||
here averaged every masked column, including the 8 next to the image, which catch spilled
|
||||
light. The decoder skips that strip and reads 127.6–128.6 on these 20 frames; the
|
||||
colour-ratio method's 124.5–127.5 is within 4 DN of it.
|
||||
- **What it cannot do:** a single neutral says nothing about how colours map, so the card
|
||||
does not change the colour profile. The chart will.
|
||||
|
||||
## Tripod grey-card results (2026-10-14)
|
||||
|
||||
- **The masked strip next to the image catches spilled light.** Columns 58–65 rise with the
|
||||
scene: 157 at 1/16 s, 206 at 1/6 s, while the columns further out stay at 127–130. The
|
||||
decoder's darkest-strip rule already excludes it: its black level stays at 127.2–128.7
|
||||
through both brackets, whatever the exposure.
|
||||
- **Black level confirmed.** Each bracket was fitted with level = black + k · time · (1 + drift
|
||||
· seconds), the drift taken from the repeated speed (the light fell 5–6 % within each 14 s
|
||||
bracket; without correcting for it the fit puts black near 120). Red, the channel least
|
||||
sensitive to any remaining exposure error, gives 128.0 and 128.2 against the decoder's
|
||||
127.9 and 128.3; green and blue give 129.9–133.1; the colour-ratio method gives
|
||||
125.5–126.2. All within 5 DN of the decoder, so no change.
|
||||
- **The sensor is linear.** With the drift corrected, every frame lies within 1.3 DN of the
|
||||
fitted line on the Auto bracket and within 4 DN on the Cloudy one.
|
||||
- **Lens falloff at f/8:** the card's level varies about ±8 % across the frame (5 × 5 cells,
|
||||
lightest at the bottom, darkest top right), part vignetting, part uneven light on the
|
||||
card. A colour chart in the middle third will see well under that.
|
||||
@@ -0,0 +1,76 @@
|
||||
# Canon CRW clean-room derivation record
|
||||
|
||||
What `craft-crw` decodes a CRW from, and the evidence that it is right. The full record was
|
||||
produced in LightCraft (`clean-room/crw/`, branch `clean-room-crw-format-decode`, revision
|
||||
`6ca439b5155820d2befeacd81ec0c6a80b2bd3bc`; the record is identical at `70c755e4`, where the decoder was ported) between 2026-10-12 and 2026-10-25; the documents below moved here
|
||||
**verbatim** when the decoder became this shared crate, and the derivation trail stays at that revision. The documents are the evidence: they are
|
||||
not rewritten after the fact, so paths they mention are LightCraft's paths of that time (the
|
||||
mapping is below).
|
||||
|
||||
The derivation trail — the starting assumptions, the plan, the container notes, the
|
||||
entropy-derivation log, the chart calibration, the probe scripts and their observational logs — is
|
||||
not part of this record. What this record holds is what the decoder is built from and what a
|
||||
reviewer needs to check it: the frozen format, its verification, the payload-layout and colour
|
||||
notes, the private-corpus record, the source log and the frozen code tables. Those documents are
|
||||
renumbered `01`–`05` in reading order.
|
||||
|
||||
## Documents
|
||||
|
||||
| File | What it is |
|
||||
|---|---|
|
||||
| [`sources.md`](sources.md) | Every URL opened and search run, with the time, what was asked and what was taken. The rules in force are at its top. |
|
||||
| [`01-payload-layout.md`](01-payload-layout.md) | The RawData payload partition established by statistics alone (no decoding): low-bits plane, 514-byte header, ff00-stuffed stream, tail. |
|
||||
| [`02-crw-format.md`](02-crw-format.md) | **The frozen format** (revision 2) and its reopening rule. The decoder implements this document. |
|
||||
| [`03-verification.md`](03-verification.md) | Full-frame exact comparison against the reference, zero tuning: 28 of 29 files, then 1,516 of 1,516 D60 files. |
|
||||
| [`04-color.md`](04-color.md) | CFA phase, active area, black and white levels, white-balance records, orientation and time: what the decoder reports beyond the samples. §3's colour matrices concern LightCraft (below). |
|
||||
| [`05-private-corpus.md`](05-private-corpus.md) | The maintainer's EOS D60 files and controlled captures (grey card, lens cap, white-balance presets) and what each set established. |
|
||||
|
||||
## Tables and logs
|
||||
|
||||
- `logs/table_sel{0,1,2}.json`: **the frozen code tables**, the source of truth that
|
||||
`tools/gen_tables.py` renders into `../src/tables.rs`. `table_sel0_d60class.json` is the
|
||||
revision-1 selector-0 table, kept for the record. These are the frozen code-table logs that ship
|
||||
with the crate: without them the frozen tables cannot be regenerated or audited.
|
||||
- `logs/private-corpus.tsv`: names, sizes and SHA-256 of the maintainer's own files. No image
|
||||
content, metadata or thumbnails are recorded anywhere here.
|
||||
- `logs/private/` (gitignored): per-file dumps of the private files, which can hold personal
|
||||
metadata. Local only.
|
||||
- The probe scripts and their observational logs (`structure-*.txt`, `p1*.txt`,
|
||||
`verification-sweep.txt`, `chart-captures.tsv`) are not part of this record.
|
||||
|
||||
## What stayed local
|
||||
|
||||
- The Python reference model and probe harness (`plan/crw/model/` in LightCraft, gitignored),
|
||||
by the maintainer's decision #4. Its findings are the entropy-coding derivation, which is not
|
||||
part of this record.
|
||||
- The reference decoder: LibRaw 0.22.2, official pre-built Windows binaries installed by the
|
||||
maintainer; only `unprocessed_raw.exe` was ever run, as a black box. The source tree shipped
|
||||
alongside it was never opened, read or searched.
|
||||
- Canon's CIFF specification PDF (copyrighted; archive URL and SHA-256 in `sources.md`).
|
||||
- The reference TIFFs (`<name>.crw.tiff`) the verification compared against; `CRAFT_CRW_ORACLE`
|
||||
points the test at a local copy.
|
||||
|
||||
## Path mapping (LightCraft at the time → craft-libs)
|
||||
|
||||
| In the documents | Now |
|
||||
|---|---|
|
||||
| `clean-room/crw/…` | `crates/crw/clean-room/…` |
|
||||
| `crates/raw/src/vendor/crw.rs` | `crates/crw/src/lib.rs` |
|
||||
| `crates/raw/src/vendor/crw_tables.rs` | `crates/crw/src/tables.rs` |
|
||||
| `crates/raw/tests/crw_oracle.rs` | `crates/crw/tests/samples.rs` (`CRAFT_CRW_ORACLE`, formerly `LIGHTCRAFT_CRW_ORACLE`) |
|
||||
| `white_from_data` (LightCraft's shared estimate) | `saturation_plateau` in `crates/crw/src/lib.rs` |
|
||||
| `xtask/src/main.rs` `RAW_SAMPLES` (corpus URLs) | `crates/crw/tests/samples.json` + `scripts/fetch-crw-samples.py` |
|
||||
| `plan/crw/model/gen_tables.py` | `crates/crw/clean-room/tools/gen_tables.py` |
|
||||
| `docs/arw6-compression.md` (the house model for a derivation) | LightCraft's document; the method is summarised in `docs/clean-room.md` of this repository |
|
||||
| `crates/raw/src/chart_table.rs`, `spectral_table.rs`, `camera_preview`, `assets/camera-profiles/` | LightCraft (colour stays in the app) |
|
||||
|
||||
## Rules that were in force
|
||||
|
||||
From `sources.md`: no raw-decoder source may be opened, fetched, searched inside
|
||||
or requested (dcraw, LibRaw, rawspeed, rawloader/rawler, dnglab, darktable, RawTherapee,
|
||||
libopenraw, ExifTool's Perl code, including licence headers, comments and mirrors). Allowed:
|
||||
published format documentation in prose, published standards, the sample files, and a reference
|
||||
decoder's binary run as a black box with every probe logged. No tuning after the freeze: a
|
||||
mismatch reopens the derivation explicitly, with its cause and a new freeze commit recorded
|
||||
(`02-crw-format.md`, revision 2, is the one reopening). The general standard this followed is
|
||||
[`docs/clean-room.md`](../../../docs/clean-room.md).
|
||||
@@ -0,0 +1 @@
|
||||
private/
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,702 @@
|
||||
{
|
||||
"first": {
|
||||
"111110": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"11110": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"101": [
|
||||
0,
|
||||
2
|
||||
],
|
||||
"00": [
|
||||
0,
|
||||
4
|
||||
],
|
||||
"1101": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"010": [
|
||||
0,
|
||||
3
|
||||
],
|
||||
"011": [
|
||||
0,
|
||||
5
|
||||
],
|
||||
"11101": [
|
||||
0,
|
||||
9
|
||||
],
|
||||
"100": [
|
||||
0,
|
||||
6
|
||||
],
|
||||
"11100": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"1100": [
|
||||
0,
|
||||
7
|
||||
],
|
||||
"1111110": [
|
||||
0,
|
||||
11
|
||||
]
|
||||
},
|
||||
"main": {
|
||||
"00": [
|
||||
0,
|
||||
3
|
||||
],
|
||||
"01": [
|
||||
0,
|
||||
4
|
||||
],
|
||||
"100": [
|
||||
0,
|
||||
2
|
||||
],
|
||||
"101": [
|
||||
0,
|
||||
5
|
||||
],
|
||||
"1100": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"1101": [
|
||||
0,
|
||||
6
|
||||
],
|
||||
"11100": [
|
||||
0,
|
||||
7
|
||||
],
|
||||
"111011": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"111100": [
|
||||
1,
|
||||
3
|
||||
],
|
||||
"111101": [
|
||||
1,
|
||||
1
|
||||
],
|
||||
"1111101": [
|
||||
0,
|
||||
9
|
||||
],
|
||||
"11111100": [
|
||||
1,
|
||||
5
|
||||
],
|
||||
"111111010": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"111111011": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"1111111010": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"1111111011": [
|
||||
15,
|
||||
0
|
||||
],
|
||||
"1111111101110100": [
|
||||
3,
|
||||
3
|
||||
],
|
||||
"1111111101110101": [
|
||||
2,
|
||||
5
|
||||
],
|
||||
"1111111101110111": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"1111111110000111": [
|
||||
5,
|
||||
2
|
||||
],
|
||||
"1111111110001010": [
|
||||
4,
|
||||
3
|
||||
],
|
||||
"1111111110001111": [
|
||||
6,
|
||||
1
|
||||
],
|
||||
"1111111110010001": [
|
||||
7,
|
||||
1
|
||||
],
|
||||
"1111111110011000": [
|
||||
5,
|
||||
3
|
||||
],
|
||||
"1111111110100010": [
|
||||
9,
|
||||
1
|
||||
],
|
||||
"1111111110110110": [
|
||||
11,
|
||||
1
|
||||
],
|
||||
"1111111111001010": [
|
||||
10,
|
||||
1
|
||||
],
|
||||
"1111111111001100": [
|
||||
13,
|
||||
1
|
||||
],
|
||||
"1111111111010101": [
|
||||
12,
|
||||
1
|
||||
],
|
||||
"1111111111011001": [
|
||||
14,
|
||||
1
|
||||
],
|
||||
"1111111111011110": [
|
||||
15,
|
||||
1
|
||||
],
|
||||
"1111111101110000": [
|
||||
3,
|
||||
1
|
||||
],
|
||||
"1111111101110110": [
|
||||
4,
|
||||
1
|
||||
],
|
||||
"1111111101111000": [
|
||||
4,
|
||||
2
|
||||
],
|
||||
"1111111101111010": [
|
||||
5,
|
||||
1
|
||||
],
|
||||
"1111111110001000": [
|
||||
5,
|
||||
5
|
||||
],
|
||||
"1111111110001110": [
|
||||
5,
|
||||
4
|
||||
],
|
||||
"1111111110110001": [
|
||||
8,
|
||||
1
|
||||
],
|
||||
"1111111111011000": [
|
||||
6,
|
||||
4
|
||||
],
|
||||
"1111111111100111": [
|
||||
6,
|
||||
3
|
||||
],
|
||||
"11111111010": [
|
||||
1,
|
||||
7
|
||||
],
|
||||
"1111111000": [
|
||||
2,
|
||||
1
|
||||
],
|
||||
"1111111100": [
|
||||
2,
|
||||
3
|
||||
],
|
||||
"1111111110000000": [
|
||||
2,
|
||||
6
|
||||
],
|
||||
"1111100": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"1111111101110001": [
|
||||
3,
|
||||
2
|
||||
],
|
||||
"111111110110": [
|
||||
2,
|
||||
4
|
||||
],
|
||||
"111010": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"1111111001": [
|
||||
1,
|
||||
6
|
||||
],
|
||||
"1111111101110010": [
|
||||
1,
|
||||
8
|
||||
],
|
||||
"1111111110000101": [
|
||||
2,
|
||||
8
|
||||
],
|
||||
"1111111111100110": [
|
||||
10,
|
||||
4
|
||||
],
|
||||
"1111111111000111": [
|
||||
6,
|
||||
5
|
||||
],
|
||||
"1111111101111001": [
|
||||
3,
|
||||
5
|
||||
],
|
||||
"1111111110111000": [
|
||||
4,
|
||||
4
|
||||
],
|
||||
"1111111110001011": [
|
||||
7,
|
||||
6
|
||||
],
|
||||
"1111111110101101": [
|
||||
4,
|
||||
5
|
||||
],
|
||||
"1111111111001111": [
|
||||
9,
|
||||
2
|
||||
],
|
||||
"1111111110111100": [
|
||||
6,
|
||||
6
|
||||
],
|
||||
"1111111110111001": [
|
||||
7,
|
||||
2
|
||||
],
|
||||
"1111111101111011": [
|
||||
3,
|
||||
6
|
||||
],
|
||||
"1111111110010100": [
|
||||
9,
|
||||
6
|
||||
],
|
||||
"1111111110101100": [
|
||||
4,
|
||||
6
|
||||
],
|
||||
"1111111110010011": [
|
||||
7,
|
||||
5
|
||||
],
|
||||
"1111111110000110": [
|
||||
2,
|
||||
7
|
||||
],
|
||||
"1111111101111101": [
|
||||
3,
|
||||
8
|
||||
],
|
||||
"1111111110011010": [
|
||||
7,
|
||||
4
|
||||
],
|
||||
"1111111101111100": [
|
||||
3,
|
||||
7
|
||||
],
|
||||
"1111111111011101": [
|
||||
11,
|
||||
3
|
||||
],
|
||||
"1111111110000011": [
|
||||
5,
|
||||
6
|
||||
],
|
||||
"1111111110001101": [
|
||||
7,
|
||||
7
|
||||
],
|
||||
"1111111110011111": [
|
||||
9,
|
||||
5
|
||||
],
|
||||
"1111111101110011": [
|
||||
1,
|
||||
9
|
||||
],
|
||||
"1111111110001001": [
|
||||
5,
|
||||
8
|
||||
],
|
||||
"1111111111100101": [
|
||||
8,
|
||||
4
|
||||
],
|
||||
"1111111110111110": [
|
||||
15,
|
||||
5
|
||||
],
|
||||
"1111111111000101": [
|
||||
6,
|
||||
2
|
||||
],
|
||||
"1111111110010101": [
|
||||
9,
|
||||
7
|
||||
],
|
||||
"1111111111010000": [
|
||||
8,
|
||||
5
|
||||
],
|
||||
"1111111111101101": [
|
||||
8,
|
||||
2
|
||||
],
|
||||
"1111111110011110": [
|
||||
4,
|
||||
8
|
||||
],
|
||||
"1111111110011001": [
|
||||
13,
|
||||
7
|
||||
],
|
||||
"1111111110101011": [
|
||||
6,
|
||||
7
|
||||
],
|
||||
"1111111110101110": [
|
||||
9,
|
||||
4
|
||||
],
|
||||
"1111111110011101": [
|
||||
4,
|
||||
7
|
||||
],
|
||||
"1111111111101011": [
|
||||
13,
|
||||
2
|
||||
],
|
||||
"1111111110110100": [
|
||||
11,
|
||||
4
|
||||
],
|
||||
"1111111111010110": [
|
||||
12,
|
||||
6
|
||||
],
|
||||
"1111111101111110": [
|
||||
2,
|
||||
9
|
||||
],
|
||||
"1111111111100010": [
|
||||
11,
|
||||
2
|
||||
],
|
||||
"1111111111110011": [
|
||||
8,
|
||||
3
|
||||
],
|
||||
"1111111110000010": [
|
||||
3,
|
||||
9
|
||||
],
|
||||
"1111111110100110": [
|
||||
11,
|
||||
5
|
||||
],
|
||||
"1111111110001100": [
|
||||
5,
|
||||
9
|
||||
],
|
||||
"1111111111110101": [
|
||||
10,
|
||||
2
|
||||
],
|
||||
"1111111110100011": [
|
||||
15,
|
||||
10
|
||||
],
|
||||
"1111111110010000": [
|
||||
15,
|
||||
9
|
||||
],
|
||||
"1111111111010011": [
|
||||
9,
|
||||
3
|
||||
],
|
||||
"1111111111111001": [
|
||||
14,
|
||||
2
|
||||
],
|
||||
"1111111110100111": [
|
||||
11,
|
||||
9
|
||||
],
|
||||
"1111111110111010": [
|
||||
13,
|
||||
9
|
||||
],
|
||||
"1111111111101010": [
|
||||
15,
|
||||
3
|
||||
],
|
||||
"1111111111000000": [
|
||||
10,
|
||||
7
|
||||
],
|
||||
"1111111111000100": [
|
||||
8,
|
||||
6
|
||||
],
|
||||
"1111111111111000": [
|
||||
12,
|
||||
2
|
||||
],
|
||||
"1111111110000100": [
|
||||
5,
|
||||
7
|
||||
],
|
||||
"1111111110100101": [
|
||||
6,
|
||||
8
|
||||
],
|
||||
"1111111111001101": [
|
||||
14,
|
||||
9
|
||||
],
|
||||
"1111111101111111": [
|
||||
7,
|
||||
9
|
||||
],
|
||||
"1111111111000110": [
|
||||
12,
|
||||
7
|
||||
],
|
||||
"1111111110111011": [
|
||||
8,
|
||||
7
|
||||
],
|
||||
"1111111111101000": [
|
||||
14,
|
||||
5
|
||||
],
|
||||
"1111111110011011": [
|
||||
11,
|
||||
6
|
||||
],
|
||||
"1111111110011100": [
|
||||
9,
|
||||
8
|
||||
],
|
||||
"1111111111100000": [
|
||||
10,
|
||||
5
|
||||
],
|
||||
"1111111111010001": [
|
||||
10,
|
||||
6
|
||||
],
|
||||
"1111111110010110": [
|
||||
4,
|
||||
9
|
||||
],
|
||||
"1111111111011100": [
|
||||
14,
|
||||
6
|
||||
],
|
||||
"1111111110101000": [
|
||||
13,
|
||||
6
|
||||
],
|
||||
"1111111110110101": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"1111111110101001": [
|
||||
15,
|
||||
7
|
||||
],
|
||||
"1111111110101111": [
|
||||
8,
|
||||
9
|
||||
],
|
||||
"1111111111000010": [
|
||||
10,
|
||||
9
|
||||
],
|
||||
"1111111111010100": [
|
||||
14,
|
||||
8
|
||||
],
|
||||
"1111111111101100": [
|
||||
12,
|
||||
4
|
||||
],
|
||||
"1111111110110011": [
|
||||
15,
|
||||
6
|
||||
],
|
||||
"1111111111101001": [
|
||||
12,
|
||||
5
|
||||
],
|
||||
"1111111110101010": [
|
||||
13,
|
||||
8
|
||||
],
|
||||
"1111111111000001": [
|
||||
7,
|
||||
3
|
||||
],
|
||||
"1111111110110111": [
|
||||
2,
|
||||
10
|
||||
],
|
||||
"1111111110000001": [
|
||||
1,
|
||||
10
|
||||
],
|
||||
"1111111111110001": [
|
||||
15,
|
||||
2
|
||||
],
|
||||
"1111111110100000": [
|
||||
6,
|
||||
9
|
||||
],
|
||||
"1111111110010111": [
|
||||
11,
|
||||
7
|
||||
],
|
||||
"1111111111111010": [
|
||||
14,
|
||||
3
|
||||
],
|
||||
"1111111111110000": [
|
||||
14,
|
||||
4
|
||||
],
|
||||
"1111111111000011": [
|
||||
10,
|
||||
8
|
||||
],
|
||||
"1111111110010010": [
|
||||
7,
|
||||
8
|
||||
],
|
||||
"1111111111011111": [
|
||||
13,
|
||||
3
|
||||
],
|
||||
"1111111111001000": [
|
||||
12,
|
||||
8
|
||||
],
|
||||
"1111111111010010": [
|
||||
14,
|
||||
7
|
||||
],
|
||||
"1111111110110010": [
|
||||
13,
|
||||
5
|
||||
],
|
||||
"1111111110111101": [
|
||||
13,
|
||||
4
|
||||
],
|
||||
"1111111111001011": [
|
||||
15,
|
||||
4
|
||||
],
|
||||
"1111111111001001": [
|
||||
12,
|
||||
9
|
||||
],
|
||||
"1111111110100100": [
|
||||
11,
|
||||
8
|
||||
],
|
||||
"1111111110110000": [
|
||||
15,
|
||||
8
|
||||
],
|
||||
"1111111111110100": [
|
||||
10,
|
||||
3
|
||||
],
|
||||
"1111111111011010": [
|
||||
4,
|
||||
10
|
||||
],
|
||||
"1111111110111111": [
|
||||
3,
|
||||
10
|
||||
],
|
||||
"1111111110100001": [
|
||||
9,
|
||||
9
|
||||
],
|
||||
"1111111111110110": [
|
||||
12,
|
||||
3
|
||||
],
|
||||
"1111111111011011": [
|
||||
6,
|
||||
10
|
||||
],
|
||||
"1111111111001110": [
|
||||
5,
|
||||
10
|
||||
],
|
||||
"1111111111101111": [
|
||||
13,
|
||||
10
|
||||
],
|
||||
"1111111111010111": [
|
||||
7,
|
||||
10
|
||||
],
|
||||
"1111111111110010": [
|
||||
12,
|
||||
10
|
||||
],
|
||||
"1111111111100011": [
|
||||
9,
|
||||
10
|
||||
],
|
||||
"1111111111101110": [
|
||||
10,
|
||||
10
|
||||
],
|
||||
"1111111111100001": [
|
||||
8,
|
||||
10
|
||||
],
|
||||
"1111111111110111": [
|
||||
14,
|
||||
10
|
||||
],
|
||||
"1111111111100100": [
|
||||
11,
|
||||
10
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,574 @@
|
||||
{
|
||||
"first": {
|
||||
"111110": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"11110": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"101": [
|
||||
0,
|
||||
2
|
||||
],
|
||||
"00": [
|
||||
0,
|
||||
4
|
||||
],
|
||||
"1101": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"010": [
|
||||
0,
|
||||
3
|
||||
],
|
||||
"011": [
|
||||
0,
|
||||
5
|
||||
],
|
||||
"11101": [
|
||||
0,
|
||||
9
|
||||
],
|
||||
"100": [
|
||||
0,
|
||||
6
|
||||
],
|
||||
"11100": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"1100": [
|
||||
0,
|
||||
7
|
||||
]
|
||||
},
|
||||
"main": {
|
||||
"00": [
|
||||
0,
|
||||
3
|
||||
],
|
||||
"01": [
|
||||
0,
|
||||
4
|
||||
],
|
||||
"100": [
|
||||
0,
|
||||
2
|
||||
],
|
||||
"101": [
|
||||
0,
|
||||
5
|
||||
],
|
||||
"1100": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"1101": [
|
||||
0,
|
||||
6
|
||||
],
|
||||
"11100": [
|
||||
0,
|
||||
7
|
||||
],
|
||||
"111011": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"111100": [
|
||||
1,
|
||||
3
|
||||
],
|
||||
"111101": [
|
||||
1,
|
||||
1
|
||||
],
|
||||
"1111101": [
|
||||
0,
|
||||
9
|
||||
],
|
||||
"11111100": [
|
||||
1,
|
||||
5
|
||||
],
|
||||
"111111010": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"111111011": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"1111111010": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"1111111011": [
|
||||
15,
|
||||
0
|
||||
],
|
||||
"1111111101110100": [
|
||||
3,
|
||||
3
|
||||
],
|
||||
"1111111101110101": [
|
||||
2,
|
||||
5
|
||||
],
|
||||
"1111111101110111": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"1111111110000111": [
|
||||
5,
|
||||
2
|
||||
],
|
||||
"1111111110001010": [
|
||||
4,
|
||||
3
|
||||
],
|
||||
"1111111110001111": [
|
||||
6,
|
||||
1
|
||||
],
|
||||
"1111111110010001": [
|
||||
7,
|
||||
1
|
||||
],
|
||||
"1111111110011000": [
|
||||
5,
|
||||
3
|
||||
],
|
||||
"1111111110100010": [
|
||||
9,
|
||||
1
|
||||
],
|
||||
"1111111110110110": [
|
||||
11,
|
||||
1
|
||||
],
|
||||
"1111111111001010": [
|
||||
10,
|
||||
1
|
||||
],
|
||||
"1111111111001100": [
|
||||
13,
|
||||
1
|
||||
],
|
||||
"1111111111010101": [
|
||||
12,
|
||||
1
|
||||
],
|
||||
"1111111111011001": [
|
||||
14,
|
||||
1
|
||||
],
|
||||
"1111111111011110": [
|
||||
15,
|
||||
1
|
||||
],
|
||||
"1111111101110000": [
|
||||
3,
|
||||
1
|
||||
],
|
||||
"1111111101110110": [
|
||||
4,
|
||||
1
|
||||
],
|
||||
"1111111101111000": [
|
||||
4,
|
||||
2
|
||||
],
|
||||
"1111111101111010": [
|
||||
5,
|
||||
1
|
||||
],
|
||||
"1111111110001000": [
|
||||
5,
|
||||
5
|
||||
],
|
||||
"1111111110001110": [
|
||||
5,
|
||||
4
|
||||
],
|
||||
"1111111110110001": [
|
||||
8,
|
||||
1
|
||||
],
|
||||
"1111111111011000": [
|
||||
6,
|
||||
4
|
||||
],
|
||||
"1111111111100111": [
|
||||
6,
|
||||
3
|
||||
],
|
||||
"11111111010": [
|
||||
1,
|
||||
7
|
||||
],
|
||||
"1111111000": [
|
||||
2,
|
||||
1
|
||||
],
|
||||
"1111111100": [
|
||||
2,
|
||||
3
|
||||
],
|
||||
"1111111110000000": [
|
||||
2,
|
||||
6
|
||||
],
|
||||
"1111100": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"1111111101110001": [
|
||||
3,
|
||||
2
|
||||
],
|
||||
"111111110110": [
|
||||
2,
|
||||
4
|
||||
],
|
||||
"111010": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"1111111001": [
|
||||
1,
|
||||
6
|
||||
],
|
||||
"1111111101110010": [
|
||||
1,
|
||||
8
|
||||
],
|
||||
"1111111110000101": [
|
||||
2,
|
||||
8
|
||||
],
|
||||
"1111111111100110": [
|
||||
10,
|
||||
4
|
||||
],
|
||||
"1111111111000111": [
|
||||
6,
|
||||
5
|
||||
],
|
||||
"1111111101111001": [
|
||||
3,
|
||||
5
|
||||
],
|
||||
"1111111110111000": [
|
||||
4,
|
||||
4
|
||||
],
|
||||
"1111111110001011": [
|
||||
7,
|
||||
6
|
||||
],
|
||||
"1111111110101101": [
|
||||
4,
|
||||
5
|
||||
],
|
||||
"1111111111001111": [
|
||||
9,
|
||||
2
|
||||
],
|
||||
"1111111110111100": [
|
||||
6,
|
||||
6
|
||||
],
|
||||
"1111111110111001": [
|
||||
7,
|
||||
2
|
||||
],
|
||||
"1111111101111011": [
|
||||
3,
|
||||
6
|
||||
],
|
||||
"1111111110010100": [
|
||||
9,
|
||||
6
|
||||
],
|
||||
"1111111110101100": [
|
||||
4,
|
||||
6
|
||||
],
|
||||
"1111111110010011": [
|
||||
7,
|
||||
5
|
||||
],
|
||||
"1111111110000110": [
|
||||
2,
|
||||
7
|
||||
],
|
||||
"1111111101111101": [
|
||||
3,
|
||||
8
|
||||
],
|
||||
"1111111110011010": [
|
||||
7,
|
||||
4
|
||||
],
|
||||
"1111111101111100": [
|
||||
3,
|
||||
7
|
||||
],
|
||||
"1111111111011101": [
|
||||
11,
|
||||
3
|
||||
],
|
||||
"1111111110000011": [
|
||||
5,
|
||||
6
|
||||
],
|
||||
"1111111110001101": [
|
||||
7,
|
||||
7
|
||||
],
|
||||
"1111111110011111": [
|
||||
9,
|
||||
5
|
||||
],
|
||||
"1111111101110011": [
|
||||
1,
|
||||
9
|
||||
],
|
||||
"1111111110001001": [
|
||||
5,
|
||||
8
|
||||
],
|
||||
"1111111111100101": [
|
||||
8,
|
||||
4
|
||||
],
|
||||
"1111111110111110": [
|
||||
15,
|
||||
5
|
||||
],
|
||||
"1111111111000101": [
|
||||
6,
|
||||
2
|
||||
],
|
||||
"1111111110010101": [
|
||||
9,
|
||||
7
|
||||
],
|
||||
"1111111111010000": [
|
||||
8,
|
||||
5
|
||||
],
|
||||
"1111111111101101": [
|
||||
8,
|
||||
2
|
||||
],
|
||||
"1111111110011110": [
|
||||
4,
|
||||
8
|
||||
],
|
||||
"1111111110011001": [
|
||||
13,
|
||||
7
|
||||
],
|
||||
"1111111110101011": [
|
||||
6,
|
||||
7
|
||||
],
|
||||
"1111111110101110": [
|
||||
9,
|
||||
4
|
||||
],
|
||||
"1111111110011101": [
|
||||
4,
|
||||
7
|
||||
],
|
||||
"1111111111101011": [
|
||||
13,
|
||||
2
|
||||
],
|
||||
"1111111110110100": [
|
||||
11,
|
||||
4
|
||||
],
|
||||
"1111111111010110": [
|
||||
12,
|
||||
6
|
||||
],
|
||||
"1111111101111110": [
|
||||
2,
|
||||
9
|
||||
],
|
||||
"1111111111100010": [
|
||||
11,
|
||||
2
|
||||
],
|
||||
"1111111111110011": [
|
||||
8,
|
||||
3
|
||||
],
|
||||
"1111111110000010": [
|
||||
3,
|
||||
9
|
||||
],
|
||||
"1111111110100110": [
|
||||
11,
|
||||
5
|
||||
],
|
||||
"1111111110001100": [
|
||||
5,
|
||||
9
|
||||
],
|
||||
"1111111111110101": [
|
||||
10,
|
||||
2
|
||||
],
|
||||
"1111111110100011": [
|
||||
15,
|
||||
10
|
||||
],
|
||||
"1111111110010000": [
|
||||
15,
|
||||
9
|
||||
],
|
||||
"1111111111010011": [
|
||||
9,
|
||||
3
|
||||
],
|
||||
"1111111111111001": [
|
||||
14,
|
||||
2
|
||||
],
|
||||
"1111111110100111": [
|
||||
11,
|
||||
9
|
||||
],
|
||||
"1111111110111010": [
|
||||
13,
|
||||
9
|
||||
],
|
||||
"1111111111101010": [
|
||||
15,
|
||||
3
|
||||
],
|
||||
"1111111111000000": [
|
||||
10,
|
||||
7
|
||||
],
|
||||
"1111111111000100": [
|
||||
8,
|
||||
6
|
||||
],
|
||||
"1111111111111000": [
|
||||
12,
|
||||
2
|
||||
],
|
||||
"1111111110000100": [
|
||||
5,
|
||||
7
|
||||
],
|
||||
"1111111110100101": [
|
||||
6,
|
||||
8
|
||||
],
|
||||
"1111111111001101": [
|
||||
14,
|
||||
9
|
||||
],
|
||||
"1111111101111111": [
|
||||
7,
|
||||
9
|
||||
],
|
||||
"1111111111000110": [
|
||||
12,
|
||||
7
|
||||
],
|
||||
"1111111110111011": [
|
||||
8,
|
||||
7
|
||||
],
|
||||
"1111111111101000": [
|
||||
14,
|
||||
5
|
||||
],
|
||||
"1111111110011011": [
|
||||
11,
|
||||
6
|
||||
],
|
||||
"1111111110011100": [
|
||||
9,
|
||||
8
|
||||
],
|
||||
"1111111111100000": [
|
||||
10,
|
||||
5
|
||||
],
|
||||
"1111111111010001": [
|
||||
10,
|
||||
6
|
||||
],
|
||||
"1111111110010110": [
|
||||
4,
|
||||
9
|
||||
],
|
||||
"1111111111011100": [
|
||||
14,
|
||||
6
|
||||
],
|
||||
"1111111110101000": [
|
||||
13,
|
||||
6
|
||||
],
|
||||
"1111111110110101": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"1111111110101001": [
|
||||
15,
|
||||
7
|
||||
],
|
||||
"1111111110101111": [
|
||||
8,
|
||||
9
|
||||
],
|
||||
"1111111111000010": [
|
||||
10,
|
||||
9
|
||||
],
|
||||
"1111111111010100": [
|
||||
14,
|
||||
8
|
||||
],
|
||||
"1111111111101100": [
|
||||
12,
|
||||
4
|
||||
],
|
||||
"1111111110110011": [
|
||||
15,
|
||||
6
|
||||
],
|
||||
"1111111111101001": [
|
||||
12,
|
||||
5
|
||||
],
|
||||
"1111111110101010": [
|
||||
13,
|
||||
8
|
||||
],
|
||||
"1111111111000001": [
|
||||
7,
|
||||
3
|
||||
],
|
||||
"1111111110110111": [
|
||||
2,
|
||||
10
|
||||
],
|
||||
"1111111110000001": [
|
||||
1,
|
||||
10
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,706 @@
|
||||
{
|
||||
"first": {
|
||||
"11111110": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"1101": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"00": [
|
||||
0,
|
||||
3
|
||||
],
|
||||
"01": [
|
||||
0,
|
||||
2
|
||||
],
|
||||
"100": [
|
||||
0,
|
||||
4
|
||||
],
|
||||
"1100": [
|
||||
0,
|
||||
5
|
||||
],
|
||||
"101": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"1110": [
|
||||
0,
|
||||
6
|
||||
],
|
||||
"11110": [
|
||||
0,
|
||||
7
|
||||
],
|
||||
"111110": [
|
||||
0,
|
||||
9
|
||||
],
|
||||
"1111110": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"111111110": [
|
||||
0,
|
||||
11
|
||||
]
|
||||
},
|
||||
"main": {
|
||||
"1111111100": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"111111010": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"01": [
|
||||
0,
|
||||
3
|
||||
],
|
||||
"101": [
|
||||
0,
|
||||
4
|
||||
],
|
||||
"00": [
|
||||
0,
|
||||
2
|
||||
],
|
||||
"100": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"11010": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"111111011": [
|
||||
2,
|
||||
3
|
||||
],
|
||||
"11101": [
|
||||
1,
|
||||
3
|
||||
],
|
||||
"11011": [
|
||||
1,
|
||||
1
|
||||
],
|
||||
"1111111010": [
|
||||
3,
|
||||
1
|
||||
],
|
||||
"11111100": [
|
||||
2,
|
||||
1
|
||||
],
|
||||
"1111100": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"1100": [
|
||||
0,
|
||||
5
|
||||
],
|
||||
"11100": [
|
||||
0,
|
||||
6
|
||||
],
|
||||
"1111011": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"111100": [
|
||||
0,
|
||||
7
|
||||
],
|
||||
"1111010": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"111111100": [
|
||||
1,
|
||||
5
|
||||
],
|
||||
"1111111101100001": [
|
||||
2,
|
||||
4
|
||||
],
|
||||
"11111111010": [
|
||||
1,
|
||||
6
|
||||
],
|
||||
"1111101": [
|
||||
0,
|
||||
9
|
||||
],
|
||||
"1111111101110001": [
|
||||
7,
|
||||
1
|
||||
],
|
||||
"1111111101100011": [
|
||||
4,
|
||||
1
|
||||
],
|
||||
"1111111011": [
|
||||
3,
|
||||
2
|
||||
],
|
||||
"1111111101101111": [
|
||||
6,
|
||||
1
|
||||
],
|
||||
"1111111101100010": [
|
||||
3,
|
||||
3
|
||||
],
|
||||
"1111111101101001": [
|
||||
5,
|
||||
1
|
||||
],
|
||||
"1111111101101011": [
|
||||
4,
|
||||
3
|
||||
],
|
||||
"1111111101100110": [
|
||||
1,
|
||||
7
|
||||
],
|
||||
"1111111101111011": [
|
||||
9,
|
||||
1
|
||||
],
|
||||
"1111111101100000": [
|
||||
15,
|
||||
0
|
||||
],
|
||||
"1111111101111111": [
|
||||
2,
|
||||
8
|
||||
],
|
||||
"1111111101100101": [
|
||||
1,
|
||||
9
|
||||
],
|
||||
"1111111101101010": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"1111111110001000": [
|
||||
13,
|
||||
1
|
||||
],
|
||||
"1111111101101000": [
|
||||
1,
|
||||
8
|
||||
],
|
||||
"1111111110000010": [
|
||||
10,
|
||||
1
|
||||
],
|
||||
"1111111110000011": [
|
||||
11,
|
||||
1
|
||||
],
|
||||
"1111111101100100": [
|
||||
4,
|
||||
2
|
||||
],
|
||||
"1111111110000100": [
|
||||
4,
|
||||
4
|
||||
],
|
||||
"1111111101111001": [
|
||||
8,
|
||||
1
|
||||
],
|
||||
"1111111101110101": [
|
||||
2,
|
||||
6
|
||||
],
|
||||
"1111111111001111": [
|
||||
6,
|
||||
7
|
||||
],
|
||||
"1111111110100001": [
|
||||
11,
|
||||
6
|
||||
],
|
||||
"1111111101101100": [
|
||||
5,
|
||||
2
|
||||
],
|
||||
"1111111101100111": [
|
||||
2,
|
||||
5
|
||||
],
|
||||
"1111111101111010": [
|
||||
2,
|
||||
7
|
||||
],
|
||||
"1111111110000111": [
|
||||
5,
|
||||
8
|
||||
],
|
||||
"1111111101111000": [
|
||||
3,
|
||||
7
|
||||
],
|
||||
"1111111110001011": [
|
||||
14,
|
||||
1
|
||||
],
|
||||
"1111111101110110": [
|
||||
3,
|
||||
8
|
||||
],
|
||||
"1111111110011111": [
|
||||
12,
|
||||
1
|
||||
],
|
||||
"1111111110110110": [
|
||||
4,
|
||||
8
|
||||
],
|
||||
"1111111111000011": [
|
||||
6,
|
||||
8
|
||||
],
|
||||
"1111111110010100": [
|
||||
4,
|
||||
6
|
||||
],
|
||||
"1111111110000000": [
|
||||
7,
|
||||
2
|
||||
],
|
||||
"1111111110001110": [
|
||||
4,
|
||||
9
|
||||
],
|
||||
"1111111101101110": [
|
||||
3,
|
||||
5
|
||||
],
|
||||
"1111111111100101": [
|
||||
11,
|
||||
2
|
||||
],
|
||||
"1111111110001100": [
|
||||
15,
|
||||
1
|
||||
],
|
||||
"1111111110001111": [
|
||||
4,
|
||||
7
|
||||
],
|
||||
"1111111101110011": [
|
||||
3,
|
||||
6
|
||||
],
|
||||
"1111111111000001": [
|
||||
9,
|
||||
2
|
||||
],
|
||||
"1111111110110111": [
|
||||
8,
|
||||
2
|
||||
],
|
||||
"1111111101101101": [
|
||||
2,
|
||||
9
|
||||
],
|
||||
"1111111110001010": [
|
||||
5,
|
||||
7
|
||||
],
|
||||
"1111111101110010": [
|
||||
6,
|
||||
2
|
||||
],
|
||||
"1111111101111110": [
|
||||
4,
|
||||
5
|
||||
],
|
||||
"1111111110011101": [
|
||||
8,
|
||||
6
|
||||
],
|
||||
"1111111110100000": [
|
||||
7,
|
||||
6
|
||||
],
|
||||
"1111111111000000": [
|
||||
12,
|
||||
6
|
||||
],
|
||||
"1111111101110100": [
|
||||
5,
|
||||
3
|
||||
],
|
||||
"1111111101111101": [
|
||||
5,
|
||||
5
|
||||
],
|
||||
"1111111111011011": [
|
||||
6,
|
||||
4
|
||||
],
|
||||
"1111111110011110": [
|
||||
6,
|
||||
5
|
||||
],
|
||||
"1111111101110000": [
|
||||
3,
|
||||
9
|
||||
],
|
||||
"1111111110000001": [
|
||||
5,
|
||||
9
|
||||
],
|
||||
"1111111110010000": [
|
||||
6,
|
||||
3
|
||||
],
|
||||
"1111111110111011": [
|
||||
10,
|
||||
6
|
||||
],
|
||||
"1111111111001110": [
|
||||
6,
|
||||
6
|
||||
],
|
||||
"1111111110101111": [
|
||||
7,
|
||||
3
|
||||
],
|
||||
"1111111111010110": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"1111111110011000": [
|
||||
7,
|
||||
8
|
||||
],
|
||||
"1111111111001101": [
|
||||
12,
|
||||
2
|
||||
],
|
||||
"1111111111001010": [
|
||||
10,
|
||||
2
|
||||
],
|
||||
"1111111110110100": [
|
||||
10,
|
||||
7
|
||||
],
|
||||
"1111111110000110": [
|
||||
5,
|
||||
4
|
||||
],
|
||||
"1111111111011101": [
|
||||
9,
|
||||
8
|
||||
],
|
||||
"1111111111011001": [
|
||||
7,
|
||||
7
|
||||
],
|
||||
"1111111110101000": [
|
||||
9,
|
||||
5
|
||||
],
|
||||
"1111111110100010": [
|
||||
9,
|
||||
6
|
||||
],
|
||||
"1111111110101101": [
|
||||
9,
|
||||
7
|
||||
],
|
||||
"1111111111011110": [
|
||||
10,
|
||||
5
|
||||
],
|
||||
"1111111110100101": [
|
||||
8,
|
||||
5
|
||||
],
|
||||
"1111111110111000": [
|
||||
8,
|
||||
4
|
||||
],
|
||||
"1111111110011011": [
|
||||
7,
|
||||
5
|
||||
],
|
||||
"1111111110010011": [
|
||||
5,
|
||||
6
|
||||
],
|
||||
"1111111111010000": [
|
||||
9,
|
||||
3
|
||||
],
|
||||
"1111111110000101": [
|
||||
6,
|
||||
9
|
||||
],
|
||||
"1111111110001001": [
|
||||
15,
|
||||
10
|
||||
],
|
||||
"1111111110010010": [
|
||||
15,
|
||||
9
|
||||
],
|
||||
"1111111111101001": [
|
||||
8,
|
||||
3
|
||||
],
|
||||
"1111111111001001": [
|
||||
14,
|
||||
9
|
||||
],
|
||||
"1111111110100100": [
|
||||
8,
|
||||
9
|
||||
],
|
||||
"1111111111011100": [
|
||||
14,
|
||||
2
|
||||
],
|
||||
"1111111111101000": [
|
||||
13,
|
||||
2
|
||||
],
|
||||
"1111111111101011": [
|
||||
13,
|
||||
3
|
||||
],
|
||||
"1111111110110011": [
|
||||
8,
|
||||
7
|
||||
],
|
||||
"1111111111100111": [
|
||||
15,
|
||||
2
|
||||
],
|
||||
"1111111110001101": [
|
||||
11,
|
||||
9
|
||||
],
|
||||
"1111111101111100": [
|
||||
7,
|
||||
9
|
||||
],
|
||||
"1111111101110111": [
|
||||
1,
|
||||
10
|
||||
],
|
||||
"1111111110110101": [
|
||||
7,
|
||||
10
|
||||
],
|
||||
"1111111111010100": [
|
||||
14,
|
||||
7
|
||||
],
|
||||
"1111111110110000": [
|
||||
11,
|
||||
7
|
||||
],
|
||||
"1111111111011000": [
|
||||
13,
|
||||
7
|
||||
],
|
||||
"1111111110110001": [
|
||||
13,
|
||||
8
|
||||
],
|
||||
"1111111111010011": [
|
||||
13,
|
||||
5
|
||||
],
|
||||
"1111111110100011": [
|
||||
13,
|
||||
6
|
||||
],
|
||||
"1111111110110010": [
|
||||
13,
|
||||
9
|
||||
],
|
||||
"1111111110010110": [
|
||||
2,
|
||||
10
|
||||
],
|
||||
"1111111110011010": [
|
||||
3,
|
||||
10
|
||||
],
|
||||
"1111111110111101": [
|
||||
5,
|
||||
10
|
||||
],
|
||||
"1111111110010111": [
|
||||
4,
|
||||
10
|
||||
],
|
||||
"1111111110101100": [
|
||||
8,
|
||||
10
|
||||
],
|
||||
"1111111111000101": [
|
||||
12,
|
||||
8
|
||||
],
|
||||
"1111111110010001": [
|
||||
6,
|
||||
10
|
||||
],
|
||||
"1111111111101010": [
|
||||
11,
|
||||
10
|
||||
],
|
||||
"1111111110011001": [
|
||||
9,
|
||||
9
|
||||
],
|
||||
"1111111110011100": [
|
||||
7,
|
||||
4
|
||||
],
|
||||
"1111111110101110": [
|
||||
11,
|
||||
8
|
||||
],
|
||||
"1111111111010101": [
|
||||
15,
|
||||
8
|
||||
],
|
||||
"1111111111010010": [
|
||||
13,
|
||||
4
|
||||
],
|
||||
"1111111110010101": [
|
||||
10,
|
||||
8
|
||||
],
|
||||
"1111111111000111": [
|
||||
14,
|
||||
5
|
||||
],
|
||||
"1111111110101001": [
|
||||
11,
|
||||
4
|
||||
],
|
||||
"1111111110111010": [
|
||||
15,
|
||||
4
|
||||
],
|
||||
"1111111111100000": [
|
||||
13,
|
||||
10
|
||||
],
|
||||
"1111111111001100": [
|
||||
14,
|
||||
3
|
||||
],
|
||||
"1111111111001011": [
|
||||
10,
|
||||
3
|
||||
],
|
||||
"1111111110111111": [
|
||||
10,
|
||||
4
|
||||
],
|
||||
"1111111111000010": [
|
||||
12,
|
||||
3
|
||||
],
|
||||
"1111111111000100": [
|
||||
11,
|
||||
5
|
||||
],
|
||||
"1111111110101011": [
|
||||
15,
|
||||
7
|
||||
],
|
||||
"1111111110100110": [
|
||||
12,
|
||||
9
|
||||
],
|
||||
"1111111111000110": [
|
||||
14,
|
||||
4
|
||||
],
|
||||
"1111111111011010": [
|
||||
12,
|
||||
4
|
||||
],
|
||||
"1111111111100010": [
|
||||
15,
|
||||
3
|
||||
],
|
||||
"1111111111100110": [
|
||||
11,
|
||||
3
|
||||
],
|
||||
"1111111111001000": [
|
||||
14,
|
||||
6
|
||||
],
|
||||
"1111111110111110": [
|
||||
9,
|
||||
4
|
||||
],
|
||||
"1111111110111100": [
|
||||
12,
|
||||
5
|
||||
],
|
||||
"1111111111100011": [
|
||||
15,
|
||||
6
|
||||
],
|
||||
"1111111111010111": [
|
||||
9,
|
||||
10
|
||||
],
|
||||
"1111111110100111": [
|
||||
15,
|
||||
5
|
||||
],
|
||||
"1111111110101010": [
|
||||
12,
|
||||
7
|
||||
],
|
||||
"1111111111100100": [
|
||||
10,
|
||||
9
|
||||
],
|
||||
"1111111111010001": [
|
||||
10,
|
||||
10
|
||||
],
|
||||
"1111111111100001": [
|
||||
14,
|
||||
8
|
||||
],
|
||||
"1111111111011111": [
|
||||
12,
|
||||
10
|
||||
],
|
||||
"1111111110111001": [
|
||||
14,
|
||||
10
|
||||
]
|
||||
},
|
||||
"reached": 6518333,
|
||||
"N": 6518336,
|
||||
"file": "CRW_0476.crw",
|
||||
"selector": 1
|
||||
}
|
||||
@@ -0,0 +1,706 @@
|
||||
{
|
||||
"first": {
|
||||
"011": [
|
||||
0,
|
||||
4
|
||||
],
|
||||
"010": [
|
||||
0,
|
||||
7
|
||||
],
|
||||
"101": [
|
||||
0,
|
||||
3
|
||||
],
|
||||
"1100": [
|
||||
0,
|
||||
9
|
||||
],
|
||||
"000": [
|
||||
0,
|
||||
6
|
||||
],
|
||||
"100": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"001": [
|
||||
0,
|
||||
5
|
||||
],
|
||||
"111110": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"1101": [
|
||||
0,
|
||||
2
|
||||
],
|
||||
"1110": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"11110": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"1111110": [
|
||||
0,
|
||||
11
|
||||
]
|
||||
},
|
||||
"main": {
|
||||
"011": [
|
||||
0,
|
||||
6
|
||||
],
|
||||
"001": [
|
||||
0,
|
||||
5
|
||||
],
|
||||
"101": [
|
||||
0,
|
||||
7
|
||||
],
|
||||
"1101": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"100": [
|
||||
0,
|
||||
2
|
||||
],
|
||||
"11100": [
|
||||
0,
|
||||
9
|
||||
],
|
||||
"1111011": [
|
||||
1,
|
||||
5
|
||||
],
|
||||
"000": [
|
||||
0,
|
||||
4
|
||||
],
|
||||
"1100": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"010": [
|
||||
0,
|
||||
3
|
||||
],
|
||||
"1111100": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"1111111100110": [
|
||||
3,
|
||||
8
|
||||
],
|
||||
"1111111100111": [
|
||||
3,
|
||||
7
|
||||
],
|
||||
"1111010": [
|
||||
1,
|
||||
1
|
||||
],
|
||||
"111111100": [
|
||||
1,
|
||||
8
|
||||
],
|
||||
"111100": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"111011": [
|
||||
1,
|
||||
3
|
||||
],
|
||||
"111111110001": [
|
||||
3,
|
||||
1
|
||||
],
|
||||
"1111111111001011": [
|
||||
6,
|
||||
1
|
||||
],
|
||||
"111111011": [
|
||||
2,
|
||||
1
|
||||
],
|
||||
"111111001": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"11111111100000": [
|
||||
4,
|
||||
1
|
||||
],
|
||||
"111010": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"11111010": [
|
||||
1,
|
||||
6
|
||||
],
|
||||
"1111111010": [
|
||||
2,
|
||||
3
|
||||
],
|
||||
"111111010": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"11111110111": [
|
||||
2,
|
||||
4
|
||||
],
|
||||
"11111011": [
|
||||
1,
|
||||
7
|
||||
],
|
||||
"1111111110100111": [
|
||||
5,
|
||||
1
|
||||
],
|
||||
"1111111111100101": [
|
||||
8,
|
||||
1
|
||||
],
|
||||
"1111111100100": [
|
||||
2,
|
||||
5
|
||||
],
|
||||
"1111111111010011": [
|
||||
7,
|
||||
1
|
||||
],
|
||||
"1111111100101": [
|
||||
3,
|
||||
3
|
||||
],
|
||||
"111111110000": [
|
||||
3,
|
||||
2
|
||||
],
|
||||
"11111110110": [
|
||||
1,
|
||||
9
|
||||
],
|
||||
"1111111110001001": [
|
||||
2,
|
||||
6
|
||||
],
|
||||
"1111111111010100": [
|
||||
9,
|
||||
1
|
||||
],
|
||||
"1111111110001011": [
|
||||
7,
|
||||
6
|
||||
],
|
||||
"1111111110010000": [
|
||||
9,
|
||||
7
|
||||
],
|
||||
"1111111111001111": [
|
||||
6,
|
||||
5
|
||||
],
|
||||
"1111111111111000": [
|
||||
10,
|
||||
1
|
||||
],
|
||||
"11111111011111": [
|
||||
2,
|
||||
7
|
||||
],
|
||||
"11111111011101": [
|
||||
5,
|
||||
6
|
||||
],
|
||||
"1111111110011011": [
|
||||
6,
|
||||
8
|
||||
],
|
||||
"1111111110111110": [
|
||||
6,
|
||||
6
|
||||
],
|
||||
"1111111110010010": [
|
||||
4,
|
||||
8
|
||||
],
|
||||
"11111111100001": [
|
||||
2,
|
||||
9
|
||||
],
|
||||
"11111111011100": [
|
||||
2,
|
||||
8
|
||||
],
|
||||
"1111111101010": [
|
||||
3,
|
||||
6
|
||||
],
|
||||
"1111111111101110": [
|
||||
13,
|
||||
1
|
||||
],
|
||||
"1111111101001": [
|
||||
3,
|
||||
5
|
||||
],
|
||||
"1111111111101100": [
|
||||
11,
|
||||
1
|
||||
],
|
||||
"11111111011010": [
|
||||
5,
|
||||
8
|
||||
],
|
||||
"1111111110100101": [
|
||||
11,
|
||||
5
|
||||
],
|
||||
"1111111110110010": [
|
||||
15,
|
||||
6
|
||||
],
|
||||
"1111111110100100": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"1111111101000": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"1111111111000001": [
|
||||
4,
|
||||
4
|
||||
],
|
||||
"1111111110001111": [
|
||||
9,
|
||||
8
|
||||
],
|
||||
"1111111110010110": [
|
||||
4,
|
||||
7
|
||||
],
|
||||
"1111111110101101": [
|
||||
8,
|
||||
7
|
||||
],
|
||||
"1111111110100010": [
|
||||
5,
|
||||
2
|
||||
],
|
||||
"1111111110011010": [
|
||||
7,
|
||||
5
|
||||
],
|
||||
"1111111111111001": [
|
||||
12,
|
||||
1
|
||||
],
|
||||
"11111111011001": [
|
||||
5,
|
||||
7
|
||||
],
|
||||
"1111111110001101": [
|
||||
1,
|
||||
10
|
||||
],
|
||||
"11111111011011": [
|
||||
5,
|
||||
9
|
||||
],
|
||||
"1111111101011": [
|
||||
3,
|
||||
9
|
||||
],
|
||||
"11111111011000": [
|
||||
7,
|
||||
9
|
||||
],
|
||||
"1111111110110101": [
|
||||
10,
|
||||
9
|
||||
],
|
||||
"1111111110110111": [
|
||||
5,
|
||||
3
|
||||
],
|
||||
"1111111110011101": [
|
||||
6,
|
||||
7
|
||||
],
|
||||
"1111111110111010": [
|
||||
4,
|
||||
3
|
||||
],
|
||||
"1111111111001110": [
|
||||
12,
|
||||
6
|
||||
],
|
||||
"1111111110101000": [
|
||||
4,
|
||||
6
|
||||
],
|
||||
"1111111110001110": [
|
||||
5,
|
||||
5
|
||||
],
|
||||
"1111111110101110": [
|
||||
4,
|
||||
5
|
||||
],
|
||||
"1111111110001010": [
|
||||
4,
|
||||
2
|
||||
],
|
||||
"111111000": [
|
||||
15,
|
||||
0
|
||||
],
|
||||
"1111111111000101": [
|
||||
12,
|
||||
8
|
||||
],
|
||||
"1111111110010100": [
|
||||
9,
|
||||
6
|
||||
],
|
||||
"1111111111011100": [
|
||||
12,
|
||||
5
|
||||
],
|
||||
"1111111110101100": [
|
||||
13,
|
||||
6
|
||||
],
|
||||
"1111111110011100": [
|
||||
11,
|
||||
6
|
||||
],
|
||||
"1111111110111001": [
|
||||
10,
|
||||
8
|
||||
],
|
||||
"1111111111010110": [
|
||||
10,
|
||||
6
|
||||
],
|
||||
"1111111111011110": [
|
||||
10,
|
||||
4
|
||||
],
|
||||
"1111111111110100": [
|
||||
8,
|
||||
2
|
||||
],
|
||||
"1111111110110011": [
|
||||
8,
|
||||
6
|
||||
],
|
||||
"1111111111010101": [
|
||||
9,
|
||||
3
|
||||
],
|
||||
"1111111111101111": [
|
||||
8,
|
||||
3
|
||||
],
|
||||
"11111111011110": [
|
||||
7,
|
||||
8
|
||||
],
|
||||
"1111111110001100": [
|
||||
9,
|
||||
9
|
||||
],
|
||||
"1111111110111111": [
|
||||
10,
|
||||
7
|
||||
],
|
||||
"1111111111101101": [
|
||||
14,
|
||||
4
|
||||
],
|
||||
"1111111111110000": [
|
||||
6,
|
||||
3
|
||||
],
|
||||
"1111111111111100": [
|
||||
14,
|
||||
1
|
||||
],
|
||||
"1111111110010001": [
|
||||
15,
|
||||
9
|
||||
],
|
||||
"1111111110110100": [
|
||||
11,
|
||||
4
|
||||
],
|
||||
"1111111110010011": [
|
||||
5,
|
||||
4
|
||||
],
|
||||
"1111111111111011": [
|
||||
10,
|
||||
2
|
||||
],
|
||||
"1111111111110101": [
|
||||
15,
|
||||
1
|
||||
],
|
||||
"1111111111100001": [
|
||||
6,
|
||||
4
|
||||
],
|
||||
"1111111111011111": [
|
||||
8,
|
||||
4
|
||||
],
|
||||
"1111111110110000": [
|
||||
9,
|
||||
5
|
||||
],
|
||||
"1111111110011000": [
|
||||
4,
|
||||
9
|
||||
],
|
||||
"1111111110111100": [
|
||||
15,
|
||||
7
|
||||
],
|
||||
"1111111111110010": [
|
||||
12,
|
||||
3
|
||||
],
|
||||
"1111111111100011": [
|
||||
11,
|
||||
3
|
||||
],
|
||||
"1111111111100110": [
|
||||
14,
|
||||
5
|
||||
],
|
||||
"1111111111010010": [
|
||||
14,
|
||||
6
|
||||
],
|
||||
"1111111111100100": [
|
||||
13,
|
||||
2
|
||||
],
|
||||
"1111111111100010": [
|
||||
10,
|
||||
5
|
||||
],
|
||||
"1111111110011110": [
|
||||
6,
|
||||
9
|
||||
],
|
||||
"1111111110001000": [
|
||||
7,
|
||||
7
|
||||
],
|
||||
"1111111111010001": [
|
||||
7,
|
||||
2
|
||||
],
|
||||
"1111111111011010": [
|
||||
6,
|
||||
2
|
||||
],
|
||||
"1111111111001001": [
|
||||
7,
|
||||
3
|
||||
],
|
||||
"1111111110100011": [
|
||||
13,
|
||||
7
|
||||
],
|
||||
"1111111111011001": [
|
||||
8,
|
||||
5
|
||||
],
|
||||
"1111111111001010": [
|
||||
4,
|
||||
10
|
||||
],
|
||||
"1111111110110001": [
|
||||
13,
|
||||
5
|
||||
],
|
||||
"1111111110111101": [
|
||||
13,
|
||||
4
|
||||
],
|
||||
"1111111110100110": [
|
||||
7,
|
||||
4
|
||||
],
|
||||
"1111111110100000": [
|
||||
11,
|
||||
8
|
||||
],
|
||||
"1111111110101011": [
|
||||
3,
|
||||
10
|
||||
],
|
||||
"1111111110100001": [
|
||||
13,
|
||||
8
|
||||
],
|
||||
"1111111110010101": [
|
||||
8,
|
||||
9
|
||||
],
|
||||
"1111111110010111": [
|
||||
11,
|
||||
7
|
||||
],
|
||||
"1111111110011111": [
|
||||
11,
|
||||
9
|
||||
],
|
||||
"1111111111001100": [
|
||||
12,
|
||||
7
|
||||
],
|
||||
"1111111111000110": [
|
||||
14,
|
||||
7
|
||||
],
|
||||
"1111111110101010": [
|
||||
15,
|
||||
8
|
||||
],
|
||||
"1111111110110110": [
|
||||
9,
|
||||
4
|
||||
],
|
||||
"1111111111110011": [
|
||||
14,
|
||||
2
|
||||
],
|
||||
"1111111110111011": [
|
||||
15,
|
||||
5
|
||||
],
|
||||
"1111111111011101": [
|
||||
11,
|
||||
2
|
||||
],
|
||||
"1111111111101001": [
|
||||
12,
|
||||
4
|
||||
],
|
||||
"1111111110101001": [
|
||||
13,
|
||||
9
|
||||
],
|
||||
"1111111111000100": [
|
||||
14,
|
||||
8
|
||||
],
|
||||
"1111111111011011": [
|
||||
15,
|
||||
3
|
||||
],
|
||||
"1111111111100111": [
|
||||
13,
|
||||
3
|
||||
],
|
||||
"1111111111101011": [
|
||||
15,
|
||||
2
|
||||
],
|
||||
"1111111111011000": [
|
||||
9,
|
||||
2
|
||||
],
|
||||
"1111111111001101": [
|
||||
15,
|
||||
4
|
||||
],
|
||||
"1111111111000000": [
|
||||
5,
|
||||
10
|
||||
],
|
||||
"1111111111000011": [
|
||||
12,
|
||||
9
|
||||
],
|
||||
"1111111110101111": [
|
||||
7,
|
||||
10
|
||||
],
|
||||
"1111111111010111": [
|
||||
13,
|
||||
10
|
||||
],
|
||||
"1111111111010000": [
|
||||
14,
|
||||
9
|
||||
],
|
||||
"1111111111110111": [
|
||||
12,
|
||||
2
|
||||
],
|
||||
"1111111111110110": [
|
||||
10,
|
||||
3
|
||||
],
|
||||
"1111111111001000": [
|
||||
6,
|
||||
10
|
||||
],
|
||||
"1111111111000111": [
|
||||
9,
|
||||
10
|
||||
],
|
||||
"1111111110111000": [
|
||||
2,
|
||||
10
|
||||
],
|
||||
"1111111111101010": [
|
||||
12,
|
||||
10
|
||||
],
|
||||
"1111111111111010": [
|
||||
14,
|
||||
3
|
||||
],
|
||||
"1111111111000010": [
|
||||
8,
|
||||
10
|
||||
],
|
||||
"1111111111100000": [
|
||||
11,
|
||||
10
|
||||
],
|
||||
"1111111111110001": [
|
||||
14,
|
||||
10
|
||||
],
|
||||
"1111111110011001": [
|
||||
15,
|
||||
10
|
||||
],
|
||||
"1111111111101000": [
|
||||
10,
|
||||
10
|
||||
]
|
||||
},
|
||||
"reached": 6518335,
|
||||
"N": 6518336,
|
||||
"file": "crw-canon-eos-300d-b.crw",
|
||||
"selector": 2
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
# CRW clean-room derivation — source log
|
||||
|
||||
Every URL opened and every web search run for this work, in order, with the time (local,
|
||||
Europe-style 24 h), what was asked, and what was taken. If a result or page exposed details of an
|
||||
existing decoder's algorithm that were not asked for, that is recorded too, with what it said.
|
||||
|
||||
Rules in force (from the task): no raw-decoder source may be opened, fetched, searched inside or
|
||||
requested — dcraw, LibRaw, rawspeed, rawloader/rawler, dnglab, darktable, RawTherapee,
|
||||
libopenraw, ExifTool's Perl code — including licence headers, comments and mirrors. Allowed:
|
||||
published format documentation in prose (ExifTool's CIFF/CRW format and tag-name pages),
|
||||
published standards, and the sample files. The branch `crw-decode` (an earlier attempt) is not
|
||||
checked out, diffed, logged or read.
|
||||
|
||||
## Local files consulted (non-web)
|
||||
|
||||
- 2026-10-12 ~19:2x — `CLAUDE.md` (project instructions; in session context from the start) and
|
||||
`docs/arw6-compression.md` (the house model for deriving and documenting a format). Taken:
|
||||
process shape (assumptions → spec + samples → Python reference model → freeze → black-box
|
||||
verification), documentation conventions, corpus table format.
|
||||
- 2026-10-12 ~19:2x — directory listing and SHA-256 of the maintainer's private
|
||||
`../RawImages/*.crw` (18 files, Canon EOS D60 era, 5.5–8.6 MB). Taken: file names, sizes,
|
||||
hashes only. No .THM files present. Contents not yet parsed.
|
||||
|
||||
## Web
|
||||
|
||||
- 2026-10-12 19:26 — **Fetch** `https://exiftool.org/TagNames/CanonRaw.html` (ExifTool's
|
||||
CanonRaw tag-name page; generated documentation, not Perl source). Asked: CIFF structure, tag
|
||||
IDs/names, compression info, spec links. Taken: tag IDs — 0x2005 RawData, 0x2007 JpgFromRaw,
|
||||
0x2008 ThumbnailImage, 0x1835 DecoderTable (subdir: DecoderTableNumber, CompressedDataOffset,
|
||||
CompressedDataLength), 0x1031 SensorInfo (subdir), 0x1810 ImageInfo (ImageWidth, ImageHeight,
|
||||
ComponentBitDepth, ColorBitDepth…), 0x1030 WhiteSample (BlackLevels at index 55), 0x10a9
|
||||
ColorBalance, 0x10ae ColorTemperature, 0x080a CanonRawMakeModel, 0x102a CanonShotInfo, 0x1834
|
||||
CanonModelID, 0x300a ImageProps, 0x2807 CameraObject, 0x300b ExifInformation, 0x0000
|
||||
NullRecord, 0x0001 FreeBytes, and others. No compression algorithm described. Links to
|
||||
`canon_raw.html` for structure. **No decoder algorithm details shown.**
|
||||
- 2026-10-12 19:27 — **Fetch** `https://exiftool.org/canon_raw.html` (Phil Harvey's prose "Canon
|
||||
CRW file format" page). Taken: 26-byte header (ByteOrder `II`/`MM` @0, HeaderLength u32 @2 =
|
||||
0x1a, signature `HEAPCCDR` @6, CRWVersion 0x00010002 @14, 8 reserved bytes @18); block =
|
||||
ValueData, DirCount u16, N×10-byte entries (tag u16, size u32, offset u32), optional
|
||||
OtherData, u32 DirStart at BlockSize−4; offsets relative to the block's ValueData start;
|
||||
values 2-byte aligned; tag masks DataLocation 0xc000 (0x0000 = ValueData, 0x4000 = in-entry,
|
||||
0x8000/0xc000 unknown), DataFormat 0x3800 (0x0000 bytes, 0x0800 ASCII string, 0x1000 u16s,
|
||||
0x1800 u32s/floats, 0x2000 structure, 0x2800/0x3000 subdirectory), TagIndex 0x07ff;
|
||||
subdirectory = a complete block inside ValueData, its BlockSize = entry Size; root block from
|
||||
HeaderLength to EOF. RawData 0x2005 noted as "the bulk of the CRW file"; JpgFromRaw 0x2007;
|
||||
no compression description. Links the original Canon CIFF spec at `http://xyrion.org/ciff/`
|
||||
(dead) and lclevy's CR2/CR3 pages. **No decoder algorithm details shown.**
|
||||
- 2026-10-12 19:28 — **Search** (WebSearch, standard): `Canon CIFF specification 1.0 rev 4 PDF
|
||||
"CIFF" camera image file format specification download`. Results included Wikipedia
|
||||
"Camera Image File Format", a freedesktop libopenraw wiki page "Canon CRW" (NOT opened — it is
|
||||
a decoder project's page; left for the maintainer to approve), an unpkg mirror of
|
||||
exiftool's canon_raw.html, PRONOM. Taken: pointer to Wikipedia for an archived spec URL.
|
||||
**No decoder algorithm details shown in the result snippets.**
|
||||
- 2026-10-12 19:28 — **Fetch attempt** `http://xyrion.org/ciff/` — failed (site dead).
|
||||
- 2026-10-12 19:29 — **Fetch** `https://en.wikipedia.org/wiki/Camera_Image_File_Format`. Taken:
|
||||
spec part names/dates (Image Data File v1.0 r4, 1997-12-24; File/Directory organization v1.0
|
||||
r3, 1998-02-09), archive URL for the PDF, camera list (EOS D30/D60/10D/300D, PowerShot Pro1,
|
||||
G1–G6, S30–S70).
|
||||
- 2026-10-12 19:29 — **Download** (curl)
|
||||
`https://web.archive.org/web/20081230095207/http://xyrion.org/ciff/CIFFspecV1R04.pdf` →
|
||||
scratchpad `ciff-spec/CIFFspecV1R04.pdf`, 99 422 bytes, SHA-256
|
||||
`416d4e49ea3f6ad2ba5ccc76b48ca100605f4a6127816c145e6dd5d6c3b51cf4`, 34 pages. Text extracted
|
||||
with a local stdlib-Python script (`pdftext.py`, kept in scratchpad; extraction output
|
||||
`extracted.txt`). This is Canon's own published "CIFF Specification on Image Data File"
|
||||
v1.0 r4 (1997-12-24); local copies (PDF + text extraction) kept in the gitignored
|
||||
`plan/crw/` (Canon-copyrighted document, not committed; re-fetchable from the archive URL,
|
||||
hash above). Taken: the heap file header struct (byteOrder, headerLength, type
|
||||
"HEAP", subType "JPGM"/"CCDR"/"TIFP"/"ARCH", version 0x00010002), heap = heap space +
|
||||
offsetTbl, offsetTblOffset as last u32 of the heap, DC_RECORD_ENTRY {typeCode u16, length
|
||||
u32, offset u32}, numRecords u16, typeCode bit fields (bits 15-14 storage: 00 kStg_InHeapSpace,
|
||||
01 kStg_InRecordEntry, 10/11 reserved; bits 13-11 dataType: 000 byte, 001 ASCII, 010 word,
|
||||
011 dword, 100 structure, 101/110 heap record, 111 reserved; bits 10-0 IDCode), masks
|
||||
kStgFormatMask 0xc000 / kDataTypeMask 0x3800 / kIDCodeMask 0x07ff, null/free records, 2-byte
|
||||
alignment rule, ImageSpec/ImageFormat/CapturedTime/ModelName/… structs, appendix type-code
|
||||
defines. The spec documents the container and common metadata only — **it does not describe
|
||||
the CRW raw-data compression** (ImageFormat fileFormat value 2 is merely "Reserved (for
|
||||
CanonCRW format)"). **No decoder algorithm details shown.**
|
||||
- 2026-10-12 19:31 — **Fetch** (curl) `https://raw.pixls.us/` and
|
||||
`https://raw.pixls.us/json/getrepository.php?set=all` (the site's own data endpoint found in
|
||||
its HTML; first tried `json/data.json`, 404). Taken: the full sample table (2016 rows);
|
||||
filtered locally for `.crw` (51 rows) and classic CIFF-era bodies; per-file licence, URL and
|
||||
SHA-256 as published by the site. CC0 CIFF-era bodies: EOS D30, D60, 10D (×3), 300D (×2),
|
||||
Digital Rebel, PowerShot G1, G5, G6. CC-BY-NC-SA only (NOT downloaded, per task rules):
|
||||
PowerShot G2, G3, Pro1, Pro70, S30, S40, S45, S50, S60, S70, S2 IS and others. Many other
|
||||
`.CRW` rows are CHDK sensor dumps from much later PowerShots (A-, SX-, ELPH-series), a
|
||||
different format. **No decoder algorithm details shown.**
|
||||
- 2026-10-12 19:32 — **Download** (Python urllib, stdlib) the 11 CC0 files above from
|
||||
`https://raw.pixls.us/getfile.php/...` into the gitignored `corpus/crw/`; every file verified
|
||||
against the site's published SHA-256 (all OK; the hashes are pinned in
|
||||
`crates/crw/tests/samples.json`).
|
||||
|
||||
## Local analysis (scripts in this directory)
|
||||
|
||||
- 2026-10-12 19:35–19:45 — `ciff_dump.py` (written here, stdlib only; container layout
|
||||
implemented from the two documented sources above) run over the 11 public files and the 18
|
||||
private files (whose dumps may contain personal metadata and are never committed). Observed
|
||||
facts are summarised in `01-payload-layout.md`. No raw-data decoding was attempted; statistics
|
||||
only. The script and its logs are not part of this record.
|
||||
- 2026-10-12 ~20:0x — P0/P1 local analysis (post-approval): the container notes written from the
|
||||
already-logged sources + corpus observations. `p1_payload.py`, `p1b_markers.py`,
|
||||
`p1c_lowbits_bounds.py` run over all 29 files. Established statistically (no decoding): the
|
||||
three payload layouts, the
|
||||
514-byte zero block at the compressed section start of every file, JPEG-style ff00 stuffing
|
||||
of every entropy stream (zero exceptions), the PowerShot `ff d9` terminator, and the exact
|
||||
low-bits region bounds on G5/G6. Documented in `01-payload-layout.md`. No external sources
|
||||
consulted in this step.
|
||||
|
||||
## Oracle (P2, from 2026-10-12)
|
||||
|
||||
- The maintainer installed LibRaw 0.22.2 (official pre-built Windows binaries) in
|
||||
a local `LibRaw-0.22.2\bin` folder. That directory tree also contains LibRaw's source
|
||||
code; **no source file there is ever opened, read or searched** — only
|
||||
`bin\unprocessed_raw.exe` is executed, as a black box, per the approved plan (§5a/§7).
|
||||
All oracle runs and probe campaigns are logged in the derivation log, which is not part of this
|
||||
record.
|
||||
|
||||
- 2026-10-13 — **Fetch** `https://raw.pixls.us/json/getrepository.php?set=all` (and the site
|
||||
root to find that endpoint): recovered the stable `getfile.php` URLs of the 11 CC0 corpus
|
||||
files by matching their SHA-256s, and re-confirmed each row's CC0 licence; URLs pinned in
|
||||
`xtask/src/main.rs` `RAW_SAMPLES`. No decoder content on these pages.
|
||||
- 2026-10-13 — **Fetch** `https://exiftool.org/TagNames/CanonRaw.html` (again) and
|
||||
`https://exiftool.org/TagNames/Canon.html` (prose tag tables only): 0x10a9 ColorBalance →
|
||||
Canon ColorBalance table (Index2; WB_RGGBLevels slot order Auto/Daylight/Shade/Cloudy/
|
||||
Tungsten/Fluorescent/Flash/Custom/Kelvin, index-29 "black levels for the D60" note),
|
||||
0x1030 WhiteSample field names, CanonShotInfo index 7 = WhiteBalance and the Canon
|
||||
WhiteBalance value enumeration, CanonModelID values. Prose documentation, no code.
|
||||
|
||||
- 2026-10-25 — **Fetch** `https://www.xrite.com/service-support/new_color_specifications_for_colorchecker_sg_and_classic_charts`
|
||||
(prose page): ColorChecker Classic charts made after November 2014 use new reference data,
|
||||
published as CGATS files. The CGATS download itself was refused to scripts (HTTP 403) and
|
||||
not worked around.
|
||||
- 2026-10-25 — **Fetch** `https://raw.githubusercontent.com/colour-science/colour/v0.4.2/colour/characterisation/datasets/colour_checkers/chromaticity_coordinates.py`
|
||||
(colour-science, BSD-3-Clause; also its rendered docs page): the 24 post-November-2014
|
||||
ColorChecker Classic L*a*b* values (D50), credited there to X-Rite's release. Reference
|
||||
data only, the chart's measured colours; no camera or decoder content. Used for the EOS D60
|
||||
colour matrices.
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regenerate crates/crw/src/tables.rs from the frozen tables in crates/crw/clean-room/logs/table_sel{0,1,2}.json.
|
||||
|
||||
Run from anywhere: `python crates/crw/clean-room/tools/gen_tables.py`. The JSON files are the
|
||||
derivation's frozen data (02-crw-format.md section 3); this script only renders them as Rust
|
||||
constants, keeping the file's doc comment. It was a local tool during the derivation in LightCraft
|
||||
(plan/crw/model/gen_tables.py) and moved here so the tables stay reproducible.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
TABLES = ROOT / "crates/crw/src/tables.rs"
|
||||
LOGS = ROOT / "crates/crw/clean-room/logs"
|
||||
|
||||
TAIL = """
|
||||
/// The tables for DecoderTable selector 0, 1 or 2.
|
||||
pub(crate) fn tables(selector: u32) -> Option<SelTables> {
|
||||
match selector {
|
||||
0 => Some(SelTables { first: FIRST_0, main: MAIN_0 }),
|
||||
1 => Some(SelTables { first: FIRST_1, main: MAIN_1 }),
|
||||
2 => Some(SelTables { first: FIRST_2, main: MAIN_2 }),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
head = TABLES.read_text(encoding="utf-8").split("#[rustfmt::skip]")[0]
|
||||
out = [head.rstrip("\n") + "\n"]
|
||||
for sel in (0, 1, 2):
|
||||
spec = json.loads((LOGS / f"table_sel{sel}.json").read_text(encoding="utf-8"))
|
||||
for part, name in (("first", "FIRST"), ("main", "MAIN")):
|
||||
rows = sorted(spec[part].items(), key=lambda kv: (len(kv[0]), kv[0]))
|
||||
out.append("#[rustfmt::skip]")
|
||||
out.append(f"const {name}_{sel}: &[(u16, u8, u8, u8)] = &[")
|
||||
line = []
|
||||
for code, (r, s) in rows:
|
||||
line.append(f"(0b{code}, {len(code)}, {r}, {s}),")
|
||||
if len(line) == 6:
|
||||
out.append(" " + " ".join(line))
|
||||
line = []
|
||||
if line:
|
||||
out.append(" " + " ".join(line))
|
||||
out.append("];")
|
||||
TABLES.write_text("\n".join(out) + TAIL, encoding="utf-8", newline="\n")
|
||||
print(f"generated {TABLES.relative_to(ROOT)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,760 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
//! Clean-room Canon CRW (CIFF heap) raw decoder: the bodies before the CR2 era (EOS D30, D60,
|
||||
//! 10D, 300D / Digital Rebel, PowerShot G-series).
|
||||
//!
|
||||
//! The container rules come from Canon's published CIFF specification and ExifTool's prose tag
|
||||
//! documentation; the compression was derived from CC0 and maintainer-owned sample files by
|
||||
//! black-box analysis against a reference decoder's command-line output, frozen in
|
||||
//! `clean-room/02-crw-format.md` and verified sample-exact on 28 of 29 corpus files plus 1,516
|
||||
//! further EOS D60 files (`clean-room/03-verification.md`). The code tables are frozen data
|
||||
//! (`tables`, generated from `clean-room/logs/table_sel{0,1,2}.json`). No other raw decoder's
|
||||
//! source was consulted; the whole derivation record is in the crate's `clean-room/` folder.
|
||||
//!
|
||||
//! - [`probe`] recognises the CIFF signature.
|
||||
//! - [`header`] reads geometry, bit depth, CFA phase, make and model, capture time, orientation
|
||||
//! and as-shot white balance without touching the entropy-coded data.
|
||||
//! - [`decode`] also decodes the sensor samples and measures the black level (masked borders)
|
||||
//! and the white level (saturation plateau or the sensor's clip level).
|
||||
//!
|
||||
//! The decoder never panics on input: every offset read from the file is bounds-checked, the heap
|
||||
//! walk is bounded by depth and by a per-file record budget, and the sample count is capped by
|
||||
//! [`Limits`]. Malformed files return [`Error::Corrupt`]; unknown decoder tables and the
|
||||
//! PowerShot G5 (whose entropy coding diverges from the derived model) return
|
||||
//! [`Error::Unsupported`].
|
||||
//!
|
||||
//! Known gaps (documented in the freeze): the PowerShot G5 is refused (its embedded JPEG is the
|
||||
//! app's fallback); PowerShot bodies without CC0 samples (G2, G3, Pro1, S-series) decode through
|
||||
//! the same model unverified; the D30 and the PowerShots carry no white-balance record the
|
||||
//! derivation deciphered, so `wb_multipliers` is `None` for them.
|
||||
//!
|
||||
//! Colour (camera matrices, profiles) is out of scope: the file carries none, and the app owns
|
||||
//! its colour model.
|
||||
|
||||
mod tables;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Why a file could not be decoded.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum Error {
|
||||
/// The bytes do not start with a CIFF `HEAPCCDR` header.
|
||||
#[error("not a Canon CRW (CIFF) file")]
|
||||
NotCrw,
|
||||
/// The file is a CRW but its structure or data is damaged.
|
||||
#[error("CRW: {0}")]
|
||||
Corrupt(String),
|
||||
/// A CRW the derived model does not cover (an unknown decoder table, the PowerShot G5).
|
||||
#[error("unsupported CRW: {0}")]
|
||||
Unsupported(String),
|
||||
/// The file declares more than [`Limits`] allow.
|
||||
#[error("CRW exceeds a limit: {0}")]
|
||||
LimitExceeded(&'static str),
|
||||
}
|
||||
|
||||
/// Caps on what a file may ask the decoder to allocate, checked against the header before
|
||||
/// anything is allocated. The struct is `#[non_exhaustive]`: start from [`Limits::default`] and
|
||||
/// set the fields you change.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub struct Limits {
|
||||
/// Most sensor samples (`width × height`) a file may declare. The decoder holds about three
|
||||
/// `u16` buffers of this size at its peak. Default: 2²³ (8.4 million) on every target; real
|
||||
/// CRW frames hold at most 7.4 million samples.
|
||||
pub max_samples: usize,
|
||||
}
|
||||
|
||||
impl Default for Limits {
|
||||
fn default() -> Self {
|
||||
Self { max_samples: 1 << 23 }
|
||||
}
|
||||
}
|
||||
|
||||
/// The 2×2 Bayer phase at sensor position (0, 0).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum Cfa {
|
||||
/// Red, green / green, blue: every verified body except the PowerShot G1.
|
||||
Rggb,
|
||||
/// Green, blue / red, green: the PowerShot G1.
|
||||
Gbrg,
|
||||
}
|
||||
|
||||
impl Cfa {
|
||||
/// The conventional four-letter name.
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Cfa::Rggb => "RGGB",
|
||||
Cfa::Gbrg => "GBRG",
|
||||
}
|
||||
}
|
||||
|
||||
/// The colour at each site of the 2×2 cell, row-major: 0 = red, 1 = green, 2 = blue.
|
||||
pub fn pattern(self) -> [u8; 4] {
|
||||
match self {
|
||||
Cfa::Rggb => [0, 1, 1, 2],
|
||||
Cfa::Gbrg => [1, 2, 0, 1],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A rectangle in sensor pixels.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub struct Rect {
|
||||
pub x: usize,
|
||||
pub y: usize,
|
||||
pub width: usize,
|
||||
pub height: usize,
|
||||
}
|
||||
|
||||
impl Rect {
|
||||
pub const fn new(x: usize, y: usize, width: usize, height: usize) -> Self {
|
||||
Self { x, y, width, height }
|
||||
}
|
||||
}
|
||||
|
||||
/// Black level per 2×2 CFA site, row-major, anchored at the active area's origin.
|
||||
///
|
||||
/// The anchor differs from [`Header::cfa`]'s: `cfa` is the phase at sensor (0, 0), `values` is
|
||||
/// indexed from the active area's origin. The active area can start at an odd sensor row (the
|
||||
/// PowerShot G1 starts at y = 15), so `values[i]` does not line up with `cfa.pattern()[i]`: read a
|
||||
/// level through [`BlackLevel::at`] rather than pairing the two by index.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct BlackLevel {
|
||||
pub values: [f32; 4],
|
||||
}
|
||||
|
||||
impl BlackLevel {
|
||||
/// The same level at every site.
|
||||
pub const fn uniform(v: f32) -> Self {
|
||||
Self { values: [v; 4] }
|
||||
}
|
||||
|
||||
/// The level at sensor position (`x`, `y`), whose CFA site is counted from `active`'s origin.
|
||||
pub fn at(&self, active: Rect, x: usize, y: usize) -> f32 {
|
||||
let px = (x as isize - active.x as isize).rem_euclid(2) as usize;
|
||||
let py = (y as isize - active.y as isize).rem_euclid(2) as usize;
|
||||
self.values.get(py * 2 + px).copied().unwrap_or(self.values[0])
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything a CRW tells about its image short of the samples: what [`header`] returns, and
|
||||
/// what [`Image::header`] carries.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Header {
|
||||
/// Sensor width in samples (SensorInfo), masked borders included.
|
||||
pub width: usize,
|
||||
/// Sensor height in samples.
|
||||
pub height: usize,
|
||||
/// Sample depth: 12 when the file carries a low-bits plane, else 10.
|
||||
pub bits: u32,
|
||||
/// The sensor area that holds the image (SensorInfo's inclusive crop); the rest is masked.
|
||||
pub active_area: Rect,
|
||||
/// Bayer phase at sensor (0, 0).
|
||||
pub cfa: Cfa,
|
||||
/// The DecoderTable number (0, 1 or 2): which frozen code tables the stream uses.
|
||||
pub selector: u32,
|
||||
/// Camera make from the MakeModel record (`Canon`).
|
||||
pub make: Option<String>,
|
||||
/// Camera model from the MakeModel record (`Canon EOS D60`).
|
||||
pub model: Option<String>,
|
||||
/// Capture time as Unix seconds on the camera's local clock (the TimeStamp record); the
|
||||
/// file carries no UTC offset. Convert with a calendar library (chrono), not by hand.
|
||||
pub capture_unix: Option<i64>,
|
||||
/// Exif orientation (1, 3, 6 or 8) from the ImageInfo rotation.
|
||||
pub exif_orientation: u8,
|
||||
/// As-shot white balance as (R, G, B) multipliers with G = 1, from the ColorBalance record
|
||||
/// slot the ShotInfo white-balance setting selects; `None` on bodies without the record
|
||||
/// (D30, PowerShots).
|
||||
pub wb_multipliers: Option<[f32; 3]>,
|
||||
}
|
||||
|
||||
/// A decoded CRW.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Image {
|
||||
pub header: Header,
|
||||
/// `width × height` sensor samples, row-major, `bits` deep (10 or 12), black not subtracted.
|
||||
pub samples: Vec<u16>,
|
||||
/// Black level measured from the masked borders.
|
||||
pub black: BlackLevel,
|
||||
/// White level: the saturation plateau, the sensor's clip level, or the full range.
|
||||
pub white: f32,
|
||||
}
|
||||
|
||||
/// Result alias for this crate.
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
fn corrupt(why: &str) -> Error {
|
||||
Error::Corrupt(why.to_string())
|
||||
}
|
||||
|
||||
/// Heap recursion bound: the deepest directory observed in real files is 4.
|
||||
const MAX_DEPTH: usize = 16;
|
||||
|
||||
/// Records visited per file, across every heap level: real files hold a few dozen. Depth alone
|
||||
/// doesn't bound the walk — a sub-heap record may point back at its own parent block, and N such
|
||||
/// records per block multiply into N^MAX_DEPTH visits — so this shared budget does.
|
||||
const MAX_RECORDS: usize = 4096;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Order {
|
||||
Le,
|
||||
Be,
|
||||
}
|
||||
|
||||
fn u16_at(b: &[u8], at: usize, o: Order) -> Option<u16> {
|
||||
let s: [u8; 2] = b.get(at..at.checked_add(2)?)?.try_into().ok()?;
|
||||
Some(match o {
|
||||
Order::Le => u16::from_le_bytes(s),
|
||||
Order::Be => u16::from_be_bytes(s),
|
||||
})
|
||||
}
|
||||
|
||||
fn u32_at(b: &[u8], at: usize, o: Order) -> Option<u32> {
|
||||
let s: [u8; 4] = b.get(at..at.checked_add(4)?)?.try_into().ok()?;
|
||||
Some(match o {
|
||||
Order::Le => u32::from_le_bytes(s),
|
||||
Order::Be => u32::from_be_bytes(s),
|
||||
})
|
||||
}
|
||||
|
||||
/// Walk a heap block (`start..start + len` of `bytes`) depth-first in record-table order,
|
||||
/// recording each in-heap record's tag (typeCode & 0x3fff), absolute offset and length — first
|
||||
/// occurrence wins. In-record values and reserved storage classes are skipped; dataType 5/6
|
||||
/// records are sub-heaps. `budget` is the number of records the whole file may still visit;
|
||||
/// the walk stops wherever it runs out.
|
||||
fn walk(bytes: &[u8], start: usize, len: usize, o: Order, depth: usize, budget: &mut usize, out: &mut BTreeMap<u16, (usize, usize)>) {
|
||||
if depth > MAX_DEPTH || len < 6 {
|
||||
return;
|
||||
}
|
||||
let Some(tbl) = start.checked_add(len).and_then(|e| e.checked_sub(4)).and_then(|at| u32_at(bytes, at, o)) else { return };
|
||||
let tbl = tbl as usize;
|
||||
// the offset table must fit between the block start and its own trailing pointer
|
||||
if tbl.checked_add(6).is_none_or(|e| e > len) {
|
||||
return;
|
||||
}
|
||||
let Some(abs_tbl) = start.checked_add(tbl) else { return };
|
||||
let Some(n) = u16_at(bytes, abs_tbl, o) else { return };
|
||||
let Some(records_end) = (n as usize).checked_mul(10).and_then(|r| tbl.checked_add(2)?.checked_add(r)) else { return };
|
||||
if records_end > len {
|
||||
return;
|
||||
}
|
||||
for k in 0..n as usize {
|
||||
let Some(left) = budget.checked_sub(1) else { return };
|
||||
*budget = left;
|
||||
let at = abs_tbl + 2 + k * 10;
|
||||
let (Some(tc), Some(rl), Some(ro)) = (u16_at(bytes, at, o), u32_at(bytes, at + 2, o), u32_at(bytes, at + 6, o)) else { return };
|
||||
if tc & 0xc000 != 0 {
|
||||
continue; // in-record value or reserved storage: nothing we need
|
||||
}
|
||||
let (rl, ro) = (rl as usize, ro as usize);
|
||||
if ro.checked_add(rl).is_none_or(|e| e > len) {
|
||||
continue;
|
||||
}
|
||||
let Some(abs) = start.checked_add(ro) else { continue };
|
||||
if matches!((tc >> 11) & 7, 5 | 6) {
|
||||
walk(bytes, abs, rl, o, depth + 1, budget, out);
|
||||
} else {
|
||||
out.entry(tc & 0x3fff).or_insert((abs, rl));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// CIFF records the decoder consumes.
|
||||
const RAW_DATA: u16 = 0x2005;
|
||||
const SENSOR_INFO: u16 = 0x1031;
|
||||
const DECODER_TABLE: u16 = 0x1835;
|
||||
const MAKE_MODEL: u16 = 0x080a;
|
||||
const IMAGE_INFO: u16 = 0x1810;
|
||||
const TIME_STAMP: u16 = 0x180e;
|
||||
const COLOR_BALANCE: u16 = 0x10a9;
|
||||
const SHOT_INFO: u16 = 0x102a;
|
||||
|
||||
/// Whether `bytes` start with a CIFF raw header: `II` or `MM`, then `HEAPCCDR` at offset 6.
|
||||
pub fn probe(bytes: &[u8]) -> bool {
|
||||
matches!(bytes.get(0..2), Some(b"II") | Some(b"MM")) && bytes.get(6..14) == Some(b"HEAPCCDR")
|
||||
}
|
||||
|
||||
/// Read the header: geometry, bit depth, CFA phase, metadata and white balance, without
|
||||
/// decoding the samples. Fails the same way [`decode`] would on the records it needs.
|
||||
pub fn header(bytes: &[u8], limits: &Limits) -> Result<Header> {
|
||||
Ok(parse(bytes, limits, false)?.header)
|
||||
}
|
||||
|
||||
/// Decode a CRW: the header plus every sensor sample, the black level and the white level.
|
||||
pub fn decode(bytes: &[u8], limits: &Limits) -> Result<Image> {
|
||||
parse(bytes, limits, true)
|
||||
}
|
||||
|
||||
fn parse(bytes: &[u8], limits: &Limits, full: bool) -> Result<Image> {
|
||||
let o = match bytes.get(0..2) {
|
||||
Some(b"II") => Order::Le,
|
||||
Some(b"MM") => Order::Be,
|
||||
_ => return Err(Error::NotCrw),
|
||||
};
|
||||
if bytes.get(6..14) != Some(b"HEAPCCDR") {
|
||||
return Err(Error::NotCrw);
|
||||
}
|
||||
let hl = u32_at(bytes, 2, o).ok_or_else(|| corrupt("truncated header"))? as usize;
|
||||
let heap_len = bytes.len().checked_sub(hl).filter(|&l| l >= 6).ok_or_else(|| corrupt("header length beyond the file"))?;
|
||||
let mut rec = BTreeMap::new();
|
||||
let mut budget = MAX_RECORDS;
|
||||
walk(bytes, hl, heap_len, o, 0, &mut budget, &mut rec);
|
||||
|
||||
let &(so, sl) = rec.get(&SENSOR_INFO).ok_or_else(|| corrupt("no SensorInfo record"))?;
|
||||
let sensor: Vec<u16> = (0..sl / 2).map_while(|i| u16_at(bytes, so + i * 2, o)).collect();
|
||||
let (w, h) = match (sensor.get(1), sensor.get(2)) {
|
||||
(Some(&w), Some(&h)) => (w as usize, h as usize),
|
||||
_ => return Err(corrupt("SensorInfo too short")),
|
||||
};
|
||||
if w == 0 || h == 0 {
|
||||
return Err(corrupt("zero sensor dimensions"));
|
||||
}
|
||||
let n = w.checked_mul(h).filter(|&n| n <= limits.max_samples).ok_or(Error::LimitExceeded("image too large"))?;
|
||||
|
||||
let &(dto, dtl) = rec.get(&DECODER_TABLE).ok_or_else(|| corrupt("no DecoderTable record"))?;
|
||||
if dtl < 16 {
|
||||
return Err(corrupt("DecoderTable too short"));
|
||||
}
|
||||
let selector = u32_at(bytes, dto, o).ok_or_else(|| corrupt("truncated DecoderTable"))?;
|
||||
let ent_off = u32_at(bytes, dto + 8, o).ok_or_else(|| corrupt("truncated DecoderTable"))? as usize;
|
||||
let tables = tables::tables(selector).ok_or_else(|| Error::Unsupported(format!("decoder table {selector}")))?;
|
||||
if ent_off > bytes.len() {
|
||||
return Err(corrupt("entropy offset beyond the file"));
|
||||
}
|
||||
|
||||
// "Canon\0<model>\0"
|
||||
let (make, model) = rec
|
||||
.get(&MAKE_MODEL)
|
||||
.and_then(|&(mo, ml)| bytes.get(mo..mo.checked_add(ml)?))
|
||||
.map(|v| {
|
||||
let mut parts = v.split(|&b| b == 0).map(|s| String::from_utf8_lossy(s).trim().to_string());
|
||||
(parts.next().unwrap_or_default(), parts.next().unwrap_or_default())
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if model.contains("PowerShot G5") {
|
||||
return Err(Error::Unsupported(
|
||||
"Canon PowerShot G5 entropy coding (a documented anomaly of the derivation, clean-room/02-crw-format.md §4); use the embedded JPEG"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
|
||||
// active area from the SensorInfo crop (inclusive left, top, right, bottom)
|
||||
let active = match (sensor.get(5), sensor.get(6), sensor.get(7), sensor.get(8)) {
|
||||
(Some(&l), Some(&t), Some(&r), Some(&b))
|
||||
if (l as usize) < (r as usize) && (r as usize) < w && (t as usize) < (b as usize) && (b as usize) < h =>
|
||||
{
|
||||
Rect::new(l as usize, t as usize, r as usize - l as usize + 1, b as usize - t as usize + 1)
|
||||
}
|
||||
_ => Rect::new(0, 0, w, h),
|
||||
};
|
||||
|
||||
let &(ro, rl) = rec.get(&RAW_DATA).ok_or_else(|| corrupt("no raw data record"))?;
|
||||
let payload =
|
||||
bytes.get(ro..ro.checked_add(rl).ok_or_else(|| corrupt("raw record size overflow"))?).ok_or_else(|| corrupt("raw data outside the file"))?;
|
||||
|
||||
// Low-bits plane discriminator, two signs of a plane: (a) its raw bytes may hold an 0xff not
|
||||
// followed by 0x00, which the stuffed entropy stream never does; (b) the zero header of
|
||||
// `ent_off` bytes sits right after it, where a file without a plane is deep in entropy data.
|
||||
// (b) catches the D60 files whose lowest bit is almost always 0, so no plane byte is 0xff.
|
||||
let q = n.div_ceil(4);
|
||||
let span = q.saturating_add(ent_off);
|
||||
let head = payload.get(..span.min(payload.len())).unwrap_or(payload);
|
||||
// Both signs need a payload long enough to hold a plane and the header behind it. A shorter
|
||||
// payload ends in the PowerShot `ff d9 00` trailer, and `ff d9` is a 0xff not followed by
|
||||
// 0x00: without this gate a flat no-plane G1 frame read as 12-bit and was then refused.
|
||||
let plane_fits = payload.len() > span;
|
||||
let header_after_plane = plane_fits && ent_off >= 64 && payload.get(q..span).is_some_and(|z| z.iter().all(|&b| b == 0));
|
||||
let has_low = header_after_plane || (plane_fits && head.windows(2).any(|p| p[0] == 0xff && p[1] != 0x00));
|
||||
let bits = if has_low { 12 } else { 10 };
|
||||
let low_len = if has_low { q } else { 0 };
|
||||
let samples = if full {
|
||||
let entropy = payload.get(low_len.saturating_add(ent_off)..).ok_or_else(|| corrupt("payload shorter than its header"))?;
|
||||
let low = has_low.then(|| payload.get(..q)).flatten();
|
||||
if has_low && low.is_none() {
|
||||
return Err(corrupt("payload shorter than its low-bits plane"));
|
||||
}
|
||||
let hi = decode_entropy(entropy, w, h, &tables)?;
|
||||
let mut samples = vec![0u16; n];
|
||||
// `samples` is documented as `bits` deep, so a damaged stream whose chain value runs past
|
||||
// the range is clamped here: a consumer indexing a `1 << bits` LUT must not see more.
|
||||
let top = ((1u32 << bits) - 1) as u16;
|
||||
for (i, (dst, &hi)) in samples.iter_mut().zip(hi.iter()).enumerate() {
|
||||
*dst = match low {
|
||||
Some(low) => {
|
||||
let lb = low.get(i >> 2).map_or(0, |&b| (b >> ((i & 3) * 2)) & 3);
|
||||
let v = (((hi as u32) << 2) | lb as u32) & 0xffff;
|
||||
v.min(u32::from(top)) as u16
|
||||
}
|
||||
None => hi.min(top),
|
||||
};
|
||||
}
|
||||
samples
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// CFA phase at sensor (0, 0): RGGB on every verified body except the PowerShot G1 (GBRG),
|
||||
// established by chroma correlation against each body's embedded JPEG (derivation log).
|
||||
let cfa = if model.contains("PowerShot G1") { Cfa::Gbrg } else { Cfa::Rggb };
|
||||
|
||||
let black = if full { black_from_borders(&samples, w, h, active) } else { BlackLevel::uniform(0.0) };
|
||||
let white = if full { white_level(&samples, bits) } else { ((1u32 << bits) - 1) as f32 };
|
||||
|
||||
// ImageInfo rotation: degrees; 270 observed on portrait files whose scene is upright after a
|
||||
// 90° counter-clockwise turn (Exif 8), so 90 is its mirror (Exif 6)
|
||||
// The record holds a signed degree count, so a file may write −90 for 270: take it modulo
|
||||
// 360 before mapping, and leave anything off-axis (0, ±360) at the identity.
|
||||
let rotation_deg =
|
||||
rec.get(&IMAGE_INFO).and_then(|&(io, il)| (il >= 16).then(|| u32_at(bytes, io + 12, o)).flatten()).map(|r| (r as i32).rem_euclid(360));
|
||||
let exif_orientation = match rotation_deg {
|
||||
Some(90) => 6,
|
||||
Some(180) => 3,
|
||||
Some(270) => 8,
|
||||
_ => 1,
|
||||
};
|
||||
|
||||
let capture_unix = rec.get(&TIME_STAMP).and_then(|&(to, tl)| (tl >= 4).then(|| u32_at(bytes, to, o)).flatten()).map(i64::from);
|
||||
|
||||
let header = Header {
|
||||
width: w,
|
||||
height: h,
|
||||
bits,
|
||||
active_area: active,
|
||||
cfa,
|
||||
selector,
|
||||
make: (!make.is_empty()).then_some(make),
|
||||
model: (!model.is_empty()).then_some(model),
|
||||
capture_unix,
|
||||
exif_orientation,
|
||||
wb_multipliers: wb_from_color_balance(bytes, &rec, o),
|
||||
};
|
||||
Ok(Image { header, samples, black, white })
|
||||
}
|
||||
|
||||
/// As-shot white-balance multipliers from the ColorBalance record (0x10a9, D60/10D/300D): a
|
||||
/// byte-length word, then (R, G, G, B) level quadruples per white-balance setting, the last
|
||||
/// quadruple being black levels (ExifTool's prose Canon/CanonRaw tag documentation names the
|
||||
/// slots; the 10D/300D order is Auto, Daylight, Shade, Cloudy, Tungsten, Fluorescent, Flash,
|
||||
/// Custom, Kelvin, and the D60's seven slots lack Shade and Kelvin). The slot is chosen by the
|
||||
/// ShotInfo (0x102a) WhiteBalance value; multipliers are `level / green`, confirmed against the
|
||||
/// cameras' own JPEG renderings (derivation log). Bodies without the record (D30, PowerShots)
|
||||
/// return `None`.
|
||||
fn wb_from_color_balance(bytes: &[u8], rec: &BTreeMap<u16, (usize, usize)>, o: Order) -> Option<[f32; 3]> {
|
||||
let &(bo, bl) = rec.get(&COLOR_BALANCE)?;
|
||||
let vals: Vec<u16> = (0..bl / 2).map_while(|i| u16_at(bytes, bo + i * 2, o)).collect();
|
||||
// quadruples after the length word; the last one is black levels
|
||||
let slots = vals.len().checked_sub(1)? / 4;
|
||||
let usable = slots.checked_sub(1)?;
|
||||
if usable == 0 {
|
||||
return None;
|
||||
}
|
||||
let setting = rec.get(&SHOT_INFO).and_then(|&(so, sl)| (sl / 2 > 7).then(|| u16_at(bytes, so + 14, o)).flatten()).unwrap_or(0);
|
||||
// Canon WhiteBalance values: 0 Auto, 1 Daylight, 2 Cloudy, 3 Tungsten, 4 Fluorescent,
|
||||
// 5 Flash, 6 Custom, 8 Shade, 9 Kelvin
|
||||
let slot = if usable >= 9 {
|
||||
match setting {
|
||||
1 => 1,
|
||||
8 => 2,
|
||||
2 => 3,
|
||||
3 => 4,
|
||||
4 => 5,
|
||||
5 => 6,
|
||||
6 => 7,
|
||||
9 => 8,
|
||||
_ => 0,
|
||||
}
|
||||
} else {
|
||||
match setting {
|
||||
s @ 1..=6 if (s as usize) < usable => s as usize,
|
||||
_ => 0,
|
||||
}
|
||||
};
|
||||
let q = vals.get(1 + slot.min(usable - 1) * 4..5 + slot.min(usable - 1) * 4)?;
|
||||
let g = (q.get(1).copied()? as f32 + q.get(2).copied()? as f32) / 2.0;
|
||||
let (r, b) = (q.first().copied()? as f32, q.get(3).copied()? as f32);
|
||||
if g <= 0.0 || r <= 0.0 || b <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
let (mr, mb) = (r / g, b / g);
|
||||
((0.2..=8.0).contains(&mr) && (0.2..=8.0).contains(&mb)).then_some([mr, 1.0, mb])
|
||||
}
|
||||
|
||||
/// Fewest samples at the top of the data that count as the sensor's clip level (hot pixels are
|
||||
/// fewer).
|
||||
const CLIP_SAMPLES: usize = 64;
|
||||
|
||||
/// White level: the saturation plateau when [`saturation_plateau`] finds one, else the top of
|
||||
/// the data when it sits just under the full range.
|
||||
///
|
||||
/// These bodies clip below the full range at low ISO: on 1,409 D60 files at ISO 100–400 no
|
||||
/// sample exceeds 4009 (most stop at 4000–4008), while ISO 800–1000 files reach 4095; the 10D
|
||||
/// and 300D samples stop at 4003–4008 too. A photo with only a small blown area has no plateau
|
||||
/// for [`saturation_plateau`], which then answered 4095 (373 of 925 ISO 100 files, 283 of 287 at
|
||||
/// ISO 400): the blown pixels sat at 0.977 of white, under the clip threshold, and kept a
|
||||
/// colour after white balance. So when at least [`CLIP_SAMPLES`] samples reach within 3 % of the
|
||||
/// full range, the level they reach is the white level, lowered by the same 1/256 band the
|
||||
/// plateau estimate uses.
|
||||
fn white_level(data: &[u16], bits: u32) -> f32 {
|
||||
let full = (1u32 << bits.clamp(1, 16)) - 1;
|
||||
let plateau = saturation_plateau(data, bits);
|
||||
if plateau < full as f32 {
|
||||
return plateau;
|
||||
}
|
||||
let mut hist = vec![0usize; full as usize + 1];
|
||||
for &v in data {
|
||||
if let Some(h) = hist.get_mut(v as usize) {
|
||||
*h += 1;
|
||||
}
|
||||
}
|
||||
let mut above = 0usize;
|
||||
let top = hist.iter().enumerate().rev().find_map(|(v, &n)| {
|
||||
above += n;
|
||||
(above >= CLIP_SAMPLES).then_some(v as u32)
|
||||
});
|
||||
match top {
|
||||
Some(top) if top < full && top as f32 >= 0.97 * full as f32 => (top - (top / 256).max(1)) as f32,
|
||||
_ => full as f32,
|
||||
}
|
||||
}
|
||||
|
||||
/// White level estimate shared by the Craft raw decoders (LightCraft's `white_from_data`): the
|
||||
/// saturation plateau if a noticeable number of samples sit at the maximum value, otherwise the
|
||||
/// full `bits` range. A few stray samples above the plateau (hot pixels) don't hide it.
|
||||
fn saturation_plateau(data: &[u16], bits: u32) -> f32 {
|
||||
let full_level = (1u32 << bits.clamp(1, 16)) - 1;
|
||||
let full = full_level as f32;
|
||||
let step = (data.len() / 2_000_000).max(1);
|
||||
// samples above the nominal range are padding / invalid, not saturation
|
||||
let mut hist = vec![0usize; full_level as usize + 1];
|
||||
for &v in data.iter().step_by(step) {
|
||||
if let Some(h) = hist.get_mut(v as usize) {
|
||||
*h += 1;
|
||||
}
|
||||
}
|
||||
let min_plateau = (data.len() / 20_000).max(16);
|
||||
// the top of the data, skipping up to half a plateau's worth of strays so a real plateau can't be skipped
|
||||
let strays = min_plateau / 2;
|
||||
let mut above = 0usize;
|
||||
let Some(mx) = hist.iter().enumerate().rev().find_map(|(v, &n)| {
|
||||
above += n * step;
|
||||
(above > strays).then_some(v as u32)
|
||||
}) else {
|
||||
return full;
|
||||
};
|
||||
let band = (mx / 256).max(1);
|
||||
let near = mx.saturating_sub(band);
|
||||
let below = near.saturating_sub(band);
|
||||
let count = |from: u32, to: u32| hist.get(from as usize..to as usize).map_or(0, |h| h.iter().sum::<usize>());
|
||||
// the strays count as saturated too
|
||||
let top = count(near, full_level + 1);
|
||||
let under = count(below, near);
|
||||
// a saturation plateau is a spike: many more samples in the top band than in the band just below it
|
||||
if mx as f32 > full * 0.5 && top * step >= min_plateau && top >= 4 * under + 4 {
|
||||
// plateau: use its lower edge so everything at saturation maps to ≥ 1.0
|
||||
near as f32
|
||||
} else {
|
||||
full
|
||||
}
|
||||
}
|
||||
|
||||
/// Black level from the masked sensor borders: the darkest flat strip among 8-column groups of
|
||||
/// the left border and 4-row groups of the top border, with the strips within 8 DN of it averaged
|
||||
/// in. A strip whose mean is ≤ 1 is clamped padding (observed on the D60's first rows), not a
|
||||
/// black reference, and bright transition strips next to the active area only raise means, so the
|
||||
/// minimum stays on true dark pixels.
|
||||
fn black_from_borders(data: &[u16], w: usize, h: usize, active: Rect) -> BlackLevel {
|
||||
let mean_of = |xs: std::ops::Range<usize>, ys: std::ops::Range<usize>| -> Option<f64> {
|
||||
let (mut sum, mut cnt) = (0f64, 0u64);
|
||||
for y in ys {
|
||||
let row = data.get(y.checked_mul(w)?..y.checked_mul(w)?.checked_add(w)?)?;
|
||||
for &v in row.get(xs.clone())? {
|
||||
sum += v as f64;
|
||||
cnt += 1;
|
||||
}
|
||||
}
|
||||
(cnt > 0).then(|| sum / cnt as f64)
|
||||
};
|
||||
let mut strips: Vec<(f64, std::ops::Range<usize>, std::ops::Range<usize>)> = Vec::new();
|
||||
let rows = active.y..active.y.saturating_add(active.height).min(h);
|
||||
let mut x = 2usize;
|
||||
while x + 8 <= active.x {
|
||||
if let Some(m) = mean_of(x..x + 8, rows.clone()) {
|
||||
strips.push((m, x..x + 8, rows.clone()));
|
||||
}
|
||||
x += 8;
|
||||
}
|
||||
let cols = active.x..active.x.saturating_add(active.width).min(w);
|
||||
let mut y = 2usize;
|
||||
while y + 4 <= active.y {
|
||||
if let Some(m) = mean_of(cols.clone(), y..y + 4) {
|
||||
strips.push((m, cols.clone(), y..y + 4));
|
||||
}
|
||||
y += 4;
|
||||
}
|
||||
let Some(darkest) = strips.iter().map(|s| s.0).filter(|&m| m > 1.0).reduce(f64::min) else {
|
||||
return BlackLevel::uniform(0.0);
|
||||
};
|
||||
// per-CFA-site means over every strip near the darkest
|
||||
let (mut sum, mut cnt) = ([0f64; 4], [0u64; 4]);
|
||||
for (m, xs, ys) in &strips {
|
||||
// clamped padding (mean ≤ 1) is no black reference: counting it dragged the D60's level
|
||||
// from 128 down to 117 and lifted every shadow
|
||||
if *m <= 1.0 || *m > darkest + 8.0 {
|
||||
continue;
|
||||
}
|
||||
for y in ys.clone() {
|
||||
let py = (y as isize - active.y as isize).rem_euclid(2) as usize;
|
||||
for x in xs.clone() {
|
||||
let px = (x as isize - active.x as isize).rem_euclid(2) as usize;
|
||||
if let Some(&v) = data.get(y * w + x) {
|
||||
sum[py * 2 + px] += v as f64;
|
||||
cnt[py * 2 + px] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if cnt.contains(&0) {
|
||||
return BlackLevel::uniform(darkest as f32);
|
||||
}
|
||||
let mut values = [0f32; 4];
|
||||
for (v, (s, c)) in values.iter_mut().zip(sum.iter().zip(&cnt)) {
|
||||
*v = (s / *c as f64) as f32;
|
||||
}
|
||||
BlackLevel { values }
|
||||
}
|
||||
|
||||
/// A code-table lookup built for 16-bit MSB-first peeks: entry = `len << 16 | run << 8 | size`,
|
||||
/// zero where no code matches.
|
||||
fn build_lut(codes: &[(u16, u8, u8, u8)]) -> Vec<u32> {
|
||||
let mut lut = vec![0u32; 1 << 16];
|
||||
for &(code, len, r, s) in codes {
|
||||
if len == 0 || len > 16 {
|
||||
continue;
|
||||
}
|
||||
let span = 1usize << (16 - len);
|
||||
let base = (code as usize) << (16 - len);
|
||||
for e in lut.iter_mut().skip(base).take(span) {
|
||||
*e = (len as u32) << 16 | (r as u32) << 8 | s as u32;
|
||||
}
|
||||
}
|
||||
lut
|
||||
}
|
||||
|
||||
/// Decode the ff00-stuffed entropy stream into the `w × h` high-plane samples, per the frozen
|
||||
/// model (`clean-room/02-crw-format.md` §2): 64-sample blocks, first/main code tables,
|
||||
/// (run, size) leaves with JPEG-rule magnitudes, EOB, block-DC carry, and two parity DPCM chains
|
||||
/// initialised to 512 and reset at every row start. A stream that ends early leaves the remaining
|
||||
/// samples at their chain values (the frozen stream-end rule).
|
||||
fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables) -> Result<Vec<u16>> {
|
||||
let d = unstuff(entropy);
|
||||
let nbits = d.len().saturating_mul(8);
|
||||
let peek16 = |pos: usize| -> u16 {
|
||||
let (byte, sh) = (pos >> 3, pos & 7);
|
||||
let b = |k: usize| d.get(byte + k).copied().unwrap_or(0) as u32;
|
||||
let w24 = b(0) << 16 | b(1) << 8 | b(2);
|
||||
((w24 >> (8 - sh)) & 0xffff) as u16
|
||||
};
|
||||
|
||||
let luts = [build_lut(tables.first), build_lut(tables.main)];
|
||||
let n = w.checked_mul(h).ok_or(Error::LimitExceeded("image too large"))?;
|
||||
let mut out = Chains { dec: vec![0u16; n], w, qi: 0, ce: 512, co: 512 };
|
||||
let mut carry: i32 = 0;
|
||||
let mut pos = 0usize;
|
||||
while out.qi < n {
|
||||
let block_end = n.min((out.qi / 64 + 1) * 64);
|
||||
let mut tok = 0u32;
|
||||
while out.qi < block_end {
|
||||
// stream exhausted: every remaining sample keeps its chain value
|
||||
if pos >= nbits.saturating_sub(1) {
|
||||
out.fill(n);
|
||||
break;
|
||||
}
|
||||
let e = luts[usize::from(tok > 0)].get(peek16(pos) as usize).copied().unwrap_or(0);
|
||||
if e == 0 {
|
||||
return Err(corrupt("undecodable entropy code"));
|
||||
}
|
||||
let (len, mut rr, s) = ((e >> 16) as usize, ((e >> 8) & 0xff) as usize, e & 0xff);
|
||||
if pos.saturating_add(len).saturating_add(s as usize) > nbits {
|
||||
out.fill(n);
|
||||
break;
|
||||
}
|
||||
pos += len;
|
||||
tok += 1;
|
||||
if s == 0 && rr == 0 && tok > 1 {
|
||||
// end of block
|
||||
out.fill(block_end);
|
||||
continue;
|
||||
}
|
||||
let diff: i32 = if s > 0 {
|
||||
let m = (peek16(pos) >> (16 - s)) as i32;
|
||||
pos += s as usize;
|
||||
// JPEG magnitude rule
|
||||
if m >= 1 << (s - 1) { m } else { m - (1 << s) + 1 }
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let mut first = out.qi.is_multiple_of(64);
|
||||
if first && rr > 0 {
|
||||
// the block's first sample covered by the skip: its diff is exactly the carry
|
||||
out.emit(carry);
|
||||
rr -= 1;
|
||||
first = false;
|
||||
}
|
||||
out.fill(block_end.min(out.qi + rr));
|
||||
if out.qi >= block_end {
|
||||
continue;
|
||||
}
|
||||
if first {
|
||||
carry = carry.wrapping_add(diff);
|
||||
out.emit(carry);
|
||||
} else {
|
||||
out.emit(diff);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out.dec)
|
||||
}
|
||||
|
||||
/// Drop the stuffing byte: an 0x00 after an 0xff.
|
||||
fn unstuff(entropy: &[u8]) -> Vec<u8> {
|
||||
let mut d = Vec::with_capacity(entropy.len());
|
||||
let mut i = 0;
|
||||
while let Some(&b) = entropy.get(i) {
|
||||
d.push(b);
|
||||
i += if b == 0xff && entropy.get(i + 1) == Some(&0x00) { 2 } else { 1 };
|
||||
}
|
||||
d
|
||||
}
|
||||
|
||||
/// The decoded samples and the two parity DPCM chains that produce them.
|
||||
struct Chains {
|
||||
dec: Vec<u16>,
|
||||
w: usize,
|
||||
qi: usize,
|
||||
ce: i32,
|
||||
co: i32,
|
||||
}
|
||||
|
||||
impl Chains {
|
||||
/// Emit one diff: row-start chain reset, parity chain update, 16-bit masked store.
|
||||
fn emit(&mut self, d: i32) {
|
||||
if self.qi.is_multiple_of(self.w) {
|
||||
self.ce = 512;
|
||||
self.co = 512;
|
||||
}
|
||||
let c = if self.qi.is_multiple_of(2) { &mut self.ce } else { &mut self.co };
|
||||
*c = c.wrapping_add(d);
|
||||
if let Some(p) = self.dec.get_mut(self.qi) {
|
||||
*p = (*c & 0xffff) as u16;
|
||||
}
|
||||
self.qi += 1;
|
||||
}
|
||||
|
||||
/// Emit zero diffs up to sample `end`.
|
||||
fn fill(&mut self, end: usize) {
|
||||
while self.qi < end {
|
||||
self.emit(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
//! Canon CRW entropy code tables, frozen clean-room data.
|
||||
//!
|
||||
//! Generated by `clean-room/tools/gen_tables.py` from `clean-room/logs/table_sel{0,1,2}.json` (the
|
||||
//! tables extracted by known-plaintext alignment and frozen at the CRW-FREEZE commit; derivation:
|
||||
//! `clean-room/02-crw-format.md` §3). Do not edit by hand; regenerate from the JSON.
|
||||
//! Each entry is `(code bits, code length, run, size)`; codes are MSB-first.
|
||||
|
||||
/// One selector's pair of canonical Huffman tables: the code starting each 64-sample
|
||||
/// block reads from `first`, every other code from `main`.
|
||||
pub(crate) struct SelTables {
|
||||
pub first: &'static [(u16, u8, u8, u8)],
|
||||
pub main: &'static [(u16, u8, u8, u8)],
|
||||
}
|
||||
|
||||
#[rustfmt::skip]
|
||||
const FIRST_0: &[(u16, u8, u8, u8)] = &[
|
||||
(0b00, 2, 0, 4), (0b010, 3, 0, 3), (0b011, 3, 0, 5), (0b100, 3, 0, 6), (0b101, 3, 0, 2), (0b1100, 4, 0, 7),
|
||||
(0b1101, 4, 0, 1), (0b11100, 5, 0, 8), (0b11101, 5, 0, 9), (0b11110, 5, 0, 0), (0b111110, 6, 0, 10), (0b1111110, 7, 0, 11),
|
||||
];
|
||||
#[rustfmt::skip]
|
||||
const MAIN_0: &[(u16, u8, u8, u8)] = &[
|
||||
(0b00, 2, 0, 3), (0b01, 2, 0, 4), (0b100, 3, 0, 2), (0b101, 3, 0, 5), (0b1100, 4, 0, 1), (0b1101, 4, 0, 6),
|
||||
(0b11100, 5, 0, 7), (0b111010, 6, 0, 8), (0b111011, 6, 1, 2), (0b111100, 6, 1, 3), (0b111101, 6, 1, 1), (0b1111100, 7, 1, 4),
|
||||
(0b1111101, 7, 0, 9), (0b11111100, 8, 1, 5), (0b111111010, 9, 2, 2), (0b111111011, 9, 0, 0), (0b1111111000, 10, 2, 1), (0b1111111001, 10, 1, 6),
|
||||
(0b1111111010, 10, 0, 10), (0b1111111011, 10, 15, 0), (0b1111111100, 10, 2, 3), (0b11111111010, 11, 1, 7), (0b111111110110, 12, 2, 4), (0b1111111101110000, 16, 3, 1),
|
||||
(0b1111111101110001, 16, 3, 2), (0b1111111101110010, 16, 1, 8), (0b1111111101110011, 16, 1, 9), (0b1111111101110100, 16, 3, 3), (0b1111111101110101, 16, 2, 5), (0b1111111101110110, 16, 4, 1),
|
||||
(0b1111111101110111, 16, 3, 4), (0b1111111101111000, 16, 4, 2), (0b1111111101111001, 16, 3, 5), (0b1111111101111010, 16, 5, 1), (0b1111111101111011, 16, 3, 6), (0b1111111101111100, 16, 3, 7),
|
||||
(0b1111111101111101, 16, 3, 8), (0b1111111101111110, 16, 2, 9), (0b1111111101111111, 16, 7, 9), (0b1111111110000000, 16, 2, 6), (0b1111111110000001, 16, 1, 10), (0b1111111110000010, 16, 3, 9),
|
||||
(0b1111111110000011, 16, 5, 6), (0b1111111110000100, 16, 5, 7), (0b1111111110000101, 16, 2, 8), (0b1111111110000110, 16, 2, 7), (0b1111111110000111, 16, 5, 2), (0b1111111110001000, 16, 5, 5),
|
||||
(0b1111111110001001, 16, 5, 8), (0b1111111110001010, 16, 4, 3), (0b1111111110001011, 16, 7, 6), (0b1111111110001100, 16, 5, 9), (0b1111111110001101, 16, 7, 7), (0b1111111110001110, 16, 5, 4),
|
||||
(0b1111111110001111, 16, 6, 1), (0b1111111110010000, 16, 15, 9), (0b1111111110010001, 16, 7, 1), (0b1111111110010010, 16, 7, 8), (0b1111111110010011, 16, 7, 5), (0b1111111110010100, 16, 9, 6),
|
||||
(0b1111111110010101, 16, 9, 7), (0b1111111110010110, 16, 4, 9), (0b1111111110010111, 16, 11, 7), (0b1111111110011000, 16, 5, 3), (0b1111111110011001, 16, 13, 7), (0b1111111110011010, 16, 7, 4),
|
||||
(0b1111111110011011, 16, 11, 6), (0b1111111110011100, 16, 9, 8), (0b1111111110011101, 16, 4, 7), (0b1111111110011110, 16, 4, 8), (0b1111111110011111, 16, 9, 5), (0b1111111110100000, 16, 6, 9),
|
||||
(0b1111111110100001, 16, 9, 9), (0b1111111110100010, 16, 9, 1), (0b1111111110100011, 16, 15, 10), (0b1111111110100100, 16, 11, 8), (0b1111111110100101, 16, 6, 8), (0b1111111110100110, 16, 11, 5),
|
||||
(0b1111111110100111, 16, 11, 9), (0b1111111110101000, 16, 13, 6), (0b1111111110101001, 16, 15, 7), (0b1111111110101010, 16, 13, 8), (0b1111111110101011, 16, 6, 7), (0b1111111110101100, 16, 4, 6),
|
||||
(0b1111111110101101, 16, 4, 5), (0b1111111110101110, 16, 9, 4), (0b1111111110101111, 16, 8, 9), (0b1111111110110000, 16, 15, 8), (0b1111111110110001, 16, 8, 1), (0b1111111110110010, 16, 13, 5),
|
||||
(0b1111111110110011, 16, 15, 6), (0b1111111110110100, 16, 11, 4), (0b1111111110110101, 16, 8, 8), (0b1111111110110110, 16, 11, 1), (0b1111111110110111, 16, 2, 10), (0b1111111110111000, 16, 4, 4),
|
||||
(0b1111111110111001, 16, 7, 2), (0b1111111110111010, 16, 13, 9), (0b1111111110111011, 16, 8, 7), (0b1111111110111100, 16, 6, 6), (0b1111111110111101, 16, 13, 4), (0b1111111110111110, 16, 15, 5),
|
||||
(0b1111111110111111, 16, 3, 10), (0b1111111111000000, 16, 10, 7), (0b1111111111000001, 16, 7, 3), (0b1111111111000010, 16, 10, 9), (0b1111111111000011, 16, 10, 8), (0b1111111111000100, 16, 8, 6),
|
||||
(0b1111111111000101, 16, 6, 2), (0b1111111111000110, 16, 12, 7), (0b1111111111000111, 16, 6, 5), (0b1111111111001000, 16, 12, 8), (0b1111111111001001, 16, 12, 9), (0b1111111111001010, 16, 10, 1),
|
||||
(0b1111111111001011, 16, 15, 4), (0b1111111111001100, 16, 13, 1), (0b1111111111001101, 16, 14, 9), (0b1111111111001110, 16, 5, 10), (0b1111111111001111, 16, 9, 2), (0b1111111111010000, 16, 8, 5),
|
||||
(0b1111111111010001, 16, 10, 6), (0b1111111111010010, 16, 14, 7), (0b1111111111010011, 16, 9, 3), (0b1111111111010100, 16, 14, 8), (0b1111111111010101, 16, 12, 1), (0b1111111111010110, 16, 12, 6),
|
||||
(0b1111111111010111, 16, 7, 10), (0b1111111111011000, 16, 6, 4), (0b1111111111011001, 16, 14, 1), (0b1111111111011010, 16, 4, 10), (0b1111111111011011, 16, 6, 10), (0b1111111111011100, 16, 14, 6),
|
||||
(0b1111111111011101, 16, 11, 3), (0b1111111111011110, 16, 15, 1), (0b1111111111011111, 16, 13, 3), (0b1111111111100000, 16, 10, 5), (0b1111111111100001, 16, 8, 10), (0b1111111111100010, 16, 11, 2),
|
||||
(0b1111111111100011, 16, 9, 10), (0b1111111111100100, 16, 11, 10), (0b1111111111100101, 16, 8, 4), (0b1111111111100110, 16, 10, 4), (0b1111111111100111, 16, 6, 3), (0b1111111111101000, 16, 14, 5),
|
||||
(0b1111111111101001, 16, 12, 5), (0b1111111111101010, 16, 15, 3), (0b1111111111101011, 16, 13, 2), (0b1111111111101100, 16, 12, 4), (0b1111111111101101, 16, 8, 2), (0b1111111111101110, 16, 10, 10),
|
||||
(0b1111111111101111, 16, 13, 10), (0b1111111111110000, 16, 14, 4), (0b1111111111110001, 16, 15, 2), (0b1111111111110010, 16, 12, 10), (0b1111111111110011, 16, 8, 3), (0b1111111111110100, 16, 10, 3),
|
||||
(0b1111111111110101, 16, 10, 2), (0b1111111111110110, 16, 12, 3), (0b1111111111110111, 16, 14, 10), (0b1111111111111000, 16, 12, 2), (0b1111111111111001, 16, 14, 2), (0b1111111111111010, 16, 14, 3),
|
||||
];
|
||||
#[rustfmt::skip]
|
||||
const FIRST_1: &[(u16, u8, u8, u8)] = &[
|
||||
(0b00, 2, 0, 3), (0b01, 2, 0, 2), (0b100, 3, 0, 4), (0b101, 3, 0, 1), (0b1100, 4, 0, 5), (0b1101, 4, 0, 0),
|
||||
(0b1110, 4, 0, 6), (0b11110, 5, 0, 7), (0b111110, 6, 0, 9), (0b1111110, 7, 0, 8), (0b11111110, 8, 0, 10), (0b111111110, 9, 0, 11),
|
||||
];
|
||||
#[rustfmt::skip]
|
||||
const MAIN_1: &[(u16, u8, u8, u8)] = &[
|
||||
(0b00, 2, 0, 2), (0b01, 2, 0, 3), (0b100, 3, 0, 1), (0b101, 3, 0, 4), (0b1100, 4, 0, 5), (0b11010, 5, 1, 2),
|
||||
(0b11011, 5, 1, 1), (0b11100, 5, 0, 6), (0b11101, 5, 1, 3), (0b111100, 6, 0, 7), (0b1111010, 7, 0, 8), (0b1111011, 7, 1, 4),
|
||||
(0b1111100, 7, 2, 2), (0b1111101, 7, 0, 9), (0b11111100, 8, 2, 1), (0b111111010, 9, 0, 0), (0b111111011, 9, 2, 3), (0b111111100, 9, 1, 5),
|
||||
(0b1111111010, 10, 3, 1), (0b1111111011, 10, 3, 2), (0b1111111100, 10, 0, 10), (0b11111111010, 11, 1, 6), (0b1111111101100000, 16, 15, 0), (0b1111111101100001, 16, 2, 4),
|
||||
(0b1111111101100010, 16, 3, 3), (0b1111111101100011, 16, 4, 1), (0b1111111101100100, 16, 4, 2), (0b1111111101100101, 16, 1, 9), (0b1111111101100110, 16, 1, 7), (0b1111111101100111, 16, 2, 5),
|
||||
(0b1111111101101000, 16, 1, 8), (0b1111111101101001, 16, 5, 1), (0b1111111101101010, 16, 3, 4), (0b1111111101101011, 16, 4, 3), (0b1111111101101100, 16, 5, 2), (0b1111111101101101, 16, 2, 9),
|
||||
(0b1111111101101110, 16, 3, 5), (0b1111111101101111, 16, 6, 1), (0b1111111101110000, 16, 3, 9), (0b1111111101110001, 16, 7, 1), (0b1111111101110010, 16, 6, 2), (0b1111111101110011, 16, 3, 6),
|
||||
(0b1111111101110100, 16, 5, 3), (0b1111111101110101, 16, 2, 6), (0b1111111101110110, 16, 3, 8), (0b1111111101110111, 16, 1, 10), (0b1111111101111000, 16, 3, 7), (0b1111111101111001, 16, 8, 1),
|
||||
(0b1111111101111010, 16, 2, 7), (0b1111111101111011, 16, 9, 1), (0b1111111101111100, 16, 7, 9), (0b1111111101111101, 16, 5, 5), (0b1111111101111110, 16, 4, 5), (0b1111111101111111, 16, 2, 8),
|
||||
(0b1111111110000000, 16, 7, 2), (0b1111111110000001, 16, 5, 9), (0b1111111110000010, 16, 10, 1), (0b1111111110000011, 16, 11, 1), (0b1111111110000100, 16, 4, 4), (0b1111111110000101, 16, 6, 9),
|
||||
(0b1111111110000110, 16, 5, 4), (0b1111111110000111, 16, 5, 8), (0b1111111110001000, 16, 13, 1), (0b1111111110001001, 16, 15, 10), (0b1111111110001010, 16, 5, 7), (0b1111111110001011, 16, 14, 1),
|
||||
(0b1111111110001100, 16, 15, 1), (0b1111111110001101, 16, 11, 9), (0b1111111110001110, 16, 4, 9), (0b1111111110001111, 16, 4, 7), (0b1111111110010000, 16, 6, 3), (0b1111111110010001, 16, 6, 10),
|
||||
(0b1111111110010010, 16, 15, 9), (0b1111111110010011, 16, 5, 6), (0b1111111110010100, 16, 4, 6), (0b1111111110010101, 16, 10, 8), (0b1111111110010110, 16, 2, 10), (0b1111111110010111, 16, 4, 10),
|
||||
(0b1111111110011000, 16, 7, 8), (0b1111111110011001, 16, 9, 9), (0b1111111110011010, 16, 3, 10), (0b1111111110011011, 16, 7, 5), (0b1111111110011100, 16, 7, 4), (0b1111111110011101, 16, 8, 6),
|
||||
(0b1111111110011110, 16, 6, 5), (0b1111111110011111, 16, 12, 1), (0b1111111110100000, 16, 7, 6), (0b1111111110100001, 16, 11, 6), (0b1111111110100010, 16, 9, 6), (0b1111111110100011, 16, 13, 6),
|
||||
(0b1111111110100100, 16, 8, 9), (0b1111111110100101, 16, 8, 5), (0b1111111110100110, 16, 12, 9), (0b1111111110100111, 16, 15, 5), (0b1111111110101000, 16, 9, 5), (0b1111111110101001, 16, 11, 4),
|
||||
(0b1111111110101010, 16, 12, 7), (0b1111111110101011, 16, 15, 7), (0b1111111110101100, 16, 8, 10), (0b1111111110101101, 16, 9, 7), (0b1111111110101110, 16, 11, 8), (0b1111111110101111, 16, 7, 3),
|
||||
(0b1111111110110000, 16, 11, 7), (0b1111111110110001, 16, 13, 8), (0b1111111110110010, 16, 13, 9), (0b1111111110110011, 16, 8, 7), (0b1111111110110100, 16, 10, 7), (0b1111111110110101, 16, 7, 10),
|
||||
(0b1111111110110110, 16, 4, 8), (0b1111111110110111, 16, 8, 2), (0b1111111110111000, 16, 8, 4), (0b1111111110111001, 16, 14, 10), (0b1111111110111010, 16, 15, 4), (0b1111111110111011, 16, 10, 6),
|
||||
(0b1111111110111100, 16, 12, 5), (0b1111111110111101, 16, 5, 10), (0b1111111110111110, 16, 9, 4), (0b1111111110111111, 16, 10, 4), (0b1111111111000000, 16, 12, 6), (0b1111111111000001, 16, 9, 2),
|
||||
(0b1111111111000010, 16, 12, 3), (0b1111111111000011, 16, 6, 8), (0b1111111111000100, 16, 11, 5), (0b1111111111000101, 16, 12, 8), (0b1111111111000110, 16, 14, 4), (0b1111111111000111, 16, 14, 5),
|
||||
(0b1111111111001000, 16, 14, 6), (0b1111111111001001, 16, 14, 9), (0b1111111111001010, 16, 10, 2), (0b1111111111001011, 16, 10, 3), (0b1111111111001100, 16, 14, 3), (0b1111111111001101, 16, 12, 2),
|
||||
(0b1111111111001110, 16, 6, 6), (0b1111111111001111, 16, 6, 7), (0b1111111111010000, 16, 9, 3), (0b1111111111010001, 16, 10, 10), (0b1111111111010010, 16, 13, 4), (0b1111111111010011, 16, 13, 5),
|
||||
(0b1111111111010100, 16, 14, 7), (0b1111111111010101, 16, 15, 8), (0b1111111111010110, 16, 8, 8), (0b1111111111010111, 16, 9, 10), (0b1111111111011000, 16, 13, 7), (0b1111111111011001, 16, 7, 7),
|
||||
(0b1111111111011010, 16, 12, 4), (0b1111111111011011, 16, 6, 4), (0b1111111111011100, 16, 14, 2), (0b1111111111011101, 16, 9, 8), (0b1111111111011110, 16, 10, 5), (0b1111111111011111, 16, 12, 10),
|
||||
(0b1111111111100000, 16, 13, 10), (0b1111111111100001, 16, 14, 8), (0b1111111111100010, 16, 15, 3), (0b1111111111100011, 16, 15, 6), (0b1111111111100100, 16, 10, 9), (0b1111111111100101, 16, 11, 2),
|
||||
(0b1111111111100110, 16, 11, 3), (0b1111111111100111, 16, 15, 2), (0b1111111111101000, 16, 13, 2), (0b1111111111101001, 16, 8, 3), (0b1111111111101010, 16, 11, 10), (0b1111111111101011, 16, 13, 3),
|
||||
];
|
||||
#[rustfmt::skip]
|
||||
const FIRST_2: &[(u16, u8, u8, u8)] = &[
|
||||
(0b000, 3, 0, 6), (0b001, 3, 0, 5), (0b010, 3, 0, 7), (0b011, 3, 0, 4), (0b100, 3, 0, 8), (0b101, 3, 0, 3),
|
||||
(0b1100, 4, 0, 9), (0b1101, 4, 0, 2), (0b1110, 4, 0, 0), (0b11110, 5, 0, 10), (0b111110, 6, 0, 1), (0b1111110, 7, 0, 11),
|
||||
];
|
||||
#[rustfmt::skip]
|
||||
const MAIN_2: &[(u16, u8, u8, u8)] = &[
|
||||
(0b000, 3, 0, 4), (0b001, 3, 0, 5), (0b010, 3, 0, 3), (0b011, 3, 0, 6), (0b100, 3, 0, 2), (0b101, 3, 0, 7),
|
||||
(0b1100, 4, 0, 1), (0b1101, 4, 0, 8), (0b11100, 5, 0, 9), (0b111010, 6, 1, 2), (0b111011, 6, 1, 3), (0b111100, 6, 1, 4),
|
||||
(0b1111010, 7, 1, 1), (0b1111011, 7, 1, 5), (0b1111100, 7, 0, 10), (0b11111010, 8, 1, 6), (0b11111011, 8, 1, 7), (0b111111000, 9, 15, 0),
|
||||
(0b111111001, 9, 0, 0), (0b111111010, 9, 2, 2), (0b111111011, 9, 2, 1), (0b111111100, 9, 1, 8), (0b1111111010, 10, 2, 3), (0b11111110110, 11, 1, 9),
|
||||
(0b11111110111, 11, 2, 4), (0b111111110000, 12, 3, 2), (0b111111110001, 12, 3, 1), (0b1111111100100, 13, 2, 5), (0b1111111100101, 13, 3, 3), (0b1111111100110, 13, 3, 8),
|
||||
(0b1111111100111, 13, 3, 7), (0b1111111101000, 13, 3, 4), (0b1111111101001, 13, 3, 5), (0b1111111101010, 13, 3, 6), (0b1111111101011, 13, 3, 9), (0b11111111011000, 14, 7, 9),
|
||||
(0b11111111011001, 14, 5, 7), (0b11111111011010, 14, 5, 8), (0b11111111011011, 14, 5, 9), (0b11111111011100, 14, 2, 8), (0b11111111011101, 14, 5, 6), (0b11111111011110, 14, 7, 8),
|
||||
(0b11111111011111, 14, 2, 7), (0b11111111100000, 14, 4, 1), (0b11111111100001, 14, 2, 9), (0b1111111110001000, 16, 7, 7), (0b1111111110001001, 16, 2, 6), (0b1111111110001010, 16, 4, 2),
|
||||
(0b1111111110001011, 16, 7, 6), (0b1111111110001100, 16, 9, 9), (0b1111111110001101, 16, 1, 10), (0b1111111110001110, 16, 5, 5), (0b1111111110001111, 16, 9, 8), (0b1111111110010000, 16, 9, 7),
|
||||
(0b1111111110010001, 16, 15, 9), (0b1111111110010010, 16, 4, 8), (0b1111111110010011, 16, 5, 4), (0b1111111110010100, 16, 9, 6), (0b1111111110010101, 16, 8, 9), (0b1111111110010110, 16, 4, 7),
|
||||
(0b1111111110010111, 16, 11, 7), (0b1111111110011000, 16, 4, 9), (0b1111111110011001, 16, 15, 10), (0b1111111110011010, 16, 7, 5), (0b1111111110011011, 16, 6, 8), (0b1111111110011100, 16, 11, 6),
|
||||
(0b1111111110011101, 16, 6, 7), (0b1111111110011110, 16, 6, 9), (0b1111111110011111, 16, 11, 9), (0b1111111110100000, 16, 11, 8), (0b1111111110100001, 16, 13, 8), (0b1111111110100010, 16, 5, 2),
|
||||
(0b1111111110100011, 16, 13, 7), (0b1111111110100100, 16, 8, 8), (0b1111111110100101, 16, 11, 5), (0b1111111110100110, 16, 7, 4), (0b1111111110100111, 16, 5, 1), (0b1111111110101000, 16, 4, 6),
|
||||
(0b1111111110101001, 16, 13, 9), (0b1111111110101010, 16, 15, 8), (0b1111111110101011, 16, 3, 10), (0b1111111110101100, 16, 13, 6), (0b1111111110101101, 16, 8, 7), (0b1111111110101110, 16, 4, 5),
|
||||
(0b1111111110101111, 16, 7, 10), (0b1111111110110000, 16, 9, 5), (0b1111111110110001, 16, 13, 5), (0b1111111110110010, 16, 15, 6), (0b1111111110110011, 16, 8, 6), (0b1111111110110100, 16, 11, 4),
|
||||
(0b1111111110110101, 16, 10, 9), (0b1111111110110110, 16, 9, 4), (0b1111111110110111, 16, 5, 3), (0b1111111110111000, 16, 2, 10), (0b1111111110111001, 16, 10, 8), (0b1111111110111010, 16, 4, 3),
|
||||
(0b1111111110111011, 16, 15, 5), (0b1111111110111100, 16, 15, 7), (0b1111111110111101, 16, 13, 4), (0b1111111110111110, 16, 6, 6), (0b1111111110111111, 16, 10, 7), (0b1111111111000000, 16, 5, 10),
|
||||
(0b1111111111000001, 16, 4, 4), (0b1111111111000010, 16, 8, 10), (0b1111111111000011, 16, 12, 9), (0b1111111111000100, 16, 14, 8), (0b1111111111000101, 16, 12, 8), (0b1111111111000110, 16, 14, 7),
|
||||
(0b1111111111000111, 16, 9, 10), (0b1111111111001000, 16, 6, 10), (0b1111111111001001, 16, 7, 3), (0b1111111111001010, 16, 4, 10), (0b1111111111001011, 16, 6, 1), (0b1111111111001100, 16, 12, 7),
|
||||
(0b1111111111001101, 16, 15, 4), (0b1111111111001110, 16, 12, 6), (0b1111111111001111, 16, 6, 5), (0b1111111111010000, 16, 14, 9), (0b1111111111010001, 16, 7, 2), (0b1111111111010010, 16, 14, 6),
|
||||
(0b1111111111010011, 16, 7, 1), (0b1111111111010100, 16, 9, 1), (0b1111111111010101, 16, 9, 3), (0b1111111111010110, 16, 10, 6), (0b1111111111010111, 16, 13, 10), (0b1111111111011000, 16, 9, 2),
|
||||
(0b1111111111011001, 16, 8, 5), (0b1111111111011010, 16, 6, 2), (0b1111111111011011, 16, 15, 3), (0b1111111111011100, 16, 12, 5), (0b1111111111011101, 16, 11, 2), (0b1111111111011110, 16, 10, 4),
|
||||
(0b1111111111011111, 16, 8, 4), (0b1111111111100000, 16, 11, 10), (0b1111111111100001, 16, 6, 4), (0b1111111111100010, 16, 10, 5), (0b1111111111100011, 16, 11, 3), (0b1111111111100100, 16, 13, 2),
|
||||
(0b1111111111100101, 16, 8, 1), (0b1111111111100110, 16, 14, 5), (0b1111111111100111, 16, 13, 3), (0b1111111111101000, 16, 10, 10), (0b1111111111101001, 16, 12, 4), (0b1111111111101010, 16, 12, 10),
|
||||
(0b1111111111101011, 16, 15, 2), (0b1111111111101100, 16, 11, 1), (0b1111111111101101, 16, 14, 4), (0b1111111111101110, 16, 13, 1), (0b1111111111101111, 16, 8, 3), (0b1111111111110000, 16, 6, 3),
|
||||
(0b1111111111110001, 16, 14, 10), (0b1111111111110010, 16, 12, 3), (0b1111111111110011, 16, 14, 2), (0b1111111111110100, 16, 8, 2), (0b1111111111110101, 16, 15, 1), (0b1111111111110110, 16, 10, 3),
|
||||
(0b1111111111110111, 16, 12, 2), (0b1111111111111000, 16, 10, 1), (0b1111111111111001, 16, 12, 1), (0b1111111111111010, 16, 14, 3), (0b1111111111111011, 16, 10, 2), (0b1111111111111100, 16, 14, 1),
|
||||
];
|
||||
/// The tables for DecoderTable selector 0, 1 or 2.
|
||||
pub(crate) fn tables(selector: u32) -> Option<SelTables> {
|
||||
match selector {
|
||||
0 => Some(SelTables { first: FIRST_0, main: MAIN_0 }),
|
||||
1 => Some(SelTables { first: FIRST_1, main: MAIN_1 }),
|
||||
2 => Some(SelTables { first: FIRST_2, main: MAIN_2 }),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,668 @@
|
||||
//! Unit tests: synthetic CIFF files built with a test-only encoder that is the inverse of the
|
||||
//! frozen model, plus the frozen tables' structural properties. The decoder's behaviour on real
|
||||
//! files is pinned in `tests/samples.rs`.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn put16(v: u16, o: Order) -> [u8; 2] {
|
||||
match o {
|
||||
Order::Le => v.to_le_bytes(),
|
||||
Order::Be => v.to_be_bytes(),
|
||||
}
|
||||
}
|
||||
fn put32(v: u32, o: Order) -> [u8; 4] {
|
||||
match o {
|
||||
Order::Le => v.to_le_bytes(),
|
||||
Order::Be => v.to_be_bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A heap block: values, record table, trailing offset-table pointer.
|
||||
fn heap(records: &[(u16, Vec<u8>)], o: Order) -> Vec<u8> {
|
||||
let mut b = Vec::new();
|
||||
let mut entries = Vec::new();
|
||||
for (tc, v) in records {
|
||||
let off = b.len();
|
||||
b.extend_from_slice(v);
|
||||
if b.len() % 2 == 1 {
|
||||
b.push(0);
|
||||
}
|
||||
entries.push((*tc, v.len() as u32, off as u32));
|
||||
}
|
||||
let tbl = b.len() as u32;
|
||||
b.extend_from_slice(&put16(entries.len() as u16, o));
|
||||
for (tc, l, off) in entries {
|
||||
b.extend_from_slice(&put16(tc, o));
|
||||
b.extend_from_slice(&put32(l, o));
|
||||
b.extend_from_slice(&put32(off, o));
|
||||
}
|
||||
b.extend_from_slice(&put32(tbl, o));
|
||||
b
|
||||
}
|
||||
|
||||
/// A complete CIFF file: 26-byte header + the root heap.
|
||||
fn ciff(records: &[(u16, Vec<u8>)], o: Order) -> Vec<u8> {
|
||||
let mut f = Vec::new();
|
||||
f.extend_from_slice(match o {
|
||||
Order::Le => b"II",
|
||||
Order::Be => b"MM",
|
||||
});
|
||||
f.extend_from_slice(&put32(26, o));
|
||||
f.extend_from_slice(b"HEAPCCDR");
|
||||
f.extend_from_slice(&put32(0x0001_0002, o));
|
||||
f.extend_from_slice(&[0; 8]);
|
||||
f.extend_from_slice(&heap(records, o));
|
||||
f
|
||||
}
|
||||
|
||||
fn sensor_info(w: u16, h: u16, crop: Option<(u16, u16, u16, u16)>, o: Order) -> Vec<u8> {
|
||||
let (l, t, r, b) = crop.unwrap_or((0, 0, 0, 0));
|
||||
[34, w, h, 0, 0, l, t, r, b, 0, 0, 0, 0, 0, 0, 0, 0].iter().flat_map(|&v| put16(v, o)).collect()
|
||||
}
|
||||
|
||||
fn decoder_table(sel: u32, o: Order) -> Vec<u8> {
|
||||
[sel, 0, 514, 0].iter().flat_map(|&v| put32(v, o)).collect()
|
||||
}
|
||||
|
||||
/// Entropy-encode a 10-bit high plane with the model's inverse: per-block first/main tables,
|
||||
/// (0, s) tokens only (the caller keeps mid-block diffs nonzero), JPEG magnitudes, carry,
|
||||
/// parity chains reset at row starts; MSB-first packing, zero padding, ff00 stuffing.
|
||||
fn encode_hi(hi: &[u16], w: usize, t: &tables::SelTables) -> Vec<u8> {
|
||||
let mut bits: Vec<bool> = Vec::new();
|
||||
let push_code = |bits: &mut Vec<bool>, tab: &[(u16, u8, u8, u8)], s: u32| {
|
||||
let &(c, l, ..) = tab.iter().find(|e| e.2 == 0 && e.3 as u32 == s).expect("(0, s) code exists");
|
||||
bits.extend((0..l).rev().map(|i| (c >> i) & 1 == 1));
|
||||
};
|
||||
let size_of = |d: i32| if d == 0 { 0 } else { 32 - d.unsigned_abs().leading_zeros() };
|
||||
let (mut ce, mut co) = (512i32, 512i32);
|
||||
let mut carry = 0i32;
|
||||
for (qi, &v) in hi.iter().enumerate() {
|
||||
if qi % w == 0 {
|
||||
ce = 512;
|
||||
co = 512;
|
||||
}
|
||||
let c = if qi % 2 == 0 { &mut ce } else { &mut co };
|
||||
let dd = v as i32 - *c;
|
||||
*c = v as i32;
|
||||
let (tab, d) = if qi % 64 == 0 {
|
||||
let d = dd - carry;
|
||||
carry = dd;
|
||||
(t.first, d)
|
||||
} else {
|
||||
assert_ne!(dd, 0, "test images avoid zero diffs mid-block");
|
||||
(t.main, dd)
|
||||
};
|
||||
let s = size_of(d);
|
||||
push_code(&mut bits, tab, s);
|
||||
if s > 0 {
|
||||
let m = if d > 0 { d } else { d + (1 << s) - 1 } as u32;
|
||||
bits.extend((0..s).rev().map(|i| (m >> i) & 1 == 1));
|
||||
}
|
||||
}
|
||||
pack_entropy(&bits)
|
||||
}
|
||||
|
||||
/// MSB-first packing into bytes, zero padding in the last byte, then ff00 stuffing.
|
||||
fn pack_entropy(bits: &[bool]) -> Vec<u8> {
|
||||
let mut stuffed = Vec::new();
|
||||
for chunk in bits.chunks(8) {
|
||||
let mut b = 0u8;
|
||||
for (i, &bit) in chunk.iter().enumerate() {
|
||||
b |= (bit as u8) << (7 - i);
|
||||
}
|
||||
stuffed.push(b);
|
||||
if b == 0xff {
|
||||
stuffed.push(0x00);
|
||||
}
|
||||
}
|
||||
stuffed
|
||||
}
|
||||
|
||||
/// One end-of-block token pair per 64-sample block: the block's first sample takes the carry
|
||||
/// (zero here), the other 63 keep their chain values. A flat frame like this compresses to a
|
||||
/// small fraction of its low-bits plane, the shape a lens-cap or blown-out frame has.
|
||||
fn encode_eob(n: usize, t: &tables::SelTables) -> Vec<u8> {
|
||||
let code = |tab: &[(u16, u8, u8, u8)]| {
|
||||
let &(c, l, ..) = tab.iter().find(|e| e.2 == 0 && e.3 == 0).expect("an end-of-block code");
|
||||
(c, l)
|
||||
};
|
||||
let (fc, fl) = code(t.first);
|
||||
let (mc, ml) = code(t.main);
|
||||
let mut bits: Vec<bool> = Vec::new();
|
||||
for _ in 0..n.div_ceil(64) {
|
||||
for (c, l) in [(fc, fl), (mc, ml)] {
|
||||
bits.extend((0..l).rev().map(|i| (c >> i) & 1 == 1));
|
||||
}
|
||||
}
|
||||
pack_entropy(&bits)
|
||||
}
|
||||
|
||||
/// A high plane whose diffs are all nonzero, bounded around 512: values follow
|
||||
/// `(3 qi) mod 5 + 1`, so same-parity neighbours differ by 1 or −4 (never 0) and
|
||||
/// row starts sit 1–5 above the reset value.
|
||||
fn test_hi(n: usize) -> Vec<u16> {
|
||||
(0..n).map(|qi| 512 + (qi as u16 * 3) % 5 + 1).collect()
|
||||
}
|
||||
|
||||
fn payload(low: Option<&[u8]>, entropy: &[u8]) -> Vec<u8> {
|
||||
let mut p = low.map(|l| l.to_vec()).unwrap_or_default();
|
||||
p.extend_from_slice(&[0; 514]);
|
||||
p.extend_from_slice(entropy);
|
||||
p.extend_from_slice(&[0, 0]);
|
||||
p
|
||||
}
|
||||
|
||||
/// The real PowerShot record shape: the payload closes with `ff d9` and a pad byte, not `00 00`.
|
||||
fn payload_with_trailer(low: Option<&[u8]>, entropy: &[u8]) -> Vec<u8> {
|
||||
let mut p = low.map(|l| l.to_vec()).unwrap_or_default();
|
||||
p.extend_from_slice(&[0; 514]);
|
||||
p.extend_from_slice(entropy);
|
||||
p.extend_from_slice(&[0xff, 0xd9, 0x00]);
|
||||
p
|
||||
}
|
||||
|
||||
/// A file whose raw record is `payload` verbatim, with `extra` records beside the usual three.
|
||||
fn file_with_payload(w: u16, h: u16, model: &str, payload: &[u8], extra: &[(u16, Vec<u8>)], o: Order) -> Vec<u8> {
|
||||
let mut props = vec![(SENSOR_INFO, sensor_info(w, h, None, o)), (DECODER_TABLE, decoder_table(0, o)), (MAKE_MODEL, make_model(model))];
|
||||
props.extend_from_slice(extra);
|
||||
ciff(&[(RAW_DATA, payload.to_vec()), (0x300a, heap(&props, o))], o)
|
||||
}
|
||||
|
||||
fn make_model(model: &str) -> Vec<u8> {
|
||||
let mut v = b"Canon\0".to_vec();
|
||||
v.extend_from_slice(model.as_bytes());
|
||||
v.push(0);
|
||||
v
|
||||
}
|
||||
|
||||
fn roundtrip_file(w: u16, h: u16, model: &str, low: Option<&[u8]>, hi: &[u16], o: Order) -> Vec<u8> {
|
||||
let tables = tables::tables(0).expect("selector 0");
|
||||
let entropy = encode_hi(hi, w as usize, &tables);
|
||||
// the realistic shape: raw data in the root, properties in a nested heap
|
||||
let props = heap(&[(SENSOR_INFO, sensor_info(w, h, None, o)), (DECODER_TABLE, decoder_table(0, o)), (MAKE_MODEL, make_model(model))], o);
|
||||
ciff(&[(RAW_DATA, payload(low, &entropy)), (0x300a, props)], o)
|
||||
}
|
||||
|
||||
fn full(bytes: &[u8]) -> Result<Image> {
|
||||
decode(bytes, &Limits::default())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_needs_the_signature() {
|
||||
assert!(probe(b"II\x1a\0\0\0HEAPCCDR\0\0"));
|
||||
assert!(probe(b"MM\0\0\0\x1aHEAPCCDR"));
|
||||
assert!(!probe(b"II\x1a\0\0\0HEAP"));
|
||||
assert!(!probe(b"II*\0\x08\0\0\0"), "a TIFF");
|
||||
assert!(!probe(b""));
|
||||
assert_eq!(full(b"II*\0\x08\0\0\0"), Err(Error::NotCrw));
|
||||
// a signature with no heap behind it is corrupt, not a panic
|
||||
assert!(matches!(full(b"II HEAPCCDR "), Err(Error::Corrupt(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrips_a_10_bit_frame() {
|
||||
let (w, h) = (16u16, 8u16);
|
||||
let hi = test_hi(128);
|
||||
let f = roundtrip_file(w, h, "Canon EOS D60", None, &hi, Order::Le);
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!((img.header.width, img.header.height, img.header.bits), (16, 8, 10));
|
||||
assert_eq!(img.samples, hi);
|
||||
assert_eq!(img.header.cfa, Cfa::Rggb);
|
||||
assert_eq!(img.header.selector, 0);
|
||||
assert_eq!(img.header.make.as_deref(), Some("Canon"));
|
||||
assert_eq!(img.header.model.as_deref(), Some("Canon EOS D60"));
|
||||
assert_eq!(img.header.exif_orientation, 1);
|
||||
assert_eq!(img.header.capture_unix, None);
|
||||
assert_eq!(img.header.wb_multipliers, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrips_a_12_bit_frame_with_a_low_bits_plane() {
|
||||
let (w, h) = (16u16, 8u16);
|
||||
let hi = test_hi(128);
|
||||
// low bits all 3: the plane is all 0xff bytes, which trips the presence rule
|
||||
let low = vec![0xffu8; 32];
|
||||
let f = roundtrip_file(w, h, "Canon EOS D60", Some(&low), &hi, Order::Le);
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!(img.header.bits, 12);
|
||||
let expected: Vec<u16> = hi.iter().map(|&v| (v << 2) | 3).collect();
|
||||
assert_eq!(img.samples, expected);
|
||||
}
|
||||
|
||||
/// A low-bits plane without a single 0xff byte (real D60 files whose lowest bit is nearly
|
||||
/// always 0) used to be taken for "no plane": the plane was read as entropy data and the
|
||||
/// file came out as 10-bit garbage. The zero header after the plane gives it away.
|
||||
#[test]
|
||||
fn a_low_bits_plane_without_ff_bytes_is_still_a_plane() {
|
||||
let hi = test_hi(128);
|
||||
for byte in [0x22u8, 0x00] {
|
||||
let low = vec![byte; 32];
|
||||
let f = roundtrip_file(16, 8, "Canon EOS D60", Some(&low), &hi, Order::Le);
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!(img.header.bits, 12);
|
||||
let expected: Vec<u16> = hi.iter().enumerate().map(|(i, &v)| (v << 2) | u16::from((byte >> ((i & 3) * 2)) & 3)).collect();
|
||||
assert_eq!(img.samples, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn big_endian_ciff_decodes_too() {
|
||||
let hi = test_hi(128);
|
||||
let f = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, Order::Be);
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!(img.samples, hi);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_reads_geometry_without_samples() {
|
||||
let hi = test_hi(128);
|
||||
let f = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, Order::Le);
|
||||
let hdr = header(&f, &Limits::default()).expect("header decodes");
|
||||
assert_eq!((hdr.width, hdr.height, hdr.bits), (16, 8, 10));
|
||||
assert_eq!(hdr, full(&f).expect("decodes").header, "the header is the same either way");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limits_cap_the_sample_count() {
|
||||
let hi = test_hi(128);
|
||||
let f = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, Order::Le);
|
||||
let mut limits = Limits { max_samples: 127 };
|
||||
assert_eq!(decode(&f, &limits), Err(Error::LimitExceeded("image too large")));
|
||||
assert!(header(&f, &limits).is_err());
|
||||
limits.max_samples = 128;
|
||||
assert!(decode(&f, &limits).is_ok());
|
||||
}
|
||||
|
||||
/// A header declaring 2²⁴ samples over a 128-sample stream is refused by the default limits
|
||||
/// before any sample-sized buffer exists: the cap must stay above every real frame (7.4 million)
|
||||
/// and below anything that would let a crafted file demand gigabytes.
|
||||
#[test]
|
||||
fn default_limits_refuse_a_crafted_giant_frame() {
|
||||
let hi = test_hi(128);
|
||||
let f = roundtrip_file(4096, 4096, "Canon EOS D60", None, &hi, Order::Le);
|
||||
assert_eq!(decode(&f, &Limits::default()), Err(Error::LimitExceeded("image too large")));
|
||||
assert_eq!(header(&f, &Limits::default()), Err(Error::LimitExceeded("image too large")));
|
||||
assert!(Limits::default().max_samples >= 7_407_040, "the cap must admit the largest real frame");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn g1_is_gbrg() {
|
||||
let hi = test_hi(128);
|
||||
let f = roundtrip_file(16, 8, "Canon PowerShot G1", None, &hi, Order::Le);
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!(img.header.cfa, Cfa::Gbrg);
|
||||
assert_eq!(img.header.cfa.pattern(), [1, 2, 0, 1]);
|
||||
assert_eq!(Cfa::Rggb.pattern(), [0, 1, 1, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn g5_is_refused_with_its_anomaly_note() {
|
||||
let hi = test_hi(128);
|
||||
let f = roundtrip_file(16, 8, "Canon PowerShot G5", None, &hi, Order::Le);
|
||||
let Err(Error::Unsupported(why)) = full(&f) else { panic!("expected Unsupported") };
|
||||
assert!(why.contains("G5"), "{why}");
|
||||
assert!(matches!(header(&f, &Limits::default()), Err(Error::Unsupported(_))));
|
||||
}
|
||||
|
||||
/// A D60 at ISO 100 clips near 4000, and a photo with a small blown area has no plateau:
|
||||
/// the white level must still come down to where the data stops, not stay at 4095. A few
|
||||
/// hot pixels, or a photo that never gets near the top, leave the full range.
|
||||
#[test]
|
||||
fn white_level_follows_a_small_clipped_area() {
|
||||
let mut data: Vec<u16> = (0..200_000).map(|i| 300 + (i % 3000) as u16).collect();
|
||||
assert_eq!(white_level(&data, 12), 4095.0, "nothing near the top");
|
||||
for v in data.iter_mut().take(100) {
|
||||
*v = 4002;
|
||||
}
|
||||
assert_eq!(white_level(&data, 12), (4002 - 15) as f32, "100 samples at the clip level");
|
||||
let mut hot: Vec<u16> = (0..200_000).map(|i| 300 + (i % 3000) as u16).collect();
|
||||
for v in hot.iter_mut().take(10) {
|
||||
*v = 4002;
|
||||
}
|
||||
assert_eq!(white_level(&hot, 12), 4095.0, "ten hot pixels are not a clip level");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn saturation_plateau_is_found_with_strays_above_it() {
|
||||
let mut d: Vec<u16> = (0..100_000).map(|i| (i % 10000) as u16).collect();
|
||||
assert_eq!(saturation_plateau(&d, 14), 16383.0);
|
||||
d.extend(std::iter::repeat_n(15000u16, 1000));
|
||||
let w = saturation_plateau(&d, 14);
|
||||
assert!((14900.0..=15000.0).contains(&w), "{w}");
|
||||
assert_eq!(saturation_plateau(&[], 12), 4095.0);
|
||||
let mut d: Vec<u16> = (0..100_000).map(|i| (2048 + i % 10000) as u16).collect();
|
||||
d.extend(std::iter::repeat_n(13583u16, 5000));
|
||||
d.extend([13700, 13900, 14100, 14426]);
|
||||
let w = saturation_plateau(&d, 14);
|
||||
assert!((13500.0..=13583.0).contains(&w), "{w}");
|
||||
}
|
||||
|
||||
/// The D60 layout: sensor rows clamped to 0 above the image, dark columns left of it. The
|
||||
/// clamped rows must not count towards the black level.
|
||||
#[test]
|
||||
fn black_level_ignores_clamped_rows() {
|
||||
let (w, h) = (64usize, 40usize);
|
||||
let active = Rect::new(24, 16, 40, 24);
|
||||
let mut data = vec![900u16; w * h];
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
data[y * w + x] = if y < 7 {
|
||||
0
|
||||
} else if x < 24 {
|
||||
128
|
||||
} else {
|
||||
900
|
||||
};
|
||||
}
|
||||
}
|
||||
let black = black_from_borders(&data, w, h, active);
|
||||
assert!(black.values.iter().all(|v| (v - 128.0).abs() < 0.5), "{:?}", black.values);
|
||||
assert_eq!(black.at(active, 24, 16), black.values[0]);
|
||||
assert_eq!(black.at(active, 25, 16), black.values[1]);
|
||||
assert_eq!(black.at(active, 24, 17), black.values[2]);
|
||||
assert_eq!(black.at(active, 23, 15), black.values[3], "sites are counted from the active origin, modulo 2");
|
||||
assert_eq!(black_from_borders(&data, w, h, Rect::new(0, 0, w, h)), BlackLevel::uniform(0.0), "no border, no level");
|
||||
}
|
||||
|
||||
/// The frozen stream-end rule: a truncated entropy stream leaves the remaining samples at
|
||||
/// their chain values instead of failing the file.
|
||||
#[test]
|
||||
fn truncated_stream_keeps_chain_values() {
|
||||
let (w, h) = (16u16, 8u16);
|
||||
let hi = test_hi(128);
|
||||
let tables = tables::tables(0).expect("selector 0");
|
||||
let entropy = encode_hi(&hi, 16, &tables);
|
||||
let cut = entropy.len() / 2;
|
||||
let props = heap(
|
||||
&[(SENSOR_INFO, sensor_info(w, h, None, Order::Le)), (DECODER_TABLE, decoder_table(0, Order::Le)), (MAKE_MODEL, make_model("Canon EOS D60"))],
|
||||
Order::Le,
|
||||
);
|
||||
let f = ciff(&[(RAW_DATA, payload(None, &entropy[..cut])), (0x300a, props)], Order::Le);
|
||||
let img = full(&f).expect("truncated stream still decodes");
|
||||
assert_eq!(img.samples.len(), 128);
|
||||
assert_eq!(&img.samples[..32], &hi[..32], "the leading samples decode exactly");
|
||||
}
|
||||
|
||||
/// Every record the decoder needs missing, bogus selectors, dimensions and offsets: errors,
|
||||
/// never panics; and every truncation of a valid file survives.
|
||||
#[test]
|
||||
fn hostile_inputs_error_cleanly() {
|
||||
let o = Order::Le;
|
||||
let hi = test_hi(128);
|
||||
let good = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, o);
|
||||
for n in 0..good.len() {
|
||||
let _ = full(&good[..n]);
|
||||
let _ = header(&good[..n], &Limits::default());
|
||||
}
|
||||
// single-byte mutations over the whole file
|
||||
for i in 0..good.len() {
|
||||
let mut bad = good.clone();
|
||||
bad[i] ^= 0xa5;
|
||||
let _ = full(&bad);
|
||||
}
|
||||
let props_no_sensor = heap(&[(DECODER_TABLE, decoder_table(0, o))], o);
|
||||
let f = ciff(&[(RAW_DATA, vec![0; 600]), (0x300a, props_no_sensor)], o);
|
||||
assert!(matches!(full(&f), Err(Error::Corrupt(_))));
|
||||
let props_bad_sel = heap(&[(SENSOR_INFO, sensor_info(16, 8, None, o)), (DECODER_TABLE, decoder_table(7, o))], o);
|
||||
let f = ciff(&[(RAW_DATA, vec![0; 600]), (0x300a, props_bad_sel)], o);
|
||||
assert!(matches!(full(&f), Err(Error::Unsupported(_))));
|
||||
let props_zero = heap(&[(SENSOR_INFO, sensor_info(0, 0, None, o)), (DECODER_TABLE, decoder_table(0, o))], o);
|
||||
let f = ciff(&[(RAW_DATA, vec![0; 600]), (0x300a, props_zero)], o);
|
||||
assert!(full(&f).is_err());
|
||||
// a heap that nests into itself must hit the depth bound, not recurse forever
|
||||
let mut self_heap = heap(&[(0x300a, vec![0; 10])], o);
|
||||
let len = self_heap.len() as u32;
|
||||
// point the nested record at the whole block
|
||||
let tbl = u32::from_le_bytes(self_heap[len as usize - 4..].try_into().unwrap()) as usize;
|
||||
self_heap[tbl + 4..tbl + 8].copy_from_slice(&len.to_le_bytes());
|
||||
self_heap[tbl + 8..tbl + 12].copy_from_slice(&0u32.to_le_bytes());
|
||||
let mut f = ciff(&[], o);
|
||||
f.truncate(26);
|
||||
f.extend_from_slice(&self_heap);
|
||||
assert!(full(&f).is_err());
|
||||
}
|
||||
|
||||
/// The frozen tables are prefix-free within each table, so the 16-bit lookup is unambiguous.
|
||||
#[test]
|
||||
fn frozen_tables_are_prefix_free() {
|
||||
for sel in 0..3 {
|
||||
let t = tables::tables(sel).expect("selector");
|
||||
for (name, tab) in [("first", t.first), ("main", t.main)] {
|
||||
let mut lut = vec![false; 1 << 16];
|
||||
for &(code, len, ..) in tab {
|
||||
assert!((1..=16).contains(&len), "sel {sel} {name}: code length {len}");
|
||||
let base = (code as usize) << (16 - len);
|
||||
for cell in lut.iter_mut().skip(base).take(1 << (16 - len)) {
|
||||
assert!(!*cell, "sel {sel} {name}: overlapping codes");
|
||||
*cell = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(tables::tables(3).is_none());
|
||||
}
|
||||
|
||||
/// Every second table codes all 162 symbols: (r, s) for r 0..=15 and s 1..=10, end of block
|
||||
/// and (15, 0). Selectors 0 and 1 lacked (7..=14, 10) and (14, 10), learned from too few
|
||||
/// files, and 223 of 1,516 real D60 files failed with "undecodable entropy code".
|
||||
#[test]
|
||||
fn second_tables_code_every_symbol() {
|
||||
for sel in 0..3 {
|
||||
let t = tables::tables(sel).expect("selector");
|
||||
let have: std::collections::BTreeSet<(u8, u8)> = t.main.iter().map(|e| (e.2, e.3)).collect();
|
||||
let mut want: std::collections::BTreeSet<(u8, u8)> = (0..16).flat_map(|r| (1..=10).map(move |s| (r, s))).collect();
|
||||
want.extend([(0, 0), (15, 0)]);
|
||||
assert_eq!(have, want, "selector {sel}");
|
||||
assert_eq!(t.main.len(), 162, "selector {sel}: one code per symbol");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn white_balance_comes_from_the_shot_info_slot() {
|
||||
let o = Order::Le;
|
||||
let hi = test_hi(128);
|
||||
// D60-shaped table: 7 slots + black levels; Daylight slot (1) = 1800, 800, 800, 960
|
||||
let mut cb: Vec<u16> = vec![66];
|
||||
cb.extend([2000u16, 760, 760, 830]);
|
||||
cb.extend([1800u16, 800, 800, 960]);
|
||||
for _ in 0..5 {
|
||||
cb.extend([1500u16, 900, 900, 1500]);
|
||||
}
|
||||
cb.extend([126u16, 127, 126, 126]);
|
||||
let cb: Vec<u8> = cb.iter().flat_map(|&v| put16(v, o)).collect();
|
||||
let mut shot: Vec<u16> = vec![54; 27];
|
||||
shot[7] = 1; // Daylight
|
||||
let shot: Vec<u8> = shot.iter().flat_map(|&v| put16(v, o)).collect();
|
||||
let tables = tables::tables(0).expect("selector 0");
|
||||
let props = heap(
|
||||
&[
|
||||
(SENSOR_INFO, sensor_info(16, 8, None, o)),
|
||||
(DECODER_TABLE, decoder_table(0, o)),
|
||||
(MAKE_MODEL, make_model("Canon EOS D60")),
|
||||
(COLOR_BALANCE, cb),
|
||||
(SHOT_INFO, shot),
|
||||
],
|
||||
o,
|
||||
);
|
||||
let f = ciff(&[(RAW_DATA, payload(None, &encode_hi(&hi, 16, &tables))), (0x300a, props)], o);
|
||||
let img = full(&f).expect("decodes");
|
||||
let wb = img.header.wb_multipliers.expect("wb");
|
||||
assert!((wb[0] - 2.25).abs() < 1e-3 && wb[1] == 1.0 && (wb[2] - 1.2).abs() < 1e-3, "{wb:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rotation_maps_to_exif_orientation_and_the_time_stamp_is_unix_seconds() {
|
||||
let o = Order::Le;
|
||||
let hi = test_hi(128);
|
||||
let tables = tables::tables(0).expect("selector 0");
|
||||
// The record holds a signed degree count, so 270 may also be written as −90.
|
||||
for (deg, want) in [(0u32, 1u8), (90, 6), (180, 3), (270, 8), (-90i32 as u32, 8), (-270i32 as u32, 6)] {
|
||||
let image_info: Vec<u8> = [3072u32, 2048, 0, deg, 12, 12, 0].iter().flat_map(|&v| put32(v, o)).collect();
|
||||
let stamp: Vec<u8> = [1083410638u32, 0, 0].iter().flat_map(|&v| put32(v, o)).collect();
|
||||
let extra = [(IMAGE_INFO, image_info), (TIME_STAMP, stamp)];
|
||||
let f = file_with_payload(16, 8, "Canon EOS 300D DIGITAL", &payload(None, &encode_hi(&hi, 16, &tables)), &extra, o);
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!(img.header.exif_orientation, want, "rotation {deg}");
|
||||
assert_eq!(img.header.capture_unix, Some(1083410638));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sensor_info_crop_is_the_active_area() {
|
||||
let o = Order::Le;
|
||||
let hi = test_hi(128);
|
||||
let tables = tables::tables(0).expect("selector 0");
|
||||
let with = |crop| {
|
||||
let props =
|
||||
heap(&[(SENSOR_INFO, sensor_info(16, 8, crop, o)), (DECODER_TABLE, decoder_table(0, o)), (MAKE_MODEL, make_model("Canon EOS D60"))], o);
|
||||
let f = ciff(&[(RAW_DATA, payload(None, &encode_hi(&hi, 16, &tables))), (0x300a, props)], o);
|
||||
full(&f).expect("decodes").header.active_area
|
||||
};
|
||||
assert_eq!(with(Some((4, 1, 15, 7))), Rect::new(4, 1, 12, 7), "inclusive left, top, right, bottom");
|
||||
assert_eq!(with(None), Rect::new(0, 0, 16, 8), "no crop: the whole sensor");
|
||||
assert_eq!(with(Some((4, 1, 16, 7))), Rect::new(0, 0, 16, 8), "a crop beyond the sensor is ignored");
|
||||
assert_eq!(with(Some((5, 1, 4, 7))), Rect::new(0, 0, 16, 8), "an inverted crop is ignored");
|
||||
}
|
||||
|
||||
/// A block whose every record is a sub-heap pointing back at the block itself: depth alone
|
||||
/// would allow N^MAX_DEPTH visits (here 64^17), the record budget ends the walk promptly.
|
||||
#[test]
|
||||
fn self_referencing_sub_heaps_terminate() {
|
||||
const N: usize = 64;
|
||||
for o in [Order::Le, Order::Be] {
|
||||
let len = (2 + N * 10 + 4) as u32;
|
||||
let mut f = Vec::new();
|
||||
f.extend_from_slice(match o {
|
||||
Order::Le => b"II",
|
||||
Order::Be => b"MM",
|
||||
});
|
||||
f.extend_from_slice(&put32(26, o));
|
||||
f.extend_from_slice(b"HEAPCCDR");
|
||||
f.extend_from_slice(&put32(0x0001_0002, o));
|
||||
f.extend_from_slice(&[0; 8]);
|
||||
f.extend_from_slice(&put16(N as u16, o)); // record table at block offset 0
|
||||
for _ in 0..N {
|
||||
f.extend_from_slice(&put16(5 << 11 | 1, o)); // storage class 5: sub-heap
|
||||
f.extend_from_slice(&put32(len, o)); // ... covering the whole parent block
|
||||
f.extend_from_slice(&put32(0, o));
|
||||
}
|
||||
f.extend_from_slice(&put32(0, o)); // trailing offset-table pointer
|
||||
let started = std::time::Instant::now();
|
||||
assert!(matches!(header(&f, &Limits::default()), Err(Error::Corrupt(_))));
|
||||
assert!(matches!(full(&f), Err(Error::Corrupt(_))));
|
||||
assert!(started.elapsed() < std::time::Duration::from_secs(5), "walk did not stay within budget");
|
||||
let mut rec = BTreeMap::new();
|
||||
let mut budget = MAX_RECORDS;
|
||||
walk(&f, 26, len as usize, o, 0, &mut budget, &mut rec);
|
||||
assert_eq!(budget, 0);
|
||||
assert!(rec.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// A no-plane G1 frame that compresses well closes with the record's real `ff d9 00` trailer and
|
||||
/// is still shorter than plane + header — the shape of a lens-cap or blown-out frame. Rule (a)
|
||||
/// read that `ff d9` as a plane byte, reported 12 bits and then refused the file.
|
||||
#[test]
|
||||
fn a_short_powershot_payload_closing_with_the_trailer_has_no_plane() {
|
||||
let o = Order::Le;
|
||||
let (w, h) = (64u16, 64u16);
|
||||
let n = w as usize * h as usize;
|
||||
let tables = tables::tables(0).expect("selector 0");
|
||||
let entropy = encode_eob(n, &tables);
|
||||
let p = payload_with_trailer(None, &entropy);
|
||||
let span = n.div_ceil(4) + 514;
|
||||
assert!(p.len() < span, "the payload must be shorter than plane + header: {} vs {span}", p.len());
|
||||
assert!(p.windows(2).any(|q| q[0] == 0xff && q[1] != 0x00), "the trailer is the unstuffed ff pair");
|
||||
|
||||
let f = file_with_payload(w, h, "Canon PowerShot G1", &p, &[], o);
|
||||
let hdr = header(&f, &Limits::default()).expect("header");
|
||||
assert_eq!(hdr.bits, 10, "the trailer is not a low-bits plane");
|
||||
// Every block is one end-of-block token, so the chain values stand (512 at each row start):
|
||||
// the frame decodes instead of being refused for a plane it does not have.
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!((img.header.bits, img.samples.iter().filter(|&&v| v == 512).count()), (10, n));
|
||||
|
||||
// The same frame without the trailer was already read as 10-bit.
|
||||
let f = file_with_payload(w, h, "Canon PowerShot G1", &payload(None, &entropy), &[], o);
|
||||
assert_eq!(header(&f, &Limits::default()).expect("header").bits, 10);
|
||||
}
|
||||
|
||||
/// An end-of-block token at the head of a block emits the carry and leaves the other 63 samples
|
||||
/// at their chain values: the runs the sample hashes cover, on a file the unit tests can read.
|
||||
#[test]
|
||||
fn end_of_block_keeps_the_chain_values() {
|
||||
let o = Order::Le;
|
||||
let (w, h) = (64u16, 64u16);
|
||||
let n = w as usize * h as usize;
|
||||
let tables = tables::tables(0).expect("selector 0");
|
||||
let f = file_with_payload(w, h, "Canon EOS D30", &payload(None, &encode_eob(n, &tables)), &[], o);
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!(img.samples.len(), n);
|
||||
assert!(img.samples.iter().all(|&v| v == 512), "every sample keeps its row's chain value");
|
||||
}
|
||||
|
||||
/// Zero sensor dimensions are damage, not a request for too much memory.
|
||||
#[test]
|
||||
fn zero_sensor_dimensions_are_corrupt_not_a_limit() {
|
||||
let f = roundtrip_file(0, 0, "Canon EOS D60", None, &[], Order::Le);
|
||||
assert!(matches!(full(&f), Err(Error::Corrupt(_))));
|
||||
assert!(matches!(header(&f, &Limits::default()), Err(Error::Corrupt(_))));
|
||||
}
|
||||
|
||||
/// A hostile stream can run the chain past the bit depth (here a "10-bit" body writing 1535).
|
||||
/// `samples` is documented as `bits` deep, so the surplus never reaches a `1 << bits` LUT.
|
||||
#[test]
|
||||
fn samples_are_clamped_to_the_declared_depth() {
|
||||
let f = roundtrip_file(2, 1, "Canon PowerShot G1", None, &[512, 1535], Order::Le);
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!((img.header.bits, img.samples.as_slice()), (10, &[512u16, 1023][..]));
|
||||
// the 12-bit merge of the same chain cannot leave 0..=4095 either
|
||||
let low = vec![0xffu8; 1];
|
||||
let f = roundtrip_file(2, 1, "Canon EOS D60", Some(&low), &[512, 1535], Order::Le);
|
||||
let img = full(&f).expect("decodes");
|
||||
assert_eq!(img.header.bits, 12);
|
||||
assert!(img.samples.iter().all(|&v| v <= 4095), "{:?}", img.samples);
|
||||
}
|
||||
|
||||
/// The two ColorBalance layouts and every Canon setting that picks a slot. Every corpus file is
|
||||
/// set to Auto, so a wrong arm would still pass on the samples alone.
|
||||
#[test]
|
||||
fn white_balance_slot_arms_follow_the_camera_setting() {
|
||||
let o = Order::Le;
|
||||
// Quadruple k is (100+k, 200, 200, 300+k): the multipliers name the slot chosen, and the last
|
||||
// quadruple is black levels, which is never a slot.
|
||||
let wb = |usable: u16, setting: u16| {
|
||||
let mut vals = vec![(usable + 1) * 8];
|
||||
for k in 0..=usable {
|
||||
vals.extend([100 + k, 200, 200, 300 + k]);
|
||||
}
|
||||
let mut bytes: Vec<u8> = vals.iter().flat_map(|&v| put16(v, o)).collect();
|
||||
let cb_len = bytes.len();
|
||||
let si_off = bytes.len();
|
||||
let mut shot = [0u16; 8];
|
||||
shot[7] = setting;
|
||||
bytes.extend(shot.iter().flat_map(|&v| put16(v, o)));
|
||||
let mut rec = BTreeMap::new();
|
||||
rec.insert(COLOR_BALANCE, (0usize, cb_len));
|
||||
rec.insert(SHOT_INFO, (si_off, 16usize));
|
||||
wb_from_color_balance(&bytes, &rec, o)
|
||||
};
|
||||
let expect = |k: u16| Some([(100 + k) as f32 / 200.0, 1.0, (300 + k) as f32 / 200.0]);
|
||||
// 10D/300D: Auto, Daylight, Shade, Cloudy, Tungsten, Fluorescent, Flash, Custom, Kelvin
|
||||
for (setting, slot) in [(0u16, 0u16), (1, 1), (8, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (9, 8), (7, 0)] {
|
||||
assert_eq!(wb(9, setting), expect(slot), "nine-slot setting {setting}");
|
||||
}
|
||||
// D60: Auto, Daylight, Cloudy, Tungsten, Fluorescent, Flash, Custom. Shade and Kelvin have no
|
||||
// slot of their own and fall back to Auto.
|
||||
for (setting, slot) in [(0u16, 0u16), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (8, 0), (9, 0)] {
|
||||
assert_eq!(wb(7, setting), expect(slot), "seven-slot setting {setting}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn errors_display_their_cause() {
|
||||
assert_eq!(Error::NotCrw.to_string(), "not a Canon CRW (CIFF) file");
|
||||
assert_eq!(corrupt("no raw data record").to_string(), "CRW: no raw data record");
|
||||
assert_eq!(Error::Unsupported("decoder table 7".into()).to_string(), "unsupported CRW: decoder table 7");
|
||||
assert_eq!(Error::LimitExceeded("image too large").to_string(), "CRW exceeds a limit: image too large");
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
//! Malformed CRW files must decode to an error, never panic: a CIFF with the records the decoder
|
||||
//! reads and a pseudo-random payload, mutated by bit flips and truncation.
|
||||
|
||||
use craft_crw::{Limits, decode, header, probe};
|
||||
use proptest::prelude::*;
|
||||
|
||||
const W: usize = 48;
|
||||
const H: usize = 20;
|
||||
|
||||
/// A Canon CRW: CIFF header, a raw-data payload of pseudo-random bytes, and a nested properties
|
||||
/// heap with the records the decoder reads.
|
||||
fn crw() -> Vec<u8> {
|
||||
let le16 = |v: u16| v.to_le_bytes();
|
||||
let le32 = |v: u32| v.to_le_bytes();
|
||||
let heap = |records: &[(u16, Vec<u8>)]| -> Vec<u8> {
|
||||
let mut b = Vec::new();
|
||||
let mut entries = Vec::new();
|
||||
for (tc, v) in records {
|
||||
entries.push((*tc, v.len() as u32, b.len() as u32));
|
||||
b.extend_from_slice(v);
|
||||
if b.len() % 2 == 1 {
|
||||
b.push(0);
|
||||
}
|
||||
}
|
||||
let tbl = b.len() as u32;
|
||||
b.extend_from_slice(&le16(entries.len() as u16));
|
||||
for (tc, l, off) in entries {
|
||||
b.extend_from_slice(&le16(tc));
|
||||
b.extend_from_slice(&le32(l));
|
||||
b.extend_from_slice(&le32(off));
|
||||
}
|
||||
b.extend_from_slice(&le32(tbl));
|
||||
b
|
||||
};
|
||||
let sensor: Vec<u8> = [34u16, W as u16, H as u16, 0, 0, 2, 2, W as u16 - 3, H as u16 - 3].iter().flat_map(|&v| le16(v)).collect();
|
||||
let dtable: Vec<u8> = [0u32, 0, 514, 0].iter().flat_map(|&v| le32(v)).collect();
|
||||
let image_info: Vec<u8> = [W as u32, H as u32, 0, 270, 12, 12, 0].iter().flat_map(|&v| le32(v)).collect();
|
||||
let mut color_balance: Vec<u16> = vec![66];
|
||||
color_balance.extend((0..8).flat_map(|i| [1800 + i * 10, 800, 800, 960 + i * 5]));
|
||||
let color_balance: Vec<u8> = color_balance.iter().flat_map(|&v| le16(v)).collect();
|
||||
let shot: Vec<u8> = [54u16; 27].iter().flat_map(|&v| le16(v)).collect();
|
||||
let props = heap(&[
|
||||
(0x1031, sensor),
|
||||
(0x1835, dtable),
|
||||
(0x080a, b"Canon\0Canon EOS D60\0".to_vec()),
|
||||
(0x1810, image_info),
|
||||
(0x180e, [1083410638u32, 0, 0].iter().flat_map(|&v| le32(v)).collect()),
|
||||
(0x10a9, color_balance),
|
||||
(0x102a, shot),
|
||||
]);
|
||||
let payload: Vec<u8> = (0..514 + 256).map(|i| (i * 41 % 253) as u8).collect();
|
||||
let root = heap(&[(0x2005, payload), (0x300a, props)]);
|
||||
let mut f = b"II".to_vec();
|
||||
f.extend_from_slice(&le32(26));
|
||||
f.extend_from_slice(b"HEAPCCDR");
|
||||
f.extend_from_slice(&le32(0x0001_0002));
|
||||
f.extend_from_slice(&[0; 8]);
|
||||
f.extend_from_slice(&root);
|
||||
f
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_synthetic_file_is_recognised() {
|
||||
let f = crw();
|
||||
assert!(probe(&f));
|
||||
let hdr = header(&f, &Limits::default()).expect("header");
|
||||
assert_eq!((hdr.width, hdr.height), (W, H));
|
||||
assert_eq!(hdr.exif_orientation, 8);
|
||||
assert_eq!(hdr.capture_unix, Some(1083410638));
|
||||
assert!(hdr.wb_multipliers.is_some());
|
||||
// pseudo-random entropy data decodes or is refused, never panics
|
||||
let _ = decode(&f, &Limits::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_truncation_and_single_byte_mutation_survives() {
|
||||
let good = crw();
|
||||
for n in 0..good.len() {
|
||||
let _ = header(&good[..n], &Limits::default());
|
||||
let _ = decode(&good[..n], &Limits::default());
|
||||
}
|
||||
for i in 0..good.len() {
|
||||
let mut bad = good.clone();
|
||||
bad[i] ^= 0xa5;
|
||||
let _ = decode(&bad, &Limits::default());
|
||||
bad[i] = 0xff;
|
||||
let _ = decode(&bad, &Limits::default());
|
||||
}
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig { cases: 2000, .. ProptestConfig::default() })]
|
||||
|
||||
#[test]
|
||||
fn mutated_files_never_panic(flips in proptest::collection::vec((any::<usize>(), any::<u8>()), 1..16), cut in any::<usize>(), max_samples in 1usize..4096) {
|
||||
let mut data = crw();
|
||||
let n = data.len();
|
||||
for (i, v) in flips {
|
||||
data[i % n] ^= v;
|
||||
}
|
||||
data.truncate(cut % (n + 1));
|
||||
let mut limits = Limits::default();
|
||||
limits.max_samples = max_samples;
|
||||
let _ = header(&data, &limits);
|
||||
let _ = decode(&data, &limits);
|
||||
let _ = decode(&data, &Limits::default());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
[
|
||||
{
|
||||
"name": "crw-canon-eos-d30.crw",
|
||||
"body": "Canon EOS D30",
|
||||
"source": "https://raw.pixls.us/getfile.php/1307/nice/Canon%20-%20EOS%20D30%20-%20RAW%20(3:2).CRW",
|
||||
"sha256": "4ee482f2b6f9f1a8d370703ea04e9f2abaa49356973a7f56b229af93a78e8a2e",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 2224, "height": 1456, "bits": 12, "selector": 1, "cfa": "RGGB",
|
||||
"active": [56, 12, 2160, 1440], "model": "Canon EOS D30", "exif_orientation": 1, "capture_unix": 1482512730,
|
||||
"wb": null, "black": [127.9338, 127.8187, 128.2763, 128.4221], "white": 4095.0,
|
||||
"samples_sha256": "8ac62f45a4ee04efebed7900eb46dd263dc4a03a96c4fe711dc41b6bec6078fe"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-eos-d60.crw",
|
||||
"body": "Canon EOS D60",
|
||||
"source": "https://raw.pixls.us/getfile.php/2010/nice/Canon%20-%20EOS%20D60%20-%20RAW%20(3:2).CRW",
|
||||
"sha256": "2b98659ecff170af1dfe44f89b493139e75e995b2979686f6ff0847cfb56b97f",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 3152, "height": 2068, "bits": 12, "selector": 0, "cfa": "RGGB",
|
||||
"active": [72, 16, 3072, 2048], "model": "Canon EOS D60", "exif_orientation": 1, "capture_unix": 1467566074,
|
||||
"wb": [2.627477, 1.0, 1.0990753], "black": [127.4332, 127.0843, 127.3606, 126.9793], "white": 3984.0,
|
||||
"samples_sha256": "e01a6bc5cb8f67cd066f68863a04050f183c41e45395e8d4bc3b5ba4393561bb"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-eos-10d-a.crw",
|
||||
"body": "Canon EOS 10D",
|
||||
"source": "https://raw.pixls.us/getfile.php/2027/nice/Canon%20-%20EOS%2010D%20-%20RAW%20(3:2).CRW",
|
||||
"sha256": "b0730eb3a67cc2281a32e0b2ee104199246928d954a68b7e5b72366514c73d02",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 3152, "height": 2068, "bits": 12, "selector": 0, "cfa": "RGGB",
|
||||
"active": [72, 16, 3072, 2048], "model": "Canon EOS 10D", "exif_orientation": 1, "capture_unix": 1228574022,
|
||||
"wb": [2.09375, 1.0, 1.4915866], "black": [127.8010, 126.4857, 127.8322, 126.5070], "white": 3983.0,
|
||||
"samples_sha256": "3aecb01756492569734885710dafd6ffc26421ac0b5a7fa1c151973d48cd389a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-eos-10d-b.crw",
|
||||
"body": "Canon EOS 10D",
|
||||
"source": "https://raw.pixls.us/getfile.php/3164/nice/Canon%20-%20EOS%2010D%20-%20RAW%20(3:2).crw",
|
||||
"sha256": "e879c94afdc64dabc88b01ce2d02628aa225259303ec71dd61f7fb0322bfea2d",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 3152, "height": 2068, "bits": 12, "selector": 2, "cfa": "RGGB",
|
||||
"active": [72, 16, 3072, 2048], "model": "Canon EOS 10D", "exif_orientation": 1, "capture_unix": 1105175848,
|
||||
"wb": [1.8184005, 1.0, 1.3529766], "black": [124.6726, 125.9032, 124.6995, 125.9359], "white": 3990.0,
|
||||
"samples_sha256": "25d8c540bace7ae9b92ab631fae3c4effee9625e388af600cdf60894764d3f92"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-eos-10d-c.crw",
|
||||
"body": "Canon EOS 10D",
|
||||
"source": "https://raw.pixls.us/getfile.php/3165/nice/Canon%20-%20EOS%2010D%20-%20RAW%20(3:2).crw",
|
||||
"sha256": "58144118854657df5cd2185df40fecf6b665fc6754fef0416c2f222b68b025c6",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 3152, "height": 2068, "bits": 12, "selector": 2, "cfa": "RGGB",
|
||||
"active": [72, 16, 3072, 2048], "model": "Canon EOS 10D", "exif_orientation": 1, "capture_unix": 1109761444,
|
||||
"wb": [1.9759471, 1.0, 1.1220685], "black": [127.7952, 128.2460, 127.8664, 128.3089], "white": 3988.0,
|
||||
"samples_sha256": "c1bc8c958de399ecb5fbaeb3bd473baabbb534c5874508f9b51c3984a1319c7f"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-eos-300d-a.crw",
|
||||
"body": "Canon EOS 300D",
|
||||
"source": "https://raw.pixls.us/getfile.php/1983/nice/Canon%20-%20EOS%20300D%20-%20RAW%20(3:2).crw",
|
||||
"sha256": "47da16e1668af5500eea63cf42cf2992d9cb3d32180b8d786f17f1e2a57f39f4",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 3152, "height": 2068, "bits": 12, "selector": 0, "cfa": "RGGB",
|
||||
"active": [72, 16, 3072, 2048], "model": "Canon EOS 300D DIGITAL", "exif_orientation": 8, "capture_unix": 1083408238,
|
||||
"wb": [1.9579327, 1.0, 1.2632211], "black": [125.1676, 124.4162, 125.1797, 124.3886], "white": 3987.0,
|
||||
"samples_sha256": "87848353618212be07370d659ceaa2ab349ba79b29c73702368233be2371de65"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-eos-300d-b.crw",
|
||||
"body": "Canon EOS 300D",
|
||||
"source": "https://raw.pixls.us/getfile.php/3162/nice/Canon%20-%20EOS%20300D%20-%20RAW%20(3:2).CRW",
|
||||
"sha256": "9e7f62239fe728046af28d0d4a8a722d35d40146a79794a5bda1775306af4383",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 3152, "height": 2068, "bits": 12, "selector": 2, "cfa": "RGGB",
|
||||
"active": [72, 16, 3072, 2048], "model": "Canon EOS 300D DIGITAL", "exif_orientation": 8, "capture_unix": 1182188238,
|
||||
"wb": [2.230908, 1.0, 1.1737823], "black": [124.8050, 124.1321, 125.1695, 124.2802], "white": 3992.0,
|
||||
"samples_sha256": "b1b61526c67948b600a03a89df0a301bc220f303bf8b70c93ea657356875a715"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-eos-digital-rebel.crw",
|
||||
"body": "Canon EOS Digital Rebel",
|
||||
"source": "https://raw.pixls.us/getfile.php/1308/nice/Canon%20-%20EOS%20Digital%20Rebel%20-%20RAW%20(3:2).CRW",
|
||||
"sha256": "d6a3131bfac390fde01b96882ec378211f04570c3252708b58f0d05b33a9fdfc",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 3152, "height": 2068, "bits": 12, "selector": 0, "cfa": "RGGB",
|
||||
"active": [72, 16, 3072, 2048], "model": "Canon EOS DIGITAL REBEL", "exif_orientation": 1, "capture_unix": 1480372394,
|
||||
"wb": [1.6094965, 1.0, 1.676627], "black": [262.0760, 260.0167, 261.7442, 260.0508], "white": 4095.0,
|
||||
"samples_sha256": "ec487fc432a31a648fc8238452976fd96115ff058bc6989352bdebdf997d99aa"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-powershot-g1.crw",
|
||||
"body": "Canon PowerShot G1",
|
||||
"source": "https://raw.pixls.us/getfile.php/2073/nice/Canon%20-%20PowerShot%20G1%20-%20RAW%20(4:3).crw",
|
||||
"sha256": "f39be5acf14057ce04527fd8d0574e57ec8ed5fed082d7bfaff763075a44ee8a",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 2144, "height": 1560, "bits": 10, "selector": 1, "cfa": "GBRG",
|
||||
"active": [24, 15, 2048, 1536], "model": "Canon PowerShot G1", "exif_orientation": 1, "capture_unix": 1065594172,
|
||||
"wb": null, "black": [28.7314, 28.7637, 28.7544, 28.6963], "white": 1023.0,
|
||||
"samples_sha256": "2316b72fd8d84cff2abbac1e56d9a6ea76997ee9a4182a040d9e261735393ba2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-powershot-g5.crw",
|
||||
"body": "Canon PowerShot G5",
|
||||
"source": "https://raw.pixls.us/getfile.php/1746/nice/Canon%20-%20PowerShot%20G5%20-%20RAW%20(4:3).CRW",
|
||||
"sha256": "bb084223d714871b99de4f549eaf00513e30a7080feeead700252c2008060196",
|
||||
"license": "CC0-1.0",
|
||||
"expect": "unsupported"
|
||||
},
|
||||
{
|
||||
"name": "crw-canon-powershot-g6.crw",
|
||||
"body": "Canon PowerShot G6",
|
||||
"source": "https://raw.pixls.us/getfile.php/4226/nice/Canon%20-%20PowerShot%20G6%20-%20RAW%20(4:3).CRW",
|
||||
"sha256": "244378e1f5660b6cc6cefc93a75ed4928d0b28aa6304ff327b61c748538f4fef",
|
||||
"license": "CC0-1.0",
|
||||
"expect": {
|
||||
"width": 3160, "height": 2344, "bits": 12, "selector": 0, "cfa": "RGGB",
|
||||
"active": [64, 24, 3072, 2304], "model": "Canon PowerShot G6", "exif_orientation": 1, "capture_unix": 1607955982,
|
||||
"wb": null, "black": [128.9415, 128.9516, 129.0142, 129.0901], "white": 4080.0,
|
||||
"samples_sha256": "17804ed2ff64873e7d24ca06c58a4c8d095aad1907affb938ebcd12b618fa7ac"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,231 @@
|
||||
//! The CC0 sample files (raw.pixls.us) pinned in `samples.json`: each decodes to exactly the
|
||||
//! sensor samples the derivation verified against the black-box reference (`clean-room/
|
||||
//! 03-verification.md`), recorded as the SHA-256 of the little-endian sample buffer, with the
|
||||
//! levels, white balance and metadata the decoder read then.
|
||||
//!
|
||||
//! The files are not in the repository (5–11 MB each): `python scripts/fetch-crw-samples.py
|
||||
//! <dir>` downloads and verifies them, and `CRAFT_CRW_SAMPLES=<dir>` points the test at them.
|
||||
//! Without the variable the test prints that it skipped and passes; `CRAFT_CRW_REQUIRE_SAMPLES=1`
|
||||
//! (CI) turns the skip into a failure.
|
||||
//!
|
||||
//! `CRAFT_CRW_ORACLE=<dir>` additionally compares every sample against `<name>.crw.tiff`
|
||||
//! references next to the files (16-bit grey TIFFs of the whole sensor, as written by the
|
||||
//! reference decoder's `unprocessed_raw -T`), the check the derivation used. Those references
|
||||
//! are not distributable and live outside the repository.
|
||||
|
||||
use craft_crw::{Error, Limits, decode, probe};
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Sample {
|
||||
name: String,
|
||||
body: String,
|
||||
sha256: String,
|
||||
expect: Expect,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Expect {
|
||||
Unsupported(String),
|
||||
Decoded(Decoded),
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Decoded {
|
||||
width: usize,
|
||||
height: usize,
|
||||
bits: u32,
|
||||
selector: u32,
|
||||
cfa: String,
|
||||
active: [usize; 4],
|
||||
model: String,
|
||||
exif_orientation: u8,
|
||||
capture_unix: i64,
|
||||
wb: Option<[f32; 3]>,
|
||||
black: [f32; 4],
|
||||
white: f32,
|
||||
samples_sha256: String,
|
||||
}
|
||||
|
||||
fn manifest() -> Vec<Sample> {
|
||||
serde_json::from_str(include_str!("samples.json")).expect("samples.json parses")
|
||||
}
|
||||
|
||||
fn hex(digest: impl AsRef<[u8]>) -> String {
|
||||
digest.as_ref().iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
fn samples_dir() -> Option<PathBuf> {
|
||||
match std::env::var_os("CRAFT_CRW_SAMPLES") {
|
||||
Some(dir) => Some(PathBuf::from(dir)),
|
||||
None if std::env::var_os("CRAFT_CRW_REQUIRE_SAMPLES").is_some_and(|v| v == "1") => {
|
||||
panic!("CRAFT_CRW_REQUIRE_SAMPLES=1 but CRAFT_CRW_SAMPLES is unset")
|
||||
}
|
||||
None => {
|
||||
eprintln!("CRAFT_CRW_SAMPLES not set; skipping the sample-file tests (scripts/fetch-crw-samples.py)");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_is_well_formed() {
|
||||
let m = manifest();
|
||||
assert_eq!(m.len(), 11);
|
||||
for s in &m {
|
||||
assert!(s.name.ends_with(".crw") && !s.name.contains('/') && !s.name.contains('\\'), "{}", s.name);
|
||||
assert_eq!(s.sha256.len(), 64, "{}", s.name);
|
||||
if let Expect::Decoded(d) = &s.expect {
|
||||
assert_eq!(d.samples_sha256.len(), 64, "{}", s.name);
|
||||
assert!(matches!(d.bits, 10 | 12), "{}", s.name);
|
||||
} else {
|
||||
assert!(s.body.contains("G5"), "only the PowerShot G5 is refused: {}", s.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_files_decode_to_the_verified_samples() {
|
||||
let Some(dir) = samples_dir() else { return };
|
||||
let mut checked = 0;
|
||||
for s in manifest() {
|
||||
let path = dir.join(&s.name);
|
||||
let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
|
||||
assert_eq!(hex(Sha256::digest(&bytes)), s.sha256, "{}: not the pinned CC0 file", s.name);
|
||||
assert!(probe(&bytes), "{}: not recognised", s.name);
|
||||
let started = std::time::Instant::now();
|
||||
let result = decode(&bytes, &Limits::default());
|
||||
let took = started.elapsed();
|
||||
match (&s.expect, result) {
|
||||
(Expect::Unsupported(kind), Err(Error::Unsupported(why))) => {
|
||||
assert_eq!(kind, "unsupported", "{}", s.name);
|
||||
assert!(why.contains("G5"), "{}: {why}", s.name);
|
||||
eprintln!("{}: refused as documented ({why})", s.name);
|
||||
}
|
||||
(Expect::Unsupported(_), other) => panic!("{}: expected Unsupported, got {other:?}", s.name.clone()),
|
||||
(Expect::Decoded(d), Ok(img)) => {
|
||||
let h = &img.header;
|
||||
assert_eq!((h.width, h.height, h.bits, h.selector), (d.width, d.height, d.bits, d.selector), "{}", s.name);
|
||||
assert_eq!(h.cfa.name(), d.cfa, "{}", s.name);
|
||||
assert_eq!([h.active_area.x, h.active_area.y, h.active_area.width, h.active_area.height], d.active, "{}", s.name);
|
||||
assert_eq!(h.model.as_deref(), Some(d.model.as_str()), "{}", s.name);
|
||||
assert_eq!(h.exif_orientation, d.exif_orientation, "{}", s.name);
|
||||
assert_eq!(h.capture_unix, Some(d.capture_unix), "{}", s.name);
|
||||
match (h.wb_multipliers, d.wb) {
|
||||
(None, None) => {}
|
||||
(Some(got), Some(want)) => {
|
||||
assert!(got.iter().zip(&want).all(|(a, b)| (a - b).abs() < 1e-4), "{}: wb {got:?} vs {want:?}", s.name)
|
||||
}
|
||||
(got, want) => panic!("{}: wb {got:?} vs {want:?}", s.name),
|
||||
}
|
||||
assert!(
|
||||
img.black.values.iter().zip(&d.black).all(|(a, b)| (a - b).abs() < 0.01),
|
||||
"{}: black {:?} vs {:?}",
|
||||
s.name,
|
||||
img.black.values,
|
||||
d.black
|
||||
);
|
||||
assert_eq!(img.white, d.white, "{}", s.name);
|
||||
assert_eq!(img.samples.len(), d.width * d.height, "{}", s.name);
|
||||
let mut hasher = Sha256::new();
|
||||
for v in &img.samples {
|
||||
hasher.update(v.to_le_bytes());
|
||||
}
|
||||
assert_eq!(hex(hasher.finalize()), d.samples_sha256, "{}: the decoded samples differ from the verified decode", s.name);
|
||||
eprintln!("{}: {}x{} {}-bit selector {} EXACT in {took:.0?}", s.name, h.width, h.height, h.bits, h.selector);
|
||||
}
|
||||
(Expect::Decoded(_), Err(e)) => panic!("{}: {e}", s.name),
|
||||
}
|
||||
checked += 1;
|
||||
}
|
||||
assert_eq!(checked, 11);
|
||||
}
|
||||
|
||||
/// Minimal reader for the reference TIFFs: one 16-bit grey image, stripped, either byte order.
|
||||
fn oracle_samples(bytes: &[u8]) -> (usize, usize, Vec<u16>) {
|
||||
let be = match &bytes[..4] {
|
||||
b"II*\0" => false,
|
||||
b"MM\0*" => true,
|
||||
other => panic!("not a TIFF: {other:?}"),
|
||||
};
|
||||
let u16_at = |at: usize| -> u16 {
|
||||
let b: [u8; 2] = bytes[at..at + 2].try_into().unwrap();
|
||||
if be { u16::from_be_bytes(b) } else { u16::from_le_bytes(b) }
|
||||
};
|
||||
let u32_at = |at: usize| -> u32 {
|
||||
let b: [u8; 4] = bytes[at..at + 4].try_into().unwrap();
|
||||
if be { u32::from_be_bytes(b) } else { u32::from_le_bytes(b) }
|
||||
};
|
||||
let ifd = u32_at(4) as usize;
|
||||
let n = u16_at(ifd) as usize;
|
||||
let (mut w, mut h, mut bits, mut spp) = (0usize, 0usize, 0u16, 1u16);
|
||||
let (mut offsets, mut counts) = (Vec::new(), Vec::new());
|
||||
for k in 0..n {
|
||||
let e = ifd + 2 + k * 12;
|
||||
let (tag, typ, cnt) = (u16_at(e), u16_at(e + 2), u32_at(e + 4) as usize);
|
||||
let size = match typ {
|
||||
3 => 2,
|
||||
4 => 4,
|
||||
_ => 1,
|
||||
};
|
||||
let values_at = if cnt * size <= 4 { e + 8 } else { u32_at(e + 8) as usize };
|
||||
let value = |i: usize| -> usize { if typ == 3 { u16_at(values_at + i * 2) as usize } else { u32_at(values_at + i * 4) as usize } };
|
||||
match tag {
|
||||
256 => w = value(0),
|
||||
257 => h = value(0),
|
||||
258 => bits = value(0) as u16,
|
||||
277 => spp = value(0) as u16,
|
||||
273 => offsets = (0..cnt).map(value).collect(),
|
||||
279 => counts = (0..cnt).map(value).collect(),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
assert_eq!((bits, spp), (16, 1), "oracle TIFF is 16-bit grey");
|
||||
let mut data = Vec::with_capacity(w * h);
|
||||
for (&o, &c) in offsets.iter().zip(&counts) {
|
||||
let strip = &bytes[o..o + c];
|
||||
data.extend(strip.as_chunks::<2>().0.iter().map(|p| if be { u16::from_be_bytes(*p) } else { u16::from_le_bytes(*p) }));
|
||||
}
|
||||
data.truncate(w * h);
|
||||
(w, h, data)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oracle_tiffs_match_every_sample() {
|
||||
let Some(dir) = std::env::var_os("CRAFT_CRW_ORACLE").map(PathBuf::from) else {
|
||||
eprintln!("CRAFT_CRW_ORACLE not set; skipping the reference-TIFF comparison");
|
||||
return;
|
||||
};
|
||||
let mut entries: Vec<PathBuf> = std::fs::read_dir(&dir).expect("oracle directory").filter_map(|e| e.ok()).map(|e| e.path()).collect();
|
||||
entries.sort();
|
||||
let mut checked = 0;
|
||||
for path in entries.iter().filter(|p| p.extension().and_then(|e| e.to_str()).is_some_and(|e| e.eq_ignore_ascii_case("crw"))) {
|
||||
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or_default();
|
||||
let reference: &Path = &path.with_extension("crw.tiff");
|
||||
if !reference.is_file() {
|
||||
eprintln!("{name}: no reference TIFF, skipped");
|
||||
continue;
|
||||
}
|
||||
let bytes = std::fs::read(path).expect("read crw");
|
||||
let img = match decode(&bytes, &Limits::default()) {
|
||||
Ok(img) => img,
|
||||
Err(Error::Unsupported(why)) if why.contains("G5") => {
|
||||
eprintln!("{name}: G5 refused as documented");
|
||||
continue;
|
||||
}
|
||||
Err(e) => panic!("{name}: {e}"),
|
||||
};
|
||||
let (w, h, oracle) = oracle_samples(&std::fs::read(reference).expect("read oracle tiff"));
|
||||
assert_eq!((img.header.width, img.header.height), (w, h), "{name}: dimensions");
|
||||
assert_eq!(oracle.len(), w * h, "{name}: oracle sample count");
|
||||
assert_eq!(img.samples.len(), w * h, "{name}: decoded sample count");
|
||||
let mismatches = img.samples.iter().zip(&oracle).filter(|(a, b)| a != b).count();
|
||||
assert_eq!(mismatches, 0, "{name}: {mismatches} mismatching samples");
|
||||
eprintln!("{name}: EXACT ({w}x{h})");
|
||||
checked += 1;
|
||||
}
|
||||
assert!(checked > 0, "no .crw files with references found in {}", dir.display());
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
# Clean-room format work in craft-libs
|
||||
|
||||
Several crates here read or write formats that other open-source projects also implement, often
|
||||
under licences the Craft apps cannot take code from (GPL, LGPL, AGPL, CDDL) or under terms that
|
||||
forbid it (Adobe's). This document is the standard a format crate follows so that every line of
|
||||
it is ours, and so that a reviewer can check that claim from the repository alone. It is the
|
||||
written form of what `craft-jpeg`, `craft-media` and `craft-crw` did; the suite-wide rules live in
|
||||
[craftrules](https://github.com/storytold/craftrules) (`standards/licensing.md`,
|
||||
`standards/never-crash.md`), which this does not replace.
|
||||
|
||||
## What "clean-room" means here
|
||||
|
||||
A clean-room crate is written only from:
|
||||
|
||||
- **published specifications and standards** (ITU-T, ISO/IEC, IETF RFCs, a vendor's own published
|
||||
format document, a registry's pages), read as documents;
|
||||
- **prose documentation** of a format or its tags (ExifTool's tag-name and format pages, a wiki
|
||||
page that describes a container), never a project's code;
|
||||
- **sample files** we may use: CC0, or owned by a contributor who consents, or produced by us.
|
||||
Not NC-licensed files, not files whose licence is unclear, never anything from inside a
|
||||
proprietary application's bundle;
|
||||
- **a reference implementation run as a black box**: an official pre-built binary of another
|
||||
decoder, fed inputs and observed for outputs, with every run logged. Its source is never
|
||||
opened, read, searched, diffed or quoted, including comments, licence headers, mirrors and
|
||||
excerpts in forum posts or AI answers.
|
||||
|
||||
Never consulted, in any form: GPL/LGPL/AGPL/CDDL decoder and encoder sources (dcraw, LibRaw,
|
||||
rawspeed, rawloader/rawler, dnglab, darktable, RawTherapee, ART, libopenraw, lensfun, ExifTool's
|
||||
Perl, FFmpeg/libav, x264/x265 and the like), Adobe's SDKs, profiles (DCP, LCP), presets, fonts,
|
||||
icons or binaries, and any earlier attempt of ours that was itself not clean.
|
||||
|
||||
## The process (what the record must show)
|
||||
|
||||
Modelled on LightCraft's ARW compression derivation and repeated for CRW
|
||||
(`crates/crw/clean-room/`). A crate that derives an undocumented format commits a `clean-room/`
|
||||
folder with:
|
||||
|
||||
1. **Starting assumptions**, written before any source is consulted, every item marked
|
||||
unverified, committed once and never edited. A reviewer diffs them against the result to see
|
||||
what was assumed and what was established.
|
||||
2. **A plan** naming the sources that will be used, the sample corpus (public files with URL,
|
||||
licence and SHA-256; private files by name, size and hash only, never content), the phases,
|
||||
the freeze point, and the questions that are the maintainer's to decide. Work past the plan
|
||||
waits for the maintainer's recorded approval.
|
||||
3. **A source log**: every URL opened and every search run, in order, with the time, what was
|
||||
asked and what was taken. If a page exposed implementation details that were not asked for,
|
||||
the log says so and what it said.
|
||||
4. **Derivation documents** in which every fact cites the experiment (script and log) or the
|
||||
document that established it, and corrections are stated, not silently applied. Statistics
|
||||
before decoding; hypotheses tested against internal consistency first; black-box probes
|
||||
logged one by one.
|
||||
5. **A freeze**: one document holding the complete derived format, with its tables as data. Its
|
||||
commit is the freeze point. Nothing in it is tuned afterwards. A mismatch found later reopens
|
||||
the derivation *explicitly*: the cause, the change and a new freeze revision are recorded in
|
||||
that document.
|
||||
6. **Verification** after the freeze, with zero tuning: every sample compared exactly against the
|
||||
reference, every file's verdict recorded, failures recorded as failures. Cross-body checks
|
||||
(tables learned on one body, exact on another) are the strongest independent evidence.
|
||||
7. **The implementation** from the frozen document, with: never-crash rules (`get()` for every
|
||||
file-derived offset, checked arithmetic, bounded recursion and allocation, no `unwrap`/`panic`),
|
||||
synthetic tests that need no sample files (a test-only encoder that inverts the model is the
|
||||
usual tool), hostile-input tests, and a sample-file test pinning the verified decode (a digest
|
||||
of the output per file) so the crate can prove it still matches without redistributing the
|
||||
references.
|
||||
8. **Table data** lives in the record as JSON and is rendered into Rust by a committed script; the
|
||||
generated file says so and is never edited by hand.
|
||||
|
||||
When the derivation was done in another repository, the crate's `clean-room/` may hold only items
|
||||
3, 5, 6 and 8, provided it pins that repository, path and revision (in its README and in `NOTICE`)
|
||||
for the starting assumptions, the plan and the derivation documents, as `craft-crw` does.
|
||||
|
||||
Scripts in the record use the Python standard library only and run under `python -I`. Local-only
|
||||
material (reference models, oracle binaries, copyrighted specification PDFs, dumps that may carry
|
||||
personal metadata) is named in the record with how to obtain it, and is gitignored.
|
||||
|
||||
## Assets and attribution
|
||||
|
||||
Sample files are never committed (sizes aside, they are other people's photographs): tests
|
||||
download them outside the checkout by pinned URL and SHA-256, licence checked (`scripts/
|
||||
fetch-*.py`), and skip cleanly without them. Each manifest, each table derived from samples and
|
||||
each log set has a row in `ATTRIBUTION.md`; code and records that came from an app are listed in
|
||||
`NOTICE` under *Source provenance* with the repository, paths and revision.
|
||||
|
||||
## When something is not clean
|
||||
|
||||
Stop, record what was seen and where (the source log), and tell the maintainer. Code written
|
||||
afterwards by the same person for the same part of the format is suspect; the maintainer decides
|
||||
whether the part is re-derived by someone else, documented as derived from that source under its
|
||||
licence terms, or dropped. Never quietly continue.
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch the pinned CC0 Canon CRW sample files for craft-crw's tests, outside the checkout.
|
||||
|
||||
The manifest is crates/crw/tests/samples.json (name, raw.pixls.us URL, SHA-256, licence). Files
|
||||
already present with the right hash are kept. Then run the tests with
|
||||
`CRAFT_CRW_SAMPLES=<destination> cargo test -p craft-crw --release --test samples`.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
MAX_BYTES = 16 * 1024 * 1024
|
||||
|
||||
|
||||
class _HttpsOnlyRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
if urllib.parse.urlsplit(newurl).scheme != "https":
|
||||
raise ValueError("refusing to follow a non-HTTPS redirect")
|
||||
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
||||
|
||||
|
||||
urllib.request.install_opener(urllib.request.build_opener(_HttpsOnlyRedirectHandler))
|
||||
|
||||
|
||||
def fetch(url):
|
||||
if urllib.parse.urlsplit(url).scheme != "https":
|
||||
raise ValueError("refusing to fetch a non-HTTPS URL")
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=120) as response:
|
||||
return response.read(MAX_BYTES + 1)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code not in (408, 429) and not 500 <= error.code <= 599:
|
||||
raise
|
||||
if attempt == 2:
|
||||
raise
|
||||
error.close()
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError):
|
||||
if attempt == 2:
|
||||
raise
|
||||
time.sleep(attempt + 1)
|
||||
raise RuntimeError("sample download retries exhausted")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("destination", type=Path)
|
||||
args = parser.parse_args()
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
destination = args.destination.resolve()
|
||||
if destination == root or root in destination.parents:
|
||||
parser.error("sample files must be stored outside the checkout (they are never committed)")
|
||||
samples = json.loads((root / "crates/crw/tests/samples.json").read_text(encoding="utf-8"))
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
for sample in samples:
|
||||
name, url, digest, licence = sample["name"], sample["source"], sample["sha256"], sample["license"]
|
||||
if Path(name).name != name or name in ("", ".", "..") or not url.startswith("https://"):
|
||||
raise ValueError("invalid sample name or source")
|
||||
if licence != "CC0-1.0":
|
||||
raise ValueError(f"{name}: only CC0 samples are fetched (manifest says {licence})")
|
||||
path = (destination / name).resolve()
|
||||
if path.parent != destination:
|
||||
raise ValueError(f"sample path escapes destination: {name}")
|
||||
if path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == digest:
|
||||
print(f"Verified {name}")
|
||||
continue
|
||||
data = fetch(url)
|
||||
if len(data) > MAX_BYTES or hashlib.sha256(data).hexdigest() != digest:
|
||||
raise ValueError(f"sample size or SHA-256 mismatch: {name}")
|
||||
fd, tmp = tempfile.mkstemp(dir=destination, prefix=f".{name}.", suffix=".tmp")
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as handle:
|
||||
handle.write(data)
|
||||
os.replace(tmp, destination / name)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
print(f"Fetched and verified {name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user