From 27fe59ab79aa8366dc9d4ad6fda1b6440114465a Mon Sep 17 00:00:00 2001 From: ".CooCoo" Date: Mon, 16 Nov 2026 18:36:03 -0800 Subject: [PATCH] craft-raw: the pinned RAF corpus list separates digest and path with a tab (#1379) --- crates/raw/docs/raf-compression.md | 2 +- crates/raw/docs/raf-corpus.sha256 | 34 +++++++++++----------- crates/raw/tests/it/main.rs | 1 + crates/raw/tests/it/raf_corpus_manifest.rs | 24 +++++++++++++++ 4 files changed, 43 insertions(+), 18 deletions(-) create mode 100644 crates/raw/tests/it/raf_corpus_manifest.rs diff --git a/crates/raw/docs/raf-compression.md b/crates/raw/docs/raf-compression.md index 5dfad256d..e19a8ba6a 100644 --- a/crates/raw/docs/raf-compression.md +++ b/crates/raw/docs/raf-compression.md @@ -30,7 +30,7 @@ Complete sensor arrays were compared sample by sample with an external decoder u All 24 files matched the reference sensor arrays exactly. This covers 13 camera bodies and both older and current compression variants. Synthetic tests also cover 12-bit streams; a real compressed 12-bit camera file has not been verified. Support is selected from the file's layout, without a camera-name allowlist. Other bodies using this layout should decode, but this is not a claim that every Fujifilm model has been tested. Older FinePix layouts without the supported raw TIFF IFD remain unsupported. -The 17 CC0 files live in the gitignored corpus. Their published SHA-256 identities are pinned in [raf-corpus.sha256](raf-corpus.sha256); their source licence records were checked as CC0 1.0. `cargo xtask corpus --download` fetches them. `cargo test -p lightcraft-raw corpus_fujifilm_compressed_samples -- --nocapture` verifies each available file's checksum before checking its reference sensor sum and position-weighted sum. The test skips absent corpus files; the full-array comparisons were performed locally. No supplied media or large binary fixtures are committed. +The 17 CC0 files live in a gitignored corpus and are not bundled. [raf-corpus.sha256](raf-corpus.sha256) pins their published SHA-256 identities: `#` comment lines, then one tab-separated row per file with the digest and the path relative to the corpus folder. Their source license records were checked as CC0 1.0. `tests/it/raf_corpus_manifest.rs` checks the list's format without the files. In LightCraft, `cargo xtask corpus --download` fetches them, and `cargo test -p lightcraft-raw corpus_fujifilm_compressed_samples -- --nocapture` verifies each available file's checksum before checking its reference sensor sum and position-weighted sum. That test skips absent corpus files; the full-array comparisons were performed locally. No supplied media or large binary fixtures are committed. Unconditional procedural tests cover Bayer/X-Trans, 12/14/16 bits, partial final stripes, quantizer changes, residual signs and escapes, malformed headers and truncated/random entropy streams. Regressions also check overflowing uncompressed row strides, unsupported modes and CFA layouts, and container/stream mode mismatches. diff --git a/crates/raw/docs/raf-corpus.sha256 b/crates/raw/docs/raf-corpus.sha256 index 0fd957283..8af0396a9 100644 --- a/crates/raw/docs/raf-corpus.sha256 +++ b/crates/raw/docs/raf-corpus.sha256 @@ -1,19 +1,19 @@ # Fujifilm RAF regression files; CC0 1.0, raw.pixls.us. # Published SHA-256 values retrieved 2026-10-07; see docs/raf-compression.md. -018781e126842596fcf3876b3fb6cd8973dc3936ccc753ee8648618be8889287 corpus/raw/raf-fuji-gfx100-3773.raf -0e7dad9819a748bb3d3be26bbf5770d4968e2d80b9cb41d97036bf7c27f7bfe7 corpus/raw/raf-fuji-gfx100-3775.raf -14bed5ac8191d6f2a7c82984e3bf6470ad4740df329293eac42b8018cc6a668a corpus/raw/raf-fuji-gfx100rf-8091.raf -a7bf8f061b603f7ef5fd29276eb6b2ac57355ca2f589a340117bfff0d03c065c corpus/raw/raf-fuji-gfx100s-4495.raf -1bb4608b61861a3d9e85b0ca6c00689b8ba4620cbe8579374d958c21ed08edf0 corpus/raw/raf-fuji-gfx100s-4503.raf -a7124d13910e0241a6b035154bc1fcdb7be4439371d043b1e1950a0251feb51e corpus/raw/raf-fuji-gfx50s-1435.raf -28f56cc9784164209e92fae209bdcc78267bd692aa2fb964595485f7e93692ab corpus/raw/raf-fuji-xe5-8509.raf -caeadb2666f6c5d9bbe32fbe692c2079b6a7e028f8514a13173f360656d08507 corpus/raw/raf-fuji-xh2-6001.raf -109be38f59963c9b8ff289ebe1f5ec9168c52f609e5b601432bd194520b4605b corpus/raw/raf-fuji-xh2-6002.raf -c76ce546e15727abb4ba201471010c554403176963a1a6b8cc017462606429e5 corpus/raw/raf-fuji-xm5-7748.raf -f756ad5be551a8b3e1b503a8a6f7e8e637fba519c0b0f8c79dd58e94b0287602 corpus/raw/raf-fuji-xt2-865.raf -a23045101b2912e64f154100d646c228c0d16585d21e95e0d1a006e537f7784c corpus/raw/raf-fuji-xt20-compressed.raf -07ea642656074834ffe5c661870a612ffa2e80fc1218a05166ee966bbe5f6fae corpus/raw/raf-fuji-xt4-3914.raf -871836938ffa695f4c41765ddb107c2703ce755f7f3ced7e10f9e04746f1734e corpus/raw/raf-fuji-xt4-3918.raf -b2a067e7fda16338f15c2532caa12b238b2415dc1c75a96a6d24d3445cfd0ae3 corpus/raw/raf-fuji-xt5-6122.raf -cddd7ae0c43f9280876e5fdcbdf8878718972ebd3d034d8affc8cfc6752d33dc corpus/raw/raf-fuji-xt5-6123.raf -3a8a9ac6d92274969aaa1f55ea669191b98bb8f588d00ec8ec17afd8b3963dea corpus/raw/raf-fuji-xt50-7807.raf +018781e126842596fcf3876b3fb6cd8973dc3936ccc753ee8648618be8889287 corpus/raw/raf-fuji-gfx100-3773.raf +0e7dad9819a748bb3d3be26bbf5770d4968e2d80b9cb41d97036bf7c27f7bfe7 corpus/raw/raf-fuji-gfx100-3775.raf +14bed5ac8191d6f2a7c82984e3bf6470ad4740df329293eac42b8018cc6a668a corpus/raw/raf-fuji-gfx100rf-8091.raf +a7bf8f061b603f7ef5fd29276eb6b2ac57355ca2f589a340117bfff0d03c065c corpus/raw/raf-fuji-gfx100s-4495.raf +1bb4608b61861a3d9e85b0ca6c00689b8ba4620cbe8579374d958c21ed08edf0 corpus/raw/raf-fuji-gfx100s-4503.raf +a7124d13910e0241a6b035154bc1fcdb7be4439371d043b1e1950a0251feb51e corpus/raw/raf-fuji-gfx50s-1435.raf +28f56cc9784164209e92fae209bdcc78267bd692aa2fb964595485f7e93692ab corpus/raw/raf-fuji-xe5-8509.raf +caeadb2666f6c5d9bbe32fbe692c2079b6a7e028f8514a13173f360656d08507 corpus/raw/raf-fuji-xh2-6001.raf +109be38f59963c9b8ff289ebe1f5ec9168c52f609e5b601432bd194520b4605b corpus/raw/raf-fuji-xh2-6002.raf +c76ce546e15727abb4ba201471010c554403176963a1a6b8cc017462606429e5 corpus/raw/raf-fuji-xm5-7748.raf +f756ad5be551a8b3e1b503a8a6f7e8e637fba519c0b0f8c79dd58e94b0287602 corpus/raw/raf-fuji-xt2-865.raf +a23045101b2912e64f154100d646c228c0d16585d21e95e0d1a006e537f7784c corpus/raw/raf-fuji-xt20-compressed.raf +07ea642656074834ffe5c661870a612ffa2e80fc1218a05166ee966bbe5f6fae corpus/raw/raf-fuji-xt4-3914.raf +871836938ffa695f4c41765ddb107c2703ce755f7f3ced7e10f9e04746f1734e corpus/raw/raf-fuji-xt4-3918.raf +b2a067e7fda16338f15c2532caa12b238b2415dc1c75a96a6d24d3445cfd0ae3 corpus/raw/raf-fuji-xt5-6122.raf +cddd7ae0c43f9280876e5fdcbdf8878718972ebd3d034d8affc8cfc6752d33dc corpus/raw/raf-fuji-xt5-6123.raf +3a8a9ac6d92274969aaa1f55ea669191b98bb8f588d00ec8ec17afd8b3963dea corpus/raw/raf-fuji-xt50-7807.raf diff --git a/crates/raw/tests/it/main.rs b/crates/raw/tests/it/main.rs index 5a9386eb0..30f331263 100644 --- a/crates/raw/tests/it/main.rs +++ b/crates/raw/tests/it/main.rs @@ -6,6 +6,7 @@ mod dng; mod dng_lines; #[cfg(feature = "line-dng")] mod dng_metadata; +mod raf_corpus_manifest; #[cfg(feature = "dng")] mod robust; mod vendor_robust; diff --git a/crates/raw/tests/it/raf_corpus_manifest.rs b/crates/raw/tests/it/raf_corpus_manifest.rs new file mode 100644 index 000000000..4cd1cfde6 --- /dev/null +++ b/crates/raw/tests/it/raf_corpus_manifest.rs @@ -0,0 +1,24 @@ +//! The pinned RAF corpus list `docs/raf-corpus.sha256` holds `#` comment lines, then one +//! `sha256path[license[url]]` row per file. The test reads the list only, not the corpus files. + +use std::collections::HashSet; + +const MANIFEST: &str = include_str!("../../docs/raf-corpus.sha256"); + +#[test] +fn raf_corpus_list_is_a_tab_separated_manifest() { + let mut paths = HashSet::new(); + let mut digests = HashSet::new(); + for line in MANIFEST.lines().filter(|l| !l.trim().is_empty() && !l.starts_with('#')) { + let cols: Vec<&str> = line.split('\t').collect(); + assert!((2..=4).contains(&cols.len()), "not a tab-separated manifest row: {line:?}"); + let (digest, path) = (cols[0], cols[1]); + assert!(digest.len() == 64 && digest.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')), "not a lowercase SHA-256 digest: {line:?}"); + assert!(path.starts_with("corpus/raw/") && path.len() > "corpus/raw/.raf".len() && path.ends_with(".raf"), "not a corpus RAF path: {line:?}"); + assert!(!path.contains(char::is_whitespace), "whitespace in the path: {line:?}"); + assert!(cols[2..].iter().all(|c| !c.is_empty()), "empty optional column: {line:?}"); + assert!(paths.insert(path), "path listed twice: {path}"); + assert!(digests.insert(digest), "digest listed twice: {digest}"); + } + assert_eq!(paths.len(), 17, "the pinned list holds the 17 CC0 RAF files"); +}