From 9399c25341289c1d523e64c92a1b95735c81933f Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:10:22 +0100 Subject: [PATCH 01/19] image-codecs: decode AVIF pixels with craft-video's pure-Rust AV1 decoder New optional `avif-decode` feature (native targets) reads still AVIF files: the HEIF/ISOBMFF item graph, grids, clap/irot/imir, alpha auxiliary images (including premultiplied), 8/10/12-bit 4:0:0, 4:2:0, 4:2:2 and 4:4:4, nclx/ICC colour, Exif and XMP, and PQ/HLG to linear F32 through cicp::linearize as the HEIF path does. The container walk follows LightCraft #3017 by @emmertex. Every size is checked against the caller's Limits before the AV1 decoder runs, the decoder's memory budget is capped by max_alloc, and the decode is wrapped in catch_unwind so a decoder panic becomes an error. Fixtures are tiny ffmpeg/libaom files; the 4:2:0 planes match libdav1d bit for bit. Adds a decode_avif fuzz target. Part of storytold/craft-libs#778. --- crates/image-codecs/Cargo.toml | 4 + crates/image-codecs/README.md | 27 +- crates/image-codecs/fuzz/Cargo.toml | 9 +- .../fuzz/fuzz_targets/decode_avif.rs | 14 + crates/image-codecs/src/cicp.rs | 2 +- crates/image-codecs/src/codecs.rs | 2 + crates/image-codecs/src/codecs/avif_decode.rs | 1239 +++++++++++++++++ crates/image-codecs/src/format.rs | 13 +- crates/image-codecs/src/lib.rs | 24 +- .../tests/fixtures/avif/README.md | 12 + .../tests/fixtures/avif/quad-420-bt709.avif | Bin 0 -> 369 bytes .../fixtures/avif/quad-420-bt709.dav1d.yuv | 1 + .../fixtures/avif/quad-444-gbr-lossless.avif | Bin 0 -> 345 bytes .../tests/fixtures/avif/quad-source.rgb | 1 + crates/image-codecs/tests/it/detect_caps.rs | 11 +- 15 files changed, 1335 insertions(+), 24 deletions(-) create mode 100644 crates/image-codecs/fuzz/fuzz_targets/decode_avif.rs create mode 100644 crates/image-codecs/src/codecs/avif_decode.rs create mode 100644 crates/image-codecs/tests/fixtures/avif/README.md create mode 100644 crates/image-codecs/tests/fixtures/avif/quad-420-bt709.avif create mode 100644 crates/image-codecs/tests/fixtures/avif/quad-420-bt709.dav1d.yuv create mode 100644 crates/image-codecs/tests/fixtures/avif/quad-444-gbr-lossless.avif create mode 100644 crates/image-codecs/tests/fixtures/avif/quad-source.rgb diff --git a/crates/image-codecs/Cargo.toml b/crates/image-codecs/Cargo.toml index df71d9860..1b422f912 100644 --- a/crates/image-codecs/Cargo.toml +++ b/crates/image-codecs/Cargo.toml @@ -15,6 +15,9 @@ corpus = [] default = [] avif = ["image/avif", "dep:ravif", "dep:rav1e", "dep:avif-serialize"] heif = ["dep:craft-heif"] +# AVIF decode: craft-video's clean-room pure-Rust AV1 decoder. Native targets only (on wasm32 a +# decoder panic aborts instead of becoming an error), so wasm32 builds leave AVIF unreadable. +avif-decode = ["dep:craft-video", "craft-video/av1-decode"] [dependencies] crc32fast = "1.5" @@ -41,6 +44,7 @@ hayro-jpeg2000 = { version = "=0.3.5", default-features = false, features = ["st [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rayon = { workspace = true } +craft-video = { workspace = true, optional = true } exr = { version = "1.73", default-features = false, features = ["rayon"] } [dev-dependencies] diff --git a/crates/image-codecs/README.md b/crates/image-codecs/README.md index 6e9e11617..1728d8308 100644 --- a/crates/image-codecs/README.md +++ b/crates/image-codecs/README.md @@ -1,9 +1,10 @@ # craft-image-codecs Codecs for flat raster image formats, written in pure Rust. This crate defines its own `Image` -type and shares its baseline and extended JPEG decoders through `craft-jpeg`, and its optional -HEIF decoder through `craft-heif`. It builds for `wasm32-unknown-unknown` and does no filesystem -I/O: decoders take `&[u8]` and encoders return `Vec`. +type and shares its baseline and extended JPEG decoders through `craft-jpeg`, its optional HEIF +decoder through `craft-heif`, and its optional AVIF decoder through `craft-video`'s AV1 decoder. +It builds for `wasm32-unknown-unknown` and does no filesystem I/O: decoders take `&[u8]` and +encoders return `Vec`. It follows the "Avoiding GIMP's hole" rules in `docs/architecture.md` §1.1: @@ -56,7 +57,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "< | Cineon | yes | yes | U16 (10-bit code values read and written exactly through U16) | Gray, RGB | no | no | no | no | no | no | no | built in | | DICOM | yes | **no** | U8, U16 | Gray, RGB | no | no | no | no | no | no | n/a | built in | | Amiga IFF (ILBM/PBM) | yes | yes (24-bit ILBM) | U8 | RGB, RGBA | yes (1-bit mask plane on write) | no | no | no | no | no | no | built in | -| AVIF | **no** | only with feature `avif` | U8 | RGB, RGBA | yes | no | no | no | no | no | yes | `image` + `ravif` | +| AVIF | with feature `avif-decode` (native only) | only with feature `avif` | U8, U16 (10/12-bit decodes to U16; PQ/HLG to linear F32) | RGB, RGBA | yes (auxiliary alpha) | read (`colr` prof/rICC) | read | read | no | no | yes | `craft-video` AV1 decoder; `image` + `ravif` to write | | HEIF/HEIC | with feature `heif` | **no** | U8, U16 (10/12-bit decodes to U16) | RGB, RGBA | yes (auxiliary alpha) | yes (`colr` prof) | yes | yes | no | no | n/a | `heic-rs` | "Native" means the data is stored and read back without conversion. Anything else is converted by @@ -177,10 +178,20 @@ embedded table (and a PNG's transparent entry) so `photocraft-io` can open them decodes to RGB. Image sequences, overlays and identity derivations, multilayer HEVC and some 4:2:2/4:4:4 streams return `CodecError::Unsupported` or `Malformed`, never wrong pixels. Writing needs an HEVC encoder and every mature one is C, so HEIF is listed in `ASYMMETRIC_EXCEPTIONS`. -* **AVIF.** Encoding uses `ravif`, which is pure Rust. Decoding - needs `dav1d`, which is C. AVIF is therefore read-unsupported, and write support is gated - behind the non-default `avif` feature. In a default build it is neither readable nor writable, - so the symmetric guarantee holds. It is listed in `ASYMMETRIC_EXCEPTIONS`. +* **AVIF.** Encoding uses `ravif`, which is pure Rust, behind the non-default `avif` feature. + Decoding (`codecs::avif_decode`, behind the non-default `avif-decode` feature) reads the HEIF + boxes itself and decodes the AV1 pictures with `craft-video`'s clean-room pure-Rust AV1 decoder: + one `av01` image or a `grid`, 8/10/12-bit, 4:2:0/4:2:2/4:4:4/monochrome, the `nclx` colour (or + the sequence header's) and an ICC profile, an `auxl` alpha plane (`prem` too), the Exif and XMP + items, and `clap`/`irot`/`imir`, which are applied unless `keep_orientation` is set, as for HEIF. + PQ and HLG open as linear light in F32, as HEIF does. Image sequences (animated AVIF), depth and + gain-map auxiliary images and layered progressive images aren't read. It is bounded like the + other decoders: the `Limits` are checked against the declared size before any picture decodes, + against every picture and every output buffer, `max_alloc` caps the AV1 decoder's picture store, + box, item and extent counts are capped, item data can't exceed the file, and a decoder panic is + `CodecError::Malformed`. wasm32 builds leave AVIF unreadable, since a panic there aborts. In a + default build AVIF is neither readable nor writable, so the symmetric guarantee holds; a build + with only one of the two features is listed in `ASYMMETRIC_EXCEPTIONS`. * **Lossy WebP.** Lossless (VP8L, via `image-webp`) is the default. `webp_lossless: false` writes a lossy file with our own clean-room VP8 key-frame encoder (`codecs::vp8`, from RFC 6386; `webp_quality` 0–100 on libwebp's scale sets the quantizer). Alpha travels losslessly diff --git a/crates/image-codecs/fuzz/Cargo.toml b/crates/image-codecs/fuzz/Cargo.toml index 3257a3953..30226c76e 100644 --- a/crates/image-codecs/fuzz/Cargo.toml +++ b/crates/image-codecs/fuzz/Cargo.toml @@ -14,7 +14,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" -craft-image-codecs = { path = "..", features = ["heif"] } +craft-image-codecs = { path = "..", features = ["heif", "avif-decode"] } # Called directly too: the codec turns its panics into errors (see decode_heif.rs). heic-rs = "=0.1.4" @@ -76,3 +76,10 @@ path = "fuzz_targets/decode_webp.rs" test = false doc = false bench = false + +[[bin]] +name = "decode_avif" +path = "fuzz_targets/decode_avif.rs" +test = false +doc = false +bench = false diff --git a/crates/image-codecs/fuzz/fuzz_targets/decode_avif.rs b/crates/image-codecs/fuzz/fuzz_targets/decode_avif.rs new file mode 100644 index 000000000..9a4b8978d --- /dev/null +++ b/crates/image-codecs/fuzz/fuzz_targets/decode_avif.rs @@ -0,0 +1,14 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use craft_image_codecs::{decode_as_with, probe_size, DecodeOptions, Format, Limits}; + +// The AVIF box reader and craft-video's AV1 decoder parse untrusted files: run with `-timeout=10` +// so a hang is reported, not just a panic. The codec turns an AV1 decoder panic into an error, so +// only the box reader's own panics would reach the fuzzer. Both orientation paths are covered. +fuzz_target!(|data: &[u8]| { + let keep_orientation = data.first().is_some_and(|b| b & 1 == 1); + let limits = Limits { max_width: 4096, max_height: 4096, max_pixels: 1 << 22, max_alloc: 256 << 20 }; + let _ = probe_size(data); + let _ = decode_as_with(Format::Avif, data, &DecodeOptions { limits, keep_orientation }); +}); diff --git a/crates/image-codecs/src/cicp.rs b/crates/image-codecs/src/cicp.rs index 290c82bc3..e6911d1b0 100644 --- a/crates/image-codecs/src/cicp.rs +++ b/crates/image-codecs/src/cicp.rs @@ -99,7 +99,7 @@ pub fn linear_to_pq(linear: f32) -> f32 { /// Relative luminance weights of the RGB primaries `code` (BT.2020's for anything else, the /// space BT.2100 defines HLG in). -fn luma_weights(code: u16) -> [f32; 3] { +pub(crate) fn luma_weights(code: u16) -> [f32; 3] { match code { 1 => [0.2126, 0.7152, 0.0722], 12 => [0.2290, 0.6917, 0.0793], diff --git a/crates/image-codecs/src/codecs.rs b/crates/image-codecs/src/codecs.rs index 0288ae22b..71db85f18 100644 --- a/crates/image-codecs/src/codecs.rs +++ b/crates/image-codecs/src/codecs.rs @@ -1,4 +1,6 @@ pub(crate) mod avif; +#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))] +pub(crate) mod avif_decode; pub(crate) mod dds; pub(crate) mod dds_bc; pub(crate) mod deep_exr; diff --git a/crates/image-codecs/src/codecs/avif_decode.rs b/crates/image-codecs/src/codecs/avif_decode.rs new file mode 100644 index 000000000..302433661 --- /dev/null +++ b/crates/image-codecs/src/codecs/avif_decode.rs @@ -0,0 +1,1239 @@ +//! AVIF (AV1 Image File Format) decoding, behind the non-default `avif-decode` feature, native +//! targets only. +//! +//! The container is HEIF (ISO/IEC 23008-12) over ISO-BMFF. This module reads the boxes itself +//! (`meta`, `pitm`, `iloc`, `iinf`, `iref`, `iprp`/`ipco`/`ipma`, `idat`), and `craft-video`'s +//! clean-room AV1 decoder (`av1-decode`, pure Rust, no `unsafe`) decodes the AV1 still pictures. +//! The container reader, colour conversion and transforms are adapted from @emmertex's LightCraft +//! AVIF importer (storytold/lightcraft#3017). +//! +//! Supported: the primary item as one `av01` image or a `grid` of them; 8-, 10- and 12-bit; +//! 4:2:0, 4:2:2, 4:4:4 and monochrome; full or limited range; the `nclx` colour description (or the +//! AV1 sequence header's) and an ICC profile (`prof` / `rICC`); an alpha auxiliary image (`auxl`, +//! premultiplied with `prem` too); the primary item's Exif and XMP items. `clap`, `irot` and `imir` +//! are applied unless the caller keeps the stored orientation, as for HEIF. PQ and HLG files +//! open as linear light in 32-bit float, as HEIF does. Not read: image sequences (animated AVIF), +//! depth and gain-map auxiliary images, layered (`lsel`/`a1lx`) progressive images. +//! +//! Bounded for hostile input: box nesting, item, property and extent counts are capped; item data +//! can't add up to more than the file; the [`Limits`] are checked against the declared size +//! before any picture is decoded, against every decoded picture and against each output buffer; +//! the AV1 decoder's picture store is capped by `max_alloc` too; and a panic in the decoder is +//! a malformed-file error. wasm32 builds leave AVIF unreadable: a panic there aborts instead of +//! unwinding. +//! +//! Sources: ISO/IEC 23008-12 (HEIF) and the AV1 Image File Format specification v1.1.0 (box +//! layouts), ITU-T H.273 (colour code points). + +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use craft_video::av1::decode as av1; + +use crate::Format; +use crate::cicp::Cicp; +use crate::error::CodecError; +use crate::image::{ChannelLayout, Image, Metadata, SampleType}; +use crate::options::Limits; + +const F: Format = Format::Avif; +/// Deepest box nesting followed (meta → iprp → ipco → property is 4). +const MAX_DEPTH: u32 = 8; +/// Most boxes, items, properties, associations or extents accepted: far above real files. +const MAX_ITEMS: usize = 1 << 16; +/// The AV1 decoder's own picture-store cap (`craft-video`'s default on 64-bit targets). +const AV1_MEMORY_BUDGET: u64 = 4 << 30; +/// The `auxC` types of an alpha plane (AVIF and the older HEVC-in-HEIF spelling). +const ALPHA_URNS: [&[u8]; 2] = [b"urn:mpeg:mpegB:cicp:systems:auxiliary:alpha", b"urn:mpeg:hevc:2015:auxid:1"]; + +type Result = std::result::Result; + +fn bad(m: impl std::fmt::Display) -> CodecError { + CodecError::malformed(F, m) +} + +/// A big-endian reader over a byte slice that never panics. +struct Rd<'a> { + b: &'a [u8], + p: usize, +} + +impl<'a> Rd<'a> { + fn new(b: &'a [u8]) -> Self { + Rd { b, p: 0 } + } + fn take(&mut self, n: usize) -> Result<&'a [u8]> { + let end = self.p.checked_add(n).ok_or_else(|| bad("box overflows"))?; + let s = self.b.get(self.p..end).ok_or_else(|| bad("box truncated"))?; + self.p = end; + Ok(s) + } + fn u8(&mut self) -> Result { + Ok(self.take(1)?.first().copied().unwrap_or(0)) + } + fn u16(&mut self) -> Result { + Ok(self.uint(2)? as u16) + } + fn u32(&mut self) -> Result { + Ok(self.uint(4)? as u32) + } + /// An unsigned big-endian number of `n` (0 to 8) bytes. + fn uint(&mut self, n: usize) -> Result { + if n > 8 { + return Err(bad("field too wide")); + } + Ok(self.take(n)?.iter().fold(0u64, |a, &x| (a << 8) | u64::from(x))) + } + /// A full box's version, its flags skipped. + fn version(&mut self) -> Result { + let v = self.u8()?; + self.take(3)?; + Ok(v) + } + /// An item ID: 16 bits in version 0 boxes, 32 bits otherwise. + fn id(&mut self, wide: bool) -> Result { + if wide { self.u32() } else { self.u16().map(u32::from) } + } + fn cstr(&mut self) -> Result<&'a [u8]> { + let rest = self.rest(); + let n = rest.iter().position(|&c| c == 0).unwrap_or(rest.len()); + let s = self.take(n)?; + if self.p < self.b.len() { + self.p += 1; + } + Ok(s) + } + fn rest(&self) -> &'a [u8] { + self.b.get(self.p..).unwrap_or(&[]) + } +} + +/// The child boxes of `b`: (type, payload). +fn boxes(b: &[u8]) -> Result> { + let mut out = Vec::new(); + let mut p = 0usize; + while p < b.len() { + let mut r = Rd::new(b.get(p..).unwrap_or(&[])); + if r.b.len() < 8 { + break; // trailing padding + } + let size32 = r.u32()?; + let t = r.take(4)?; + let ty = [t[0], t[1], t[2], t[3]]; + let (hdr, size) = match size32 { + 0 => (8usize, r.b.len() as u64), + 1 => (16, r.uint(8)?), + s => (8, u64::from(s)), + }; + let size = usize::try_from(size).map_err(|_| bad("box too large"))?; + if size < hdr || size > r.b.len() { + return Err(bad(format!("box {} has a bad size", String::from_utf8_lossy(&ty)))); + } + out.push((ty, r.b.get(hdr..size).unwrap_or(&[]))); + if out.len() > MAX_ITEMS { + return Err(bad("too many boxes")); + } + p = p.checked_add(size).ok_or_else(|| bad("box overflows"))?; + } + Ok(out) +} + +#[derive(Clone, Debug, Default)] +struct Extent { + offset: u64, + length: u64, +} + +#[derive(Clone, Debug, Default)] +struct Loc { + method: u8, + base: u64, + extents: Vec, +} + +#[derive(Clone, Copy, Debug)] +enum Transform { + /// Clean aperture: width, height, horizontal and vertical offset of the centre. + Clap { w: f64, h: f64, dx: f64, dy: f64 }, + /// Anticlockwise rotation by `n` × 90°. + Rot(u8), + /// Mirror, as HEIF amendment 2, libheif and libavif read it: 0 = top and bottom exchanged, + /// 1 = left and right exchanged. + Mir(u8), +} + +#[derive(Clone, Debug)] +enum Prop { + Av1c(Vec), + Nclx(Cicp), + Icc(Vec), + Ispe(u32, u32), + AuxC(Vec), + T(Transform), + Other, +} + +struct Item { + id: u32, + kind: [u8; 4], + /// `mime` items: the content type. + content_type: Option>, +} + +#[derive(Default)] +struct Container<'a> { + file: &'a [u8], + primary: u32, + items: Vec, + locs: Vec<(u32, Loc)>, + idat: &'a [u8], + props: Vec, + /// item → 1-based property indices, in order. + assoc: Vec<(u32, Vec)>, + /// (type, from, to…) + refs: Vec<([u8; 4], u32, Vec)>, + /// `iloc` extents read so far, across all items (capped at [`MAX_ITEMS`]). + extents: usize, +} + +fn parse(file: &[u8]) -> Result> { + let top = boxes(file)?; + let meta = top.iter().find(|(t, _)| t == b"meta").map(|(_, b)| *b).ok_or_else(|| { + if top.iter().any(|(t, _)| t == b"moov") { + CodecError::unsupported(F, "AVIF image sequences (animated AVIF) aren't read") + } else { + bad("no meta box") + } + })?; + let mut c = Container { file, ..Default::default() }; + let meta = meta.get(4..).ok_or_else(|| bad("meta box truncated"))?; + for (t, b) in boxes(meta)? { + let mut r = Rd::new(b); + match &t { + b"pitm" => { + let v = r.version()?; + c.primary = r.id(v != 0)?; + } + b"iloc" => parse_iloc(&mut r, &mut c)?, + b"iinf" => parse_iinf(&mut r, &mut c)?, + b"iref" => parse_iref(&mut r, &mut c)?, + b"idat" => c.idat = b, + b"iprp" => parse_iprp(b, &mut c, 0)?, + _ => {} + } + } + if c.primary == 0 { + return Err(bad("no primary item")); + } + Ok(c) +} + +fn parse_iinf(r: &mut Rd, c: &mut Container) -> Result<()> { + let v = r.version()?; + let _count = r.id(v != 0)?; + for (t, b) in boxes(r.rest())? { + if &t != b"infe" { + continue; + } + let mut r = Rd::new(b); + let v = r.version()?; + if v < 2 { + continue; + } + let id = r.id(v != 2)?; + r.u16()?; // protection index + let kind = r.take(4)?; + let kind = [kind[0], kind[1], kind[2], kind[3]]; + let _name = r.cstr()?; + let content_type = (&kind == b"mime").then(|| r.cstr().ok().map(<[u8]>::to_vec)).flatten(); + c.items.push(Item { id, kind, content_type }); + if c.items.len() > MAX_ITEMS { + return Err(bad("too many items")); + } + } + Ok(()) +} + +fn parse_iref(r: &mut Rd, c: &mut Container) -> Result<()> { + let v = r.version()?; + for (t, b) in boxes(r.rest())? { + let mut r = Rd::new(b); + let from = r.id(v != 0)?; + let n = r.u16()?; + let mut to = Vec::with_capacity(usize::from(n).min(r.rest().len() / 2 + 1)); + for _ in 0..n { + to.push(r.id(v != 0)?); + } + c.refs.push((t, from, to)); + if c.refs.len() > MAX_ITEMS { + return Err(bad("too many item references")); + } + } + Ok(()) +} + +fn parse_iloc(r: &mut Rd, c: &mut Container) -> Result<()> { + let v = r.version()?; + if v > 2 { + return Err(CodecError::unsupported(F, "iloc version")); + } + let a = r.u8()?; + let b = r.u8()?; + let (off_size, len_size, base_size) = (usize::from(a >> 4), usize::from(a & 15), usize::from(b >> 4)); + let index_size = if v >= 1 { usize::from(b & 15) } else { 0 }; + let count = if v < 2 { u32::from(r.u16()?) } else { r.u32()? }; + for _ in 0..count { + let id = r.id(v >= 2)?; + let method = if v >= 1 { (r.u16()? & 15) as u8 } else { 0 }; + r.u16()?; // data reference index + let base = r.uint(base_size)?; + let n = r.u16()?; + let mut extents = Vec::with_capacity(usize::from(n).min(64)); + for _ in 0..n { + r.uint(index_size)?; + let offset = r.uint(off_size)?; + let length = r.uint(len_size)?; + extents.push(Extent { offset, length }); + // Zero-width offset and length fields read no bytes, so the file size alone doesn't + // bound this. + c.extents += 1; + if c.extents > MAX_ITEMS { + return Err(bad("too many item extents")); + } + } + c.locs.push((id, Loc { method, base, extents })); + if c.locs.len() > MAX_ITEMS { + return Err(bad("too many items")); + } + } + Ok(()) +} + +fn parse_iprp(b: &[u8], c: &mut Container, depth: u32) -> Result<()> { + if depth > MAX_DEPTH { + return Err(bad("boxes nested too deep")); + } + for (t, b) in boxes(b)? { + match &t { + b"ipco" => { + for (t, b) in boxes(b)? { + c.props.push(parse_prop(&t, b)?); + if c.props.len() > MAX_ITEMS { + return Err(bad("too many properties")); + } + } + } + b"ipma" => parse_ipma(b, c)?, + _ => {} + } + } + Ok(()) +} + +fn parse_ipma(b: &[u8], c: &mut Container) -> Result<()> { + let mut r = Rd::new(b); + let v = r.u8()?; + let flags = r.take(3)?; + let wide = flags.get(2).is_some_and(|f| f & 1 != 0); + let n = r.u32()?; + for _ in 0..n { + let id = r.id(v != 0)?; + let k = r.u8()?; + let mut list = Vec::with_capacity(usize::from(k)); + for _ in 0..k { + let idx = if wide { usize::from(r.u16()? & 0x7fff) } else { usize::from(r.u8()? & 0x7f) }; + list.push(idx); + } + c.assoc.push((id, list)); + if c.assoc.len() > MAX_ITEMS { + return Err(bad("too many property associations")); + } + } + Ok(()) +} + +fn parse_prop(t: &[u8; 4], b: &[u8]) -> Result { + let mut r = Rd::new(b); + Ok(match t { + b"av1C" => Prop::Av1c(b.get(4..).unwrap_or(&[]).to_vec()), + b"colr" => match r.take(4)? { + b"nclx" => Prop::Nclx(Cicp { primaries: r.u16()?, transfer: r.u16()?, matrix: r.u16()?, full_range: r.u8()? & 0x80 != 0 }), + b"prof" | b"rICC" => Prop::Icc(r.rest().to_vec()), + _ => Prop::Other, + }, + b"ispe" => { + r.version()?; + Prop::Ispe(r.u32()?, r.u32()?) + } + b"auxC" => { + r.version()?; + Prop::AuxC(r.cstr()?.to_vec()) + } + b"irot" => Prop::T(Transform::Rot(r.u8()? & 3)), + b"imir" => Prop::T(Transform::Mir(r.u8()? & 1)), + b"clap" => { + let mut q = || -> Result { + let n = r.u32()? as i32; + let d = r.u32()? as i32; + if d == 0 { + return Err(bad("clap with a zero denominator")); + } + Ok(f64::from(n) / f64::from(d)) + }; + let (w, h, dx, dy) = (q()?, q()?, q()?, q()?); + Prop::T(Transform::Clap { w, h, dx, dy }) + } + _ => Prop::Other, + }) +} + +impl Container<'_> { + fn item_type(&self, id: u32) -> Option<[u8; 4]> { + self.items.iter().find(|t| t.id == id).map(|t| t.kind) + } + fn props_of(&self, id: u32) -> impl Iterator { + self.assoc.iter().filter(move |a| a.0 == id).flat_map(|a| a.1.iter()).filter_map(|&i| i.checked_sub(1).and_then(|i| self.props.get(i))) + } + fn data(&self, id: u32) -> Result> { + let loc = self.locs.iter().find(|l| l.0 == id).map(|l| &l.1).ok_or_else(|| bad(format!("item {id} has no location")))?; + let src = match loc.method { + 0 => self.file, + 1 => self.idat, + _ => return Err(CodecError::unsupported(F, "item data by reference to another item")), + }; + let mut out = Vec::new(); + for e in &loc.extents { + let start = loc.base.checked_add(e.offset).ok_or_else(|| bad("item offset overflows"))?; + let start = usize::try_from(start).map_err(|_| bad("item offset out of range"))?; + let end = if e.length == 0 { + src.len() + } else { + usize::try_from(e.length).ok().and_then(|l| start.checked_add(l)).ok_or_else(|| bad("item length overflows"))? + }; + let chunk = src.get(start..end).ok_or_else(|| bad("item data runs past the end of the file"))?; + // Every byte comes from the file, so a real item is never larger than it; repeated or + // overlapping extents that add up to more are refused before they allocate. + if out.len().saturating_add(chunk.len()) > self.file.len() { + return Err(bad("item data is larger than the file")); + } + out.extend_from_slice(chunk); + } + Ok(out) + } + fn refs_from(&self, kind: &[u8; 4], from: u32) -> Vec { + self.refs.iter().filter(|r| &r.0 == kind && r.1 == from).flat_map(|r| r.2.iter().copied()).collect() + } + /// Items with a `kind` reference to `to`. + fn refs_to(&self, kind: &[u8; 4], to: u32) -> impl Iterator { + self.refs.iter().filter(move |r| &r.0 == kind && r.2.contains(&to)).map(|r| r.1) + } + fn ispe(&self, id: u32) -> Option<(u32, u32)> { + self.props_of(id).find_map(|p| if let Prop::Ispe(w, h) = p { Some((*w, *h)) } else { None }) + } + fn transforms(&self, id: u32) -> Vec { + self.props_of(id).filter_map(|p| if let Prop::T(t) = p { Some(*t) } else { None }).collect() + } + fn nclx(&self, id: u32) -> Option { + self.props_of(id).find_map(|p| if let Prop::Nclx(n) = p { Some(*n) } else { None }) + } + fn icc(&self, id: u32) -> Option> { + self.props_of(id).find_map(|p| if let Prop::Icc(b) = p { Some(b.clone()) } else { None }) + } + /// The primary item's alpha plane: an auxiliary image (`auxl`) whose `auxC` says alpha. + fn alpha_item(&self) -> Option { + self.refs_to(b"auxl", self.primary).find(|&id| { + matches!(self.item_type(id), Some(t) if &t == b"av01" || &t == b"grid") + && self.props_of(id).any(|p| matches!(p, Prop::AuxC(urn) if ALPHA_URNS.contains(&urn.as_slice()))) + }) + } + /// The primary item's colour is premultiplied by its alpha (`prem` from colour to alpha). + fn premultiplied(&self, alpha: u32) -> bool { + self.refs_from(b"prem", self.primary).contains(&alpha) + } + /// The first item of a metadata kind that describes (`cdsc`) the primary item. + fn metadata_item(&self, is: impl Fn(&Item) -> bool) -> Option { + let described: Vec = self.refs_to(b"cdsc", self.primary).collect(); + self.items.iter().filter(|i| is(i)).map(|i| i.id).find(|id| described.contains(id)) + } +} + +/// The size of an image item (`av01` or `grid`) before its transforms. +fn item_size(c: &Container, id: u32) -> Result<(u32, u32)> { + let kind = c.item_type(id).ok_or_else(|| bad("an image item is not described"))?; + let (w, h) = match (c.ispe(id), &kind) { + (_, b"grid") => grid_header(&c.data(id)?)?.2, + (Some(s), b"av01") => s, + (None, b"av01") => return Err(bad("an AV1 image item has no size (ispe)")), + _ => return Err(CodecError::unsupported(F, "the image item is not an AV1 image or a grid")), + }; + if w == 0 || h == 0 { + return Err(bad("zero dimension")); + } + Ok((w, h)) +} + +/// The size after `clap` / `irot`, from a stored size. +fn displayed(mut w: u32, mut h: u32, ts: &[Transform]) -> (u32, u32) { + for t in ts { + match *t { + Transform::Clap { w: cw, h: ch, .. } if cw >= 1.0 && ch >= 1.0 && cw.is_finite() && ch.is_finite() => { + w = (cw.round() as u32).clamp(1, w); + h = (ch.round() as u32).clamp(1, h); + } + Transform::Rot(n) if n % 2 == 1 => std::mem::swap(&mut w, &mut h), + _ => {} + } + } + (w, h) +} + +/// (rows, columns, output size) of a `grid` item. +fn grid_header(d: &[u8]) -> Result<(u32, u32, (u32, u32))> { + let mut r = Rd::new(d); + r.u8()?; // version + let flags = r.u8()?; + let rows = u32::from(r.u8()?) + 1; + let cols = u32::from(r.u8()?) + 1; + let n = if flags & 1 != 0 { 4 } else { 2 }; + let w = r.uint(n)? as u32; + let h = r.uint(n)? as u32; + Ok((rows, cols, (w, h))) +} + +/// The upright size [`decode`] returns (`keep_orientation` off), from the boxes alone. +pub(crate) fn probe(bytes: &[u8], limits: &Limits) -> Result<(u32, u32)> { + let c = parse(bytes)?; + let (w, h) = item_size(&c, c.primary)?; + limits.check_size(w, h)?; + Ok(displayed(w, h, &c.transforms(c.primary))) +} + +/// One decoded AV1 picture. +struct Pic { + w: usize, + h: usize, + bits: u8, + ssx: usize, + ssy: usize, + mono: bool, + planes: [Vec; 3], + cicp: Cicp, +} + +fn av1_error(e: av1::Error) -> CodecError { + match e { + av1::Error::Unsupported(w) => CodecError::unsupported(F, format!("AV1: {w}")), + other => bad(format!("AV1: {other}")), + } +} + +/// Decodes the `av01` item `id`, refusing pictures over `limits`. +fn decode_av1(c: &Container, id: u32, limits: &Limits) -> Result { + if c.item_type(id) != Some(*b"av01") { + return Err(CodecError::unsupported(F, "an image tile is not an AV1 image")); + } + // A tile's declared size counts before its picture is decoded too. + if let Some((w, h)) = c.ispe(id) { + limits.check(w, h, ChannelLayout::Rgb, SampleType::U16)?; + } + let data = c.data(id)?; + let config = c.props_of(id).find_map(|p| if let Prop::Av1c(o) = p { Some(o.clone()) } else { None }); + let mut dec = av1::Decoder::new(); + dec.set_memory_budget(limits.max_alloc.min(AV1_MEMORY_BUDGET)); + let mut pics = catch_unwind(AssertUnwindSafe(|| -> std::result::Result, av1::Error> { + let mut pics = Vec::new(); + // The sequence header normally travels in the item too; the av1C copy covers files + // where it doesn't. + if let Some(cfg) = config.filter(|o| !o.is_empty()) { + pics.extend(dec.decode(&cfg)?); + } + pics.extend(dec.decode(&data)?); + pics.extend(dec.flush_result()?); + Ok(pics) + })) + .map_err(|_| bad("the AV1 decoder failed on this file"))? + .map_err(av1_error)?; + if pics.is_empty() { + return Err(bad("the AV1 data holds no picture")); + } + let p = pics.swap_remove(0); + limits.check(p.width, p.height, ChannelLayout::Rgb, SampleType::U16)?; + let (w, h) = (p.width as usize, p.height as usize); + let (ssx, ssy) = (usize::from(p.subsampling_x.min(1)), usize::from(p.subsampling_y.min(1))); + let need = w.checked_mul(h).ok_or_else(|| bad("picture too large"))?; + let cneed = ((w + ssx) >> ssx) * ((h + ssy) >> ssy); + if need == 0 || p.planes[0].len() < need || (!p.mono_chrome && (p.planes[1].len() < cneed || p.planes[2].len() < cneed)) { + return Err(bad("AV1 picture planes are short")); + } + let cicp = Cicp { + primaries: u16::from(p.color_primaries), + transfer: u16::from(p.transfer_characteristics), + matrix: u16::from(p.matrix_coefficients), + full_range: p.full_range, + }; + Ok(Pic { w, h, bits: p.bit_depth.clamp(8, 16), ssx, ssy, mono: p.mono_chrome, planes: p.planes, cicp }) +} + +/// Kr, Kb of an H.273 matrix code point; `None` = identity (GBR). +fn coefficients(matrix: u16, primaries: u16) -> Result> { + Ok(match matrix { + 0 => None, + 1 => Some((0.2126, 0.0722)), + 4 => Some((0.30, 0.11)), + 7 => Some((0.212, 0.087)), + 9 | 10 => Some((0.2627, 0.0593)), + // Chromaticity-derived non-constant luminance. + 12 => { + let [kr, _, kb] = crate::cicp::luma_weights(primaries); + Some((kr, kb)) + } + // 2 (unspecified: AVIF readers assume BT.601), 5, 6 (BT.601) and reserved values. + 2 | 5 | 6 | 3 | 15.. => Some((0.299, 0.114)), + _ => return Err(CodecError::unsupported(F, format!("matrix coefficients {matrix} (YCgCo, constant luminance or ICtCp)"))), + }) +} + +/// A chroma sample at luma position (x, y), bilinear between chroma sites centred on their +/// luma pairs. +#[inline] +fn chroma_at(plane: &[u16], cw: usize, ch: usize, ssx: usize, ssy: usize, x: usize, y: usize) -> f32 { + let pos = |v: usize, ss: usize, n: usize| -> (usize, usize, f32) { + if ss == 0 { + return (v.min(n - 1), v.min(n - 1), 0.0); + } + let f = ((v as f32 + 0.5) / 2.0 - 0.5).max(0.0); + let i = f as usize; + (i.min(n - 1), (i + 1).min(n - 1), f - i as f32) + }; + let (x0, x1, fx) = pos(x, ssx, cw); + let (y0, y1, fy) = pos(y, ssy, ch); + let s = |xx: usize, yy: usize| f32::from(plane.get(yy * cw + xx).copied().unwrap_or(0)); + let top = s(x0, y0) * (1.0 - fx) + s(x1, y0) * fx; + let bot = s(x0, y1) * (1.0 - fx) + s(x1, y1) * fx; + top * (1.0 - fy) + bot * fy +} + +/// The sample scaling of a picture: luma offset and range, chroma range and midpoint. +struct Levels { + yoff: f32, + yrange: f32, + crange: f32, + mid: f32, +} + +impl Levels { + fn new(bits: u8, full_range: bool) -> Self { + let max = ((1u32 << bits) - 1) as f32; + let scale = f32::from(1u16 << (bits - 8).min(8)); + let (yoff, yrange, crange) = if full_range { (0.0, max, max) } else { (16.0 * scale, 219.0 * scale, 224.0 * scale) }; + Levels { yoff, yrange, crange, mid: f32::from(1u16 << (bits - 1).min(15)) } + } +} + +fn to_u16(v: f32) -> u16 { + (v.clamp(0.0, 1.0) * 65535.0 + 0.5) as u16 +} + +/// Where a decoded picture goes: an interleaved 16-bit canvas of `ch` channels. +struct Canvas { + w: usize, + h: usize, + ch: usize, + data: Vec, +} + +impl Canvas { + fn new(w: u32, h: u32, layout: ChannelLayout, limits: &Limits) -> Result { + limits.check(w, h, layout, SampleType::U16)?; + let (w, h, ch) = (w as usize, h as usize, layout.channels()); + let len = w.checked_mul(h).and_then(|n| n.checked_mul(ch)).ok_or_else(|| bad("image too large"))?; + Ok(Canvas { w, h, ch, data: vec![0; len] }) + } + + /// The picture's YCbCr as non-linear R'G'B' into channels 0..3, its top-left at (x0, y0). + fn put_colour(&mut self, p: &Pic, cicp: Cicp, x0: usize, y0: usize) -> Result<()> { + let lv = Levels::new(p.bits, cicp.full_range); + let k = coefficients(cicp.matrix, cicp.primaries)?; + let (cw, chh) = ((p.w + p.ssx) >> p.ssx, (p.h + p.ssy) >> p.ssy); + for y in 0..p.h.min(self.h.saturating_sub(y0)) { + for x in 0..p.w.min(self.w.saturating_sub(x0)) { + let luma = (f32::from(p.planes[0][y * p.w + x]) - lv.yoff) / lv.yrange; + let (cb, cr) = if p.mono { + (lv.mid, lv.mid) + } else { + (chroma_at(&p.planes[1], cw, chh, p.ssx, p.ssy, x, y), chroma_at(&p.planes[2], cw, chh, p.ssx, p.ssy, x, y)) + }; + let rgb = match k { + // Identity (GBR): Y = G, Cb = B, Cr = R, all three at the luma levels. + None => [(cr - lv.yoff) / lv.yrange, luma, (cb - lv.yoff) / lv.yrange], + Some((kr, kb)) => { + let (cb, cr) = ((cb - lv.mid) / lv.crange, (cr - lv.mid) / lv.crange); + let r = luma + 2.0 * (1.0 - kr) * cr; + let b = luma + 2.0 * (1.0 - kb) * cb; + [r, (luma - kr * r - kb * b) / (1.0 - kr - kb), b] + } + }; + let o = ((y0 + y) * self.w + x0 + x) * self.ch; + for (d, v) in self.data[o..o + 3].iter_mut().zip(rgb) { + *d = to_u16(v); + } + } + } + Ok(()) + } + + /// The picture's luma as alpha into channel 3, its top-left at (x0, y0). + fn put_alpha(&mut self, p: &Pic, x0: usize, y0: usize) { + let lv = Levels::new(p.bits, p.cicp.full_range); + for y in 0..p.h.min(self.h.saturating_sub(y0)) { + for x in 0..p.w.min(self.w.saturating_sub(x0)) { + let a = (f32::from(p.planes[0][y * p.w + x]) - lv.yoff) / lv.yrange; + self.data[((y0 + y) * self.w + x0 + x) * self.ch + 3] = to_u16(a); + } + } + } +} + +/// Decodes the image item `id` (one `av01` or a `grid` of them) into `canvas`, through `put`. +/// Returns the first picture's bit depth and colour description. +fn render( + c: &Container, + id: u32, + limits: &Limits, + canvas: &mut Canvas, + put: &mut dyn FnMut(&mut Canvas, &Pic, usize, usize) -> Result<()>, +) -> Result<(u8, Cicp)> { + if c.item_type(id) != Some(*b"grid") { + let p = decode_av1(c, id, limits)?; + if (p.w, p.h) != (canvas.w, canvas.h) { + return Err(bad("the AV1 picture's size differs from its item's")); + } + put(canvas, &p, 0, 0)?; + return Ok((p.bits, p.cicp)); + } + let (rows, cols, _) = grid_header(&c.data(id)?)?; + let tiles = c.refs_from(b"dimg", id); + if tiles.is_empty() || tiles.len() as u64 != u64::from(rows) * u64::from(cols) { + return Err(bad("grid tile count doesn't match its rows and columns")); + } + let mut first: Option<(usize, usize, u8, Cicp)> = None; + for (i, &t) in tiles.iter().enumerate() { + let p = decode_av1(c, t, limits)?; + let (tw, th, ..) = *first.get_or_insert((p.w, p.h, p.bits, p.cicp)); + if (p.w, p.h) != (tw, th) { + return Err(bad("grid tiles differ in size")); + } + if tw.saturating_mul(cols as usize) < canvas.w || th.saturating_mul(rows as usize) < canvas.h { + return Err(bad("grid tiles don't cover the grid")); + } + put(canvas, &p, (i % cols as usize) * tw, (i / cols as usize) * th)?; + } + first.map(|(_, _, bits, cicp)| (bits, cicp)).ok_or_else(|| bad("grid without tiles")) +} + +/// Applies the container's transforms in order to an interleaved canvas. +fn apply_transforms(mut cv: Canvas, ts: &[Transform]) -> Canvas { + for t in ts { + cv = match *t { + Transform::Clap { w, h, dx, dy } => clap(cv, w, h, dx, dy), + Transform::Rot(n) => (0..n).fold(cv, |cv, _| rotate(cv)), + Transform::Mir(axis) => mirror(cv, axis), + }; + } + cv +} + +fn clap(cv: Canvas, cw: f64, chh: f64, dx: f64, dy: f64) -> Canvas { + if !(cw >= 1.0 && chh >= 1.0 && cw.is_finite() && chh.is_finite() && dx.is_finite() && dy.is_finite()) { + return cv; + } + let (w, h, ch) = (cv.w, cv.h, cv.ch); + let nw = (cw.round() as usize).clamp(1, w); + let nh = (chh.round() as usize).clamp(1, h); + // The centre of the aperture is the image centre plus the offset. + let x0 = ((w as f64 - 1.0) / 2.0 + dx - (nw as f64 - 1.0) / 2.0).round().clamp(0.0, (w - nw) as f64) as usize; + let y0 = ((h as f64 - 1.0) / 2.0 + dy - (nh as f64 - 1.0) / 2.0).round().clamp(0.0, (h - nh) as f64) as usize; + let mut data = Vec::with_capacity(nw * nh * ch); + for y in y0..y0 + nh { + data.extend_from_slice(&cv.data[(y * w + x0) * ch..(y * w + x0 + nw) * ch]); + } + Canvas { w: nw, h: nh, ch, data } +} + +/// 90° anticlockwise: (x, y) → (y, w - 1 - x), the new size h × w. +fn rotate(cv: Canvas) -> Canvas { + let (w, h, ch) = (cv.w, cv.h, cv.ch); + let mut data = vec![0u16; cv.data.len()]; + for y in 0..h { + for x in 0..w { + let d = ((w - 1 - x) * h + y) * ch; + let s = (y * w + x) * ch; + data[d..d + ch].copy_from_slice(&cv.data[s..s + ch]); + } + } + Canvas { w: h, h: w, ch, data } +} + +fn mirror(mut cv: Canvas, axis: u8) -> Canvas { + let (w, h, ch) = (cv.w, cv.h, cv.ch); + if axis == 0 { + for y in 0..h / 2 { + let (top, bottom) = cv.data.split_at_mut((h - 1 - y) * w * ch); + top[y * w * ch..(y + 1) * w * ch].swap_with_slice(&mut bottom[..w * ch]); + } + } else { + for row in cv.data.chunks_exact_mut(w * ch) { + for x in 0..w / 2 { + for c in 0..ch { + row.swap(x * ch + c, (w - 1 - x) * ch + c); + } + } + } + } + cv +} + +/// Straight colour from colour premultiplied by alpha. +fn unpremultiply(cv: &mut Canvas) { + for px in cv.data.as_chunks_mut::<4>().0 { + let a = u32::from(px[3]); + if a == 0 { + continue; + } + for v in &mut px[..3] { + *v = (u32::from(*v) * 65535 / a).min(65535) as u16; + } + } +} + +/// The primary item's Exif (TIFF structure, its offset prefix dropped) and XMP. +fn read_metadata(c: &Container) -> (Option>, Option) { + let exif = c.metadata_item(|i| &i.kind == b"Exif").and_then(|id| c.data(id).ok()).and_then(|d| { + let offset = usize::try_from(u32::from_be_bytes(d.get(..4)?.try_into().ok()?)).ok()?; + let tiff = d.get(4usize.checked_add(offset)?..)?; + (tiff.starts_with(b"II*\0") || tiff.starts_with(b"MM\0*")).then(|| tiff.to_vec()) + }); + let xmp = c + .metadata_item(|i| &i.kind == b"mime" && i.content_type.as_deref() == Some(b"application/rdf+xml")) + .and_then(|id| c.data(id).ok()) + .map(|d| String::from_utf8_lossy(&d).trim_end_matches('\0').to_owned()); + (exif, xmp) +} + +/// Decodes the primary image. `keep_orientation` leaves the container's `clap`, `irot` and +/// `imir` unapplied (the stored pixels), as for HEIF. +pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result { + let c = parse(bytes)?; + let id = c.primary; + let (w, h) = item_size(&c, id)?; + let alpha = c.alpha_item().filter(|&a| item_size(&c, a).is_ok_and(|s| s == (w, h))); + let layout = if alpha.is_some() { ChannelLayout::Rgba } else { ChannelLayout::Rgb }; + // The declared size, before any picture is decoded. + let mut canvas = Canvas::new(w, h, layout, limits)?; + let file_cicp = c.nclx(id); + let mut colour = |cv: &mut Canvas, p: &Pic, x: usize, y: usize| cv.put_colour(p, file_cicp.unwrap_or(p.cicp), x, y); + let (bits, stream_cicp) = render(&c, id, limits, &mut canvas, &mut colour)?; + if let Some(a) = alpha { + render(&c, a, limits, &mut canvas, &mut |cv, p, x, y| { + cv.put_alpha(p, x, y); + Ok(()) + })?; + if c.premultiplied(a) { + unpremultiply(&mut canvas); + } + } + if !keep_orientation { + canvas = apply_transforms(canvas, &c.transforms(id)); + } + let (w, h) = (canvas.w as u32, canvas.h as u32); + let image = Image::from_u16(w, h, layout, &canvas.data)?; + drop(canvas); + let mut img = if bits > 8 { image } else { image.convert(layout, SampleType::U8) }; + let cicp = file_cicp.unwrap_or(stream_cicp); + img.icc = c.icc(id); + img.cicp = Some(cicp); + let (exif, xmp) = read_metadata(&c); + img.meta = Metadata { + exif: exif.map(|e| if keep_orientation { e } else { crate::orientation::upright_exif(&e).into_owned() }), + xmp: xmp.map(|x| if keep_orientation { x } else { crate::orientation::upright_xmp(&x).into_owned() }), + ..Default::default() + }; + // PQ or HLG: linear light in 32-bit float, as for HEIF. + if cicp.is_hdr() { + limits.check(w, h, layout, SampleType::F32)?; + return crate::cicp::linearize(img, cicp); + } + Ok(img) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::DecodeOptions; + + const QUAD_420: &[u8] = include_bytes!("../../tests/fixtures/avif/quad-420-bt709.avif"); + const QUAD_420_DAV1D: &[u8] = include_bytes!("../../tests/fixtures/avif/quad-420-bt709.dav1d.yuv"); + const QUAD_GBR_LOSSLESS: &[u8] = include_bytes!("../../tests/fixtures/avif/quad-444-gbr-lossless.avif"); + const QUAD_SOURCE: &[u8] = include_bytes!("../../tests/fixtures/avif/quad-source.rgb"); + + fn rgb8(img: &Image) -> Vec { + img.to_rgba8().as_chunks::<4>().0.iter().flat_map(|p| [p[0], p[1], p[2]]).collect() + } + + fn mean_abs_diff(a: &[u8], b: &[u8]) -> f64 { + assert_eq!(a.len(), b.len()); + a.iter().zip(b).map(|(&x, &y)| (f64::from(x) - f64::from(y)).abs()).sum::() / a.len() as f64 + } + + /// A box of type `t` around `body`. + fn bx(t: &[u8; 4], body: &[u8]) -> Vec { + let mut v = ((body.len() + 8) as u32).to_be_bytes().to_vec(); + v.extend_from_slice(t); + v.extend_from_slice(body); + v + } + + /// A full box: version 0, no flags. + fn fbx(t: &[u8; 4], body: &[u8]) -> Vec { + let mut b = vec![0, 0, 0, 0]; + b.extend_from_slice(body); + bx(t, &b) + } + + /// An `infe` (version 2) for item `id` of `kind`. + fn infe(id: u16, kind: &[u8; 4]) -> Vec { + let mut b = vec![2, 0, 0, 0]; + b.extend_from_slice(&id.to_be_bytes()); + b.extend_from_slice(&[0, 0]); + b.extend_from_slice(kind); + b.push(0); + bx(b"infe", &b) + } + + fn ispe(w: u32, h: u32) -> Vec { + let mut b = w.to_be_bytes().to_vec(); + b.extend_from_slice(&h.to_be_bytes()); + fbx(b"ispe", &b) + } + + /// The AV1 data and `av1C` of an encoded file's primary item. + fn av1_of(file: &[u8]) -> (Vec, Vec) { + let c = parse(file).unwrap(); + let cfg = c.props_of(c.primary).find_map(|p| if let Prop::Av1c(o) = p { Some(o.clone()) } else { None }).unwrap(); + (c.data(c.primary).unwrap(), cfg) + } + + /// An AVIF of `items` (id, type, data, property indices) with `props`, `refs` and primary 1. + /// All item data sits in `idat` (construction method 1). + /// An item: ID, type, data and 1-based property indices. + type TestItem<'a> = (u16, &'a [u8; 4], &'a [u8], &'a [u8]); + /// An item reference: type, from, to. + type TestRef<'a> = (&'a [u8; 4], u16, &'a [u16]); + + fn build(items: &[TestItem], props: &[Vec], refs: &[TestRef]) -> Vec { + let mut iinf = vec![0, 0, 0, 0]; + iinf.extend_from_slice(&(items.len() as u16).to_be_bytes()); + let mut iloc = vec![1, 0, 0, 0, 0x44, 0x00]; + iloc.extend_from_slice(&(items.len() as u16).to_be_bytes()); + let mut ipma = vec![0, 0, 0, 0]; + ipma.extend_from_slice(&(items.len() as u32).to_be_bytes()); + let mut idat = Vec::new(); + for &(id, kind, data, assoc) in items { + iinf.extend(infe(id, kind)); + iloc.extend_from_slice(&id.to_be_bytes()); + iloc.extend_from_slice(&[0, 1, 0, 0, 0, 1]); // method 1, data ref 0, one extent + iloc.extend_from_slice(&(idat.len() as u32).to_be_bytes()); + iloc.extend_from_slice(&(data.len() as u32).to_be_bytes()); + idat.extend_from_slice(data); + ipma.extend_from_slice(&id.to_be_bytes()); + ipma.push(assoc.len() as u8); + ipma.extend(assoc.iter().map(|&p| p | 0x80)); + } + let mut iref = vec![0, 0, 0, 0]; + for &(kind, from, to) in refs { + let mut b = from.to_be_bytes().to_vec(); + b.extend_from_slice(&(to.len() as u16).to_be_bytes()); + to.iter().for_each(|t| b.extend_from_slice(&t.to_be_bytes())); + iref.extend(bx(kind, &b)); + } + let mut meta = vec![0, 0, 0, 0]; + meta.extend(fbx(b"hdlr", b"\0\0\0\0pict\0\0\0\0\0\0\0\0\0\0\0\0\0")); + meta.extend(fbx(b"pitm", &[0, 1])); + meta.extend(bx(b"iinf", &iinf)); + meta.extend(bx(b"iloc", &iloc)); + meta.extend(bx(b"iref", &iref)); + meta.extend(bx(b"iprp", &[bx(b"ipco", &props.concat()), bx(b"ipma", &ipma)].concat())); + meta.extend(bx(b"idat", &idat)); + let mut file = bx(b"ftyp", b"avif\0\0\0\0avifmif1miaf"); + file.extend(bx(b"meta", &meta)); + file + } + + #[test] + fn libaom_420_limited_range_matches_dav1d() { + let img = crate::decode(QUAD_420).unwrap(); + assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (33, 17, ChannelLayout::Rgb, SampleType::U8)); + assert_eq!(img.cicp.map(|c| (c.matrix, c.full_range)), Some((1, false)), "BT.709 matrix, limited range"); + assert_eq!(crate::probe_size(QUAD_420).unwrap(), (33, 17)); + // The AV1 picture is libdav1d's, sample for sample. + let c = parse(QUAD_420).unwrap(); + let p = decode_av1(&c, c.primary, &Limits::default()).unwrap(); + assert_eq!((p.w, p.h, p.bits, p.ssx, p.ssy, p.mono), (33, 17, 8, 1, 1, false)); + let planes: Vec = p.planes.iter().flatten().map(|&v| v as u8).collect(); + assert_eq!(planes, QUAD_420_DAV1D); + let ours = rgb8(&img); + // The quadrant centres are the source's colours, give or take the lossy coding. + for (x, y) in [(5usize, 4usize), (27, 4), (5, 13), (27, 13)] { + let i = (y * 33 + x) * 3; + let d = mean_abs_diff(&ours[i..i + 3], &QUAD_SOURCE[i..i + 3]); + assert!(d < 12.0, "({x}, {y}): {:?} vs {:?}", &ours[i..i + 3], &QUAD_SOURCE[i..i + 3]); + } + } + + #[test] + fn libaom_lossless_identity_matrix_is_exact() { + let img = crate::decode(QUAD_GBR_LOSSLESS).unwrap(); + assert_eq!((img.width(), img.height(), img.sample_type()), (33, 17, SampleType::U8)); + assert_eq!(img.cicp.map(|c| (c.matrix, c.full_range)), Some((0, true))); + assert_eq!(rgb8(&img), QUAD_SOURCE); + } + + #[test] + fn limits_are_checked_before_decoding() { + let tight = |max_pixels, max_alloc| DecodeOptions { limits: Limits { max_pixels, max_alloc, ..Limits::default() }, ..Default::default() }; + assert!(matches!(crate::decode_with(QUAD_420, &tight(33 * 17 - 1, u64::MAX)), Err(CodecError::LimitExceeded(_)))); + // The 16-bit intermediate counts: 33 × 17 RGB at 2 bytes. + assert!(matches!(crate::decode_with(QUAD_420, &tight(u64::MAX, 33 * 17 * 6 - 1)), Err(CodecError::LimitExceeded(_)))); + assert!(crate::decode_with(QUAD_420, &tight(33 * 17, Limits::default().max_alloc)).is_ok()); + // `max_alloc` is the AV1 decoder's picture-store budget too. + assert!(crate::decode_with(QUAD_420, &tight(33 * 17, 33 * 17 * 6)).is_err()); + // A declared size over the limits fails from the boxes alone. + let (av1, cfg) = av1_of(QUAD_420); + let huge = build(&[(1, b"av01", &av1, &[1, 2])], &[ispe(1 << 20, 1 << 20), bx(b"av1C", &[&[0x81, 0, 0, 0][..], &cfg].concat())], &[]); + assert!(matches!(crate::decode(&huge), Err(CodecError::LimitExceeded(_)))); + assert!(matches!(crate::probe_size(&huge), Err(CodecError::LimitExceeded(_)))); + } + + #[test] + fn a_picture_that_differs_from_its_ispe_is_refused() { + let (av1, cfg) = av1_of(QUAD_420); + let file = build(&[(1, b"av01", &av1, &[1, 2])], &[ispe(32, 17), bx(b"av1C", &[&[0x81, 0, 0, 0][..], &cfg].concat())], &[]); + assert!(matches!(crate::decode(&file), Err(CodecError::Malformed { .. }))); + } + + #[test] + fn a_grid_is_assembled_from_its_tiles_and_cropped() { + // Two copies of the 33 × 17 picture side by side, cropped to 60 × 15 by the grid. + let (av1, cfg) = av1_of(QUAD_GBR_LOSSLESS); + let av1c = bx(b"av1C", &[&[0x81, 0, 0, 0][..], &cfg].concat()); + let grid = [0u8, 0, 0, 1, 0, 60, 0, 15]; + let file = build( + &[(1, b"grid", &grid, &[1]), (2, b"av01", &av1, &[2, 3]), (3, b"av01", &av1, &[2, 3])], + &[ispe(60, 15), ispe(33, 17), av1c, fbx(b"pixi", &[3, 8, 8, 8])], + &[(b"dimg", 1, &[2, 3])], + ); + assert_eq!(crate::probe_size(&file).unwrap(), (60, 15)); + let img = crate::decode(&file).unwrap(); + assert_eq!((img.width(), img.height()), (60, 15)); + let got = rgb8(&img); + for y in 0..15 { + for x in 0..60 { + let s = (y * 33 + x % 33) * 3; + assert_eq!(&got[(y * 60 + x) * 3..][..3], &QUAD_SOURCE[s..s + 3], "({x}, {y})"); + } + } + // Tiles that can't cover the grid are refused. + let wide = [0u8, 0, 0, 1, 0, 70, 0, 15]; + let file = build( + &[(1, b"grid", &wide, &[1]), (2, b"av01", &av1, &[2, 3]), (3, b"av01", &av1, &[2, 3])], + &[ispe(70, 15), ispe(33, 17), bx(b"av1C", &[&[0x81, 0, 0, 0][..], &cfg].concat())], + &[(b"dimg", 1, &[2, 3])], + ); + assert!(matches!(crate::decode(&file), Err(CodecError::Malformed { .. }))); + } + + #[test] + fn container_rotation_and_mirror_are_applied_unless_kept() { + let (av1, cfg) = av1_of(QUAD_GBR_LOSSLESS); + let av1c = bx(b"av1C", &[&[0x81, 0, 0, 0][..], &cfg].concat()); + let file = build(&[(1, b"av01", &av1, &[1, 2, 3, 4])], &[ispe(33, 17), av1c, bx(b"irot", &[1]), bx(b"imir", &[0])], &[]); + assert_eq!(crate::probe_size(&file).unwrap(), (17, 33)); + let img = crate::decode(&file).unwrap(); + assert_eq!((img.width(), img.height()), (17, 33)); + let got = rgb8(&img); + // Rotated 90° anticlockwise, then top and bottom exchanged: a transpose, (x, y) from (y, x). + for y in 0..33 { + for x in 0..17 { + let s = (x * 33 + y) * 3; + assert_eq!(&got[(y * 17 + x) * 3..][..3], &QUAD_SOURCE[s..s + 3], "({x}, {y})"); + } + } + let kept = crate::decode_with(&file, &DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap(); + assert_eq!((kept.width(), kept.height(), rgb8(&kept)), (33, 17, QUAD_SOURCE.to_vec())); + } + + #[test] + fn transforms_rotate_mirror_and_crop() { + // 3 × 2, one channel, value = index + let cv = || Canvas { w: 3, h: 2, ch: 1, data: (0..6).collect() }; + let r = apply_transforms(cv(), &[Transform::Rot(1)]); + // anticlockwise: the right column becomes the top row + assert_eq!((r.w, r.h, r.data), (2, 3, vec![2, 5, 1, 4, 0, 3])); + // axis 0 exchanges top and bottom, axis 1 left and right (libheif's and libavif's reading) + assert_eq!(apply_transforms(cv(), &[Transform::Mir(0)]).data, vec![3, 4, 5, 0, 1, 2]); + assert_eq!(apply_transforms(cv(), &[Transform::Mir(1)]).data, vec![2, 1, 0, 5, 4, 3]); + let c = apply_transforms(cv(), &[Transform::Clap { w: 1.0, h: 1.0, dx: 1.0, dy: 0.0 }]); + assert_eq!((c.w, c.h), (1, 1)); + assert!(c.data == [2] || c.data == [5]); + assert_eq!(displayed(3, 2, &[Transform::Rot(3)]), (2, 3)); + // A non-finite or empty aperture is ignored. + assert_eq!(apply_transforms(cv(), &[Transform::Clap { w: f64::NAN, h: 1.0, dx: 0.0, dy: 0.0 }]).data.len(), 6); + } + + #[test] + fn hostile_files_are_errors_not_panics() { + for file in [QUAD_420, QUAD_GBR_LOSSLESS] { + // every truncation + for n in 0..file.len() { + let _ = crate::probe_size(&file[..n]); + let _ = crate::decode(&file[..n]); + } + // byte flips through the container and the AV1 data + for i in 0..file.len() { + for x in [0x01u8, 0x80, 0xff] { + let mut m = file.to_vec(); + m[i] ^= x; + let _ = crate::probe_size(&m); + let _ = crate::decode(&m); + } + } + } + // box sizes that point far outside the file + let mut b = bx(b"ftyp", b"avif\0\0\0\0avif"); + b.extend_from_slice(b"\xff\xff\xff\xffmeta\0\0\0\0"); + assert!(crate::decode(&b).is_err()); + let mut b = bx(b"ftyp", b"avif\0\0\0\0avif"); + b.extend_from_slice(b"\0\0\0\x01meta\xff\xff\xff\xff\xff\xff\xff\xff"); + assert!(crate::probe_size(&b).is_err()); + } + + #[test] + fn an_animated_avif_is_unsupported() { + let mut b = bx(b"ftyp", b"avis\0\0\0\0avis"); + b.extend(bx(b"moov", &[])); + assert!(matches!(crate::decode(&b), Err(CodecError::Unsupported { format: Format::Avif, .. }))); + } + + #[test] + fn zero_width_iloc_extents_are_capped() { + // iloc v2 with zero-byte offset/length/base/index fields: each extent costs no input bytes + let mut iloc = vec![2, 0, 0, 0, 0, 0]; + iloc.extend_from_slice(&3u32.to_be_bytes()); + for id in 1..=3u32 { + iloc.extend_from_slice(&id.to_be_bytes()); + iloc.extend_from_slice(&[0, 0, 0, 0, 0xff, 0xff]); // method, data ref, 65535 extents + } + let mut meta = vec![0, 0, 0, 0]; + meta.extend(bx(b"iloc", &iloc)); + let mut b = bx(b"ftyp", b"avif\0\0\0\0avif"); + b.extend(bx(b"meta", &meta)); + assert!(matches!(crate::decode(&b), Err(CodecError::Malformed { message, .. }) if message.contains("extents"))); + } + + #[test] + fn item_data_larger_than_the_file_is_refused() { + // one item whose 1000 extents each repeat the whole file (length 0 = to the end) + let mut iloc = vec![1, 0, 0, 0, 0x44, 0]; + iloc.extend_from_slice(&1u16.to_be_bytes()); + iloc.extend_from_slice(&[0, 1, 0, 0, 0, 0]); // id 1, method 0, data ref 0 + iloc.extend_from_slice(&1000u16.to_be_bytes()); + for _ in 0..1000 { + iloc.extend_from_slice(&[0; 8]); + } + let mut meta = vec![0, 0, 0, 0]; + meta.extend(fbx(b"pitm", &[0, 1])); + meta.extend(bx(b"iloc", &iloc)); + let mut b = bx(b"ftyp", b"avif\0\0\0\0avif"); + b.extend(bx(b"meta", &meta)); + let c = parse(&b).unwrap(); + assert!(matches!(c.data(1), Err(CodecError::Malformed { message, .. }) if message.contains("larger than the file"))); + } + + #[test] + fn unsupported_matrices_are_errors() { + for m in [8u16, 11, 13, 14] { + assert!(matches!(coefficients(m, 1), Err(CodecError::Unsupported { .. })), "{m}"); + } + assert_eq!(coefficients(0, 1).unwrap(), None); + assert!(coefficients(12, 9).unwrap().is_some()); + } + + #[cfg(feature = "avif")] + mod with_the_encoder { + use super::*; + use crate::codecs::avif::{HdrTransfer, encode_avif_hdr, encode_avif_with_speed}; + + /// A smooth test card with a saturated band. + fn card(w: usize, h: usize, ch: usize) -> Vec { + (0..w * h) + .flat_map(|i| { + let (x, y) = (i % w, i / w); + let px = + [(x * 255 / (w - 1)) as u8, (y * 255 / (h - 1)) as u8, if x < w / 4 { 230 } else { 40 }, (255 - x * 200 / (w - 1)) as u8]; + px.into_iter().take(ch) + }) + .collect() + } + + #[test] + fn ravif_rgb_round_trip_is_10_bit() { + let (w, h) = (48, 32); + let src = Image::from_u8(w as u32, h as u32, ChannelLayout::Rgb, card(w, h, 3)).unwrap(); + let file = encode_avif_with_speed(&src, 95, 10).unwrap(); + let img = crate::decode(&file).unwrap(); + assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (48, 32, ChannelLayout::Rgb, SampleType::U16)); + let diff = mean_abs_diff(&rgb8(&img), &card(w, h, 3)); + assert!(diff < 2.5, "mean difference {diff}"); + } + + #[test] + fn ravif_alpha_round_trips() { + let (w, h) = (40, 24); + let src = Image::from_u8(w as u32, h as u32, ChannelLayout::Rgba, card(w, h, 4)).unwrap(); + let file = encode_avif_with_speed(&src, 95, 10).unwrap(); + let img = crate::decode(&file).unwrap(); + assert_eq!(img.layout(), ChannelLayout::Rgba); + let got = img.to_rgba8(); + let want = card(w, h, 4); + let alpha = |v: &[u8]| v.iter().skip(3).step_by(4).copied().collect::>(); + let diff = mean_abs_diff(&alpha(&got), &alpha(&want)); + assert!(diff < 2.0, "alpha mean difference {diff}"); + } + + #[test] + fn exif_is_read_from_its_item() { + let mut src = Image::from_u8(16, 8, ChannelLayout::Rgb, card(16, 8, 3)).unwrap(); + let exif = b"II*\0\x08\0\0\0\0\0\0\0\0\0".to_vec(); + src.meta.exif = Some(exif.clone()); + let file = encode_avif_with_speed(&src, 80, 10).unwrap(); + let img = crate::decode_with(&file, &DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap(); + assert_eq!(img.meta.exif.as_deref(), Some(exif.as_slice())); + } + + #[test] + fn hdr_round_trips_keep_highlights_above_white() { + let (w, h) = (32usize, 16usize); + // linear BT.2020: left half grey 0.18, right half 4 × SDR white + let rgb: Vec = (0..w * h).flat_map(|i| if i % w < w / 2 { [0.18f32; 3] } else { [4.0; 3] }).collect(); + for t in [HdrTransfer::Pq, HdrTransfer::Hlg] { + let file = encode_avif_hdr(w as u32, h as u32, &rgb, 98, 10, None, t).unwrap(); + let img = crate::decode(&file).unwrap(); + assert_eq!(img.sample_type(), SampleType::F32, "{t:?}"); + assert_eq!(img.cicp.map(|c| c.transfer), Some(Cicp::TRANSFER_LINEAR), "{t:?}"); + let v = img.to_f32_samples().unwrap(); + let at = |x: usize| v[(8 * w + x) * 3 + 1]; + assert!((at(4) / 0.18 - 1.0).abs() < 0.08, "{t:?}: grey {}", at(4)); + assert!((at(w - 4) / 4.0 - 1.0).abs() < 0.08, "{t:?}: highlight {}", at(w - 4)); + } + } + } +} diff --git a/crates/image-codecs/src/format.rs b/crates/image-codecs/src/format.rs index ecaf945cd..fbdb412b1 100644 --- a/crates/image-codecs/src/format.rs +++ b/crates/image-codecs/src/format.rs @@ -26,7 +26,8 @@ pub enum Format { OpenExr, /// Radiance RGBE. Hdr, - /// Encode-only behind the `avif` feature; see [`ASYMMETRIC_EXCEPTIONS`]. + /// Encoded behind the `avif` feature and decoded behind `avif-decode` (native targets); see + /// [`ASYMMETRIC_EXCEPTIONS`] for a build with only one of them. Avif, /// HEIF/HEIC (HEVC-coded): read-only; see [`ASYMMETRIC_EXCEPTIONS`]. Heif, @@ -89,8 +90,10 @@ pub const ASYMMETRIC_EXCEPTIONS: &[(Format, &str)] = &[ ), ( Format::Avif, - "AVIF encode uses ravif (pure Rust) but decoding requires dav1d (C); read stays unsupported \ - until a pure-Rust AV1 decoder is viable. Only enabled with the non-default `avif` feature.", + "AVIF encode (ravif and rav1e, pure Rust) is behind the non-default `avif` feature and AVIF \ + decode (craft-video's clean-room pure-Rust AV1 decoder) behind the non-default \ + `avif-decode` feature, native targets only (on wasm32 a decoder panic would abort). A build \ + with only one of them reads or writes AVIF, not both.", ), (Format::Jxl, "Lossless JPEG XL export uses the pure-Rust zune-jpegxl encoder; JPEG XL reading is tracked separately and is not yet available."), ( @@ -298,7 +301,9 @@ pub fn caps(format: Format) -> FormatCaps { Format::Qoi => base, Format::OpenExr => FormatCaps { depths: &[S::F16, S::F32], layouts: RGB_GRAY, ..base }, Format::Hdr => FormatCaps { depths: &[S::F32], layouts: &[L::Rgb], alpha: false, lossy: true, ..base }, - Format::Avif => FormatCaps { read: false, write: cfg!(feature = "avif"), lossy: true, ..base }, + Format::Avif => { + FormatCaps { read: cfg!(all(feature = "avif-decode", not(target_arch = "wasm32"))), write: cfg!(feature = "avif"), lossy: true, ..base } + } Format::Heif => { FormatCaps { read: cfg!(feature = "heif"), write: false, depths: &[S::U8, S::U16], icc: true, exif: true, xmp: true, lossy: true, ..base } } diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index 6516b9ff2..a4480b2b8 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -106,7 +106,11 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res Format::Heif => heif::decode(bytes, l, opts.keep_orientation), Format::Jp2 => jp2::decode(bytes, l), Format::Jxl => Err(CodecError::unsupported(format, "JPEG XL reading is not yet available; export is supported")), - Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr | Format::Avif => via_image::decode(format, bytes, l), + #[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))] + Format::Avif => codecs::avif_decode::decode(bytes, l, opts.keep_orientation), + #[cfg(not(all(feature = "avif-decode", not(target_arch = "wasm32"))))] + Format::Avif => via_image::decode(format, bytes, l), + Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr => via_image::decode(format, bytes, l), }?; finish_format(format, bytes, img, opts) } @@ -163,10 +167,11 @@ fn finish_format(format: Format, bytes: &[u8], mut img: Image, opts: &DecodeOpti } // Turn the pixels upright, like Photoshop: a TIFF records it in the decoded page's own // directory, the others in their EXIF block. The metadata is rewritten to Orientation = 1 - // on the way. HEIF keeps it in its container, and its decoder has already applied it. + // on the way. HEIF and AVIF keep it in their container, and their decoders have already + // applied it. let o = match format { Format::Tiff => tiff::orientation(bytes, None), - Format::Heif => 1, + Format::Heif | Format::Avif => 1, Format::Dpx => dpx::orientation(bytes), Format::Cineon => dpx::cineon_orientation(bytes), _ => img.meta.exif.as_deref().map_or(1, exif_orientation), @@ -195,10 +200,11 @@ pub fn decode_png_rows>( } /// The upright pixel size [`decode`] would return, read from the file's header without decoding -/// pixels, for WebP, OpenEXR (flat) and the GIF/BMP/TGA/ICO/QOI/HDR family. It runs the same +/// pixels, for WebP, OpenEXR (flat), AVIF (with `avif-decode`) and the GIF/BMP/TGA/ICO/QOI/HDR family. It runs the same /// header reads and size checks as the decoder (default [`Limits`]) and swaps the size for an -/// EXIF orientation of 5–8, so wherever [`decode`] succeeds the two agree. Other formats (and -/// deep EXR or AVIF) return [`CodecError::Unsupported`]: decode those to learn their size. +/// EXIF orientation of 5–8 (AVIF: its container's `clap` and `irot`), so wherever [`decode`] +/// succeeds the two agree. Other formats (and deep EXR) return [`CodecError::Unsupported`]: decode +/// those to learn their size. pub fn probe_size(bytes: &[u8]) -> Result<(u32, u32), CodecError> { let format = detect(bytes).ok_or(CodecError::UnknownFormat)?; if !caps(format).read { @@ -208,7 +214,11 @@ pub fn probe_size(bytes: &[u8]) -> Result<(u32, u32), CodecError> { let (w, h, exif) = match format { Format::WebP => webp::probe(bytes, l)?, Format::OpenExr => exr::probe(bytes, l).map(|(w, h)| (w, h, None))?, - Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr | Format::Avif => via_image::probe(format, bytes, l)?, + #[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))] + Format::Avif => codecs::avif_decode::probe(bytes, l).map(|(w, h)| (w, h, None))?, + #[cfg(not(all(feature = "avif-decode", not(target_arch = "wasm32"))))] + Format::Avif => via_image::probe(format, bytes, l)?, + Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr => via_image::probe(format, bytes, l)?, _ => return Err(CodecError::unsupported(format, "no header-only size probe")), }; let o = exif.as_deref().map_or(1, exif_orientation); diff --git a/crates/image-codecs/tests/fixtures/avif/README.md b/crates/image-codecs/tests/fixtures/avif/README.md new file mode 100644 index 000000000..b8b3a8c2a --- /dev/null +++ b/crates/image-codecs/tests/fixtures/avif/README.md @@ -0,0 +1,12 @@ +# AVIF decode fixtures + +Small AVIF files from an independent encoder (libaom through FFmpeg 7.1), for `src/codecs/avif_decode.rs`. + +- `quad-source.rgb`: the 33×17 source, raw RGB24 four-colour quadrants (red, green, blue, near-white). +- `quad-420-bt709.avif`: 8-bit 4:2:0, limited range, BT.709 matrix, odd size (FFmpeg leaves the primaries and transfer unspecified): + `ffmpeg -f rawvideo -pix_fmt rgb24 -s 33x17 -i quad-source.rgb -c:v libaom-av1 -crf 12 -still-picture 1 -pix_fmt yuv420p -colorspace bt709 -color_primaries bt709 -color_trc iec61966-2-1 -color_range tv quad-420-bt709.avif` +- `quad-420-bt709.dav1d.yuv`: that file's AV1 picture as libdav1d decodes it, planar 8-bit 4:2:0 (Y, then Cb, then Cr): + `ffmpeg -c:v libdav1d -i quad-420-bt709.avif -f rawvideo -pix_fmt yuv420p quad-420-bt709.dav1d.yuv` +- `quad-444-gbr-lossless.avif`: lossless 8-bit 4:4:4 with the identity (GBR) matrix, full range: + `ffmpeg -f rawvideo -pix_fmt rgb24 -s 33x17 -i quad-source.rgb -c:v libaom-av1 -crf 0 -still-picture 1 -pix_fmt gbrp -colorspace rgb -color_range pc quad-444-gbr-lossless.avif`. + It decodes to `quad-source.rgb` exactly. diff --git a/crates/image-codecs/tests/fixtures/avif/quad-420-bt709.avif b/crates/image-codecs/tests/fixtures/avif/quad-420-bt709.avif new file mode 100644 index 0000000000000000000000000000000000000000..419196cb178743782cf147ea50d111e2b8642d42 GIT binary patch literal 369 zcmZQzU{FXasVqn=%S>Yc0uY^>nP!-qnV9D5Xy^nK`k9+rk_eIm0{x7XoFWL5fuSHX zxdg@r(E&hyX;G?2VqQv4Dp(a?L1sxVSP3JLl*`P?PX_T^7#J9V5JWOCDgx;MAlAst z%mX_CEF%Tvq%r}8LA=B=14HNhoP4mES(ybz1we7{%!1^6AT5$vT#yP<2m~Or1c6wf zAhRM9B*4tU!2zUr63Yyo8yR>&Lc+=UIYoKNITZ{{KnOIQfk8a8AU6@j00N-HSQwd_ zS%A!l+?2!;E=~zih2|6v4i*6;XO{y&vw`U4k2c-I-AWygkL2ynp7_}AP4*4enEOQm q)z5b1%rN?KQ|H$?;i_Mo{+`I#w8y3Pf9fWWdacYc0uY^>nP!-qnV9D5Xy^zO`k9+rk_eIm0{x7XoFWL5fuSHX zxdg@r(E&hyX;G?2VqQv4Dp(a?L1sxVSP3JLl*`P?PX_T^7#J9V5JWOCDgtQ>AlAst z%mX_CEF%Tvq%r}8LA=B=14HNhoP4mES(ybz1we7{%!1^6AT5$vT#yP<2m~Or1c6wf zAhRM9B*4tU!2zUr63Yyo8x?>o1_t5e{G6h^jRAvYzlgp0#MRG~S8BZJXM2k7n(sn>EClKWl Date: Sun, 15 Nov 2026 18:08:34 +0100 Subject: [PATCH 02/19] craft-raw: one lookup per Huffman symbol and a fixed predictor per run in LJ92 and NEF/RW2 decoding Co-Authored-By: Claude Sonnet 5.5 --- crates/raw/src/huffman.rs | 107 ++++++++++++++++++--- crates/raw/src/ljpeg.rs | 169 ++++++++++++++++++++++++++-------- crates/raw/src/ljpeg/tests.rs | 88 ++++++++++++++++++ 3 files changed, 311 insertions(+), 53 deletions(-) diff --git a/crates/raw/src/huffman.rs b/crates/raw/src/huffman.rs index cce27237b..85b242be1 100644 --- a/crates/raw/src/huffman.rs +++ b/crates/raw/src/huffman.rs @@ -4,16 +4,19 @@ use crate::{RawError, Result}; -/// A canonical-by-table Huffman decoder with a 12-bit lookup. +/// A canonical-by-table Huffman decoder with a 12-bit lookup that also holds the difference when code and +/// additional bits fit in 12 bits. pub(crate) struct Huffman { - /// Indexed by the next 12 bits: (symbol, length); length 0 = invalid. - lut: Vec<(u8, u8)>, + /// Indexed by the next 12 bits. Per entry: bits 0-7 the symbol, bits 8-11 the code length (0 = invalid), bits + /// 12-15 the length of code plus additional bits when both fit in 12 bits (0 = they do not), bits 16-31 the + /// signed difference in that case. + lut: Box<[u32; 4096]>, } impl Huffman { /// From `(left-aligned 12-bit code, length)` per symbol. pub fn new(codes: &[(u16, u8)]) -> Result { - let mut lut = vec![(0u8, 0u8); 4096]; + let mut lut = Box::new([0u32; 4096]); for (sym, &(code, len)) in codes.iter().enumerate() { if len == 0 { continue; @@ -21,10 +24,19 @@ impl Huffman { if len > 12 || sym > 16 { return Err(RawError::Unsupported(format!("PEF Huffman code of length {len}"))); } + let (len, s) = (len as u32, sym as u32); let first = (code >> (12 - len) << (12 - len)) as usize; - for e in &mut lut[first..first + (1 << (12 - len))] { - if e.1 == 0 { - *e = (sym as u8, len); + for (i, e) in lut[first..first + (1 << (12 - len))].iter_mut().enumerate() { + if (*e >> 8) & 15 == 0 { + let mut entry = (len << 8) | s; + if s == 0 { + entry |= len << 12; + } else if len + s <= 12 { + let v = (((first + i) >> (12 - len - s)) & ((1 << s) - 1)) as i32; + let d = if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v }; + entry |= ((len + s) << 12) | ((d as i16 as u16 as u32) << 16); + } + *e = entry; } } } @@ -85,21 +97,90 @@ impl<'a> Bits<'a> { } } -/// Decode one difference: Huffman category then T.81 additional bits. +/// Decode one difference: Huffman category then T.81 additional bits, in one lookup when both fit in 12 bits. #[inline] pub(crate) fn diff(bits: &mut Bits, h: &Huffman) -> Option { - let (sym, len) = h.lut[bits.peek(12) as usize]; + let e = h.lut[(bits.peek(12) & 0xfff) as usize]; + let len = (e >> 8) & 15; if len == 0 { return None; } - bits.skip(len as u32); - let s = sym as u32; - if s == 0 { - return Some(0); + let total = (e >> 12) & 15; + if total > 0 { + bits.skip(total); + return Some((e as i32) >> 16); } + bits.skip(len); + let s = e & 0xff; if s > 16 { return None; } let v = bits.get(s) as i32; Some(if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Canonical left-aligned 12-bit codes for symbols 0..lens.len() (shorter codes first, ties by symbol). + fn canonical(lens: &[u8]) -> Vec<(u16, u8)> { + let mut order: Vec = (0..lens.len()).collect(); + order.sort_by_key(|&s| (lens[s], s)); + let mut codes = vec![(0u16, 0u8); lens.len()]; + let (mut code, mut prev) = (0u32, 0u8); + for s in order { + code <<= lens[s] - prev; + prev = lens[s]; + codes[s] = ((code << (12 - lens[s])) as u16, lens[s]); + code += 1; + } + codes + } + + fn bit(src: &[u8], i: usize) -> u32 { + src.get(i / 8).map_or(0, |b| ((b >> (7 - i % 8)) & 1) as u32) + } + + /// Reference: match the code bit by bit, then read the category's additional bits (T.81 F.1.2.1). + fn reference(src: &[u8], pos: &mut usize, codes: &[(u16, u8)]) -> Option { + let window: u32 = (0..12).fold(0, |a, k| (a << 1) | bit(src, *pos + k)); + let (sym, len) = codes.iter().enumerate().find_map(|(s, &(c, l))| { + (l > 0 && (c as u32 >> (12 - l)) == window >> (12 - l)).then_some((s as u32, l as usize)) + })?; + *pos += len; + if sym == 0 { + return Some(0); + } + let v = (0..sym as usize).fold(0u32, |a, k| (a << 1) | bit(src, *pos + k)) as i32; + *pos += sym as usize; + Some(if v < 1 << (sym - 1) { v - (1 << sym) + 1 } else { v }) + } + + #[test] + fn diff_matches_bitwise_reference_for_every_category() { + // categories 0..=16, code lengths up to 12 so that short codes combine with their bits and long ones do not + let lens = [2u8, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12]; + let codes = canonical(&lens); + let h = Huffman::new(&codes).unwrap(); + let mut s = 0x1234_5678_9abc_def0u64; + for _ in 0..40 { + let src: Vec = (0..400) + .map(|_| { + s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (s >> 40) as u8 + }) + .collect(); + let mut bits = Bits::new(&src); + let mut pos = 0; + while pos < src.len() * 8 - 64 { + let want = reference(&src, &mut pos, &codes); + assert_eq!(diff(&mut bits, &h), want); + if want.is_none() { + break; + } + assert_eq!(bits.consumed_bits(), pos); + } + } + } +} diff --git a/crates/raw/src/ljpeg.rs b/crates/raw/src/ljpeg.rs index 56b07a533..ab27ed57f 100644 --- a/crates/raw/src/ljpeg.rs +++ b/crates/raw/src/ljpeg.rs @@ -32,15 +32,17 @@ fn err(s: &str) -> RawError { /// Huffman decoding table (T.81 Annex C "BITS"/"HUFFVAL"). #[derive(Clone, Debug)] pub struct Huffman { - /// For peeks of `LOOKUP` bits: (code length, symbol); length 0 = slow path. - fast: Vec<(u8, u8)>, + /// Indexed by the next `LOOKUP` bits; length 0 = slow path. Per entry: bits 0-7 the symbol, bits 8-11 the code + /// length, bits 12-15 the length of code plus extra bits when both fit in the lookup (0 = they do not), and + /// bits 16-31 the signed difference in that case. + fast: Vec, maxcode: [i32; 18], valptr: [i32; 17], mincode: [i32; 17], values: Vec, } -const LOOKUP: u32 = 10; +const LOOKUP: u32 = 12; impl Huffman { /// Build from the 16 code-length counts and the symbol list. @@ -52,7 +54,7 @@ impl Huffman { let mut maxcode = [-1i32; 18]; let mut valptr = [0i32; 17]; let mut mincode = [0i32; 17]; - let mut fast = vec![(0u8, 0u8); 1 << LOOKUP]; + let mut fast = vec![0u32; 1 << LOOKUP]; let mut code = 0i32; let mut k = 0usize; for len in 1..=16usize { @@ -63,9 +65,10 @@ impl Huffman { if len as u32 <= LOOKUP { let shift = LOOKUP - len as u32; let base = (code as usize) << shift; + let ssss = values[k] as u32; for i in 0..(1usize << shift) { if let Some(slot) = fast.get_mut(base + i) { - *slot = (len as u8, values[k]); + *slot = fast_entry(len as u32, ssss, base + i); } } } @@ -84,12 +87,41 @@ impl Huffman { #[inline] pub fn decode(&self, br: &mut B) -> Result { - let peek = br.peek(LOOKUP); - let (len, sym) = self.fast[peek as usize]; + let e = self.fast[br.peek(LOOKUP) as usize]; + let len = (e >> 8) & 15; if len > 0 { - br.consume(len as u32); - return Ok(sym); + br.consume(len); + return Ok(e as u8); } + self.decode_slow(br) + } + + /// One symbol and its T.81 difference in a single lookup when code and extra bits fit in `LOOKUP` bits; the same + /// as [`Huffman::decode`] followed by [`diff_value`]. `strict`: a category above 16 is an error (subsampled + /// scans) instead of reading 16 bits. + #[inline] + pub fn decode_diff(&self, br: &mut B, strict: bool) -> Result { + let e = self.fast[br.peek(LOOKUP) as usize]; + let len = (e >> 8) & 15; + let total = (e >> 12) & 15; + if total > 0 { + br.consume(total); + return Ok((e as i32) >> 16); + } + let ssss = if len > 0 { + br.consume(len); + e as u8 + } else { + self.decode_slow(br)? + }; + if strict && ssss > 16 { + return Err(err("invalid difference category")); + } + Ok(diff_value(br, ssss)) + } + + #[cold] + fn decode_slow(&self, br: &mut B) -> Result { let mut code = br.peek(LOOKUP + 1) as i32; let mut len = LOOKUP as usize + 1; while len <= 16 && code > self.maxcode[len] { @@ -105,6 +137,21 @@ impl Huffman { } } +/// Lookup entry for a code of `len` bits (`len <= LOOKUP`) naming category `ssss`, at window position `index` +/// (the code is the top `len` bits of the `LOOKUP`-bit window). See [`Huffman::fast`]. +fn fast_entry(len: u32, ssss: u32, index: usize) -> u32 { + let base = (len << 8) | ssss; + if ssss == 0 { + return base | (len << 12); + } + if ssss > 11 || len + ssss > LOOKUP { + return base; + } + let v = ((index >> (LOOKUP - len - ssss)) & ((1 << ssss) - 1)) as i32; + let d = if v < (1 << (ssss - 1)) { v - (1 << ssss) + 1 } else { v }; + base | ((len + ssss) << 12) | ((d as i16 as u16 as u32) << 16) +} + /// A source of entropy-coded bits, first bit most significant. pub trait Bits { /// The next `n` (1..=32) bits without consuming them; zeros past the end. @@ -614,11 +661,7 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) - Prediction::Sequential if mx == 0 && dx == 0 && dy == 0 => row_first[c], Prediction::Sequential => last[c], }; - let ssss = tables[c].decode(&mut br)?; - if ssss > 16 { - return Err(err("invalid difference category")); - } - let value = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16; + let value = ((pred + tables[c].decode_diff(&mut br, true)?) & 0xffff) as u16; plane[i] = value; last[c] = i32::from(value); if mx == 0 && dx == 0 && dy == 0 { @@ -642,6 +685,44 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) - Ok(FrameSubsampled { width: h.width, height: h.height, vertical_subsampling: vertical, planes }) } +/// T.81 Table H.1 predictor `pred` (1..=7; 0 = none) from the left, above and above-left neighbours. +#[inline(always)] +fn predict(pred: u8, ra: i32, rb: i32, rc: i32) -> i32 { + match pred { + 0 => 0, + 1 => ra, + 2 => rb, + 3 => rc, + 4 => ra + rb - rc, + 5 => ra + ((rb - rc) >> 1), + 6 => rb + ((ra - rc) >> 1), + _ => (ra + rb) >> 1, + } +} + +/// Decode the pixels `x0..x1` (`x0 >= 1`) of one row with the fixed predictor `P`. `prev` is empty on the first +/// line, which only predicts from the left (`P = 1`). +#[inline(always)] +fn decode_run( + tables: &[&Huffman], + br: &mut BitReader<'_>, + cur: &mut [u16], + prev: &[u16], + nc: usize, + x0: usize, + x1: usize, +) -> Result<(), RawError> { + for x in x0..x1 { + for (c, table) in tables.iter().enumerate() { + let i = x * nc + c; + let ra = cur[i - nc] as i32; + let p = if P <= 1 { predict(P, ra, 0, 0) } else { predict(P, ra, prev[i] as i32, prev[i - nc] as i32) }; + cur[i] = ((p + table.decode_diff(br, false)?) & 0xffff) as u16; + } + } + Ok(()) +} + /// Decode a lossless JPEG stream. `max_samples` bounds the allocation (hostile headers). pub fn decode(d: &[u8], max_samples: usize) -> Result { let h = parse_header(d)?; @@ -680,19 +761,23 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result { let (prev_rows, cur_rows) = out.split_at_mut(y * row_len); let cur = &mut cur_rows[..row_len]; let prev = if y > 0 { &prev_rows[(y - 1) * row_len..] } else { &[][..] }; - for x in 0..w { - if h.restart > 0 { - if restart_left == 0 { - if !br.restart() { - return Err(err("missing restart marker")); - } - restart_left = h.restart; - first_line = true; - reset_pending = true; + let mut x = 0; + while x < w { + if h.restart > 0 && restart_left == 0 { + if !br.restart() { + return Err(err("missing restart marker")); } - restart_left -= 1; + restart_left = h.restart; + first_line = true; + reset_pending = true; } - #[allow(clippy::needless_range_loop)] // `c` indexes the tables and the sample position alike + // pixels up to the next restart marker or the end of the row + let mut end = w; + if h.restart > 0 { + end = w.min(x + restart_left); + restart_left -= end - x; + } + // the first pixel of the run has the special predictions (start of the interval, first column) for c in 0..nc { let i = x * nc + c; let p = if reset_pending { @@ -702,24 +787,28 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result { } else if x == 0 || pred == 0 { if pred == 0 { 0 } else { prev[i] as i32 } } else { - let ra = cur[i - nc] as i32; - let rb = prev[i] as i32; - let rc = prev[i - nc] as i32; - match pred { - 1 => ra, - 2 => rb, - 3 => rc, - 4 => ra + rb - rc, - 5 => ra + ((rb - rc) >> 1), - 6 => rb + ((ra - rc) >> 1), - _ => (ra + rb) >> 1, - } + predict(pred, cur[i - nc] as i32, prev[i] as i32, prev[i - nc] as i32) }; - let ssss = tables[c].decode(&mut br)?; - let diff = diff_value(&mut br, ssss); - cur[i] = ((p + diff) & mask) as u16; + cur[i] = ((p + tables[c].decode_diff(&mut br, false)?) & mask) as u16; } reset_pending = false; + // the rest of the run: the predictor is fixed, so it is chosen once here, not per sample + if first_line { + // the first line of an interval predicts from the left neighbour + decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?; + } else { + match pred { + 0 => decode_run::<0>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 1 => decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 2 => decode_run::<2>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 3 => decode_run::<3>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 4 => decode_run::<4>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 5 => decode_run::<5>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 6 => decode_run::<6>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + _ => decode_run::<7>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + } + } + x = end; } first_line = false; if br.overrun > 64 * 8 { diff --git a/crates/raw/src/ljpeg/tests.rs b/crates/raw/src/ljpeg/tests.rs index c5c6ccf25..17791ea8b 100644 --- a/crates/raw/src/ljpeg/tests.rs +++ b/crates/raw/src/ljpeg/tests.rs @@ -125,6 +125,94 @@ fn roundtrip_extremes_and_restarts() { } } +/// Re-label an encoded stream as `precision` bits with point transform `pt`: the encoder codes the samples at +/// `precision - pt` bits, which is what a transformed scan holds. +fn with_point_transform(mut enc: Vec, precision: u8, pt: u8) -> Vec { + enc[6] = precision; // SOF3 precision follows SOI, marker and length + let sos = enc.windows(2).position(|w| w == [0xff, 0xda]).unwrap(); + let seg_len = u16::from_be_bytes([enc[sos + 2], enc[sos + 3]]) as usize; + enc[sos + 2 + seg_len - 1] = pt; + enc +} + +#[test] +fn roundtrip_point_transform_all_predictors() { + for pred in 1..=7u8 { + for comps in [1usize, 2, 3] { + for (bits, pt) in [(12u8, 1u8), (14, 2), (16, 4), (8, 3)] { + let (w, h) = (11, 6); + let data = noise(w * h * comps, (bits - pt) as u32, pred as u64 * 17 + comps as u64); + let enc = with_point_transform(encode(&data, w, h, comps, bits - pt, pred, 0), bits, pt); + let f = decode(&enc, usize::MAX).unwrap(); + assert_eq!((f.precision, f.point_transform, f.predictor), (bits, pt, pred)); + let want: Vec = data.iter().map(|&v| v << pt).collect(); + assert_eq!(f.data, want, "pred {pred} comps {comps} bits {bits} pt {pt}"); + } + } + } + // with restart intervals that end mid-row + let data = noise(20 * 9 * 2, 10, 9); + for rst in [3usize, 7, 20] { + let enc = with_point_transform(encode(&data, 20, 9, 2, 10, 4, rst), 12, 2); + let want: Vec = data.iter().map(|&v| v << 2).collect(); + assert_eq!(decode(&enc, usize::MAX).unwrap().data, want, "restart {rst}"); + } +} + +#[test] +fn roundtrip_skewed_categories_with_long_codes() { + // category k with probability 2^-(k+1): the table gets codes longer than the lookup, rare 16-bit differences + let mut s = 0x9e37_79b9_7f4a_7c15u64; + let mut last = 0u32; + let data: Vec = (0..256 * 256) + .map(|_| { + s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + let r = (s >> 32) as u32; + let cat = r.trailing_zeros().min(16); + let mag = (((s >> 8) as u32) & ((1u32 << cat) - 1)) | (1 << cat) >> 1; + let d = if s & 1 == 0 { mag } else { 0u32.wrapping_sub(mag) }; + last = last.wrapping_add(d) & 0xffff; + last as u16 + }) + .collect(); + for pred in [1u8, 4, 7] { + let enc = encode(&data, 256, 256, 1, 16, pred, 0); + assert_eq!(decode(&enc, usize::MAX).unwrap().data, data, "pred {pred}"); + } +} + +/// `decode_diff` is `decode` followed by `diff_value`, for tables with codes inside and beyond the lookup. +#[test] +fn decode_diff_matches_decode_and_diff_value() { + for lens in [[2u8, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12], [2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 16]] { + let mut counts = [0u8; 16]; + for &l in &lens { + counts[l as usize - 1] += 1; + } + let values: Vec = (0..17).collect(); // lengths are already ascending + let t = Huffman::new(&counts, &values).unwrap(); + let mut s = 77u64; + let src: Vec = (0..6000) + .map(|_| { + s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + ((s >> 40) as u8).min(0xfe) + }) + .collect(); + let (mut a, mut b) = (BitReader::new(&src), BitReader::new(&src)); + for n in 0..8000 { + let want = t.decode(&mut a).map(|ssss| diff_value(&mut a, ssss)); + let got = t.decode_diff(&mut b, false); + assert_eq!(got.ok(), want.ok(), "symbol {n}"); + assert_eq!(a.peek(32), b.peek(32), "symbol {n}"); + if want.is_err() { + // step past the invalid code so the walk goes on + a.consume(1); + b.consume(1); + } + } + } +} + #[test] fn constant_image_single_symbol_table() { let data = vec![1234u16; 16 * 4]; From df89f6b6b7534c717debfbcb19b287a14aba16fb Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 15 Nov 2026 18:13:19 +0100 Subject: [PATCH 03/19] craft-raw: test tables where code plus extra bits fill the lookup, canonical reference, clippy allow Co-Authored-By: Claude Sonnet 5.5 --- crates/raw/src/huffman.rs | 50 ++++++++++++++++++++++++++++------- crates/raw/src/ljpeg.rs | 1 + crates/raw/src/ljpeg/tests.rs | 41 +++++++++++++++++++++++----- 3 files changed, 76 insertions(+), 16 deletions(-) diff --git a/crates/raw/src/huffman.rs b/crates/raw/src/huffman.rs index 85b242be1..4f35f64bb 100644 --- a/crates/raw/src/huffman.rs +++ b/crates/raw/src/huffman.rs @@ -145,9 +145,10 @@ mod tests { /// Reference: match the code bit by bit, then read the category's additional bits (T.81 F.1.2.1). fn reference(src: &[u8], pos: &mut usize, codes: &[(u16, u8)]) -> Option { let window: u32 = (0..12).fold(0, |a, k| (a << 1) | bit(src, *pos + k)); - let (sym, len) = codes.iter().enumerate().find_map(|(s, &(c, l))| { - (l > 0 && (c as u32 >> (12 - l)) == window >> (12 - l)).then_some((s as u32, l as usize)) - })?; + let (sym, len) = codes + .iter() + .enumerate() + .find_map(|(s, &(c, l))| (l > 0 && (c as u32 >> (12 - l)) == window >> (12 - l)).then_some((s as u32, l as usize)))?; *pos += len; if sym == 0 { return Some(0); @@ -157,13 +158,25 @@ mod tests { Some(if v < 1 << (sym - 1) { v - (1 << sym) + 1 } else { v }) } - #[test] - fn diff_matches_bitwise_reference_for_every_category() { - // categories 0..=16, code lengths up to 12 so that short codes combine with their bits and long ones do not - let lens = [2u8, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12]; - let codes = canonical(&lens); + /// `diff` against the reference on random streams (invalid codes are stepped over) and on every (code, extra + /// bits) pair of every symbol. + fn check_table(lens: &[u8]) { + let codes = canonical(lens); let h = Huffman::new(&codes).unwrap(); + for (sym, &(code, len)) in codes.iter().enumerate() { + for v in 0..1u64 << sym { + let (len, n) = (len as u32, sym as u32); + let word = (((code >> (12 - len)) as u64) << n | v) << (64 - len - n); + let src = word.to_be_bytes(); + let mut pos = 0; + let want = reference(&src, &mut pos, &codes); + let mut bits = Bits::new(&src); + assert_eq!(diff(&mut bits, &h), want, "symbol {sym} bits {v:#b}"); + assert_eq!(bits.consumed_bits(), pos); + } + } let mut s = 0x1234_5678_9abc_def0u64; + let (mut invalid, mut long) = (0, 0); for _ in 0..40 { let src: Vec = (0..400) .map(|_| { @@ -174,13 +187,32 @@ mod tests { let mut bits = Bits::new(&src); let mut pos = 0; while pos < src.len() * 8 - 64 { + let before = pos; let want = reference(&src, &mut pos, &codes); assert_eq!(diff(&mut bits, &h), want); if want.is_none() { - break; + // step over the invalid code + invalid += 1; + pos += 1; + bits.skip(1); + } else { + long += (pos - before > 12) as u32; } assert_eq!(bits.consumed_bits(), pos); } } + assert!(long > 0, "{invalid} invalid codes, no symbol longer than the lookup"); + } + + #[test] + fn diff_matches_bitwise_reference_for_every_category() { + // categories 0..=16, code lengths up to 12 so that short codes combine with their bits and long ones do not + check_table(&[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12]); + } + + #[test] + fn diff_matches_reference_when_code_and_bits_fill_the_lookup_exactly() { + // category 8 with a 4-bit code and category 7 with a 5-bit code: len + category == 12 + check_table(&[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12]); } } diff --git a/crates/raw/src/ljpeg.rs b/crates/raw/src/ljpeg.rs index ab27ed57f..498a8302f 100644 --- a/crates/raw/src/ljpeg.rs +++ b/crates/raw/src/ljpeg.rs @@ -778,6 +778,7 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result { restart_left -= end - x; } // the first pixel of the run has the special predictions (start of the interval, first column) + #[allow(clippy::needless_range_loop)] // `c` indexes the tables and the sample position alike for c in 0..nc { let i = x * nc + c; let p = if reset_pending { diff --git a/crates/raw/src/ljpeg/tests.rs b/crates/raw/src/ljpeg/tests.rs index 17791ea8b..a77425d40 100644 --- a/crates/raw/src/ljpeg/tests.rs +++ b/crates/raw/src/ljpeg/tests.rs @@ -181,15 +181,42 @@ fn roundtrip_skewed_categories_with_long_codes() { } } -/// `decode_diff` is `decode` followed by `diff_value`, for tables with codes inside and beyond the lookup. +/// Canonical decoding from the code lengths alone, bit by bit (T.81 Annex F, Figure F.16): the reference for the +/// lookup tables. +fn canonical_symbol(br: &mut BitReader, lens: &[u8]) -> Option { + let window = br.peek(16); + let (mut first, mut index) = (0u32, 0usize); + for len in 1..=16u8 { + let code = window >> (16 - len); + let n = lens.iter().filter(|&&l| l == len).count() as u32; + if code < first + n { + br.consume(len as u32); + return Some((index + (code - first) as usize) as u8); + } + index += n as usize; + first = (first + n) << 1; + } + None +} + +/// `decode_diff` against canonical decoding and `diff_value`, for tables with codes inside and beyond the lookup +/// and with `len + category` equal to the lookup width. #[test] -fn decode_diff_matches_decode_and_diff_value() { - for lens in [[2u8, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12], [2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 16]] { +fn decode_diff_matches_canonical_decoding() { + let tables: [[u8; 17]; 3] = [ + [2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12], + [2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 16], + [2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12], + ]; + for lens in tables { let mut counts = [0u8; 16]; for &l in &lens { counts[l as usize - 1] += 1; } - let values: Vec = (0..17).collect(); // lengths are already ascending + // symbols in code order: by length, ties by symbol + let mut values: Vec = (0..17).collect(); + values.sort_by_key(|&v| (lens[v as usize], v)); + let sorted: Vec = values.iter().map(|&v| lens[v as usize]).collect(); let t = Huffman::new(&counts, &values).unwrap(); let mut s = 77u64; let src: Vec = (0..6000) @@ -200,11 +227,11 @@ fn decode_diff_matches_decode_and_diff_value() { .collect(); let (mut a, mut b) = (BitReader::new(&src), BitReader::new(&src)); for n in 0..8000 { - let want = t.decode(&mut a).map(|ssss| diff_value(&mut a, ssss)); + let want = canonical_symbol(&mut a, &sorted).map(|i| diff_value(&mut a, values[i as usize])); let got = t.decode_diff(&mut b, false); - assert_eq!(got.ok(), want.ok(), "symbol {n}"); + assert_eq!(got.ok(), want, "symbol {n}"); assert_eq!(a.peek(32), b.peek(32), "symbol {n}"); - if want.is_err() { + if want.is_none() { // step past the invalid code so the walk goes on a.consume(1); b.consume(1); From a05083e2f02ff88f0729f3ef61ee2e6eaef24d04 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 17:37:40 +0000 Subject: [PATCH 04/19] Expose original AVIF sample depth during decode --- crates/image-codecs/src/codecs/avif_decode.rs | 8 ++++++-- crates/image-codecs/src/lib.rs | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/image-codecs/src/codecs/avif_decode.rs b/crates/image-codecs/src/codecs/avif_decode.rs index 302433661..4dd15fd7d 100644 --- a/crates/image-codecs/src/codecs/avif_decode.rs +++ b/crates/image-codecs/src/codecs/avif_decode.rs @@ -822,6 +822,10 @@ fn read_metadata(c: &Container) -> (Option>, Option) { /// Decodes the primary image. `keep_orientation` leaves the container's `clap`, `irot` and /// `imir` unapplied (the stored pixels), as for HEIF. pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result { + decode_with_depth(bytes, limits, keep_orientation).map(|(image, _)| image) +} + +pub(crate) fn decode_with_depth(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<(Image, u8)> { let c = parse(bytes)?; let id = c.primary; let (w, h) = item_size(&c, id)?; @@ -860,9 +864,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> R // PQ or HLG: linear light in 32-bit float, as for HEIF. if cicp.is_hdr() { limits.check(w, h, layout, SampleType::F32)?; - return crate::cicp::linearize(img, cicp); + return crate::cicp::linearize(img, cicp).map(|image| (image, bits)); } - Ok(img) + Ok((img, bits)) } #[cfg(test)] diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index a4480b2b8..86c4290b8 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -115,6 +115,14 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res finish_format(format, bytes, img, opts) } +/// Decode AVIF pixels and report their original AV1 bit depth (8, 10 or 12). +/// The returned image may use a wider sample type, particularly for linear HDR. +#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))] +pub fn decode_avif_with_depth(bytes: &[u8], opts: &DecodeOptions) -> Result<(Image, u8), CodecError> { + let (image, depth) = codecs::avif_decode::decode_with_depth(bytes, &opts.limits, opts.keep_orientation)?; + Ok((finish_format(Format::Avif, bytes, image, opts)?, depth)) +} + /// What [`decode_as_with`] does with a PNG decoded in steps by [`PngDecodeRows`] (`img` is its /// output): the resolution from the metadata, the upright orientation and the final limit check, /// so the result equals [`decode_as_with`]'s on the same file. From 6325cd286bd87effa3020b01d6b9d289d4f7c055 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 17:37:54 +0000 Subject: [PATCH 05/19] Test AVIF source depth and decoded-image parity --- crates/image-codecs/src/codecs/avif_decode.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/image-codecs/src/codecs/avif_decode.rs b/crates/image-codecs/src/codecs/avif_decode.rs index 4dd15fd7d..eaeafb65d 100644 --- a/crates/image-codecs/src/codecs/avif_decode.rs +++ b/crates/image-codecs/src/codecs/avif_decode.rs @@ -1194,6 +1194,9 @@ mod tests { let src = Image::from_u8(w as u32, h as u32, ChannelLayout::Rgb, card(w, h, 3)).unwrap(); let file = encode_avif_with_speed(&src, 95, 10).unwrap(); let img = crate::decode(&file).unwrap(); + let (with_depth, depth) = crate::decode_avif_with_depth(&file, &crate::DecodeOptions::default()).unwrap(); + assert_eq!(depth, 10); + assert_eq!(with_depth, img); assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (48, 32, ChannelLayout::Rgb, SampleType::U16)); let diff = mean_abs_diff(&rgb8(&img), &card(w, h, 3)); assert!(diff < 2.5, "mean difference {diff}"); From 02ab4e01ef584345c4c5fc43d3bad2e9fdfc28bf Mon Sep 17 00:00:00 2001 From: bflatastic Date: Sun, 15 Nov 2026 15:15:31 -0500 Subject: [PATCH 06/19] Share bounded USD Crate LZ4 compression Adapt TextureCraft bdbb52c9 single-block framing with checked worst-case output sizing and fallible allocation. Preserve original compressed wire bytes. --- crates/usd/README.md | 2 +- crates/usd/src/compress.rs | 76 ++++++++++++++++++++++++++++++++++++++ crates/usd/src/lib.rs | 3 ++ 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 crates/usd/src/compress.rs diff --git a/crates/usd/README.md b/crates/usd/README.md index 9e9635185..fcf585760 100644 --- a/crates/usd/README.md +++ b/crates/usd/README.md @@ -2,6 +2,6 @@ Bounded wire primitives for USD Crate, independent of application scene and material models. -Provides bounded integer-delta array decoding and Crate-framed LZ4 decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and export remain outside this extraction. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation. +Provides bounded integer-delta array decoding and Crate-framed LZ4 compression/decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and full scene export remain outside this extraction. Compression uses a caller cap on worst-case reserved output and fallible allocation. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation. Adapted from TextureCraft `crates/formats/src/mesh_import/usd/codec.rs`, introduced by bflatastic in `bdbb52c9b66b221a55b370105b42f485a76e6fc7`. The LZ4 backend is MIT-licensed `lz4_flex`. diff --git a/crates/usd/src/compress.rs b/crates/usd/src/compress.rs new file mode 100644 index 000000000..b92fa3be8 --- /dev/null +++ b/crates/usd/src/compress.rs @@ -0,0 +1,76 @@ +//! Bounded single-block USD Crate LZ4 writing. + +/// Compression admission, allocation or backend failure. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Input or worst-case output exceeds the supported/caller budget. + #[error("USD Crate LZ4 compression exceeds budget")] + Limit, + /// The bounded output reservation failed. + #[error("cannot allocate USD Crate LZ4 output")] + Allocation, + /// The backend could not encode into the preflighted output buffer. + #[error("USD Crate LZ4 compression: {0}")] + Encode(#[from] lz4_flex::block::CompressError), +} + +fn capacity(input: usize, max_output_bytes: usize) -> Result { + // Keep inputs below the LZ4 single-block limit and preflight the backend's + // 20 + input*110/100 sizing arithmetic before calling it, including the count byte. + if input > 0x7e00_0000 { + return Err(Error::Limit); + } + input + .checked_mul(110) + .map(|n| n / 100) + .and_then(|n| n.checked_add(21)) + .filter(|n| *n <= max_output_bytes && *n <= isize::MAX as usize) + .ok_or(Error::Limit) +} + +/// Write one Crate-framed LZ4 block, beginning with a zero chunk-count byte. +/// +/// `max_output_bytes` bounds the reserved output including framing. Admission uses +/// the backend's worst-case size, so even highly compressible input must fit that bound. +/// Allocation is fallible; backend scratch memory is not included in the output cap. +/// Uses TextureCraft's original LZ4 backend/configuration and produces the same block bytes. +pub fn compress(input: &[u8], max_output_bytes: usize) -> Result, Error> { + let capacity = capacity(input.len(), max_output_bytes)?; + let mut output = Vec::new(); + output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?; + output.resize(capacity, 0); + let target = output.get_mut(1..).ok_or(Error::Limit)?; + let written = lz4_flex::block::compress_into(input, target)?; + output.truncate(written.checked_add(1).ok_or(Error::Limit)?); + Ok(output) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn original_wire_bytes_and_roundtrip() { + for size in [0, 1, 15, 16, 255, 4096, 65536] { + for repeated in [false, true] { + let input: Vec = (0..size).map(|i| if repeated { 17 } else { (i % 251) as u8 }).collect(); + let mut old = vec![0]; + old.extend(lz4_flex::block::compress(&input)); + let encoded = compress(&input, 1 << 20).unwrap(); + assert_eq!(encoded, old); + assert_eq!(crate::decompress(&encoded, size).unwrap(), input); + } + } + } + + #[test] + fn checks_worst_case_before_allocation() { + let input = [0; 100]; + assert!(matches!(compress(&input, 130), Err(Error::Limit))); + assert!(compress(&input, 131).is_ok()); + assert!(capacity(usize::MAX, usize::MAX).is_err()); + assert!(capacity(0x7e00_0001, usize::MAX).is_err()); + assert_eq!(capacity(0, 21).unwrap(), 21); + } +} diff --git a/crates/usd/src/lib.rs b/crates/usd/src/lib.rs index edaeb89a2..33c7f3484 100644 --- a/crates/usd/src/lib.rs +++ b/crates/usd/src/lib.rs @@ -5,6 +5,9 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +/// Bounded single-block Crate LZ4 compression. +pub mod compress; + /// Bounded Crate integer delta arrays. pub mod integers; From a99c8f8e366363ac5ff2f1eee535751610c2c82c Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:26:02 +0100 Subject: [PATCH 07/19] raw: leave LinearizationTable unapplied when OpcodeList1 is present DNG chapter 7 runs OpcodeList1 on the stored codes, then the LinearizationTable maps them to sensor values. decode was applying the table first, so a develop pipeline that later applied list 1 saw already linearized samples. When list 1 is present the table is now carried on RawImage and left unapplied (`linearized` stays false). apply_linearization maps it after the list. Files without list 1 are unchanged. The dng-only build already compiles on main (d16ee792 gated cr3::orientation). A comment at that gate records the E0433 failure mode. Part of #778. --- crates/raw/README.md | 2 +- crates/raw/src/arw.rs | 1 + crates/raw/src/cr2.rs | 1 + crates/raw/src/cr2/sraw.rs | 1 + crates/raw/src/cr3.rs | 2 ++ crates/raw/src/crw.rs | 1 + crates/raw/src/dng.rs | 44 +++++++++++++++-------- crates/raw/src/dngwrite.rs | 9 +++++ crates/raw/src/erf.rs | 1 + crates/raw/src/kodak.rs | 1 + crates/raw/src/lib.rs | 37 ++++++++++++++++--- crates/raw/src/mef.rs | 1 + crates/raw/src/mrw.rs | 1 + crates/raw/src/nef.rs | 2 ++ crates/raw/src/orf.rs | 1 + crates/raw/src/pef.rs | 1 + crates/raw/src/raf.rs | 1 + crates/raw/src/rw2.rs | 1 + crates/raw/src/srw.rs | 1 + crates/raw/src/tests.rs | 1 + crates/raw/tests/it/dng.rs | 68 ++++++++++++++++++++++++++++++++++- crates/raw/tests/it/robust.rs | 1 + 22 files changed, 158 insertions(+), 21 deletions(-) diff --git a/crates/raw/README.md b/crates/raw/README.md index e15e355c1..f0faf0670 100644 --- a/crates/raw/README.md +++ b/crates/raw/README.md @@ -33,7 +33,7 @@ The format layer ends where arithmetic on the picture begins. |---|---| | Recognising containers (`probe`), walking TIFF/IFD structure on `craft-tiff` | Black subtraction, white scaling, cropping to the active area | | Decompressing the samples: unpacking, lossless JPEG, lossy JPEG, Deflate, JPEG XL | Demosaicing, binned previews, hot-pixel and highlight handling | -| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values) | Raising samples stuck below the stated white level (`lift_clipped_samples`) | +| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values; left unapplied when `OpcodeList1` is present, so the develop pipeline can run that list on the stored codes first) | Raising samples stuck below the stated white level (`lift_clipped_samples`) | | Black and white levels, CFA layout, active area, default crop, orientation, pixel aspect | The DNG colour model's arithmetic (illuminant interpolation, camera to working space) | | Colour tags as data: matrices, calibration illuminants, as-shot neutral / white | White balance, Kelvin scales, spectral or fitted camera colour | | Opcode lists parsed and written (`opcodes`), profile tables and gain table maps as data | Applying opcodes (gain maps, warps, bad pixels), profile look tables, tone curves | diff --git a/crates/raw/src/arw.rs b/crates/raw/src/arw.rs index 1b25aacda..389c464a5 100644 --- a/crates/raw/src/arw.rs +++ b/crates/raw/src/arw.rs @@ -852,6 +852,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Result Result = table.iter().map(|&x| x.min(65535) as u16).collect(); - let last = table.len() - 1; - v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]); - linearized = true; - table_max = table.iter().copied().max().filter(|&m| m > 0); - } + let table_max = linearization_table.as_deref().and_then(|t| t.iter().copied().max().filter(|&m| m > 0)); // a reduced copy's own levels, where it states them let levels = |tag| if pixels.contains(tag) { pixels } else { raw }; @@ -442,10 +436,6 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f None }; - let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default(); - let opcodes = - crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) }; - let color = color_data(ifd0, raw, tiff.order); let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2)) && color.profile.gain_table_map.is_none(); @@ -465,6 +455,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f color, wb_multipliers: None, linearized, + linearization_table, pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()), opcodes, camera: Camera::from_ifd0(ifd0), @@ -477,6 +468,31 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f Ok(img) } +/// Parse opcode lists and, for integer data, the LinearizationTable. +/// +/// OpcodeList1 runs on the stored codes, then the table maps them to sensor values (DNG 1.7 +/// chapter 7). Applying opcodes is the develop pipeline's work, so a non-empty list 1 leaves the +/// table unapplied (`linearized` is false) and available on the image. +fn linearization_and_opcodes(pixels: &Ifd, raw: &Ifd, data: &mut RawData) -> (bool, Option>, crate::OpcodeLists) { + let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default(); + let opcodes = + crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) }; + let linearization_table = if matches!(data, RawData::U16(_)) { + pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty()).map(|table| table.iter().map(|&x| x.min(65535) as u16).collect::>()) + } else { + None + }; + let mut linearized = false; + if opcodes.list1.is_empty() + && let (RawData::U16(v), Some(table)) = (data, linearization_table.as_deref()) + { + let last = table.len() - 1; + v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]); + linearized = true; + } + (linearized, linearization_table, opcodes) +} + /// CinemaDNG `TimeCodes` (CinemaDNG Image Data Format Specification 1.0, tag 51043). const TIME_CODES: u16 = 51043; /// CinemaDNG `FrameRate` (tag 51044). diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index 8953f94ea..14471387f 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -159,6 +159,14 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { } else { ifd.set(t::WHITE_LEVEL, Value::Long(white)); } + // A table still to be applied (OpcodeList1 has to run on the stored codes first) is written + // with the samples. Once the samples have been mapped, writing the tag would apply it twice + // on the next read of a file without list 1. + if !raw.linearized + && let Some(table) = raw.linearization_table.as_deref().filter(|t| !t.is_empty()) + { + ifd.set(t::LINEARIZATION_TABLE, Value::Short(table.to_vec())); + } let a = raw.active_area; ifd.set(t::ACTIVE_AREA, Value::Long(vec![a.y as u32, a.x as u32, (a.y + a.height) as u32, (a.x + a.width) as u32])); ifd.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![raw.crop.x as u32, raw.crop.y as u32])); @@ -456,6 +464,7 @@ mod tests { color: crate::ColorData::default(), wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera: Default::default(), diff --git a/crates/raw/src/erf.rs b/crates/raw/src/erf.rs index fcdfa43cd..b7dd9d8be 100644 --- a/crates/raw/src/erf.rs +++ b/crates/raw/src/erf.rs @@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Option { } // A damaged/unsupported pixel IFD must not hide usable orientation metadata from preview // fallback. CR3 keeps IFD0 in its CMT1 box; TIFF-based raws keep it at the file start. + // The `cr3` module exists only with the `cr3` feature: this gate is required so + // `cargo check -p craft-raw --no-default-features --features dng` builds (E0433 otherwise). #[cfg(feature = "cr3")] if probe(bytes) == Some(RawFormat::Cr3) { return cr3::orientation(bytes); @@ -1041,6 +1045,12 @@ pub struct RawImage { pub wb_multipliers: Option<[f32; 3]>, /// Whether a `LinearizationTable` was applied to `data`. pub linearized: bool, + /// The file's `LinearizationTable`, when present. `data` has already been mapped through it when + /// [`Self::linearized`] is true. When `linearized` is false and this is `Some`, the table was left + /// unapplied so `OpcodeList1` can run on the stored samples first (DNG chapter 7); call + /// [`Self::apply_linearization`] after that list. + #[serde(default)] + pub linearization_table: Option>, /// Height of a sensor pixel over its width (1.0 for square pixels). The D1X and D1H write 2.0: their frame is /// 4028 x 1324 and the picture is 3:2 only once the rows are doubled. A developer stretches the rows by /// this factor. @@ -1119,6 +1129,23 @@ impl RawImage { self.check(true) } + /// Map integer `data` through [`Self::linearization_table`] when the table has not yet been applied. + /// No-op when [`Self::linearized`] is already true, the table is missing, or the samples are not + /// integers. Call this after `OpcodeList1`, which DNG applies to the stored codes (chapter 7). + pub fn apply_linearization(&mut self) { + if self.linearized { + return; + } + let Some(table) = self.linearization_table.as_deref().filter(|t| !t.is_empty()) else { + return; + }; + if let RawData::U16(v) = &mut self.data { + let last = table.len() - 1; + v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]); + self.linearized = true; + } + } + fn check(&self, samples: bool) -> Result<()> { if self.width == 0 || self.height == 0 || !(1..=4).contains(&self.cpp) { return Err(RawError::Corrupt("bad dimensions".into())); diff --git a/crates/raw/src/mef.rs b/crates/raw/src/mef.rs index 36ba18e0f..1180fe063 100644 --- a/crates/raw/src/mef.rs +++ b/crates/raw/src/mef.rs @@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Result Result Result Result Result Result, cpp: usize) -> RawImage { color, wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera, @@ -1177,12 +1178,77 @@ fn white_level_defaults_to_the_linearization_tables_maximum() { }; let img = build(None); assert!(img.linearized); + assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice())); assert_eq!(img.data, RawData::U16(codes.iter().map(|&c| table[c as usize]).collect())); assert_eq!(img.white, vec![51000.0], "255 × 200, the table's top"); // a stated level still wins assert_eq!(build(Some(40000)).white, vec![40000.0]); } +/// DNG chapter 7: OpcodeList1 runs on the stored codes, then the LinearizationTable maps them to +/// sensor values. When list 1 is present, decode leaves the table unapplied so the develop pipeline +/// can keep that order. Applying the table first would give a different picture. +#[test] +fn opcode_list1_runs_on_stored_samples_before_linearization() { + let (w, h) = (4, 2); + let stored = vec![2u16; w * h]; + // table: 2 → 20, 7 → 30, 20 → 99 (so the two orders disagree) + let mut table = vec![0u16; 32]; + table[2] = 20; + table[7] = 30; + table[20] = 99; + // MapTable: 2 → 7, 20 → 8 + let mut map: Vec = (0..32).collect(); + map[2] = 7; + map[20] = 8; + let area = Area { top: 0, left: 0, bottom: h as u32, right: w as u32, plane: 0, planes: 1, row_pitch: 1, col_pitch: 1 }; + let list1 = opcodes::write_list(&[Opcode::MapTable { area, table: map.clone() }]); + + let mut raw = base_ifd(w, h, 16, 1, true); + raw.set(t::COMPRESSION, Value::Short(vec![compression::NONE])); + raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![u16_bytes(&stored, ByteOrder::Little)] }); + raw.set(t::LINEARIZATION_TABLE, Value::Short(table.clone())); + raw.set(t::OPCODE_LIST_1, Value::Undefined(list1)); + raw.set(t::WHITE_LEVEL, Value::Long(vec![100])); + let img = decode(&dng(raw, ByteOrder::Little)).unwrap(); + assert!(!img.linearized, "list 1 is present, so the table must wait"); + assert_eq!(img.data, RawData::U16(stored.clone())); + assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice())); + assert_eq!(img.opcodes.list1.len(), 1); + + let apply_map = |samples: &mut [u16]| { + for s in samples { + let i = (*s as usize).min(map.len() - 1); + *s = map[i]; + } + }; + + let mut correct = img.clone(); + if let RawData::U16(v) = &mut correct.data { + apply_map(v); + } + correct.apply_linearization(); + assert!(correct.linearized); + // 2 → 7 (MapTable) → 30 (linearization) + assert_eq!(correct.data, RawData::U16(vec![30; w * h])); + + let mut wrong = img.clone(); + wrong.apply_linearization(); + if let RawData::U16(v) = &mut wrong.data { + apply_map(v); + } + // 2 → 20 (linearization) → 8 (MapTable) + assert_eq!(wrong.data, RawData::U16(vec![8; w * h])); + assert_ne!(correct.data, wrong.data); + + // the unapplied table and list 1 survive a write / decode round trip + let back = decode(&write_dng(&img, &DngWriteOptions { compression: DngCompression::Uncompressed, ..Default::default() }).unwrap()).unwrap(); + assert!(!back.linearized); + assert_eq!(back.data, RawData::U16(stored)); + assert_eq!(back.linearization_table.as_deref(), Some(table.as_slice())); + assert_eq!(back.opcodes.list1.len(), 1); +} + /// PhotoCraft: `AsShotWhiteXY` is a chromaticity, so both coordinates lie inside (0, 1). #[test] fn an_as_shot_white_outside_the_chromaticity_range_is_ignored() { diff --git a/crates/raw/tests/it/robust.rs b/crates/raw/tests/it/robust.rs index 5ad1d1b32..28640a803 100644 --- a/crates/raw/tests/it/robust.rs +++ b/crates/raw/tests/it/robust.rs @@ -22,6 +22,7 @@ fn sample(comp: DngCompression) -> Vec { color: ColorData::default(), wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera: Default::default(), From f7655fb3ea59d6b21681700c0ff880823f5bb709 Mon Sep 17 00:00:00 2001 From: bflatastic Date: Sun, 15 Nov 2026 15:29:04 -0500 Subject: [PATCH 08/19] Extract bounded USD Crate integer delta encoding Adapt TextureCraft bdbb52c9b66b221a55b370105b42f485a76e6fc7 integer writer with exact wire oracle. Preserve common-delta tie-breaking and signed wrapping; cap combined output and scratch reservations and reserve fallibly. Co-authored-by: Dylan M. Taylor --- crates/usd/src/integers.rs | 144 +++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/crates/usd/src/integers.rs b/crates/usd/src/integers.rs index c50dd0e0f..0d8e67df4 100644 --- a/crates/usd/src/integers.rs +++ b/crates/usd/src/integers.rs @@ -80,6 +80,72 @@ pub fn decode(data: &[u8], count: usize, width: Width, max_output_bytes: usize) Ok(output) } +/// Encode integers with their original signed wrapping and largest-delta tie-break policy. +/// +/// `max_bytes` bounds the combined worst-case output reservation and sorted delta scratch. +/// Both reservations are fallible. Sorting uses no extra heap storage; input storage and +/// allocator bookkeeping are outside this budget. Empty input emits no bytes. +pub fn encode(values: &[i64], width: Width, max_bytes: usize) -> Result, Error> { + if values.is_empty() { + return Ok(Vec::new()); + } + let size = width.bytes(); + let codes_len = values.len().div_ceil(4); + let capacity = values.len().checked_mul(size).and_then(|n| n.checked_add(size)).and_then(|n| n.checked_add(codes_len)).ok_or(Error::Limit)?; + let scratch = values.len().checked_mul(8).ok_or(Error::Limit)?; + if capacity > isize::MAX as usize || scratch > isize::MAX as usize || capacity.checked_add(scratch).is_none_or(|n| n > max_bytes) { + return Err(Error::Limit); + } + let wrap = |v: i64| match width { + Width::I32 => i64::from(v as i32), + Width::I64 => v, + }; + let mut deltas = Vec::new(); + deltas.try_reserve_exact(values.len()).map_err(|_| Error::Allocation)?; + let mut previous = 0i64; + for &value in values { + deltas.push(wrap(value.wrapping_sub(previous))); + previous = value; + } + deltas.sort_unstable(); + let mut best = (0usize, 0i64); + let mut at = 0; + while at < deltas.len() { + let value = deltas[at]; + let start = at; + while at < deltas.len() && deltas[at] == value { + at += 1; + } + best = best.max((at - start, value)); + } + let common = best.1; + let mut output = Vec::new(); + output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?; + output.extend_from_slice(&common.to_le_bytes()[..size]); + output.resize(size + codes_len, 0); + previous = 0; + for (i, &value) in values.iter().enumerate() { + let delta = wrap(value.wrapping_sub(previous)); + previous = value; + let small = |bits: u32| delta >= -(1i64 << (bits - 1)) && delta < (1i64 << (bits - 1)); + let code: u8 = if delta == common { + 0 + } else if small(size as u32 * 2) { + 1 + } else if small(size as u32 * 4) { + 2 + } else { + 3 + }; + if code != 0 { + let bytes = (size >> (3 - usize::from(code))).max(1); + output.extend_from_slice(&delta.to_le_bytes()[..bytes]); + } + output[size + i / 4] |= code << (2 * (i % 4)); + } + Ok(output) +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { @@ -125,3 +191,81 @@ mod tests { } } } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod encode_tests { + use super::*; + fn oracle(values: &[i64], width: usize) -> Vec { + if values.is_empty() { + return Vec::new(); + } + let wrap = |v: i64| if width == 4 { i64::from(v as i32) } else { v }; + let mut prev = 0i64; + let deltas: Vec = values + .iter() + .map(|&v| { + let d = wrap(v.wrapping_sub(prev)); + prev = v; + d + }) + .collect(); + let mut counts = std::collections::HashMap::new(); + for d in &deltas { + *counts.entry(*d).or_insert(0usize) += 1; + } + let common = counts.into_iter().max_by_key(|&(v, c)| (c, v)).map_or(0, |(v, _)| v); + let mut out = Vec::with_capacity(width + values.len().div_ceil(4) + values.len() * width); + out.extend_from_slice(&common.to_le_bytes()[..width]); + let codes_at = out.len(); + out.resize(codes_at + values.len().div_ceil(4), 0); + for (i, &d) in deltas.iter().enumerate() { + let small = |bits: u32| d >= -(1i64 << (bits - 1)) && d < (1i64 << (bits - 1)); + let code: u8 = if d == common { + 0 + } else if small(width as u32 * 2) { + 1 + } else if small(width as u32 * 4) { + 2 + } else { + 3 + }; + if code > 0 { + let w = (width >> (3 - code as usize)).max(1); + out.extend_from_slice(&d.to_le_bytes()[..w]); + } + if let Some(b) = out.get_mut(codes_at + i / 4) { + *b |= code << (2 * (i % 4)); + } + } + out + } + + #[test] + fn matches_original_bytes_and_wrapping() { + for width in [Width::I32, Width::I64] { + for n in [0, 1, 2, 3, 4, 5, 127, 1024] { + let mut seed = 17u64; + let values: Vec = (0..n) + .map(|_| { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + seed as i64 + }) + .collect(); + let bytes = encode(&values, width, 32768).unwrap(); + assert_eq!(bytes, oracle(&values, width.bytes())); + let expected: Vec = values.iter().map(|&v| if width == Width::I32 { i64::from(v as i32) } else { v }).collect(); + assert_eq!(decode(&bytes, n, width, 32768).unwrap(), expected); + } + for values in [vec![1, 3, 4, 6], vec![i64::MAX, i64::MIN, 0], vec![7; 32]] { + assert_eq!(encode(&values, width, 4096).unwrap(), oracle(&values, width.bytes())); + } + } + } + #[test] + fn combined_reservation_budget_is_enforced() { + assert_eq!(encode(&[], Width::I64, 0).unwrap(), Vec::::new()); + assert!(matches!(encode(&[1], Width::I32, 16), Err(Error::Limit))); + assert!(encode(&[1], Width::I32, 17).is_ok()); + } +} From 02224df6cb51563b4cade129399dc2b197a4bee9 Mon Sep 17 00:00:00 2001 From: Daniel Voshart <205798369+voshart@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:48:09 -0500 Subject: [PATCH 09/19] Import LightCraft HTTP URL parser --- crates/http-url/src/lib.rs | 126 +++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 crates/http-url/src/lib.rs diff --git a/crates/http-url/src/lib.rs b/crates/http-url/src/lib.rs new file mode 100644 index 000000000..3babf1b13 --- /dev/null +++ b/crates/http-url/src/lib.rs @@ -0,0 +1,126 @@ +/// A parsed `http://` or `https://` URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Url { + pub tls: bool, + pub host: String, + pub port: u16, + /// Path and query, starting with `/`. + pub path: String, +} + +impl Url { + pub fn parse(s: &str) -> Result { + let bad = |why: &str| HttpError::BadUrl(format!("{s}: {why}")); + let s = s.trim(); + if s.chars().any(|c| c.is_control() || c == ' ') { + return Err(bad("contains spaces or control characters")); + } + let (tls, rest) = if let Some(r) = s.strip_prefix("https://") { + (true, r) + } else if let Some(r) = s.strip_prefix("http://") { + (false, r) + } else { + return Err(bad("only http:// and https:// are supported")); + }; + let rest = rest.split('#').next().unwrap_or_default(); + let (authority, path) = match rest.find(['/', '?']) { + Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()), + None => (rest, ""), + }; + let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") }; + if authority.contains('@') { + return Err(bad("user names and passwords in URLs are not supported")); + } + let (host, port) = if let Some(v6) = authority.strip_prefix('[') { + let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?; + let host = v6.get(..end).unwrap_or_default(); + let after = v6.get(end + 1..).unwrap_or_default(); + (host.to_string(), after.strip_prefix(':')) + } else { + match authority.rsplit_once(':') { + Some((h, p)) => (h.to_string(), Some(p)), + None => (authority.to_string(), None), + } + }; + let port = match port { + Some(p) => p.parse::().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?, + None if tls => 443, + None => 80, + }; + if host.is_empty() || !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | ':' | '_')) { + return Err(bad("bad host name")); + } + Ok(Url { tls, host: host.to_ascii_lowercase(), port, path }) + } + + /// `Location` of a redirect, resolved against this URL. + pub fn join(&self, location: &str) -> Result { + let l = location.trim(); + if l.starts_with("http://") || l.starts_with("https://") { + return Url::parse(l); + } + // another scheme (`ftp:`, `file:`, `data:`…) + if let Some((scheme, _)) = l.split_once(':') + && !scheme.is_empty() + && scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + { + return Err(HttpError::BadUrl(format!("redirect to an unsupported location `{scheme}:`"))); + } + let scheme = if self.tls { "https" } else { "http" }; + let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() }; + let base = format!("{scheme}://{host}:{}", self.port); + if let Some(rest) = l.strip_prefix("//") { + return Url::parse(&format!("{scheme}://{rest}")); + } + if l.starts_with('/') { + return Url::parse(&format!("{base}{l}")); + } + // relative to the current directory + let dir = self.path.split('?').next().unwrap_or("/"); + let dir = dir.rsplit_once('/').map_or("", |(d, _)| d); + Url::parse(&format!("{base}{dir}/{l}")) + } + + fn host_header(&self) -> String { + let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() }; + let default = if self.tls { 443 } else { 80 }; + if self.port == default { host } else { format!("{host}:{}", self.port) } + } + + /// The URL without its query (for messages: a query may hold a signed token). + pub fn display(&self) -> String { + let scheme = if self.tls { "https" } else { "http" }; + format!("{scheme}://{}{}", self.host_header(), self.path.split('?').next().unwrap_or("/")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_urls() { + let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap(); + assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "cdn.example.com", 443, "/models/sam3/model.safetensors?sig=1")); + assert_eq!(u.display(), "https://cdn.example.com/models/sam3/model.safetensors"); + let u = Url::parse("http://127.0.0.1:8080").unwrap(); + assert_eq!((u.tls, u.port, u.path.as_str()), (false, 8080, "/")); + let u = Url::parse("http://[::1]:9/a").unwrap(); + assert_eq!((u.host.as_str(), u.port), ("::1", 9)); + for bad in ["ftp://x/y", "https://", "https://a b/", "https://u:p@h/", "http://h:0/", "http://h:99999/", "https://h/\r\nX: y", "file:///etc"] + { + assert!(Url::parse(bad).is_err(), "{bad}"); + } + } + + #[test] + fn resolves_redirects() { + let u = Url::parse("https://a.example/x/y/file?q").unwrap(); + assert_eq!(u.join("https://b.example/z").unwrap().host, "b.example"); + assert_eq!(u.join("/root").unwrap().path, "/root"); + assert_eq!(u.join("other").unwrap().path, "/x/y/other"); + assert_eq!(u.join("//c.example/p").unwrap().host, "c.example"); + assert!(u.join("ftp://nope").is_err()); + } + +} From d577b23c9c5b796161d6eec275dd9afd8163a24d Mon Sep 17 00:00:00 2001 From: Daniel Voshart <205798369+voshart@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:53:07 -0500 Subject: [PATCH 10/19] Preserve bounded URLs and query-safe error messages --- crates/http-url/src/lib.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/http-url/src/lib.rs b/crates/http-url/src/lib.rs index 3babf1b13..bae96da5b 100644 --- a/crates/http-url/src/lib.rs +++ b/crates/http-url/src/lib.rs @@ -10,7 +10,10 @@ pub struct Url { impl Url { pub fn parse(s: &str) -> Result { - let bad = |why: &str| HttpError::BadUrl(format!("{s}: {why}")); + let bad = |why: &str| HttpError::BadUrl(why.to_string()); + if s.len() > 16 * 1024 { + return Err(bad("URL exceeds 16 KiB")); + } let s = s.trim(); if s.chars().any(|c| c.is_control() || c == ' ') { return Err(bad("contains spaces or control characters")); @@ -98,6 +101,12 @@ impl Url { mod tests { use super::*; + #[test] + fn rejects_oversized_and_control_character_urls() { + assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err()); + assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err()); + } + #[test] fn parses_urls() { let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap(); From 07fa210956f57ae31c09515c49fc3e59120a03b6 Mon Sep 17 00:00:00 2001 From: MarekSpodBiedry <73122968+MarekSpodBiedry@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:53:07 -0500 Subject: [PATCH 11/19] Preserve HTTPS redirect downgrade rejection --- crates/http-url/src/lib.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/http-url/src/lib.rs b/crates/http-url/src/lib.rs index bae96da5b..5c80f3c4c 100644 --- a/crates/http-url/src/lib.rs +++ b/crates/http-url/src/lib.rs @@ -60,7 +60,11 @@ impl Url { pub fn join(&self, location: &str) -> Result { let l = location.trim(); if l.starts_with("http://") || l.starts_with("https://") { - return Url::parse(l); + let url = Url::parse(l)?; + if self.tls && !url.tls { + return Err(HttpError::BadUrl("redirect from https to http refused".into())); + } + return Ok(url); } // another scheme (`ftp:`, `file:`, `data:`…) if let Some((scheme, _)) = l.split_once(':') @@ -130,6 +134,11 @@ mod tests { assert_eq!(u.join("other").unwrap().path, "/x/y/other"); assert_eq!(u.join("//c.example/p").unwrap().host, "c.example"); assert!(u.join("ftp://nope").is_err()); + // no downgrade from https to http, however the redirect is written + assert!(u.join("http://a.example/z").is_err()); + let plain = Url::parse("http://127.0.0.1:8080/a").unwrap(); + assert!(!plain.join("http://127.0.0.1:9/b").unwrap().tls); + assert!(plain.join("https://b.example/z").unwrap().tls); } } From 837e4d365a3ce56ac21b2fc95ebbaeeee8629b35 Mon Sep 17 00:00:00 2001 From: NetterSilver <133535268+Silvertree2010@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:48:09 -0500 Subject: [PATCH 12/19] Preserve URL normalization and malformed authority regressions --- crates/http-url/src/lib.rs | 112 +++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/crates/http-url/src/lib.rs b/crates/http-url/src/lib.rs index 5c80f3c4c..5bde118a3 100644 --- a/crates/http-url/src/lib.rs +++ b/crates/http-url/src/lib.rs @@ -8,6 +8,43 @@ pub struct Url { pub path: String, } +/// Remove `.` and `..` segments from the path part of `path` (RFC 3986 section 5.2.4); the query is kept as is. +fn remove_dot_segments(path: &str) -> String { + let (p, query) = match path.find('?') { + Some(i) => (path.get(..i).unwrap_or_default(), path.get(i..).unwrap_or_default()), + None => (path, ""), + }; + let mut out: Vec<&str> = Vec::new(); + let mut segments = p.split('/').skip(1).peekable(); + let mut trailing_slash = false; + while let Some(seg) = segments.next() { + let last = segments.peek().is_none(); + match seg { + "." => trailing_slash = last, + ".." => { + out.pop(); + trailing_slash = last; + } + _ => { + out.push(seg); + trailing_slash = false; + } + } + } + let mut result = format!("/{}", out.join("/")); + if trailing_slash && !result.ends_with('/') { + result.push('/'); + } + result.push_str(query); + result +} + +/// `s` without the leading `prefix`, matched ASCII case-insensitively (RFC 3986 §3.1). +fn strip_scheme<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { + let head = s.get(..prefix.len())?; + head.eq_ignore_ascii_case(prefix).then(|| s.get(prefix.len()..).unwrap_or_default()) +} + impl Url { pub fn parse(s: &str) -> Result { let bad = |why: &str| HttpError::BadUrl(why.to_string()); @@ -18,9 +55,9 @@ impl Url { if s.chars().any(|c| c.is_control() || c == ' ') { return Err(bad("contains spaces or control characters")); } - let (tls, rest) = if let Some(r) = s.strip_prefix("https://") { + let (tls, rest) = if let Some(r) = strip_scheme(s, "https://") { (true, r) - } else if let Some(r) = s.strip_prefix("http://") { + } else if let Some(r) = strip_scheme(s, "http://") { (false, r) } else { return Err(bad("only http:// and https:// are supported")); @@ -31,6 +68,7 @@ impl Url { None => (rest, ""), }; let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") }; + let path = remove_dot_segments(&path); if authority.contains('@') { return Err(bad("user names and passwords in URLs are not supported")); } @@ -38,6 +76,9 @@ impl Url { let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?; let host = v6.get(..end).unwrap_or_default(); let after = v6.get(end + 1..).unwrap_or_default(); + if !after.is_empty() && !after.starts_with(':') { + return Err(bad("unexpected text after IPv6 address")); + } (host.to_string(), after.strip_prefix(':')) } else { match authority.rsplit_once(':') { @@ -46,7 +87,8 @@ impl Url { } }; let port = match port { - Some(p) => p.parse::().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?, + Some(p) if p.bytes().all(|b| b.is_ascii_digit()) => p.parse::().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?, + Some(_) => return Err(bad("bad port")), None if tls => 443, None => 80, }; @@ -59,7 +101,7 @@ impl Url { /// `Location` of a redirect, resolved against this URL. pub fn join(&self, location: &str) -> Result { let l = location.trim(); - if l.starts_with("http://") || l.starts_with("https://") { + if strip_scheme(l, "http://").is_some() || strip_scheme(l, "https://").is_some() { let url = Url::parse(l)?; if self.tls && !url.tls { return Err(HttpError::BadUrl("redirect from https to http refused".into())); @@ -82,8 +124,16 @@ impl Url { if l.starts_with('/') { return Url::parse(&format!("{base}{l}")); } + // empty or fragment-only: the same document + if l.is_empty() || l.starts_with('#') { + return Ok(self.clone()); + } // relative to the current directory let dir = self.path.split('?').next().unwrap_or("/"); + // a query-only reference keeps the whole path, file name included (RFC 3986 §5.2.2) + if l.starts_with('?') { + return Url::parse(&format!("{base}{dir}{l}")); + } let dir = dir.rsplit_once('/').map_or("", |(d, _)| d); Url::parse(&format!("{base}{dir}/{l}")) } @@ -105,12 +155,46 @@ impl Url { mod tests { use super::*; + #[test] + fn fragment_only_or_empty_location_keeps_file_name_and_query() { + let base = Url::parse("https://a.example/x/y/file?q").unwrap(); + for l in ["#top", "", " "] { + let u = base.join(l).unwrap(); + assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/x/y/file?q"), "{l:?}"); + } + } + #[test] fn rejects_oversized_and_control_character_urls() { assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err()); assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err()); } + #[test] + fn join_query_only_keeps_file_name() { + let u = Url::parse("https://a.example/x/y/file?q=1").unwrap(); + assert_eq!(u.join("?page=2").unwrap().path, "/x/y/file?page=2"); + } + + #[test] + fn join_removes_dot_segments() { + let base = Url::parse("https://a.example/x/y/file").unwrap(); + for (loc, want) in + [("../z", "/x/z"), ("./z", "/x/y/z"), ("../../../z", "/z"), ("/a/./b/../c", "/a/c"), ("../z?q=1", "/x/z?q=1"), ("..", "/x/")] + { + assert_eq!(base.join(loc).unwrap().path, want, "{loc}"); + } + } + + #[test] + fn url_scheme_is_matched_case_insensitively() { + let u = Url::parse("HTTPS://A.example/z").unwrap(); + assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/z")); + assert!(Url::parse("Http://a.example/").is_ok()); + let j = Url::parse("https://a.example/start").unwrap().join("HTTPS://cdn.example/file").unwrap(); + assert_eq!((j.tls, j.host.as_str(), j.path.as_str()), (true, "cdn.example", "/file")); + } + #[test] fn parses_urls() { let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap(); @@ -126,6 +210,26 @@ mod tests { } } + #[test] + fn rejects_signed_ports() { + for bad in ["http://example.com:+80/x", "http://example.com:-80/x", "http://[::1]:+80/x"] { + assert!(Url::parse(bad).is_err(), "{bad}"); + } + assert!(Url::parse("http://a.test/").unwrap().join("http://example.com:+80/x").is_err()); + assert_eq!(Url::parse("http://example.com:80/x").unwrap().port, 80); + } + + #[test] + fn rejects_text_after_a_bracketed_ipv6_host() { + for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] { + assert!(matches!(Url::parse(bad), Err(HttpError::BadUrl(_))), "{bad}"); + } + let u = Url::parse("http://[::1]:8080/x").unwrap(); + assert_eq!((u.host.as_str(), u.port), ("::1", 8080)); + let u = Url::parse("http://[::1]/x").unwrap(); + assert_eq!((u.host.as_str(), u.port), ("::1", 80)); + } + #[test] fn resolves_redirects() { let u = Url::parse("https://a.example/x/y/file?q").unwrap(); From c247a085be0b2db5456f1f9178024d6a12e4cc57 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 16:50:07 -0500 Subject: [PATCH 13/19] Expose dependency-free HTTP URL syntax with bounded redirect input --- Cargo.lock | 4 ++++ Cargo.toml | 1 + crates/http-url/Cargo.toml | 11 +++++++++ crates/http-url/README.md | 25 ++++++++++++++++++++ crates/http-url/src/lib.rs | 47 ++++++++++++++++++++++++++++++-------- 5 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 crates/http-url/Cargo.toml create mode 100644 crates/http-url/README.md diff --git a/Cargo.lock b/Cargo.lock index b3325f790..b36240d6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1879,6 +1879,10 @@ dependencies = [ "proptest", ] +[[package]] +name = "craft-http-url" +version = "0.1.0" + [[package]] name = "craft-i18n" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 0c9729e00..1b409247e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ rust-version = "1.95" repository = "https://github.com/storytold/craft-libs" [workspace.dependencies] +craft-http-url = { path = "crates/http-url" } craft-smime = { path = "crates/smime" } craft-ber = { path = "crates/ber" } craft-proto = { path = "crates/proto" } diff --git a/crates/http-url/Cargo.toml b/crates/http-url/Cargo.toml new file mode 100644 index 000000000..f23d9bbc7 --- /dev/null +++ b/crates/http-url/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "craft-http-url" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license = "Apache-2.0" +repository.workspace = true +description = "Bounded HTTP URL parsing and redirect resolution without network access" + +[lints] +workspace = true diff --git a/crates/http-url/README.md b/crates/http-url/README.md new file mode 100644 index 000000000..cc3dc1bf5 --- /dev/null +++ b/crates/http-url/README.md @@ -0,0 +1,25 @@ +# craft-http-url + +HTTP/HTTPS URL parsing, dot-segment removal, relative redirects, HTTPS downgrade rejection, +and query-redacted display. No dependencies or network operations. Parsing rejects inputs +larger than 16 KiB and redirects are bounded before concatenation. + +This is not an SSRF filter, DNS resolver or HTTP client. Local/private hosts are allowed. +Applications keep destination permission, authentication, TLS, request/response limits and +redirect-count policy. Public fields preserve the original API; manually constructed values +must not be treated as validated URLs. + +## Attribution + +Extracted from LightCraft `crates/fetch/src/http.rs` at +`2dcb258ba40738230a07484a95eb27dc84ed66fe` (Apache-2.0). +Original parser: Daniel Voshart, `7fa7506db46d69d609e85e39b338f33786ed0293`. +HTTPS downgrade protection: MarekSpodBiedry, `4a91a1c218bf3d0ac49cc73ebb60621a21ae55e2`. +URL normalization, scheme, fragment and authority fixes: NetterSilver, including +`edc369565eef92161b56b8f1e39ef53629c05489`, +`78f0e6e650444886069f4d4ae4e6593f221b98d4`, +`cee93496587d31c514aaeced3e6b8027cfdb41a0`, +`dee5f7910ad8c24cf28bab528f8d007550af2b23`, +`c3a5d9b7f1f837dc14c01942e3d0293ecbf36345` and +`13849307a9343504339d68b086f1a32cdcbbb457`. +Source-author commits are retained separately in this extraction. diff --git a/crates/http-url/src/lib.rs b/crates/http-url/src/lib.rs index 5bde118a3..1e3e5875f 100644 --- a/crates/http-url/src/lib.rs +++ b/crates/http-url/src/lib.rs @@ -1,3 +1,24 @@ +//! Bounded HTTP/HTTPS URL parsing and redirect resolution, extracted from LightCraft. +//! +//! This is syntax handling, not a network permission or SSRF policy. It permits local/private +//! hosts and performs no DNS, socket or TLS operations. Callers own destination allowlists, +//! credential forwarding, redirect counts, connection deadlines and response limits. +//! Only parsed values have validated fields; constructing public fields bypasses parsing. + +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +/// A URL or redirect cannot be represented by this bounded HTTP URL syntax. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UrlError(pub String); + +impl std::fmt::Display for UrlError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} +impl std::error::Error for UrlError {} + /// A parsed `http://` or `https://` URL. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Url { @@ -46,8 +67,8 @@ fn strip_scheme<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { } impl Url { - pub fn parse(s: &str) -> Result { - let bad = |why: &str| HttpError::BadUrl(why.to_string()); + pub fn parse(s: &str) -> Result { + let bad = |why: &str| UrlError(why.to_string()); if s.len() > 16 * 1024 { return Err(bad("URL exceeds 16 KiB")); } @@ -99,12 +120,16 @@ impl Url { } /// `Location` of a redirect, resolved against this URL. - pub fn join(&self, location: &str) -> Result { + pub fn join(&self, location: &str) -> Result { + // Reject before concatenating a hostile Location with the base URL. + if location.len() > 16 * 1024 || self.host.len().saturating_add(self.path.len()) > 16 * 1024 { + return Err(UrlError("URL exceeds 16 KiB".into())); + } let l = location.trim(); if strip_scheme(l, "http://").is_some() || strip_scheme(l, "https://").is_some() { let url = Url::parse(l)?; if self.tls && !url.tls { - return Err(HttpError::BadUrl("redirect from https to http refused".into())); + return Err(UrlError("redirect from https to http refused".into())); } return Ok(url); } @@ -113,7 +138,7 @@ impl Url { && !scheme.is_empty() && scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) { - return Err(HttpError::BadUrl(format!("redirect to an unsupported location `{scheme}:`"))); + return Err(UrlError(format!("redirect to an unsupported location `{scheme}:`"))); } let scheme = if self.tls { "https" } else { "http" }; let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() }; @@ -138,7 +163,8 @@ impl Url { Url::parse(&format!("{base}{dir}/{l}")) } - fn host_header(&self) -> String { + /// Authority formatted for an HTTP Host field. + pub fn host_header(&self) -> String { let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() }; let default = if self.tls { 443 } else { 80 }; if self.port == default { host } else { format!("{host}:{}", self.port) } @@ -153,7 +179,8 @@ impl Url { #[cfg(test)] mod tests { - use super::*; + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::{Url, UrlError}; #[test] fn fragment_only_or_empty_location_keeps_file_name_and_query() { @@ -166,6 +193,9 @@ mod tests { #[test] fn rejects_oversized_and_control_character_urls() { + let base = Url::parse("https://example.com/path").unwrap(); + assert!(base.join(&format!("#{}", "x".repeat(16 * 1024))).is_err()); + assert!(base.join(&"x".repeat(16 * 1024 + 1)).is_err()); assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err()); assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err()); } @@ -222,7 +252,7 @@ mod tests { #[test] fn rejects_text_after_a_bracketed_ipv6_host() { for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] { - assert!(matches!(Url::parse(bad), Err(HttpError::BadUrl(_))), "{bad}"); + assert!(matches!(Url::parse(bad), Err(UrlError(_))), "{bad}"); } let u = Url::parse("http://[::1]:8080/x").unwrap(); assert_eq!((u.host.as_str(), u.port), ("::1", 8080)); @@ -244,5 +274,4 @@ mod tests { assert!(!plain.join("http://127.0.0.1:9/b").unwrap().tls); assert!(plain.join("https://b.example/z").unwrap().tls); } - } From 17db8defc24dc0ae1811f8700a58bb474dd270a9 Mon Sep 17 00:00:00 2001 From: NetterSilver <133535268+Silvertree2010@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:31:18 -0500 Subject: [PATCH 14/19] cms: preserve every numeric token when reading Cube domains Adapt the strict-token correction from storytold/lightcraft#3464, original commit 77bb537552d59142814eaa65cfea4626c33d6dfc, to the existing shared Cube reader. --- crates/cms/src/lutfile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/cms/src/lutfile.rs b/crates/cms/src/lutfile.rs index b7cc99bf3..8b3a8495f 100644 --- a/crates/cms/src/lutfile.rs +++ b/crates/cms/src/lutfile.rs @@ -176,7 +176,7 @@ pub fn parse_cube(text: &str) -> Result { let mut it = line.split_whitespace(); let key = it.next().unwrap_or(""); let nums = |it: std::str::SplitWhitespace<'_>| -> Result<[f32; 3], LutError> { - let v: Vec = it.filter_map(|s| s.parse().ok()).collect(); + let v: Vec = it.map(|s| s.parse().ok()).collect::>().ok_or_else(|| LutError(format!("bad line `{line}`")))?; if v.len() == 3 { Ok([v[0], v[1], v[2]]) } else { err(format!("bad line `{line}`")) } }; match key { From 33852afad44d968f2c78a0fef7f4011e76aa7044 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 15 Nov 2026 17:31:18 -0500 Subject: [PATCH 15/19] cms: share bounded three-dimensional Cube profile parsing Move LightCraft profile-reader semantics (origin 4a490db2) into the existing shared Cube engine. Preserve optional titles, finite endpoints and caller-selected limits; leave application interpolation and registry policies in the consumer. Integration and resource-bound adaptations by Dylan M. Taylor. --- crates/cms/src/lutfile.rs | 164 ++++++++++++++++++++++++++++---------- 1 file changed, 122 insertions(+), 42 deletions(-) diff --git a/crates/cms/src/lutfile.rs b/crates/cms/src/lutfile.rs index 8b3a8495f..1dff864c1 100644 --- a/crates/cms/src/lutfile.rs +++ b/crates/cms/src/lutfile.rs @@ -150,62 +150,142 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> { } } -/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table. -pub fn parse_cube(text: &str) -> Result { - let mut title = String::new(); - let (mut size3, mut size1) = (0usize, 0usize); - let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]); - let mut rows: Vec = Vec::new(); - for line in text.lines() { - // Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an - // unquoted title's comment must not turn that comment into part of the title. - if let Some(quoted) = line - .trim_start() - .strip_prefix("TITLE") - .filter(|rest| rest.starts_with(char::is_whitespace)) - .and_then(|rest| rest.trim_start().strip_prefix('"')) +/// A bounded, three-dimensional Cube profile with RGB rows in file order. +/// Unlike [`parse_cube`], this preserves the optional title and does not resample 1D curves. +#[derive(Clone, Debug, PartialEq)] +pub struct Cube3d { + pub title: Option, + pub size: usize, + pub data: Vec<[f32; 3]>, + pub domain_min: [f32; 3], + pub domain_max: [f32; 3], +} + +/// Read the simple 3D profile dialect used by camera-profile consumers. +/// +/// `max_size` must be in 2..=256; the row budget is checked before each allocation. +/// Whole-line comments and literal titles are preserved. Unknown alphabetic directives +/// are ignored; 1D tables are refused. Finite domains are returned unchanged so the +/// consumer retains its existing policy for equal or reversed endpoints. +/// Numeric fields must consist of exactly three finite numbers (never silently skip tokens). +pub fn parse_cube_3d(text: &str, max_size: usize) -> Result { + let p = read_cube(text, Some(max_size))?; + if p.size3 == 0 { + return err("not a 3D .cube LUT (no LUT_3D_SIZE)"); + } + let expected = p.size3.checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?; + if p.rows.len() != expected { + return err(format!("expected {expected} entries, found {}", p.rows.len())); + } + Ok(Cube3d { title: p.title, size: p.size3, data: p.rows, domain_min: p.min, domain_max: p.max }) +} + +struct CubeRows { + title: Option, + size3: usize, + size1: usize, + min: [f32; 3], + max: [f32; 3], + rows: Vec<[f32; 3]>, +} + +fn cube_triplet(text: &str) -> Result<[f32; 3], LutError> { + let mut it = text.split_whitespace(); + let mut row = [0.0; 3]; + for value in &mut row { + *value = it.next().and_then(|s| s.parse::().ok()).filter(|v| v.is_finite()).ok_or_else(|| LutError(format!("bad line `{text}`")))?; + } + if it.next().is_some() { + return err(format!("bad line `{text}`")); + } + Ok(row) +} + +fn read_cube(text: &str, profile_max: Option) -> Result { + if profile_max.is_some_and(|n| !(2..=256).contains(&n)) { + return err("3D profile size limit must be in 2..=256"); + } + let mut p = CubeRows { title: None, size3: 0, size1: 0, min: [0.0; 3], max: [1.0; 3], rows: Vec::new() }; + let row_limit = profile_max.unwrap_or(256).checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?; + for raw in text.lines() { + let profile = profile_max.is_some(); + let line = if profile { raw.trim() } else { raw.split('#').next().unwrap_or("").trim() }; + // The standard dialect protects '#' inside quoted titles; profile titles are literal. + if !profile + && let Some(quoted) = + raw.trim_start().strip_prefix("TITLE").filter(|r| r.starts_with(char::is_whitespace)).and_then(|r| r.trim_start().strip_prefix('"')) { - let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") }; - title = value.to_string(); + let Some((title, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") }; + p.title = Some(title.to_string()); continue; } - let line = line.split('#').next().unwrap_or("").trim(); - if line.is_empty() { + if line.is_empty() || line.starts_with('#') { continue; } - let mut it = line.split_whitespace(); - let key = it.next().unwrap_or(""); - let nums = |it: std::str::SplitWhitespace<'_>| -> Result<[f32; 3], LutError> { - let v: Vec = it.map(|s| s.parse().ok()).collect::>().ok_or_else(|| LutError(format!("bad line `{line}`")))?; - if v.len() == 3 { Ok([v[0], v[1], v[2]]) } else { err(format!("bad line `{line}`")) } + let (key, rest) = if profile { + ["TITLE", "LUT_3D_SIZE", "LUT_1D_SIZE", "DOMAIN_MIN", "DOMAIN_MAX"] + .into_iter() + .find_map(|key| line.strip_prefix(key).map(|rest| (key, rest.trim()))) + .unwrap_or_else(|| line.split_once(char::is_whitespace).unwrap_or((line, ""))) + } else { + line.split_once(char::is_whitespace).map_or((line, ""), |(key, rest)| (key, rest.trim())) }; match key { - "TITLE" => title = line[5..].trim().trim_matches('"').to_string(), - "LUT_3D_SIZE" => size3 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_3D_SIZE".into()))?, - "LUT_1D_SIZE" => size1 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?, - "DOMAIN_MIN" => dmin = nums(it)?, - "DOMAIN_MAX" => dmax = nums(it)?, - "LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" => { - let v: Vec = it.filter_map(|s| s.parse().ok()).collect(); - if v.len() == 2 { - dmin = [v[0]; 3]; - dmax = [v[1]; 3]; + "TITLE" => p.title = Some(rest.trim_matches('"').to_string()), + "LUT_3D_SIZE" => { + let value = if profile { rest } else { rest.split_whitespace().next().unwrap_or("") }; + p.size3 = value.parse().map_err(|_| LutError(format!("bad LUT_3D_SIZE `{value}`")))?; + let limit = profile_max.unwrap_or(MAX_SIZE); + if !(2..=limit).contains(&p.size3) { + return err(format!("LUT_3D_SIZE {} out of range", p.size3)); } } - k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => { - let mut v = vec![k.parse::().map_err(|_| LutError(format!("bad number in `{line}`")))?]; - for value in it { - v.push(value.parse::().map_err(|_| LutError(format!("bad number in `{line}`")))?); + "LUT_1D_SIZE" => { + if profile { + return err("1D LUTs are not supported (only 3D .cube files)"); } - if v.len() != 3 { - return err(format!("bad data line `{line}`")); + p.size1 = rest.split_whitespace().next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?; + if p.size1 > row_limit { + return err("1D LUT exceeds row budget"); + } + } + "DOMAIN_MIN" => p.min = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MIN".into()))?, + "DOMAIN_MAX" => p.max = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MAX".into()))?, + "LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" if !profile => { + let mut it = rest.split_whitespace(); + let min = it.next().and_then(|s| s.parse::().ok()); + let max = it.next().and_then(|s| s.parse::().ok()); + if let (Some(min), Some(max), None) = (min, max, it.next()) { + p.min = [min; 3]; + p.max = [max; 3]; + } + } + _ => { + let numeric = key.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.'); + let nonfinite = key.parse::().is_ok(); + if numeric || nonfinite || (profile && !key.starts_with(|c: char| c.is_ascii_alphabetic())) { + let row = cube_triplet(line)?; + if p.rows.len() >= row_limit { + return err("LUT exceeds row budget"); + } + // Explicit fallible growth prevents capacity growth past the configured budget. + if p.rows.len() == p.rows.capacity() { + let grow = (p.rows.capacity().max(1)).min(row_limit - p.rows.len()); + p.rows.try_reserve_exact(grow).map_err(|_| LutError("LUT allocation failed".into()))?; + } + p.rows.push(row); } - rows.extend(v); } - // Unknown keywords (e.g. Resolve's LUT_IN_VIDEO_RANGE) are ignored, as the spec allows. - _ => {} } } + Ok(p) +} + +/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table. +pub fn parse_cube(text: &str) -> Result { + let p = read_cube(text, None)?; + let (title, size3, size1, dmin, dmax) = (p.title.unwrap_or_default(), p.size3, p.size1, p.min, p.max); + let rows: Vec = p.rows.into_iter().flatten().collect(); let domain_ok = (0..3).all(|k| dmin[k] < dmax[k] && dmin[k].is_finite() && dmax[k].is_finite()); if (size3 > 0 || size1 >= 2) && !domain_ok { return err(format!("DOMAIN_MIN {:?} must lie below DOMAIN_MAX {:?}", dmin, dmax)); From 753c676f7bc66e9665727712999a6962a81f5c02 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 17:31:18 -0500 Subject: [PATCH 16/19] test(cms): cover bounded Cube profile compatibility and token rejection --- crates/cms/src/lutfile.rs | 49 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/cms/src/lutfile.rs b/crates/cms/src/lutfile.rs index 1dff864c1..e2e0a5980 100644 --- a/crates/cms/src/lutfile.rs +++ b/crates/cms/src/lutfile.rs @@ -1466,3 +1466,52 @@ LUT_3D_SIZE 2 assert_eq!(back.data, [0.0, 0.25, 1.0].repeat(8)); } } + +#[cfg(test)] +mod profile_cube_tests { + use super::*; + + fn cube(prefix: &str) -> String { + format!("{prefix}\nLUT_3D_SIZE 2\n{}", "0.25 0.5 0.75\n".repeat(8)) + } + + #[test] + fn profile_preserves_optional_literal_title_and_domains() { + let no_title = parse_cube_3d(&cube(""), 256).unwrap(); + assert_eq!(no_title.title, None); + let value = parse_cube_3d(&cube("TITLE \"look # literal\"\nDOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1"), 256).unwrap(); + assert_eq!(value.title.as_deref(), Some("look # literal")); + assert_eq!(value.domain_min, [1.0, 0.0, 2.0]); + assert_eq!(value.domain_max, [0.0, 0.0, 1.0]); + assert_eq!(value.data, vec![[0.25, 0.5, 0.75]; 8]); + assert_eq!(parse_cube_3d(&cube("TITLE \"\""), 256).unwrap().title.as_deref(), Some("")); + assert!(parse_cube(&cube("DOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1")).is_err()); + } + + #[test] + fn profile_limits_are_checked_without_large_allocations() { + assert!(parse_cube_3d("LUT_3D_SIZE 256", 256).unwrap_err().0.contains("16777216")); + assert!(parse_cube_3d("LUT_3D_SIZE 257", 256).unwrap_err().0.contains("out of range")); + assert!(parse_cube_3d("LUT_3D_SIZE 256", 129).unwrap_err().0.contains("out of range")); + assert!(parse_cube_3d(&cube(""), usize::MAX).is_err()); + assert!(parse_cube_3d(&cube(""), 1).is_err()); + assert!(parse_cube_3d(&format!("{}0 0 0\n", cube("")), 2).unwrap_err().0.contains("budget")); + assert!(parse_cube_3d("LUT_1D_SIZE 2\n0 0 0\n1 1 1", 256).is_err()); + } + + #[test] + fn every_numeric_token_must_be_valid_and_finite() { + for row in ["0 ignored 0 0", "0 0 0 ignored", "NaN 0 0", "inf 0 0", "0 0 -inf", "1e39 0 0", "0 0", "0 0 0 0"] { + assert!(parse_cube_3d(&format!("LUT_3D_SIZE 2\n{row}\n{}", "0 0 0\n".repeat(7)), 256).is_err(), "{row}"); + assert!(parse_cube_3d(&cube(&format!("DOMAIN_MIN {row}")), 256).is_err(), "{row}"); + } + } + + #[test] + fn standard_reader_keeps_quoted_comments_and_1d_expansion() { + assert_eq!(parse_cube(&cube("TITLE \"look # preserved\" # comment")).unwrap().title, "look # preserved"); + let value = parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1").unwrap(); + assert_eq!(value.size, 33); + assert!(parse_cube_3d(&cube("DOMAIN_MIN 0 0 0 # not a whole-line comment"), 256).is_err()); + } +} From f0fa27110d43519832df9a7a60962bef25daa273 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 15 Nov 2026 17:44:34 -0500 Subject: [PATCH 17/19] Preserve attributed print raster writer as shared extension source --- crates/pdf/src/write/raster_print.rs | 520 +++++++++++++++++++++++++++ 1 file changed, 520 insertions(+) create mode 100644 crates/pdf/src/write/raster_print.rs diff --git a/crates/pdf/src/write/raster_print.rs b/crates/pdf/src/write/raster_print.rs new file mode 100644 index 000000000..abce4061e --- /dev/null +++ b/crates/pdf/src/write/raster_print.rs @@ -0,0 +1,520 @@ +//! A small PDF writer for print output: pages of a given size in points, each with raster images +//! placed on it, tagged with their ICC profile (`ICCBased`) so the numbers mean the same colour in +//! every reader and RIP, Flate-compressed (lossless) at 8 or 16 bits per component, or carried as +//! JPEG (`DCTDecode`) where small files matter more than exact samples. Optionally an output intent: +//! the printing condition (the printer / paper profile) the file was made for, as PDF/X readers +//! expect it in the catalog. Profiles shared by several images are written once. +//! +//! The file is PDF 1.6 (16-bit images need 1.5): classic cross-reference table, no object streams, +//! deterministic (no dates or IDs), so the same pages always give the same bytes. +//! +//! Sources: ISO 32000-1:2008 (§7.3 objects, §7.5 file structure, §8.6.5.5 ICCBased colour spaces, +//! §8.9 images, §14.11.5 output intents). Own design. + +use std::collections::HashMap; +use std::sync::Arc; + +/// Pages per document: a print job beyond this is a mistake, not a book. +pub const MAX_PAGES: usize = 1000; +/// Images per page. +pub const MAX_IMAGES_PER_PAGE: usize = 256; +/// Largest page side in points (200 inches): more than any printer's roll. +pub const MAX_PAGE_POINTS: f64 = 14_400.0; +/// Largest image, in pixels per side. +pub const MAX_IMAGE_SIDE: u32 = 65_535; + +/// How an image's samples are stored. +#[derive(Clone, Debug, PartialEq)] +pub enum ImageData { + /// Interleaved samples, `bits` (8 or 16) per component, 16-bit values big-endian: compressed + /// losslessly in the file. + Raw { bits: u8, samples: Vec }, + /// A baseline JPEG stream, copied into the file as it is. + Jpeg(Vec), + /// Samples already compressed with zlib (see [`PdfImage::deflated`]), `bits` per component. + Flate { bits: u8, data: Vec }, +} + +/// A raster image and the colour space its samples are in. +#[derive(Clone, Debug, PartialEq)] +pub struct PdfImage { + pub width: u32, + pub height: u32, + /// Components per pixel: 1 (grey), 3 (RGB) or 4 (CMYK). + pub channels: u8, + pub data: ImageData, + /// The ICC profile the samples are in; without one the image is device colour. + pub icc: Option>>, +} + +impl PdfImage { + /// An image from interleaved 16-bit samples. + pub fn from_u16(width: u32, height: u32, channels: u8, samples: &[u16], icc: Option>>) -> PdfImage { + let bytes = samples.iter().flat_map(|v| v.to_be_bytes()).collect(); + PdfImage { width, height, channels, data: ImageData::Raw { bits: 16, samples: bytes }, icc } + } + + /// The same image with its samples compressed now (checked first), so a long print doesn't + /// hold every page's uncompressed samples until the file is written. + pub fn deflated(self) -> Result { + self.check()?; + let data = match self.data { + ImageData::Raw { bits, samples } => ImageData::Flate { bits, data: deflate(&samples) }, + other => other, + }; + Ok(PdfImage { data, ..self }) + } + + fn check(&self) -> Result<(), String> { + if self.width == 0 || self.height == 0 || self.width > MAX_IMAGE_SIDE || self.height > MAX_IMAGE_SIDE { + return Err(format!("an image of {} × {} pixels can't be written", self.width, self.height)); + } + if !matches!(self.channels, 1 | 3 | 4) { + return Err(format!("images have 1, 3 or 4 components, not {}", self.channels)); + } + match &self.data { + ImageData::Raw { bits, samples } => { + if !matches!(bits, 8 | 16) { + return Err(format!("images have 8 or 16 bits per component, not {bits}")); + } + let want = u64::from(self.width) * u64::from(self.height) * u64::from(self.channels) * u64::from(bits / 8); + if samples.len() as u64 != want { + return Err(format!("the image needs {want} bytes of samples, not {}", samples.len())); + } + } + ImageData::Jpeg(data) => { + if !data.starts_with(&[0xFF, 0xD8]) { + return Err("the image isn't a JPEG stream".into()); + } + } + ImageData::Flate { bits, data } => { + if !matches!(bits, 8 | 16) || data.is_empty() { + return Err("a compressed image has 8 or 16 bits per component and some data".into()); + } + } + } + Ok(()) + } +} + +/// Where an image goes on its page, in points from the page's bottom-left corner. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Placement { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +/// One page: its size in points and the images on it, drawn in order (later ones on top). +#[derive(Clone, Debug, PartialEq)] +pub struct PdfPage { + pub width: f64, + pub height: f64, + pub images: Vec<(PdfImage, Placement)>, +} + +/// The printing condition the document is made for (PDF/X-style output intent). +#[derive(Clone, Debug, PartialEq)] +pub struct OutputIntent { + /// The printer / paper profile. + pub icc: Arc>, + /// Its components: 1, 3 or 4. + pub channels: u8, + /// A name for the condition (the profile's description). + pub condition: String, +} + +/// Document-level settings. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PdfInfo { + pub title: Option, + pub output_intent: Option, +} + +fn finite(v: f64) -> bool { + v.is_finite() +} + +/// A number as PDF writes it: up to four decimals, no exponent, no trailing zeros. +fn num(v: f64) -> String { + let s = format!("{:.4}", if finite(v) { v } else { 0.0 }); + let s = s.trim_end_matches('0').trim_end_matches('.'); + if s == "-0" || s.is_empty() { "0".into() } else { s.into() } +} + +/// A text string: plain ASCII as a literal string, anything else as UTF-16BE with a byte order mark. +fn text(s: &str) -> String { + if s.bytes().all(|b| (0x20..0x7f).contains(&b)) { + let mut out = String::with_capacity(s.len() + 2); + out.push('('); + for c in s.chars() { + if matches!(c, '(' | ')' | '\\') { + out.push('\\'); + } + out.push(c); + } + out.push(')'); + out + } else { + let mut out = String::from("'); + out + } +} + +struct Writer { + out: Vec, + offsets: Vec, +} + +impl Writer { + /// Reserve an object number (filled later by [`Writer::object`]). + fn reserve(&mut self) -> usize { + self.offsets.push(0); + self.offsets.len() + } + + fn object(&mut self, id: usize, dict: &str, stream: Option<&[u8]>) { + if let Some(slot) = self.offsets.get_mut(id.wrapping_sub(1)) { + *slot = self.out.len(); + } + self.out.extend_from_slice(format!("{id} 0 obj\n{dict}").as_bytes()); + if let Some(data) = stream { + self.out.extend_from_slice(b"\nstream\n"); + self.out.extend_from_slice(data); + self.out.extend_from_slice(b"\nendstream"); + } + self.out.extend_from_slice(b"\nendobj\n"); + } +} + +fn deflate(data: &[u8]) -> Vec { + miniz_oxide::deflate::compress_to_vec_zlib(data, 6) +} + +/// Write a document. Errors name the first page or image that can't be written. +pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result, String> { + if pages.is_empty() { + return Err("there are no pages to write".into()); + } + if pages.len() > MAX_PAGES { + return Err(format!("a document holds at most {MAX_PAGES} pages")); + } + for (n, page) in pages.iter().enumerate() { + let n = n + 1; + if !(finite(page.width) && finite(page.height)) + || page.width <= 0.0 + || page.height <= 0.0 + || page.width > MAX_PAGE_POINTS + || page.height > MAX_PAGE_POINTS + { + return Err(format!("page {n} has no usable size")); + } + if page.images.len() > MAX_IMAGES_PER_PAGE { + return Err(format!("page {n} has more than {MAX_IMAGES_PER_PAGE} images")); + } + for (image, at) in &page.images { + image.check().map_err(|e| format!("page {n}: {e}"))?; + if ![at.x, at.y, at.width, at.height].into_iter().all(finite) || at.width <= 0.0 || at.height <= 0.0 { + return Err(format!("page {n}: an image has no usable placement")); + } + } + } + if let Some(intent) = &info.output_intent + && !matches!(intent.channels, 1 | 3 | 4) + { + return Err("the output intent's profile has 1, 3 or 4 components".into()); + } + + let mut w = Writer { out: Vec::new(), offsets: Vec::new() }; + w.out.extend_from_slice(b"%PDF-1.6\n%\xE2\xE3\xCF\xD3\n"); + let catalog = w.reserve(); + let pages_id = w.reserve(); + let info_id = w.reserve(); + + // ICC profiles, each written once (keyed by content) with its component count + let mut profiles: HashMap, usize> = HashMap::new(); + let mut profile = |w: &mut Writer, icc: &Arc>, n: u8| -> usize { + if let Some(&id) = profiles.get(icc.as_slice()) { + return id; + } + let id = w.reserve(); + let data = deflate(icc); + let alternate = match n { + 1 => "/DeviceGray", + 4 => "/DeviceCMYK", + _ => "/DeviceRGB", + }; + w.object(id, &format!("<< /N {n} /Alternate {alternate} /Filter /FlateDecode /Length {} >>", data.len()), Some(&data)); + profiles.insert(icc.to_vec(), id); + id + }; + + let intent_id = info.output_intent.as_ref().map(|intent| profile(&mut w, &intent.icc, intent.channels)); + + let mut kids = Vec::with_capacity(pages.len()); + for page in pages { + let page_id = w.reserve(); + let mut xobjects = String::new(); + let mut content = String::new(); + for (k, (image, at)) in page.images.iter().enumerate() { + let space = match &image.icc { + Some(icc) => format!("[/ICCBased {} 0 R]", profile(&mut w, icc, image.channels)), + None => match image.channels { + 1 => "/DeviceGray".into(), + 4 => "/DeviceCMYK".into(), + _ => "/DeviceRGB".into(), + }, + }; + let id = w.reserve(); + let (bits, filter, data): (u8, &str, std::borrow::Cow<[u8]>) = match &image.data { + ImageData::Raw { bits, samples } => (*bits, "/FlateDecode", deflate(samples).into()), + ImageData::Flate { bits, data } => (*bits, "/FlateDecode", data.as_slice().into()), + ImageData::Jpeg(jpeg) => (8, "/DCTDecode", jpeg.as_slice().into()), + }; + w.object( + id, + &format!( + "<< /Type /XObject /Subtype /Image /Width {} /Height {} /ColorSpace {space} /BitsPerComponent {bits} /Filter {filter} /Length {} >>", + image.width, + image.height, + data.len() + ), + Some(&data), + ); + xobjects.push_str(&format!("/Im{k} {id} 0 R ")); + content.push_str(&format!("q {} 0 0 {} {} {} cm /Im{k} Do Q\n", num(at.width), num(at.height), num(at.x), num(at.y))); + } + let content_id = w.reserve(); + let stream = deflate(content.as_bytes()); + w.object(content_id, &format!("<< /Filter /FlateDecode /Length {} >>", stream.len()), Some(&stream)); + w.object( + page_id, + &format!( + "<< /Type /Page /Parent {pages_id} 0 R /MediaBox [0 0 {} {}] /Resources << /XObject << {xobjects}>> >> /Contents {content_id} 0 R >>", + num(page.width), + num(page.height) + ), + None, + ); + kids.push(format!("{page_id} 0 R")); + } + w.object(pages_id, &format!("<< /Type /Pages /Count {} /Kids [{}] >>", kids.len(), kids.join(" ")), None); + + let intents = match (&info.output_intent, intent_id) { + (Some(intent), Some(id)) => format!( + " /OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX /OutputConditionIdentifier (Custom) /Info {} /DestOutputProfile {id} 0 R >>]", + text(&intent.condition) + ), + _ => String::new(), + }; + w.object(catalog, &format!("<< /Type /Catalog /Pages {pages_id} 0 R{intents} >>"), None); + let title = info.title.as_deref().map(|t| format!(" /Title {}", text(t))).unwrap_or_default(); + w.object(info_id, &format!("<< /Producer (LightCraft){title} >>"), None); + + let xref = w.out.len(); + let mut table = format!("xref\n0 {}\n0000000000 65535 f \n", w.offsets.len() + 1); + for offset in &w.offsets { + table.push_str(&format!("{offset:010} 00000 n \n")); + } + table.push_str(&format!("trailer\n<< /Size {} /Root {catalog} 0 R /Info {info_id} 0 R >>\nstartxref\n{xref}\n%%EOF\n", w.offsets.len() + 1)); + w.out.extend_from_slice(table.as_bytes()); + Ok(w.out) +} + +#[cfg(test)] +mod tests { + use super::*; + use lightcraft_codecs::NamedSpace; + + /// The body of object `id` (dictionary and raw stream bytes), found through the xref table. + fn object(pdf: &[u8], id: usize) -> (String, Vec) { + // byte offsets: the streams are binary, so search the bytes, not a lossy string + let start = pdf.windows(10).rposition(|w| w == b"startxref\n").unwrap() + 10; + let tail = std::str::from_utf8(&pdf[start..]).unwrap(); + let xref: usize = tail.lines().next().unwrap().trim().parse().unwrap(); + let table = std::str::from_utf8(&pdf[xref..]).unwrap(); + let line = table.lines().nth(2 + id).unwrap(); + let offset: usize = line[..10].parse().unwrap(); + let head = format!("{id} 0 obj\n"); + assert_eq!(&pdf[offset..offset + head.len()], head.as_bytes(), "xref entry for {id}"); + let rest = &pdf[offset + head.len()..]; + let end = rest.windows(7).position(|w| w == b"\nendobj").unwrap(); + let body = &rest[..end]; + match body.windows(8).position(|w| w == b"\nstream\n") { + Some(p) => { + let data = &body[p + 8..body.len() - "\nendstream".len()]; + (String::from_utf8_lossy(&body[..p]).into_owned(), data.to_vec()) + } + None => (String::from_utf8_lossy(body).into_owned(), vec![]), + } + } + + fn inflate(data: &[u8]) -> Vec { + miniz_oxide::inflate::decompress_to_vec_zlib(data).unwrap() + } + + /// Object ids of every image XObject, in file order. + fn images(pdf: &[u8]) -> Vec { + let s = String::from_utf8_lossy(pdf); + s.match_indices(" 0 obj\n<< /Type /XObject /Subtype /Image").map(|(i, _)| s[..i].rsplit('\n').next().unwrap().parse().unwrap()).collect() + } + + fn reference(dict: &str, key: &str) -> usize { + let at = dict.find(key).unwrap() + key.len(); + dict[at..].trim_start().split(' ').next().unwrap().parse().unwrap() + } + + fn srgb() -> Arc> { + Arc::new(lightcraft_codecs::icc::write_named(NamedSpace::Srgb)) + } + + fn page(images: Vec<(PdfImage, Placement)>) -> PdfPage { + PdfPage { width: 612.0, height: 792.0, images } + } + + fn full(w: f64, h: f64) -> Placement { + Placement { x: 0.0, y: 0.0, width: w, height: h } + } + + #[test] + fn a_16_bit_icc_image_round_trips() { + let samples: Vec = (0..2 * 3 * 3).map(|i| (i * 3851) as u16).collect(); + let img = PdfImage::from_u16(3, 2, 3, &samples, Some(srgb())); + let pdf = write(&[page(vec![(img, full(612.0, 792.0))])], &PdfInfo::default()).unwrap(); + assert!(pdf.starts_with(b"%PDF-1.6\n")); + assert!(pdf.ends_with(b"%%EOF\n")); + let ids = images(&pdf); + assert_eq!(ids.len(), 1); + let (dict, data) = object(&pdf, ids[0]); + assert!(dict.contains("/Width 3 /Height 2") && dict.contains("/BitsPerComponent 16") && dict.contains("/FlateDecode"), "{dict}"); + let back: Vec = inflate(&data).chunks(2).map(|b| u16::from_be_bytes([b[0], b[1]])).collect(); + assert_eq!(back, samples); + let (profile, icc) = object(&pdf, reference(&dict, "/ICCBased")); + assert!(profile.contains("/N 3") && profile.contains("/Alternate /DeviceRGB"), "{profile}"); + assert_eq!(inflate(&icc), *srgb()); + assert_eq!(lightcraft_codecs::icc::parse(&inflate(&icc)).and_then(|i| i.named), Some(NamedSpace::Srgb)); + } + + #[test] + fn pages_sizes_placements_and_a_shared_profile() { + let px = |v: u16| PdfImage::from_u16(1, 1, 3, &[v, v, v], Some(srgb())); + let at = Placement { x: 36.0, y: 72.5, width: 288.0, height: 216.0 }; + let pages = [page(vec![(px(1), at), (px(2), full(612.0, 792.0))]), PdfPage { width: 841.89, height: 595.28, images: vec![(px(3), at)] }]; + let pdf = write(&pages, &PdfInfo::default()).unwrap(); + let s = String::from_utf8_lossy(&pdf); + assert!(s.contains("/Type /Pages /Count 2"), "two pages"); + assert!(s.contains("/MediaBox [0 0 612 792]") && s.contains("/MediaBox [0 0 841.89 595.28]")); + // the sRGB profile is written once for all three images + assert_eq!(s.matches("/N 3 /Alternate").count(), 1); + let first: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect(); + assert!(first.windows(2).all(|w| w[0] == w[1]), "{first:?}"); + // placements reach the content streams + let contents: Vec = s + .match_indices("/Contents ") + .map(|(i, _)| s[i + 10..].split(' ').next().unwrap().parse::().unwrap()) + .map(|id| String::from_utf8(inflate(&object(&pdf, id).1)).unwrap()) + .collect(); + assert_eq!(contents[0], "q 288 0 0 216 36 72.5 cm /Im0 Do Q\nq 612 0 0 792 0 0 cm /Im1 Do Q\n"); + assert_eq!(write(&pages, &PdfInfo::default()).unwrap(), pdf, "deterministic"); + } + + #[test] + fn output_intent_and_title() { + let cmyk_like = srgb(); + let info = PdfInfo { + title: Some("Garden – prints (2)".into()), + output_intent: Some(OutputIntent { icc: cmyk_like.clone(), channels: 3, condition: "Glossy (Photo) \\ test".into() }), + }; + let img = PdfImage::from_u16(1, 1, 3, &[0, 0, 0], Some(cmyk_like)); + let pdf = write(&[page(vec![(img, full(10.0, 10.0))])], &info).unwrap(); + let (catalog, _) = object(&pdf, 1); + assert!(catalog.contains("/OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX"), "{catalog}"); + assert!(catalog.contains("/Info (Glossy \\(Photo\\) \\\\ test)"), "{catalog}"); + // the intent and the image share one profile object + let dest = reference(&catalog, "/DestOutputProfile"); + assert_eq!(reference(&object(&pdf, images(&pdf)[0]).0, "/ICCBased"), dest); + let (info_dict, _) = object(&pdf, 3); + assert!(info_dict.contains("/Title = ids.iter().map(|&id| object(&pdf, id)).collect(); + assert!(dicts[0].0.contains("/ColorSpace /DeviceCMYK") && dicts[0].0.contains("/BitsPerComponent 8")); + assert!(dicts[1].0.contains("/ColorSpace /DeviceGray")); + assert!(dicts[2].0.contains("/DCTDecode") && dicts[2].0.contains("/DeviceRGB")); + assert_eq!(dicts[2].1, jpeg, "JPEG streams are copied as they are"); + } + + #[test] + fn compressing_early_writes_the_same_file() { + let samples: Vec = (0..4 * 3 * 3).map(|i| (i * 977) as u16).collect(); + let raw = PdfImage::from_u16(4, 3, 3, &samples, Some(srgb())); + let early = raw.clone().deflated().unwrap(); + assert!(matches!(early.data, ImageData::Flate { bits: 16, .. })); + let at = full(100.0, 75.0); + let a = write(&[page(vec![(raw, at)])], &PdfInfo::default()).unwrap(); + let b = write(&[page(vec![(early, at)])], &PdfInfo::default()).unwrap(); + assert_eq!(a, b); + let mut short = PdfImage::from_u16(4, 3, 3, &samples, None); + short.width = 5; + assert!(short.deflated().is_err(), "checked before compressing"); + } + + #[test] + fn hostile_documents_are_errors() { + let ok = || PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None); + let at = full(10.0, 10.0); + let bad = |pages: Vec| write(&pages, &PdfInfo::default()).is_err(); + assert!(bad(vec![])); + assert!(bad(vec![PdfPage { width: f64::NAN, height: 10.0, images: vec![] }])); + assert!(bad(vec![PdfPage { width: -1.0, height: 10.0, images: vec![] }])); + assert!(bad(vec![PdfPage { width: 1e9, height: 10.0, images: vec![] }])); + assert!(bad(vec![page(vec![(ok(), Placement { x: f64::INFINITY, ..at })])])); + assert!(bad(vec![page(vec![(ok(), Placement { width: 0.0, ..at })])])); + let mut short = ok(); + short.data = ImageData::Raw { bits: 16, samples: vec![0; 5] }; + assert!(bad(vec![page(vec![(short, at)])])); + let mut empty = ok(); + empty.width = 0; + assert!(bad(vec![page(vec![(empty, at)])])); + let mut odd = ok(); + odd.channels = 2; + assert!(bad(vec![page(vec![(odd, at)])])); + let mut not_jpeg = ok(); + not_jpeg.data = ImageData::Jpeg(vec![1, 2, 3]); + assert!(bad(vec![page(vec![(not_jpeg, at)])])); + let too_many = (0..MAX_PAGES + 1).map(|_| page(vec![])).collect(); + assert!(bad(too_many)); + let info = PdfInfo { title: None, output_intent: Some(OutputIntent { icc: srgb(), channels: 2, condition: String::new() }) }; + assert!(write(&[page(vec![])], &info).is_err()); + } + + #[test] + fn numbers_and_strings() { + assert_eq!(num(612.0), "612"); + assert_eq!(num(72.5), "72.5"); + assert_eq!(num(-0.00001), "0"); + assert_eq!(num(f64::NAN), "0"); + assert_eq!(num(1.0 / 3.0), "0.3333"); + assert_eq!(text("a(b)c\\"), "(a\\(b\\)c\\\\)"); + assert_eq!(text("é"), ""); + } +} From 2651dbeb2b13f621db23f65347380e7f26e316f0 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 17:48:04 -0500 Subject: [PATCH 18/19] Expose bounded multi-image print PDF serialization with per-image ICC --- crates/pdf/src/write/raster.rs | 4 + crates/pdf/src/write/raster_print.rs | 252 +++++++++++++++++++++------ 2 files changed, 199 insertions(+), 57 deletions(-) diff --git a/crates/pdf/src/write/raster.rs b/crates/pdf/src/write/raster.rs index 1aa64ec3c..bcc32c23f 100644 --- a/crates/pdf/src/write/raster.rs +++ b/crates/pdf/src/write/raster.rs @@ -3,6 +3,10 @@ //! Callers map their page and color models to these neutral records. This module owns PDF object //! layout, image streams, ICC/Separation resources, page labels, marks, xref and trailer syntax. +/// Multi-image 8/16-bit print pages with per-image ICC resources and explicit byte budgets. +#[path = "raster_print.rs"] +pub mod print; + use std::collections::BTreeSet; use std::fmt::Write as _; use std::io::Write as _; diff --git a/crates/pdf/src/write/raster_print.rs b/crates/pdf/src/write/raster_print.rs index abce4061e..9866d73e1 100644 --- a/crates/pdf/src/write/raster_print.rs +++ b/crates/pdf/src/write/raster_print.rs @@ -8,10 +8,17 @@ //! The file is PDF 1.6 (16-bit images need 1.5): classic cross-reference table, no object streams, //! deterministic (no dates or IDs), so the same pages always give the same bytes. //! +//! Encoded JPEG and Flate payloads are copied, not decoded or verified against declared +//! dimensions. Callers own their encoding validity and ICC/profile-channel compatibility. +//! An output intent describes a printing condition; this API does not certify PDF/X. +//! Byte limits bound accounted input and emitted output, not peak process memory or +//! caller-owned image construction. This is not the streaming slideshow writer. +//! //! Sources: ISO 32000-1:2008 (§7.3 objects, §7.5 file structure, §8.6.5.5 ICCBased colour spaces, //! §8.9 images, §14.11.5 output intents). Own design. use std::collections::HashMap; +use std::io::Write as _; use std::sync::Arc; /// Pages per document: a print job beyond this is a mistake, not a book. @@ -23,6 +30,19 @@ pub const MAX_PAGE_POINTS: f64 = 14_400.0; /// Largest image, in pixels per side. pub const MAX_IMAGE_SIDE: u32 = 65_535; +/// Serialization budgets; input records already belong to the caller. These are not a +/// process-wide peak-memory guarantee. Repeated references count toward the input budget. +#[derive(Clone, Copy, Debug)] +pub struct Limits { + pub input_bytes: usize, + pub output_bytes: usize, +} +impl Default for Limits { + fn default() -> Self { + Self { input_bytes: 512 * 1024 * 1024, output_bytes: 512 * 1024 * 1024 } + } +} + /// How an image's samples are stored. #[derive(Clone, Debug, PartialEq)] pub enum ImageData { @@ -58,8 +78,11 @@ impl PdfImage { /// hold every page's uncompressed samples until the file is written. pub fn deflated(self) -> Result { self.check()?; + if matches!(&self.data, ImageData::Raw { samples, .. } if samples.len() > Limits::default().input_bytes) { + return Err("PDF input byte limit exceeded".into()); + } let data = match self.data { - ImageData::Raw { bits, samples } => ImageData::Flate { bits, data: deflate(&samples) }, + ImageData::Raw { bits, samples } => ImageData::Flate { bits, data: deflate(&samples, Limits::default().output_bytes)? }, other => other, }; Ok(PdfImage { data, ..self }) @@ -130,6 +153,8 @@ pub struct OutputIntent { pub struct PdfInfo { pub title: Option, pub output_intent: Option, + /// Application name, not a PDF/X conformance claim. + pub producer: String, } fn finite(v: f64) -> bool { @@ -157,47 +182,89 @@ fn text(s: &str) -> String { out.push(')'); out } else { - let mut out = String::from("'); - out + super::pdf_unicode_string(s) } } struct Writer { out: Vec, offsets: Vec, + limit: usize, } - impl Writer { - /// Reserve an object number (filled later by [`Writer::object`]). - fn reserve(&mut self) -> usize { - self.offsets.push(0); - self.offsets.len() + fn put(&mut self, bytes: &[u8]) -> Result<(), String> { + let end = self.out.len().checked_add(bytes.len()).ok_or("PDF output size overflow")?; + if end > self.limit { + return Err("PDF output byte limit exceeded".into()); + } + self.out.try_reserve(bytes.len()).map_err(|_| "PDF output allocation failed")?; + self.out.extend_from_slice(bytes); + Ok(()) } - - fn object(&mut self, id: usize, dict: &str, stream: Option<&[u8]>) { - if let Some(slot) = self.offsets.get_mut(id.wrapping_sub(1)) { - *slot = self.out.len(); - } - self.out.extend_from_slice(format!("{id} 0 obj\n{dict}").as_bytes()); + fn reserve(&mut self) -> Result { + self.offsets.try_reserve(1).map_err(|_| "PDF object allocation failed")?; + self.offsets.push(0); + Ok(self.offsets.len()) + } + fn object(&mut self, id: usize, dict: &str, stream: Option<&[u8]>) -> Result<(), String> { + let pos = self.out.len(); + *self.offsets.get_mut(id.checked_sub(1).ok_or("invalid PDF object")?).ok_or("invalid PDF object")? = pos; + self.put(format!("{id} 0 obj\n{dict}").as_bytes())?; if let Some(data) = stream { - self.out.extend_from_slice(b"\nstream\n"); - self.out.extend_from_slice(data); - self.out.extend_from_slice(b"\nendstream"); + self.put(b"\nstream\n")?; + self.put(data)?; + self.put(b"\nendstream")?; } - self.out.extend_from_slice(b"\nendobj\n"); + self.put(b"\nendobj\n") } } -fn deflate(data: &[u8]) -> Vec { - miniz_oxide::deflate::compress_to_vec_zlib(data, 6) +fn deflate(data: &[u8], max: usize) -> Result, String> { + struct Sink { + bytes: Vec, + max: usize, + } + impl std::io::Write for Sink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.len() > self.max.saturating_sub(self.bytes.len()) { + return Err(std::io::Error::other("PDF compressed byte limit exceeded")); + } + self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?; + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut encoder = flate2::write::ZlibEncoder::new(Sink { bytes: Vec::new(), max }, flate2::Compression::new(6)); + encoder.write_all(data).map_err(|e| e.to_string())?; + Ok(encoder.finish().map_err(|e| e.to_string())?.bytes) } /// Write a document. Errors name the first page or image that can't be written. pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result, String> { + write_with_limits(pages, info, Limits::default()) +} + +/// Write with explicit total input and serialized output budgets. ICC bytes are opaque: +/// callers validate their profile semantics; resources are keyed by bytes AND channels. +pub fn write_with_limits(pages: &[PdfPage], info: &PdfInfo, limits: Limits) -> Result, String> { + let mut input = 0usize; + let mut account = |n: usize| -> Result<(), String> { + input = input.checked_add(n).ok_or("PDF input size overflow")?; + if input > limits.input_bytes { + return Err("PDF input byte limit exceeded".into()); + } + Ok(()) + }; + account(info.producer.len())?; + account(info.title.as_ref().map_or(0, String::len))?; + if let Some(intent) = &info.output_intent { + account(intent.icc.len())?; + account(intent.condition.len())?; + } + if pages.is_empty() { return Err("there are no pages to write".into()); } @@ -218,6 +285,13 @@ pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result, String> { return Err(format!("page {n} has more than {MAX_IMAGES_PER_PAGE} images")); } for (image, at) in &page.images { + account(match &image.data { + ImageData::Raw { samples, .. } => samples.len(), + ImageData::Jpeg(data) | ImageData::Flate { data, .. } => data.len(), + })?; + if let Some(icc) = &image.icc { + account(icc.len())?; + } image.check().map_err(|e| format!("page {n}: {e}"))?; if ![at.x, at.y, at.width, at.height].into_iter().all(finite) || at.width <= 0.0 || at.height <= 0.0 { return Err(format!("page {n}: an image has no usable placement")); @@ -230,49 +304,50 @@ pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result, String> { return Err("the output intent's profile has 1, 3 or 4 components".into()); } - let mut w = Writer { out: Vec::new(), offsets: Vec::new() }; - w.out.extend_from_slice(b"%PDF-1.6\n%\xE2\xE3\xCF\xD3\n"); - let catalog = w.reserve(); - let pages_id = w.reserve(); - let info_id = w.reserve(); + let mut w = Writer { out: Vec::new(), offsets: Vec::new(), limit: limits.output_bytes }; + w.put(b"%PDF-1.6\n%\xE2\xE3\xCF\xD3\n")?; + let catalog = w.reserve()?; + let pages_id = w.reserve()?; + let info_id = w.reserve()?; // ICC profiles, each written once (keyed by content) with its component count - let mut profiles: HashMap, usize> = HashMap::new(); - let mut profile = |w: &mut Writer, icc: &Arc>, n: u8| -> usize { - if let Some(&id) = profiles.get(icc.as_slice()) { - return id; + let mut profiles: HashMap<(Arc>, u8), usize> = HashMap::new(); + let mut profile = |w: &mut Writer, icc: &Arc>, n: u8| -> Result { + if let Some(&id) = profiles.get(&(Arc::clone(icc), n)) { + return Ok(id); } - let id = w.reserve(); - let data = deflate(icc); + let id = w.reserve()?; + let data = deflate(icc, limits.output_bytes)?; let alternate = match n { 1 => "/DeviceGray", 4 => "/DeviceCMYK", _ => "/DeviceRGB", }; - w.object(id, &format!("<< /N {n} /Alternate {alternate} /Filter /FlateDecode /Length {} >>", data.len()), Some(&data)); - profiles.insert(icc.to_vec(), id); - id + w.object(id, &format!("<< /N {n} /Alternate {alternate} /Filter /FlateDecode /Length {} >>", data.len()), Some(&data))?; + profiles.try_reserve(1).map_err(|_| "PDF profile allocation failed")?; + profiles.insert((Arc::clone(icc), n), id); + Ok(id) }; - let intent_id = info.output_intent.as_ref().map(|intent| profile(&mut w, &intent.icc, intent.channels)); + let intent_id = info.output_intent.as_ref().map(|intent| profile(&mut w, &intent.icc, intent.channels)).transpose()?; let mut kids = Vec::with_capacity(pages.len()); for page in pages { - let page_id = w.reserve(); + let page_id = w.reserve()?; let mut xobjects = String::new(); let mut content = String::new(); for (k, (image, at)) in page.images.iter().enumerate() { let space = match &image.icc { - Some(icc) => format!("[/ICCBased {} 0 R]", profile(&mut w, icc, image.channels)), + Some(icc) => format!("[/ICCBased {} 0 R]", profile(&mut w, icc, image.channels)?), None => match image.channels { 1 => "/DeviceGray".into(), 4 => "/DeviceCMYK".into(), _ => "/DeviceRGB".into(), }, }; - let id = w.reserve(); + let id = w.reserve()?; let (bits, filter, data): (u8, &str, std::borrow::Cow<[u8]>) = match &image.data { - ImageData::Raw { bits, samples } => (*bits, "/FlateDecode", deflate(samples).into()), + ImageData::Raw { bits, samples } => (*bits, "/FlateDecode", deflate(samples, limits.output_bytes)?.into()), ImageData::Flate { bits, data } => (*bits, "/FlateDecode", data.as_slice().into()), ImageData::Jpeg(jpeg) => (8, "/DCTDecode", jpeg.as_slice().into()), }; @@ -285,13 +360,13 @@ pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result, String> { data.len() ), Some(&data), - ); + )?; xobjects.push_str(&format!("/Im{k} {id} 0 R ")); content.push_str(&format!("q {} 0 0 {} {} {} cm /Im{k} Do Q\n", num(at.width), num(at.height), num(at.x), num(at.y))); } - let content_id = w.reserve(); - let stream = deflate(content.as_bytes()); - w.object(content_id, &format!("<< /Filter /FlateDecode /Length {} >>", stream.len()), Some(&stream)); + let content_id = w.reserve()?; + let stream = deflate(content.as_bytes(), limits.output_bytes)?; + w.object(content_id, &format!("<< /Filter /FlateDecode /Length {} >>", stream.len()), Some(&stream))?; w.object( page_id, &format!( @@ -300,10 +375,10 @@ pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result, String> { num(page.height) ), None, - ); + )?; kids.push(format!("{page_id} 0 R")); } - w.object(pages_id, &format!("<< /Type /Pages /Count {} /Kids [{}] >>", kids.len(), kids.join(" ")), None); + w.object(pages_id, &format!("<< /Type /Pages /Count {} /Kids [{}] >>", kids.len(), kids.join(" ")), None)?; let intents = match (&info.output_intent, intent_id) { (Some(intent), Some(id)) => format!( @@ -312,9 +387,9 @@ pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result, String> { ), _ => String::new(), }; - w.object(catalog, &format!("<< /Type /Catalog /Pages {pages_id} 0 R{intents} >>"), None); + w.object(catalog, &format!("<< /Type /Catalog /Pages {pages_id} 0 R{intents} >>"), None)?; let title = info.title.as_deref().map(|t| format!(" /Title {}", text(t))).unwrap_or_default(); - w.object(info_id, &format!("<< /Producer (LightCraft){title} >>"), None); + w.object(info_id, &format!("<< /Producer {}{title} >>", text(&info.producer)), None)?; let xref = w.out.len(); let mut table = format!("xref\n0 {}\n0000000000 65535 f \n", w.offsets.len() + 1); @@ -322,14 +397,14 @@ pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result, String> { table.push_str(&format!("{offset:010} 00000 n \n")); } table.push_str(&format!("trailer\n<< /Size {} /Root {catalog} 0 R /Info {info_id} 0 R >>\nstartxref\n{xref}\n%%EOF\n", w.offsets.len() + 1)); - w.out.extend_from_slice(table.as_bytes()); + w.put(table.as_bytes())?; Ok(w.out) } #[cfg(test)] +#[allow(clippy::unwrap_used)] mod tests { use super::*; - use lightcraft_codecs::NamedSpace; /// The body of object `id` (dictionary and raw stream bytes), found through the xref table. fn object(pdf: &[u8], id: usize) -> (String, Vec) { @@ -355,7 +430,12 @@ mod tests { } fn inflate(data: &[u8]) -> Vec { - miniz_oxide::inflate::decompress_to_vec_zlib(data).unwrap() + { + use std::io::Read; + let mut out = Vec::new(); + flate2::read::ZlibDecoder::new(data).read_to_end(&mut out).unwrap(); + out + } } /// Object ids of every image XObject, in file order. @@ -370,7 +450,7 @@ mod tests { } fn srgb() -> Arc> { - Arc::new(lightcraft_codecs::icc::write_named(NamedSpace::Srgb)) + Arc::new(b"opaque RGB profile fixture".to_vec()) } fn page(images: Vec<(PdfImage, Placement)>) -> PdfPage { @@ -397,7 +477,6 @@ mod tests { let (profile, icc) = object(&pdf, reference(&dict, "/ICCBased")); assert!(profile.contains("/N 3") && profile.contains("/Alternate /DeviceRGB"), "{profile}"); assert_eq!(inflate(&icc), *srgb()); - assert_eq!(lightcraft_codecs::icc::parse(&inflate(&icc)).and_then(|i| i.named), Some(NamedSpace::Srgb)); } #[test] @@ -428,6 +507,7 @@ mod tests { let cmyk_like = srgb(); let info = PdfInfo { title: Some("Garden – prints (2)".into()), + producer: "Fixture".into(), output_intent: Some(OutputIntent { icc: cmyk_like.clone(), channels: 3, condition: "Glossy (Photo) \\ test".into() }), }; let img = PdfImage::from_u16(1, 1, 3, &[0, 0, 0], Some(cmyk_like)); @@ -503,7 +583,11 @@ mod tests { assert!(bad(vec![page(vec![(not_jpeg, at)])])); let too_many = (0..MAX_PAGES + 1).map(|_| page(vec![])).collect(); assert!(bad(too_many)); - let info = PdfInfo { title: None, output_intent: Some(OutputIntent { icc: srgb(), channels: 2, condition: String::new() }) }; + let info = PdfInfo { + producer: String::new(), + title: None, + output_intent: Some(OutputIntent { icc: srgb(), channels: 2, condition: String::new() }), + }; assert!(write(&[page(vec![])], &info).is_err()); } @@ -517,4 +601,58 @@ mod tests { assert_eq!(text("a(b)c\\"), "(a\\(b\\)c\\\\)"); assert_eq!(text("é"), ""); } + #[test] + fn distinct_profiles_and_component_counts_are_not_conflated() { + let common = Arc::new(b"opaque profile A".to_vec()); + let different = Arc::new(b"opaque profile B".to_vec()); + let image = |channels, profile| PdfImage { + width: 1, + height: 1, + channels, + data: ImageData::Raw { bits: 8, samples: vec![7; channels as usize] }, + icc: Some(profile), + }; + let pages = [page(vec![ + (image(3, common.clone()), full(1.0, 1.0)), + (image(4, common.clone()), full(1.0, 1.0)), + (image(3, different.clone()), full(1.0, 1.0)), + (image(3, common.clone()), full(1.0, 1.0)), + ])]; + let pdf = write(&pages, &PdfInfo::default()).unwrap(); + let refs: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect(); + assert_ne!(refs[0], refs[1]); + assert_ne!(refs[0], refs[2]); + assert_eq!(refs[0], refs[3]); + for (i, channels, bytes) in [(0, 3, common.as_slice()), (1, 4, common.as_slice()), (2, 3, different.as_slice())] { + let (dict, data) = object(&pdf, refs[i]); + assert!(dict.contains(&format!("/N {channels}"))); + assert_eq!(inflate(&data), bytes); + } + } + + #[test] + fn gray_rgb_cmyk_raw_depths_preserve_samples() { + for channels in [1, 3, 4] { + for bits in [8, 16] { + let bytes: Vec = (0..channels * (bits / 8)).collect(); + let image = PdfImage { width: 1, height: 1, channels, data: ImageData::Raw { bits, samples: bytes.clone() }, icc: None }; + let pdf = write(&[page(vec![(image, full(1.0, 1.0))])], &PdfInfo::default()).unwrap(); + let (dict, data) = object(&pdf, images(&pdf)[0]); + assert!(dict.contains(&format!("/BitsPerComponent {bits}"))); + assert_eq!(inflate(&data), bytes); + } + } + } + + #[test] + fn input_and_output_budgets_fail_without_partial_success() { + let image = PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None); + let pages = [page(vec![(image, full(1.0, 1.0))])]; + let info = PdfInfo::default(); + let expected = write(&pages, &info).unwrap(); + assert_eq!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() }).unwrap(), expected); + assert!(write_with_limits(&pages, &info, Limits { input_bytes: 5, output_bytes: expected.len() }).is_err()); + assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() - 1 }).is_err()); + assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: 0 }).is_err()); + } } From 920c745f4217c04c5d881d2c721d537d26239408 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:13:47 +0100 Subject: [PATCH 19/19] raw: initialize linearization_table in the dngwrite test image --- crates/raw/src/dngwrite.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index 3a715085a..384152e66 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -465,6 +465,7 @@ mod tests { color: crate::ColorData::default(), wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera: Default::default(),