diff --git a/NOTICE b/NOTICE index 3bca3ae0d..eb22134a2 100644 --- a/NOTICE +++ b/NOTICE @@ -93,7 +93,7 @@ crates/raw (craft-raw) is moved from LightCraft, with behaviour ported from two - LightCraft, crates/raw (src/lib.rs, dng.rs, dngwrite.rs, tiffraw.rs, unpack.rs, ljpeg.rs, jxl.rs, opcodes.rs, profile.rs, gaintable.rs, semantic.rs, preview.rs, vendor/crw.rs, vendor/hasselblad.rs, vendor.rs (shared level helpers), vendor/mrw.rs, vendor/erf.rs, vendor/mef.rs, vendor/kodak.rs, vendor/pef.rs, - vendor/srw.rs, vendor/cr2.rs, vendor/cr2/sraw.rs, kelvin_presets.rs (the preset ordering), tests/it/vendor_robust.rs and their tests) and crates/meta/src/mrw.rs (the MRW block container), + vendor/srw.rs, vendor/cr2.rs, vendor/cr2/sraw.rs, vendor/nef.rs, vendor/nefc.rs, vendor/nef_lens.rs, docs/nikon-lens-corrections.md, kelvin_presets.rs (the preset ordering), tests/it/vendor_robust.rs and their tests) and crates/meta/src/mrw.rs (the MRW block container), revision 5ff567a4a2e067bef1b6bce152c11e7f727ccbaf. https://github.com/storytold/lightcraft Copyright (c) 2026 ArtCraft Team and the LightCraft contributors. diff --git a/README.md b/README.md index 5b51e5ebd..951fc1ab9 100644 --- a/README.md +++ b/README.md @@ -134,8 +134,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " Result { RawFormat::Mef => mef::decode(bytes, mode, limits), #[cfg(feature = "cr2")] RawFormat::Cr2 => cr2::decode(bytes, mode, limits), + #[cfg(feature = "nef")] + RawFormat::Nef | RawFormat::Nrw => nef::decode(bytes, mode, limits), #[cfg(feature = "pef")] RawFormat::Pef => pef::decode(bytes, mode, limits), #[cfg(feature = "srw")] diff --git a/crates/raw/src/nef.rs b/crates/raw/src/nef.rs new file mode 100644 index 000000000..86ef0e1b2 --- /dev/null +++ b/crates/raw/src/nef.rs @@ -0,0 +1,1510 @@ +//! Nikon NEF / NRW — uncompressed and Huffman-compressed (lossless, lossy) variants. +//! +//! Sources: TIFF 6.0 (the raw image is a standard CFA SubIFD), Laurent Clévy's NEF structure notes (prose: IFD +//! layout, SubIFDs, maker note header) and the ExifTool Nikon tag-name documentation (`0x000c` WB_RBLevels, +//! `0x003d` BlackLevel, `0x0096` NEFLinearizationTable). The Huffman-compressed data (compression 34713) is decoded +//! by [`crate::nefc`], which documents its clean-room sources; "lossy after split" files are decoded when their strip follows the +//! rule documented there; files it can't decode are reported as [`RawError::Unsupported`] and their embedded previews still work. So are High Efficiency NEFs +//! (HE / HE★, Z 8, Z 9, Z 6III, Z f): they keep compression 34713 but carry a wavelet codestream that starts with +//! the JPEG XS markers SOC + CAP (ISO/IEC 21122-1, `FF10 FF50`) and have no `0x0096` table (storytold/lightcraft#193). +//! Strips labelled 34713 without a `0x0096` +//! linearization table are packed uncompressed data in several measured layouts ([`PackedLayout`]) or, when the strip +//! is exactly three bytes per pixel, Nikon small raw ([`small_raw`]); the D1X's genuinely Huffman-coded strip stays +//! refused. + +use crate::nefc; +use crate::shared::white_from_data; +use crate::tiffraw::{Packing, check_image, read_image}; +use crate::{BlackLevel, Camera, Cfa, ColorData, Limits, Mode, OpcodeLists, Orientation, RawData, RawError, RawFormat, RawImage, Rect, Result}; +use craft_tiff::image::ImageInfo; +use craft_tiff::tags::{self as t, photometric}; +use craft_tiff::{ByteOrder, Ifd, Tiff, makernote}; + +const WB_RB_LEVELS: u16 = 0x000c; +const BLACK_LEVEL: u16 = 0x003d; +const LINEARIZATION_TABLE: u16 = 0x0096; +const CROP_AREA: u16 = 0x0045; +/// Maker note NEFCompression (ExifTool's Nikon tag documentation): `0x0093`, or in newer bodies the `u16` at +/// byte 10 of `0x0051`. 6 is packed 12-bit, 10 packed 14-bit, 13 and 14 High Efficiency and High Efficiency★. +const NEF_COMPRESSION: u16 = 0x0093; +const NEF_COMPRESSION_RECORD: u16 = 0x0051; + +fn nef_compression(mn: Option<&makernote::MakerNote>) -> Option { + let m = mn?; + m.ifd.u64(NEF_COMPRESSION).or_else(|| { + let v: [u8; 2] = m.ifd.bytes(NEF_COMPRESSION_RECORD)?.get(10..12)?.try_into().ok()?; + Some(u64::from(m.order.u16(v))) + }) +} +/// JPEG XS start of codestream (SOC, `FF10`) followed by its mandatory capabilities marker (CAP, `FF50`). +const JPEG_XS_START: [u8; 4] = [0xff, 0x10, 0xff, 0x50]; +/// Why a High Efficiency NEF shows its embedded preview: the UI shows the part before " (". +pub(crate) const HIGH_EFFICIENCY: &str = + "Nikon High Efficiency NEF (HE / HE★, a JPEG XS-based codec with no public specification; shoot RAW Lossless compressed to edit the raw data)"; +/// Maker note `0x0014`, the "NRW" data block of Coolpix raws: `"NRW "` + a 4-character version, then fields. In +/// version `0104` the `u32` at byte `0x20` is the black level in sample units (200 in the four 12-bit files here +/// that have 200 as their darkest samples, 0 in the one whose samples reach 0). +const NRW_DATA: u16 = 0x0014; +const NRW_BLACK_AT: usize = 0x20; + +/// The NRW data block, when the maker note has one. +fn nrw_data(mn: Option<&makernote::MakerNote>) -> Option<(&[u8], ByteOrder)> { + let m = mn?; + m.ifd.bytes(NRW_DATA).filter(|b| b.starts_with(b"NRW ")).map(|b| (b, m.order)) +} + +/// Black level from the NRW data block, bounded to a quarter of the sample range so an unknown version's field +/// can't turn into a large offset. +fn nrw_black(mn: Option<&makernote::MakerNote>, bits: u32) -> Option { + let (b, order) = nrw_data(mn)?; + let v = order.u32(b.get(NRW_BLACK_AT..NRW_BLACK_AT + 4)?.try_into().ok()?); + (u64::from(v) < (1u64 << bits.min(16)) / 4).then_some(v as f32) +} + +/// Nikon CropArea is `[left, top, width, height]` in sensor pixels (maker-note tag documentation). +/// Keep the active area/CFA origin unchanged: the default crop is applied after demosaicing. +fn default_crop(mn: Option<&makernote::MakerNote>, w: usize, h: usize) -> Rect { + let fallback = Rect::new(0, 0, w, h); + let Some(values) = mn.and_then(|m| m.ifd.u64s(CROP_AREA)) else { return fallback }; + let [x, y, width, height] = values.as_slice() else { return fallback }; + if *width == 0 || *height == 0 || x.checked_add(*width).is_none_or(|v| v > w as u64) || y.checked_add(*height).is_none_or(|v| v > h as u64) { + return fallback; + } + Rect::new(*x as usize, *y as usize, *width as usize, *height as usize) +} + +/// A body that records no CropArea, measured on its own files: the image area inside the sensor data (where the +/// camera's full-size JPEG sits when the two are aligned) and, when the data's own top would be wrong, the level +/// where its channels clip. +struct MeasuredBody { + model: &'static str, + /// Sensor data size and sample depth of the measured files. + size: (usize, usize), + bits: u32, + /// Image area in sensor pixels: x, y, width, height. + crop: (usize, usize, usize, usize), + white: Option, +} + +/// * **NIKON D4** (148 uncompressed 14-bit NEFs from one body, 2023–2026, ISO 100–10000): the data is 4992 × 3292, +/// with 2 columns of optical black left of the image and 42 right of it, then 8 empty ones; the camera's +/// 4928 × 3280 JPEG sits at (8, 6). Without this area the optical black shows as purple stripes on both sides. +/// Green saturates at 15546–15672 while red and blue run on to 15687–16383, so the data's own top (~16300) leaves +/// clipped highlights magenta; one level of 15520 for every channel turns them white and keeps the white balance. +/// * **NIKON D7100** (storytold/lightcraft#1248; 14-bit NEFs from the public zjqc/photographing-art repository, with their +/// Lightroom Classic exports): the data holds picture, not masked lines, beyond the camera's frame. In full frame +/// it is 6036 × 4020 and the camera's JPEG is 6000 × 4000; in the 1.3x crop mode it is 4812 × 3212 and the JPEG +/// 4800 × 3200. Fitting renders of the full data to Lightroom's exports (feature matches, homography) puts +/// Lightroom's frame, the camera JPEG's size, at (18, 10) and (6, 6): within 0.5 px mean on DSC_1316, DSC_2763 +/// and DSC_1486. Without it, renders are larger than Lightroom's and its crops land up to 6 px off (18 px in +/// full frame). +/// * **NIKON D90** (550 compressed 12-bit NEFs from one body, 10 measured): the data is 4352 × 2868 and its last 42 +/// columns (from 4310) are filler at a flat ~254 whatever the exposure, above the dark image samples (black 0), so +/// the masked-column test keeps them and they develop as a pink band on the right. The camera's 4288 × 2848 JPEG +/// sits at (10, 10). Green saturates at 3840–3851 while red and blue run on to 4095; one level of 3830 keeps +/// clipped highlights white. +/// * **NIKON D7000** and **NIKON D5100** (and the Coolpix A, the same readout): the data is 4992 × 3280, the picture its +/// first 4948 columns, then 36 masked ones at one level (~600, with a black level of 0) and 8 of 0. The masked +/// columns sit above the dark-column trim's threshold, so without this area they show as a coloured bar on the right +/// edge (the top in portrait; storytold/lightcraft#976). The camera's 4928 × 3264 JPEG sits at (8, 8): in the D5100's 3 CC0 +/// samples, and in the D7000's one clean one, with the D5100 and Coolpix A as siblings (see below). +/// * **The other bodies** below keep masked or dark lines inside the whole data (a survey of every Nikon raw on raw.pixls.us): +/// each crop is its camera JPEG's frame as `cargo xtask raw-frame` measures it on raw.pixls.us CC0 samples (the +/// D200, D3200 and Nikon 1 V1 also on other publicly posted sample files), with at least 3 clean +/// files agreeing: of the model (bit depths pooled: both read out the same area), or, with at least +/// one of its own, together with models of the same readout (data and JPEG size) that agree on the same frame +/// (*siblings*). The D4's 12-bit mode reads out the 14-bit area measured above; the Df, the same readout, agrees on +/// (8, 6) in 6 files. Bodies whose crop holds only picture beyond the camera frame have no entry (no change). +const MEASURED_BODIES: &[MeasuredBody] = &[ + MeasuredBody { model: "NIKON D4", size: (4992, 3292), bits: 14, crop: (8, 6, 4928, 3280), white: Some(15520.0) }, + // the 12-bit samples are the 14-bit ones shifted right by two, and so is the level: 15520 >> 2 + MeasuredBody { model: "NIKON D4", size: (4992, 3292), bits: 12, crop: (8, 6, 4928, 3280), white: Some(3880.0) }, + MeasuredBody { model: "NIKON D7100", size: (6036, 4020), bits: 14, crop: (18, 10, 6000, 4000), white: None }, + MeasuredBody { model: "NIKON D7100", size: (4812, 3212), bits: 14, crop: (6, 6, 4800, 3200), white: None }, + MeasuredBody { model: "NIKON D90", size: (4352, 2868), bits: 12, crop: (10, 10, 4288, 2848), white: Some(3830.0) }, + MeasuredBody { model: "NIKON D7000", size: (4992, 3280), bits: 14, crop: (8, 8, 4928, 3264), white: None }, + MeasuredBody { model: "NIKON D7000", size: (4992, 3280), bits: 12, crop: (8, 8, 4928, 3264), white: None }, + MeasuredBody { model: "NIKON D5100", size: (4992, 3280), bits: 14, crop: (8, 8, 4928, 3264), white: None }, + // siblings: D7000 (4), D5100 (3); masked: right 36 + MeasuredBody { model: "COOLPIX A", size: (4992, 3280), bits: 14, crop: (8, 8, 4928, 3264), white: None }, + // 6 files each; masked: left 2, right 2 + MeasuredBody { model: "NIKON D3", size: (4288, 2844), bits: 12, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D3", size: (4288, 2844), bits: 14, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D3S", size: (4288, 2844), bits: 12, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D3S", size: (4288, 2844), bits: 14, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D700", size: (4288, 2844), bits: 12, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D700", size: (4288, 2844), bits: 14, crop: (15, 6, 4256, 2832), white: None }, + // 6 and 4 files; masked: right up to 32 + MeasuredBody { model: "NIKON D300", size: (4352, 2868), bits: 12, crop: (15, 10, 4288, 2848), white: None }, + MeasuredBody { model: "NIKON D300", size: (4352, 2868), bits: 14, crop: (15, 10, 4288, 2848), white: None }, + MeasuredBody { model: "NIKON D300S", size: (4352, 2868), bits: 12, crop: (15, 10, 4288, 2848), white: None }, + MeasuredBody { model: "NIKON D300S", size: (4352, 2868), bits: 14, crop: (15, 10, 4288, 2848), white: None }, + // D2X (1) and D2Xs (2) as siblings; dark: right 1 + MeasuredBody { model: "NIKON D2Xs", size: (4320, 2868), bits: 12, crop: (16, 10, 4288, 2848), white: None }, + // D3000, D40X, D60, D80: one file each, siblings; masked: right 4, bottom 3 + MeasuredBody { model: "NIKON D3000", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + MeasuredBody { model: "NIKON D40X", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + MeasuredBody { model: "NIKON D60", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + MeasuredBody { model: "NIKON D80", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + // 3 files; masked: left 1, right 3 (the D80's data size, framed elsewhere) + MeasuredBody { model: "NIKON D200", size: (3904, 2616), bits: 12, crop: (16, 12, 3872, 2592), white: None }, + // 22 files; masked: right 46 + MeasuredBody { model: "NIKON D3200", size: (6080, 4012), bits: 12, crop: (8, 6, 6016, 4000), white: None }, + // 8 clean of 19, the rest ~0.3 px between pixels on the same frame; dark: bottom 2 + MeasuredBody { model: "NIKON 1 V1", size: (3904, 2606), bits: 12, crop: (12, 6, 3872, 2592), white: None }, + // Best guesses, short of the evidence rule; every frame below keeps the masked lines out by several pixels. + // D3100: 3 of 4 clean files at (24, 6), the 4th at (22, 7); masked: left 6, right 22 + MeasuredBody { model: "NIKON D3100", size: (4672, 3084), bits: 12, crop: (24, 6, 4608, 3072), white: None }, + // D5000: 4 of 5 clean files at (11, 10), the 5th at (11, 12); the D90's sensor, whose own files also align at + // x = 11; masked: right 42 (the same filler as the D90's) + MeasuredBody { model: "NIKON D5000", size: (4352, 2868), bits: 12, crop: (11, 10, 4288, 2848), white: None }, + // D40: 2 of 2 clean files at (16, 9); masked: right 1. The D50 / D70 / D70s share its data size but not its frame + MeasuredBody { model: "NIKON D40", size: (3040, 2014), bits: 12, crop: (16, 9, 3008, 2000), white: None }, + // D50, D70, D70s (one readout): one clean D50 and one clean D70 file at (15, 7); the D70s's 2 files align + // between pixels next to it, at (15.4, 6.9) and (15.6, 6.4); masked: right 1 + MeasuredBody { model: "NIKON D50", size: (3040, 2014), bits: 12, crop: (15, 7, 3008, 2000), white: None }, + MeasuredBody { model: "NIKON D70", size: (3040, 2014), bits: 12, crop: (15, 7, 3008, 2000), white: None }, + MeasuredBody { model: "NIKON D70s", size: (3040, 2014), bits: 12, crop: (15, 7, 3008, 2000), white: None }, + // 3 files; dark: bottom 4 + MeasuredBody { model: "NIKON D4S", size: (4936, 3288), bits: 12, crop: (4, 4, 4928, 3280), white: None }, + MeasuredBody { model: "NIKON D4S", size: (4936, 3288), bits: 14, crop: (4, 4, 4928, 3280), white: None }, + // 4 files; dark: bottom up to 4 + MeasuredBody { model: "NIKON D5600", size: (6016, 4016), bits: 12, crop: (8, 8, 6000, 4000), white: None }, + MeasuredBody { model: "NIKON D5600", size: (6016, 4016), bits: 14, crop: (8, 8, 6000, 4000), white: None }, + // 4 files each; masked: right 8 + MeasuredBody { model: "NIKON D600", size: (6080, 4028), bits: 12, crop: (8, 6, 6016, 4016), white: None }, + MeasuredBody { model: "NIKON D600", size: (6080, 4028), bits: 14, crop: (8, 6, 6016, 4016), white: None }, + MeasuredBody { model: "NIKON D610", size: (6080, 4028), bits: 12, crop: (8, 6, 6016, 4016), white: None }, + MeasuredBody { model: "NIKON D610", size: (6080, 4028), bits: 14, crop: (8, 6, 6016, 4016), white: None }, + // D800: 6 files; D800E: 2, with the D800 as sibling; masked: right 14 (storytold/lightcraft#914, the top in portrait) + MeasuredBody { model: "NIKON D800", size: (7424, 4924), bits: 12, crop: (8, 6, 7360, 4912), white: None }, + MeasuredBody { model: "NIKON D800", size: (7424, 4924), bits: 14, crop: (8, 6, 7360, 4912), white: None }, + MeasuredBody { model: "NIKON D800E", size: (7424, 4924), bits: 14, crop: (8, 6, 7360, 4912), white: None }, + // 3 files; dark: bottom 1 + MeasuredBody { model: "NIKON D810", size: (7380, 4928), bits: 12, crop: (12, 8, 7360, 4912), white: None }, + MeasuredBody { model: "NIKON D810", size: (7380, 4928), bits: 14, crop: (12, 8, 7360, 4912), white: None }, + // 6 files; masked: left 2, right 42 (the D4's readout) + MeasuredBody { model: "NIKON Df", size: (4992, 3292), bits: 12, crop: (8, 6, 4928, 3280), white: None }, + MeasuredBody { model: "NIKON Df", size: (4992, 3292), bits: 14, crop: (8, 6, 4928, 3280), white: None }, +]; + +fn measured_body(model: &str, w: usize, h: usize, bits: u32) -> Option<&'static MeasuredBody> { + MEASURED_BODIES.iter().find(|b| b.model == model.trim() && b.size == (w, h) && b.bits == bits) +} + +/// Pixel height over width from the raw IFD's resolution tags: the D1X and D1H write 600 x 300 per inch for a +/// 4028 x 1324 frame, rows twice as tall as the columns are wide. Only ratios from 1.5 to 4 count (a frame +/// squashed 3:1 or 2:1), the rest is rounding or a stray tag. +pub(crate) fn pixel_aspect_of(raw: &Ifd) -> f32 { + match (raw.f64(t::X_RESOLUTION), raw.f64(t::Y_RESOLUTION)) { + (Some(x), Some(y)) if x > 0.0 && y > 0.0 && (1.5..=4.0).contains(&(x / y)) => (x / y).round() as f32, + _ => 1.0, + } +} + +fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> { + tiff.all_ifds() + .into_iter() + .filter(|i| i.u16(t::PHOTOMETRIC) == Some(photometric::CFA)) + .max_by_key(|i| i.u64(t::IMAGE_WIDTH).unwrap_or(0).saturating_mul(i.u64(t::IMAGE_LENGTH).unwrap_or(0))) +} + +/// Rows the masked-column test reads, from the top of the frame. The header probe ([`Mode::Header`]) decodes +/// only these (a compressed strip is one sequential Huffman stream, so it can stop here; an uncompressed one reads +/// just its first strips), and the full decode judges the active width from the same rows, so +/// [`crate::probe_info`] equals [`RawImage::info`] of the decoded image (storytold/lightcraft#708). On every corpus NEF the +/// answer is the same for 8 to 512 rows as for the whole frame: the masked columns decode to 0 (compressed) or +/// ~19 (uncompressed) while live columns sit at the black level (~150 at 12 bit, ~600 at 14 bit) or above. +pub(crate) const MASK_ROWS: usize = 128; + +/// Width without the optically masked columns some bodies append on the right (the D5100 and D7000 write 8, with +/// no maker-note tag that says so, in compressed and uncompressed files alike): trailing columns (at most 64) whose +/// mean over the first [`MASK_ROWS`] rows of `d` is below 1% of the nominal white level and a tenth of the image +/// interior. Kept even for CFA phase. +pub(crate) fn trailing_masked_columns(d: &[u16], w: usize, bits: u32) -> usize { + let rows = d.len().checked_div(w).unwrap_or(0).min(MASK_ROWS); + if w < 128 || rows == 0 { + return w; + } + let col_mean = |x: usize| (0..rows).map(|y| d.get(y * w + x).copied().unwrap_or(0) as f64).sum::() / rows as f64; + let interior = (w / 4..w * 3 / 4).step_by(w / 64).map(col_mean).sum::() / 32.0; + let white = ((1u32 << bits.clamp(1, 16)) - 1) as f64; + let dark = (white * 0.01).min(interior * 0.1); + let mut aw = w; + while aw > w - 64 && col_mean(aw - 1) < dark { + aw -= 1; + } + aw & !1 +} + +pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result { + let tiff = Tiff::parse(bytes)?; + let ifd0 = tiff.ifds.first().ok_or_else(|| RawError::Corrupt("TIFF without IFDs".into()))?; + let raw = raw_ifd(&tiff).ok_or_else(|| RawError::Unsupported("NEF without a CFA image IFD".into()))?; + let info = raw.image()?; + let (w, h) = (info.width as usize, info.height as usize); + let bits = info.bits() as u32; + let make = ifd0.string(t::MAKE).unwrap_or_default(); + let mn = + tiff.exif().and_then(|e| e.get(t::MAKER_NOTE)).and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &make)); + if let Some(strip) = small_raw_strip(&info, bytes.len(), mn.as_ref()) { + return small_raw(bytes, &tiff, &info, strip, mode, limits); + } + // the header probe reads only the rows the masked-column test needs + let rows = match mode { + Mode::Full => h, + Mode::Header => h.min(MASK_ROWS), + }; + let data = if info.compression == t::compression::NIKON { + compressed(bytes, &info, mn.as_ref(), rows, limits)? + } else { + uncompressed(bytes, &info, tiff.order, mn.as_ref(), rows, limits)? + }; + let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float NEF".into())) }; + + // Maker note 0x003d is in 14-bit units whatever the sample depth: 12-bit files from the D750, D780, D850, + // D7500 and Z 50 store 600 / 1008 / 400 / 400 / 1008 while their darkest samples are 150 / ~252 / 99 / 99 / ~250 + // (our measurements on CC0 raw.pixls.us samples; the same bodies' 14-bit files match the tag as is). + let black_scale = if bits == 12 { 0.25 } else { 1.0 }; + let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() { + Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => BlackLevel { + repeat_rows: 2, + repeat_cols: 2, + values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(), + ..Default::default() + }, + _ => BlackLevel::uniform(nrw_black(mn.as_ref(), bits).unwrap_or_else(|| be16_black(&info, bytes.len() as u64, mn.as_ref(), samples))), + }; + let wb = mn + .as_ref() + .and_then(|m| m.ifd.f64s(WB_RB_LEVELS)) + .filter(|v| v.len() >= 2 && v[0] > 0.1 && v[1] > 0.1 && v[0] < 10.0 && v[1] < 10.0) + .map(|v| [v[0] as f32, 1.0, v[1] as f32]); + let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) { + (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() }, + _ => Cfa::bayer_static("RGGB"), + }; + let model = ifd0.string(t::MODEL).unwrap_or_default(); + let body = measured_body(&model, w, h, bits); + let active_w = trailing_masked_columns(samples, w, bits); + // the white level is measured from the whole image (or known for a measured body); the probe (no samples in its + // result) takes the nominal one + let white = match (body.and_then(|b| b.white), mode) { + (Some(white), _) => white, + (None, Mode::Full) => white_from_data(samples, bits), + (None, Mode::Header) => ((1u32 << bits.clamp(1, 16)) - 1) as f32, + }; + let crop = match body { + Some(&MeasuredBody { crop: (x, y, cw, ch), .. }) if x + cw <= active_w && y + ch <= h => Rect::new(x, y, cw, ch), + _ => default_crop(mn.as_ref(), active_w, h), + }; + let pixel_aspect = pixel_aspect_of(raw); + let img = RawImage { + format: RawFormat::Nef, + width: w, + height: h, + cpp: 1, + data: match mode { + Mode::Full => data, + Mode::Header => RawData::U16(Vec::new()), + }, + cfa: Some(cfa), + bits, + black, + white: vec![white], + active_area: Rect::new(0, 0, active_w, h), + crop, + orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)), + color: ColorData::default(), + wb_multipliers: wb, + linearized: false, + pixel_aspect, + // Nikon's in-file distortion correction, about the optical centre (which follows the sensor shift) + opcodes: OpcodeLists { + list3: crate::nef_lens::distortion(&model, raw, Rect::new(0, 0, active_w, h), crop).into_iter().collect(), + ..Default::default() + }, + camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(), + cinema: Default::default(), + }; + img.validate_for(mode)?; + Ok(img) +} + +/// Uncompressed strips: 16-bit words or 12-bit packed, told apart by the strip size. Packed 12-bit data is an MSB-first +/// byte stream, except in files with an NRW data block (Coolpix), where it is MSB-first inside little-endian 32-bit +/// words (the rows are whole words). Reads the first `rows` rows (the strips that hold them) of the `info.height`. +fn uncompressed( + bytes: &[u8], + info: &ImageInfo, + order: ByteOrder, + mn: Option<&makernote::MakerNote>, + rows: usize, + limits: &Limits, +) -> Result { + let (w, h) = (info.width as usize, info.height as usize); + let bits = info.bits() as u32; + let row_samples = w * info.samples_per_pixel as usize; + let chunks = info.chunks(bytes.len() as u64); + let rows_in_first = chunks.first().map(|c| c.height as usize).unwrap_or(h).max(1); + let bytes_per_row = chunks.first().map(|c| c.len as usize / rows_in_first).unwrap_or(0); + let packing = if bytes_per_row >= row_samples * 2 { + Packing::Word16 + } else if bits == 12 && bytes_per_row * 8 >= row_samples * 12 && bytes_per_row * 8 < row_samples * 13 { + if nrw_data(mn).is_some() && bytes_per_row.is_multiple_of(4) { Packing::Words32Msb } else { Packing::Msb } + } else if info.compression == 1 && bits != 8 && bits != 16 { + return Err(RawError::Unsupported(format!("NEF uncompressed packing ({bytes_per_row} bytes per {row_samples}-sample row)"))); + } else { + Packing::Msb + }; + if rows >= h { + return read_image(bytes, info, order, packing, limits); + } + // the whole image's limits apply either way; then only the strips holding the first `rows` rows are unpacked + check_image(bytes, info, limits)?; + let top = ImageInfo { height: rows as u32, ..info.clone() }; + read_image(bytes, &top, order, packing, limits) +} + +/// Packed layouts of strips that are labelled compression 34713 but have no linearization table (maker note +/// `0x0096`): they hold plain packed samples, not Huffman codes. The layouts were measured from CC0 raw.pixls.us +/// files by a clean-room black-box analysis (strip byte-count identity, same-colour smoothness and correlation with +/// the camera JPEG against at least eight alternative packings). Every layout is a single strip with +/// `stride = StripByteCounts / height`, selected only when `stride * height` equals the strip size exactly. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PackedLayout { + /// LSB-first bit stream of `bits` per sample (12: 3 bytes per 2 samples, 14: 7 bytes per 4), rows byte aligned. + /// 12-bit rows are `3w/2` bytes (older bodies) or padded to 16 bytes (Z 6, D6); 14-bit rows are `7w/4` bytes + /// padded to 16 bytes. + Lsb { stride: usize }, + /// D100: 16-byte chunks of fifteen data bytes holding ten MSB-first 12-bit samples, plus one zero pad byte. The + /// rows carry more samples than the IFD width (3040 against 3034); the extra columns are not image. + Chunked16 { stride: usize }, + /// Coolpix NRW: one big-endian 16-bit word per pixel, rows `2w` bytes with no header or padding; the 12-bit sample + /// is the upper 12 bits of the word (the low nibble is zero, rarely 8). + Be16, +} + +fn packed_layout(w: usize, h: usize, bits: u32, strip: u64, stated: bool) -> Option { + if w == 0 || h == 0 || !matches!(bits, 12 | 14) { + return None; + } + let strip = usize::try_from(strip).ok()?; + if !strip.is_multiple_of(h) { + return None; + } + let stride = strip / h; + let tight = w.checked_mul(bits as usize)?.div_ceil(8); + let tight = if bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4) { tight } else { return None }; + let padded = tight.checked_next_multiple_of(16)?; + let lsb_ok = match bits { + 12 => stride == tight || stride == padded, + _ => stride == padded || stated && stride == tight, + }; + if lsb_ok { + return Some(PackedLayout::Lsb { stride }); + } + None +} + +fn d100_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option { + let strip = usize::try_from(strip).ok()?; + if bits != 12 || h == 0 || !strip.is_multiple_of(h) { + return None; + } + let stride = strip / h; + let per_row = stride / 16 * 10; + (stride.is_multiple_of(16) && per_row >= w && per_row < w.checked_add(10)?).then_some(PackedLayout::Chunked16 { stride }) +} + +/// 12-bit strips of exactly `2 * w * h` bytes: big-endian 16-bit words (Coolpix P330, P7700, P7800 NRW, found by a +/// clean-room black-box analysis of the CC0 files). Tried after the other layouts, whose strides (`1.5w` to +/// `1.5w + 15`, or ten samples per 16 bytes) only reach `2w` for rows narrower than 40 samples, so it can't take a +/// strip they handle. +fn be16_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option { + let want = w.checked_mul(h)?.checked_mul(2)?; + (bits == 12 && w > 0 && h > 0 && usize::try_from(strip).ok()? == want).then_some(PackedLayout::Be16) +} + +/// Black level of the big-endian 16-bit layout, which carries neither `0x003d` nor an NRW block and has no masked +/// pixels (the borders of the frame are image). The darkest samples of the three measured bodies are 214 (P330), +/// 217 (P7700) and 201 (P7800, the only one with a dark enough frame: its per-channel 1st percentile is 209 / 225 / +/// 223), and the other 12-bit Coolpix NRWs state 200 in their NRW block; so 200, never above the darkest sample. +const BE16_BLACK: f32 = 200.0; + +fn be16_black(info: &ImageInfo, file_len: u64, mn: Option<&makernote::MakerNote>, samples: &[u16]) -> f32 { + let has_table = mn.is_some_and(|m| m.ifd.bytes(LINEARIZATION_TABLE).is_some()); + let single = matches!(info.chunks(file_len).as_slice(), [c] if be16_layout(info.width as usize, info.height as usize, info.bits() as u32, c.len).is_some()); + if info.compression != t::compression::NIKON || has_table || info.samples_per_pixel != 1 || !single { + return 0.0; + } + let darkest = samples.iter().copied().min().unwrap_or(0); + BE16_BLACK.min(f32::from(darkest)) +} + +/// Unpack a [`PackedLayout`] strip into `w * h` samples. Short input leaves the missing samples zero. +fn unpack_packed(src: &[u8], w: usize, h: usize, bits: u32, layout: PackedLayout) -> Vec { + use rayon::prelude::*; + let mut out = vec![0u16; w * h]; + let stride = match layout { + PackedLayout::Lsb { stride } | PackedLayout::Chunked16 { stride } => stride, + PackedLayout::Be16 => w.saturating_mul(2), + }; + out.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let Some(line) = src.get(y * stride..) else { return }; + let line = &line[..stride.min(line.len())]; + match layout { + PackedLayout::Lsb { .. } => crate::unpack::unpack_lsb(line, bits, row), + PackedLayout::Be16 => { + let (words, _) = line.as_chunks::<2>(); + for (dst, word) in row.iter_mut().zip(words) { + *dst = u16::from_be_bytes(*word) >> 4; + } + } + PackedLayout::Chunked16 { .. } => { + let mut samples = vec![0u16; line.len() / 16 * 10]; + for (i, dst) in samples.chunks_mut(10).enumerate() { + if let Some(chunk) = line.get(i * 16..i * 16 + 15) { + crate::unpack::unpack_msb(chunk, 12, dst); + } + } + let n = row.len().min(samples.len()); + row[..n].copy_from_slice(&samples[..n]); + } + } + }); + out +} + +/// Nikon Huffman-compressed strip (see [`crate::nefc`]); the decode table is maker note `0x0096`. Without a table the +/// strip may be packed uncompressed data (see [`PackedLayout`]). +fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>, rows: usize, limits: &Limits) -> Result { + let starts_with = |magic: &[u8]| { + let start = info.chunks(bytes.len() as u64).first().and_then(|c| usize::try_from(c.offset).ok()); + start.and_then(|o| bytes.get(o..o.checked_add(magic.len())?)) == Some(magic) + }; + // the maker note's own compression mode decides first: Z bodies label packed data 34713 and may still carry + // a 0x0096 table + let mode = nef_compression(mn); + if starts_with(&JPEG_XS_START) || matches!(mode, Some(13 | 14)) { + return Err(RawError::Unsupported(HIGH_EFFICIENCY.into())); + } + if let Some(depth) = match mode { + Some(6) => Some(12), + Some(10) => Some(14), + _ => None, + } { + if info.bits() as u32 != depth { + return Err(RawError::Unsupported(format!("Nikon packed {depth}-bit NEF with {}-bit samples", info.bits()))); + } + return packed(bytes, info, rows, true, limits); + } + let Some(table) = mn.and_then(|m| Some((m.ifd.bytes(LINEARIZATION_TABLE)?, m.order))) else { + return packed(bytes, info, rows, false, limits); + }; + if info.samples_per_pixel != 1 { + return Err(RawError::Unsupported(format!("Nikon compressed NEF with {} samples per pixel", info.samples_per_pixel))); + } + let bits = info.bits() as u32; + if u64::from(info.width).saturating_mul(u64::from(info.height)) > limits.samples() as u64 { + return Err(RawError::Limit("image too large")); + } + let table = nefc::parse_table(table.0, table.1, bits)?; + // one strip in every sample; several would be contiguous parts of the same bit stream + let chunks = info.chunks(bytes.len() as u64); + let (Some(first), Some(last)) = (chunks.first(), chunks.last()) else { return Err(RawError::Corrupt("NEF: no image data".into())) }; + let end = last.offset.saturating_add(last.len).min(bytes.len() as u64); + let src = usize::try_from(first.offset).ok().zip(usize::try_from(end).ok()).and_then(|(a, b)| bytes.get(a..b)); + let src = src.ok_or_else(|| RawError::Corrupt("NEF: image data outside the file".into()))?; + Ok(RawData::U16(nefc::decode_rows(src, info.width as usize, info.height as usize, rows, bits, &table)?)) +} + +/// A "compressed" strip of packed samples: the strip size must match a measured layout exactly. `stated` is set +/// when the maker note names the packed mode, which also allows 14-bit rows without padding. +fn packed(bytes: &[u8], info: &ImageInfo, rows: usize, stated: bool, limits: &Limits) -> Result { + let (w, h) = (info.width as usize, info.height as usize); + let bits = info.bits() as u32; + let chunks = info.chunks(bytes.len() as u64); + let layout = match chunks.as_slice() { + [c] if info.samples_per_pixel == 1 => { + packed_layout(w, h, bits, c.len, stated).or_else(|| d100_layout(w, h, bits, c.len)).or_else(|| be16_layout(w, h, bits, c.len)) + } + _ => None, + }; + let (Some(layout), [c]) = (layout, chunks.as_slice()) else { + return Err(RawError::Unsupported("Nikon compressed NEF without a linearization table (maker note 0x96)".into())); + }; + let src = craft_tiff::image::chunk_bytes(bytes, c) + .filter(|src| u64::try_from(src.len()).ok() == Some(c.len)) + .ok_or_else(|| RawError::Corrupt("NEF: incomplete image strip".into()))?; + if (w as u64).saturating_mul(h as u64) > limits.samples() as u64 { + return Err(RawError::Limit("image too large")); + } + Ok(RawData::U16(unpack_packed(src, w, rows.min(h), bits, layout))) +} + +/// Nikon small raw (sRAW), measured clean-room from the CC0 raw.pixls.us D810 (2036) and D4S (2227) samples. The raw SubIFD is labelled compression 34713, +/// 12 bits, one CFA sample per pixel, but it carries no linearization table (maker note `0x0096`) and its single strip +/// is exactly `3 * width * height` bytes. It holds a YCbCr 4:2:2 image of the IFD size, not a CFA mosaic (the CFA +/// tags are dummies): +/// +/// - the strip is an LSB-first 12-bit stream (`s0 = b0 | (b1 & 0x0f) << 8`, `s1 = b1 >> 4 | b2 << 4`), a row is `3 * width` +/// bytes with no padding, i.e. `2 * width` samples; +/// - the samples come in groups of four, `[Y_left, Y_right, Cb, Cr]`, for the pixel pair `(2k, 2k + 1)`: chroma is +/// halved horizontally and not vertically; +/// - luma is linear and already black-subtracted (the darkest samples are near 0) and never uses the top two bits +/// (maxima 873 and 1004 of 4095), so the white level is 1023; chroma is centred on 2048 (fitted neutral 2045-2049). +/// +/// The chroma is a camera colour-difference scale, not JFIF Cb/Cr. In linear terms the stored values behave as +/// `R - Y = R_GAIN * (Cr - 2048)` and `B - Y = B_GAIN * (Cb - 2048)` on the luma scale: regressing the camera JPEG's +/// linear `(R - Y) / Y` and `(B - Y) / Y` on `(Cr - 2048) / Y` and `(Cb - 2048) / Y` over mid-tone pixels gave slopes +/// 4.92 / 5.09 (D810, correlation 0.94 / 0.88) and 3.52 / 3.73 (D4S, 0.98 / 0.97); the gains below are the rounded +/// means. They are approximate (the two bodies differ by about 30 %), the luma is not tone-mapped, and the data is +/// treated as already white balanced (neutral D4S areas have `Cb`, `Cr` within 2 of 2048 and equal JPEG R, G, B). G +/// follows from the luma equation with the BT.601 weights. The chroma is upsampled horizontally with linear +/// interpolation, centred between the two pixels of a pair. Output is linear RGB scaled by 16 (to a 14-bit range), +/// like Sony's downsized ARWs. +const SMALL_RAW_WHITE: f32 = 1023.0; +const SMALL_RAW_SCALE: f32 = 16.0; +const SMALL_RAW_CHROMA_ZERO: f32 = 2048.0; +const SMALL_RAW_R_GAIN: f32 = 4.2; +const SMALL_RAW_B_GAIN: f32 = 4.4; + +/// The strip of a small raw: compression 34713, 12 bits, no maker note `0x0096`, one strip of exactly +/// `3 * width * height` bytes (even width). +fn small_raw_strip(info: &ImageInfo, file_len: usize, mn: Option<&makernote::MakerNote>) -> Option { + if info.compression != t::compression::NIKON || info.bits() != 12 || info.samples_per_pixel != 1 { + return None; + } + if mn.is_some_and(|m| m.ifd.bytes(LINEARIZATION_TABLE).is_some()) { + return None; + } + let (w, h) = (info.width as u64, info.height as u64); + let chunks = info.chunks(file_len as u64); + match chunks.as_slice() { + [c] if w.is_multiple_of(2) && w > 0 && h > 0 && w.checked_mul(h).and_then(|n| n.checked_mul(3)) == Some(c.len) => Some(*c), + _ => None, + } +} + +/// Decode one row of a small raw (`3 * width` bytes) into `3 * width` linear RGB values on the 14-bit scale. +/// Short input reads as zero bytes. +fn small_raw_row(src: &[u8], w: usize, out: &mut [u16]) { + let Some(n) = w.checked_mul(3).filter(|n| *n <= out.len()) else { return }; + if w == 0 || !w.is_multiple_of(2) { + return; + } + let mut s = vec![0u16; 2 * w]; + crate::unpack::unpack_lsb(&src[..src.len().min(n)], 12, &mut s); + let pairs = w / 2; + let chroma = |k: usize| { + let g = &s[4 * k + 2..4 * k + 4]; + (f32::from(g[0]) - SMALL_RAW_CHROMA_ZERO, f32::from(g[1]) - SMALL_RAW_CHROMA_ZERO) + }; + for k in 0..pairs { + let (cb, cr) = chroma(k); + for side in 0..2 { + let (nb, nr) = if side == 0 { chroma(k.saturating_sub(1)) } else { chroma((k + 1).min(pairs - 1)) }; + let (u, v) = (0.75 * cb + 0.25 * nb, 0.75 * cr + 0.25 * nr); + let y = f32::from(s[4 * k + side]); + let r = y + SMALL_RAW_R_GAIN * v; + let b = y + SMALL_RAW_B_GAIN * u; + let g = (y - 0.299 * r - 0.114 * b) / 0.587; + let dst = (2 * k + side) * 3; + for (i, value) in [r, g, b].into_iter().enumerate() { + out[dst + i] = (value * SMALL_RAW_SCALE).round().clamp(0.0, 65535.0) as u16; + } + } + } +} + +fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::image::Chunk, mode: Mode, limits: &Limits) -> Result { + use rayon::prelude::*; + let (w, h) = (info.width as usize, info.height as usize); + let stride = w.checked_mul(3).ok_or(RawError::Limit("image too large"))?; + let count = stride.checked_mul(h).filter(|n| *n > 0 && *n <= limits.samples()).ok_or(RawError::Limit("image too large"))?; + let src = craft_tiff::image::chunk_bytes(bytes, &strip) + .filter(|src| u64::try_from(src.len()).ok() == Some(strip.len)) + .ok_or_else(|| RawError::Corrupt("NEF: incomplete image strip".into()))?; + let data = match mode { + Mode::Full => { + let mut data = vec![0u16; count]; + data.par_chunks_mut(stride).enumerate().for_each(|(y, out)| { + small_raw_row(src.get(y * stride..).unwrap_or(&[]), w, out); + }); + data + } + Mode::Header => Vec::new(), + }; + let img = RawImage { + format: RawFormat::Nef, + width: w, + height: h, + cpp: 3, + data: RawData::U16(data), + cfa: None, + bits: 14, + black: BlackLevel::uniform(0.0), + white: vec![SMALL_RAW_WHITE * SMALL_RAW_SCALE], + active_area: Rect::new(0, 0, w, h), + crop: Rect::new(0, 0, w, h), + orientation: Orientation::from_exif(tiff.ifds.first().and_then(|i| i.u16(t::ORIENTATION)).unwrap_or(1)), + color: ColorData::default(), + // the stored colour is already white balanced (see above) + wb_multipliers: Some([1.0; 3]), + linearized: false, + pixel_aspect: 1.0, + opcodes: OpcodeLists::default(), + camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(), + cinema: Default::default(), + }; + img.validate_for(mode)?; + Ok(img) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter, Value}; + + fn nef(compression: u16, bits: u16, strips: Vec>, w: u32, h: u32, rps: u32) -> Vec { + nef_with_note(compression, bits, strips, w, h, rps, None) + } + + /// [`nef`] with an optional maker note (`Nikon\0` v2 header + embedded big-endian TIFF holding `note`). + fn nef_with_note(compression: u16, bits: u16, strips: Vec>, w: u32, h: u32, rps: u32, note: Option) -> Vec { + nef_in(ByteOrder::Big, compression, bits, strips, w, h, rps, note) + } + + /// [`nef_with_note`] in either byte order (file and maker note). + #[allow(clippy::too_many_arguments)] + fn nef_in(order: ByteOrder, compression: u16, bits: u16, strips: Vec>, w: u32, h: u32, rps: u32, note: Option) -> Vec { + nef_model_in(order, "NIKON TEST", compression, bits, strips, w, h, rps, note) + } + + /// [`nef_in`] from camera `model`. + #[allow(clippy::too_many_arguments)] + fn nef_model_in( + order: ByteOrder, + model: &str, + compression: u16, + bits: u16, + strips: Vec>, + w: u32, + h: u32, + rps: u32, + note: Option, + ) -> Vec { + let mut raw = IfdBuilder::new(); + raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0])); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![w])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![h])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![bits])); + raw.set(t::COMPRESSION, Value::Short(vec![compression])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])); + raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2])); + raw.set(t::CFA_PATTERN_EP, Value::Byte(vec![2, 1, 1, 0])); + raw.set_image(ImageData::Strips { rows_per_strip: rps, strips }); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("NIKON CORPORATION".into())); + ifd0.set(t::MODEL, Value::Ascii(model.into())); + if let Some(mn) = note { + let mut bytes = b"Nikon\0\x02\x10\0\0".to_vec(); + bytes.extend(TiffWriter::new(order, false).write(&[mn]).unwrap()); + let mut exif = IfdBuilder::new(); + exif.set(t::MAKER_NOTE, Value::Undefined(bytes)); + ifd0.set_child(t::EXIF_IFD, exif); + } + ifd0.add_sub_ifd(raw); + TiffWriter::new(order, false).write(&[ifd0]).unwrap() + } + + /// A Z 6II NEF carrying NikonNEFInfo gets Nikon's distortion correction as one warp, in the header probe and + /// the full decode alike, and keeps it through DNG export; another body with the same data gets none. + #[test] + fn nikon_in_file_distortion_becomes_a_warp() { + use crate::nef_lens::tests::{distortion_block, nef_info_blob, offset_block}; + let (w, h) = (48u32, 32u32); + let file = |model: &str| { + let words: Vec = (0..w * h).flat_map(|i| (600 + (i % 97) as u16 * 50).to_be_bytes()).collect(); + let mut raw = IfdBuilder::new(); + raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0])); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![w])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![h])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![14])); + raw.set(t::COMPRESSION, Value::Short(vec![1])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])); + raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2])); + raw.set(t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2])); + let info = nef_info_blob(&[ + (0x0005, distortion_block(3, 21.6, 4, [0.039895, 0.019211, -0.127218], -0.025043)), + (0x0008, offset_block(1.5, -0.75)), + ]); + raw.set(0xc7d5, Value::Undefined(info)); + raw.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![words] }); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("NIKON CORPORATION".into())); + ifd0.set(t::MODEL, Value::Ascii(model.into())); + ifd0.add_sub_ifd(raw); + TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap() + }; + let bytes = file("NIKON Z 6_2"); + let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap(); + let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap(); + assert_eq!(header.info(), full.info()); + assert_eq!(header.opcodes.list3, full.opcodes.list3); + let [crate::Opcode::WarpRectilinear { planes, center }] = full.opcodes.list3.as_slice() else { panic!("{:?}", full.opcodes.list3) }; + assert!((planes[0][0] - 1.0 / (1.0 - 0.025043)).abs() < 1e-5, "{planes:?}"); + assert!((center[0] * 47.0 - (23.5 + 1.5)).abs() < 0.01 && (center[1] * 31.0 - (15.5 - 0.75)).abs() < 0.01, "{center:?}"); + #[cfg(feature = "dng")] + { + let dng = crate::write_dng(&full, &Default::default()).unwrap(); + assert_eq!(crate::decode(&dng).unwrap().opcodes.list3, full.opcodes.list3); + } + assert!(decode(&file("NIKON D850"), Mode::Full, &Limits::default()).unwrap().opcodes.list3.is_empty()); + } + + #[test] + fn preview_color_space_is_read_from_the_nikon_note() { + for (value, expected) in [(1, Some(crate::PreviewColorSpace::Srgb)), (2, Some(crate::PreviewColorSpace::AdobeRgb)), (3, None)] { + let mut mn = IfdBuilder::new(); + mn.set(0x001e, Value::Short(vec![value])); + let bytes = nef_with_note(1, 14, vec![vec![0; 128]], 8, 8, 8, Some(mn)); + assert_eq!(crate::embedded_preview_color_space(&bytes), expected); + } + assert_eq!(crate::embedded_preview_color_space(&nef(1, 14, vec![vec![0; 128]], 8, 8, 8)), None); + assert_eq!(crate::embedded_preview_color_space(b"not a TIFF"), None); + } + + /// storytold/lightcraft#778: any make's raw falls back to its EXIF ColorSpace, with Uncalibrated + InteropIndex "R03" read as + /// Adobe RGB; the Nikon note wins; a DNG is left alone. + #[test] + fn preview_color_space_falls_back_to_the_exif_color_space() { + let file = |make: &str, cs: u16, interop: Option<&str>, note: Option, dng: bool| { + let mut exif = IfdBuilder::new(); + exif.set(0xA001, Value::Short(vec![cs])); + if let Some(i) = interop { + let mut io = IfdBuilder::new(); + io.set(0x0001, Value::Ascii(i.into())); + exif.set_child(t::INTEROP_IFD, io); + } + if let Some(v) = note { + let mut mn = IfdBuilder::new(); + mn.set(0x001e, Value::Short(vec![v])); + let mut bytes = b"Nikon".to_vec(); + bytes.extend(TiffWriter::new(ByteOrder::Little, false).write(&[mn]).unwrap()); + exif.set(t::MAKER_NOTE, Value::Undefined(bytes)); + } + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii(make.into())); + if dng { + ifd0.set(t::DNG_VERSION, Value::Byte(vec![1, 4, 0, 0])); + } + ifd0.set_child(t::EXIF_IFD, exif); + TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap() + }; + use crate::PreviewColorSpace::{AdobeRgb, Srgb}; + let f = crate::embedded_preview_color_space; + assert_eq!(f(&file("SONY", 0xFFFF, Some("R03"), None, false)), Some(AdobeRgb)); + assert_eq!(f(&file("SONY", 1, Some("R98"), None, false)), Some(Srgb)); + assert_eq!(f(&file("SONY", 0xFFFF, Some("R98"), None, false)), None); + assert_eq!(f(&file("SONY", 0xFFFF, None, None, false)), None); + assert_eq!(f(&file("Canon", 0xFFFF, Some("R03"), None, false)), Some(AdobeRgb)); + assert_eq!(f(&file("NIKON CORPORATION", 1, Some("R98"), Some(2), false)), Some(AdobeRgb)); + assert_eq!(f(&file("NIKON CORPORATION", 0xFFFF, Some("R03"), None, false)), Some(AdobeRgb)); + assert_eq!(f(&file("Canon", 0xFFFF, Some("R03"), None, true)), None); + } + + #[test] + fn nikon_crop_area_preserves_sensor_origin_and_rejects_invalid_rectangles() { + let (w, h) = (32, 24); + let words: Vec = (0..w * h).flat_map(|i| (100 + i as u16).to_be_bytes()).collect(); + for (values, expected) in [ + (vec![2, 4, 28, 16], Rect::new(2, 4, 28, 16)), + (vec![31, 4, 28, 16], Rect::new(0, 0, 32, 24)), + (vec![2, 4, 0, 16], Rect::new(0, 0, 32, 24)), + (vec![u32::MAX, 4, 28, 16], Rect::new(0, 0, 32, 24)), + (vec![2, 4, 28], Rect::new(0, 0, 32, 24)), + ] { + let mut mn = IfdBuilder::new(); + mn.set(CROP_AREA, Value::Long(values)); + let bytes = nef_with_note(1, 14, vec![words.clone()], w, h, h, Some(mn)); + let raw = crate::decode(&bytes).unwrap(); + assert_eq!(raw.crop, expected); + assert_eq!(raw.active_area, Rect::new(0, 0, 32, 24)); + assert_eq!((raw.crop.width, raw.crop.height), (expected.width, expected.height)); + assert_eq!(raw.info().developed_size(), (expected.width, expected.height)); + } + } + + /// The D4 records no CropArea: its measured image area and saturation level apply to files of its data size and + /// depth only. + #[test] + fn measured_d4_image_area_and_white_level() { + let (w, h) = (4992usize, 3292usize); + let decode_as = |model: &str, bits: u16| { + let mut nef = nef(1, bits, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32); + let at = nef.windows(10).position(|b| b == b"NIKON TEST").unwrap(); + let name = format!("{model:<10}"); + nef[at..at + 10].copy_from_slice(name.as_bytes()); + crate::decode(&nef).unwrap() + }; + let d4 = decode_as("NIKON D4", 14); + assert_eq!(d4.crop, Rect::new(8, 6, 4928, 3280)); + assert_eq!(d4.white, vec![15520.0]); + assert_eq!((d4.crop.width, d4.crop.height), (4928, 3280)); + assert_eq!(d4.info().developed_size(), (4928, 3280)); + // the 12-bit mode reads out the same area, and its level is the 14-bit one shifted like its samples + let d4_12 = decode_as("NIKON D4", 12); + assert_eq!(d4_12.crop, Rect::new(8, 6, 4928, 3280)); + assert_eq!(d4_12.white, vec![3880.0]); + let other = decode_as("NIKON D5", 14); + assert_eq!(other.crop, Rect::new(0, 0, w, h)); + assert_ne!(other.white, vec![15520.0]); + } + + /// storytold/lightcraft#976: the D7000 and D5100 record no CropArea, and their 36 masked columns at ~600 were developed as a + /// coloured bar. Their measured area applies to the measured models, data sizes and depths only. + #[test] + fn measured_d7000_and_d5100_image_area() { + let (w, h) = (4992usize, 3280usize); + let decode_as = |model: &str, bits: u16| { + let nef = nef_model_in(ByteOrder::Big, model, 1, bits, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32, None); + crate::decode(&nef).unwrap() + }; + for (model, bits) in [("NIKON D7000", 14), ("NIKON D7000", 12), ("NIKON D5100", 14)] { + let raw = decode_as(model, bits); + assert_eq!(raw.crop, Rect::new(8, 8, 4928, 3264), "{model} {bits}-bit"); + assert_eq!((raw.crop.width, raw.crop.height), (4928, 3264)); + assert_eq!(raw.info().developed_size(), (4928, 3264)); + } + // another model, or the D5100's unmeasured 12-bit mode, keeps the whole data + for (model, bits) in [("NIKON D7100", 14), ("NIKON D5100", 12), ("NIKON D7000 ", 16)] { + assert_eq!(decode_as(model, bits).crop, Rect::new(0, 0, w, h), "{model} {bits}-bit"); + } + } + + /// The D200, D3200 and Nikon 1 V1 keep masked or dark lines in their whole data (the D200 shares the D80's data + /// size but frames it elsewhere), measured on public sample files beyond the CC0 ones; the D3100, D5000, D40, D50, + /// D70 and D70s get best-guess frames (the D40 shares the D50 / D70 / D70s data size but not their frame). + #[test] + fn measured_older_dx_bodies_image_area() { + let decode_as = |model: &str, (w, h): (usize, usize)| { + let nef = nef_model_in(ByteOrder::Big, model, 1, 12, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32, None); + crate::decode(&nef).unwrap() + }; + for (model, size, crop) in [ + ("NIKON D200", (3904, 2616), Rect::new(16, 12, 3872, 2592)), + ("NIKON D80", (3904, 2616), Rect::new(14, 9, 3872, 2592)), + ("NIKON D3200", (6080, 4012), Rect::new(8, 6, 6016, 4000)), + ("NIKON 1 V1", (3904, 2606), Rect::new(12, 6, 3872, 2592)), + ("NIKON D3100", (4672, 3084), Rect::new(24, 6, 4608, 3072)), + ("NIKON D5000", (4352, 2868), Rect::new(11, 10, 4288, 2848)), + ("NIKON D40", (3040, 2014), Rect::new(16, 9, 3008, 2000)), + ("NIKON D50", (3040, 2014), Rect::new(15, 7, 3008, 2000)), + ("NIKON D70", (3040, 2014), Rect::new(15, 7, 3008, 2000)), + ("NIKON D70s", (3040, 2014), Rect::new(15, 7, 3008, 2000)), + ] { + assert_eq!(decode_as(model, size).crop, crop, "{model}"); + } + // the V1's area on the V2's data size: no entry, the whole data + assert_eq!(decode_as("NIKON 1 V1", (4620, 3084)).crop, Rect::new(0, 0, 4620, 3084)); + } + + /// Every measured entry is a crop inside its data, once per model, data size and depth, with a white level (if + /// any) inside its bit range. + #[test] + fn measured_bodies_are_consistent() { + let mut seen = std::collections::HashSet::new(); + for b in MEASURED_BODIES { + let (x, y, w, h) = b.crop; + assert!(w > 0 && h > 0 && x + w <= b.size.0 && y + h <= b.size.1, "{}: crop outside its data", b.model); + assert!(seen.insert((b.model, b.size, b.bits)), "{} {:?} {}-bit listed twice", b.model, b.size, b.bits); + assert!(matches!(b.bits, 12 | 14), "{}: {}-bit", b.model, b.bits); + assert!(b.white.is_none_or(|v| v > 0.0 && v < (1u32 << b.bits) as f32), "{}: white level", b.model); + assert_eq!(b.model.trim(), b.model, "{}: model names are matched trimmed", b.model); + } + } + + /// storytold/lightcraft#1248: the D7100 records no CropArea, and its data holds picture beyond Lightroom's frame in both + /// image areas (full frame and 1.3x crop), so renders came out larger and Lightroom crops landed off. + #[test] + fn measured_d7100_image_area() { + let decode_as = |model: &str, (w, h): (usize, usize)| { + let mut nef = nef(1, 14, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32); + // the fixture's model is "NIKON TEST" plus its NUL: 11 bytes to overwrite + let at = nef.windows(11).position(|b| b == b"NIKON TEST\0").unwrap(); + let name = format!("{model:\0<11}"); + nef[at..at + 11].copy_from_slice(name.as_bytes()); + crate::decode(&nef).unwrap() + }; + for (size, crop) in [((6036, 4020), Rect::new(18, 10, 6000, 4000)), ((4812, 3212), Rect::new(6, 6, 4800, 3200))] { + let d7100 = decode_as("NIKON D7100", size); + assert_eq!(d7100.crop, crop); + assert_eq!((d7100.crop.width, d7100.crop.height), (crop.width, crop.height)); + assert_ne!(d7100.white, vec![15520.0]); + assert_eq!(d7100.info().developed_size(), (crop.width, crop.height)); + assert_eq!(decode_as("NIKON D7200", size).crop, Rect::new(0, 0, size.0, size.1)); + } + } + + /// The D90's last 42 columns are filler at a flat ~254, above its dark image samples: without the measured + /// area they stay in the image and develop as a pink band on the right. + #[test] + fn measured_d90_drops_the_filler_columns() { + let (w, h) = (4352usize, 2868usize); + let mut px = vec![40u16; w * h]; + for row in px.chunks_mut(w) { + row[4310..].fill(254); + } + let mut nef = nef(1, 12, vec![px.iter().flat_map(|v| v.to_le_bytes()).collect()], w as u32, h as u32, h as u32); + let at = nef.windows(10).position(|b| b == b"NIKON TEST").unwrap(); + nef[at..at + 10].copy_from_slice(b"NIKON D90 "); + let d90 = crate::decode(&nef).unwrap(); + assert_eq!(d90.crop, Rect::new(10, 10, 4288, 2848)); + assert!(d90.crop.x + d90.crop.width <= 4310, "the filler columns are outside the image"); + assert_eq!(d90.white, vec![3830.0]); + assert_eq!((d90.crop.width, d90.crop.height), (4288, 2848)); + } + + /// storytold/lightcraft#708: the active width is judged from the first [`MASK_ROWS`] rows in both modes, so the header probe + /// (which reads only those rows' strips) and the full decode agree whether the masked columns are there or not. + #[test] + fn masked_columns_are_judged_from_the_top_rows_in_both_modes() { + let (w, h, rps) = (256usize, MASK_ROWS + 64, 8u32); + let file = |masked_rows: std::ops::Range| { + let words: Vec = (0..w * h) + .map(|i| if masked_rows.contains(&(i / w)) && i % w >= w - 8 { 0u16 } else { 1000 + (i % 7) as u16 }) + .flat_map(|v| v.to_be_bytes()) + .collect(); + nef(1, 14, words.chunks(w * 2 * rps as usize).map(|c| c.to_vec()).collect(), w as u32, h as u32, rps) + }; + for (masked_rows, expected_width) in [(0..h, w - 8), (0..MASK_ROWS, w - 8), (MASK_ROWS..h, w), (0..0, w)] { + let bytes = file(masked_rows.clone()); + let full = crate::decode(&bytes).unwrap(); + assert_eq!(full.active_area.width, expected_width, "masked rows {masked_rows:?}"); + assert_eq!(full.crop.width, expected_width); + assert_eq!(crate::probe_info(&bytes).unwrap(), full.info(), "masked rows {masked_rows:?}"); + } + // the shared rule on its own: nominal white, trailing columns at most 64, even width + let d: Vec = (0..w * 4).map(|i| if i % w >= w - 9 { 0 } else { 600 }).collect(); + assert_eq!(trailing_masked_columns(&d, w, 14), w - 10, "nine dark columns trim to an even width"); + assert_eq!(trailing_masked_columns(&d, w, 12), w - 10); + assert_eq!(trailing_masked_columns(&d[..w - 1], w, 14), w, "less than a row of samples leaves the width alone"); + assert_eq!(trailing_masked_columns(&[0; 64], 64, 14), 64, "narrow images are left alone"); + assert_eq!(trailing_masked_columns(&[], 0, 14), 0); + } + + #[test] + fn black_level_tag_is_in_14_bit_units() { + let (w, h) = (16usize, 4usize); + let words: Vec = (0..w * h).flat_map(|i| (100 + i as u16).to_be_bytes()).collect(); + let note = || { + let mut mn = IfdBuilder::new(); + mn.set(BLACK_LEVEL, Value::Short(vec![400, 404, 408, 412])); + mn + }; + let black = |bits| crate::decode(&nef_with_note(1, bits, vec![words.clone()], w as u32, h as u32, h as u32, Some(note()))).unwrap().black; + assert_eq!(black(12).values, vec![100.0, 101.0, 102.0, 103.0]); + assert_eq!(black(14).values, vec![400.0, 404.0, 408.0, 412.0]); + } + + #[test] + fn uncompressed_word16_and_packed12() { + let (w, h) = (16usize, 6usize); + let px: Vec = (0..w * h).map(|i| (i * 131 % 4096) as u16).collect(); + let words: Vec = px.iter().flat_map(|v| v.to_be_bytes()).collect(); + let bytes = nef(1, 12, words.chunks(w * 2 * 3).map(|c| c.to_vec()).collect(), w as u32, h as u32, 3); + assert_eq!(crate::probe(&bytes), Some(RawFormat::Nef)); + let r = crate::decode(&bytes).unwrap(); + assert_eq!(r.data, RawData::U16(px.clone())); + assert_eq!(r.cfa.as_ref().unwrap().name(), "BGGR"); + // 12-bit MSB packed + let mut packed = Vec::new(); + for pair in px.chunks(2) { + let (a, b) = (pair[0] as u32, pair[1] as u32); + let v = (a << 12) | b; + packed.extend_from_slice(&[(v >> 16) as u8, (v >> 8) as u8, v as u8]); + } + let bytes = nef(1, 12, vec![packed], w as u32, h as u32, h as u32); + assert_eq!(crate::decode(&bytes).unwrap().data, RawData::U16(px)); + } + + /// An NRW data block (`"NRW 0104"`, fields from byte 8) with `black` at byte `0x20`, as a maker note. + fn nrw_note(black: u32) -> IfdBuilder { + let mut block = b"NRW 0104".to_vec(); + block.resize(0x20, 0); + block.extend_from_slice(&black.to_le_bytes()); + block.resize(0x40, 0); + let mut mn = IfdBuilder::new(); + mn.set(NRW_DATA, Value::Undefined(block)); + mn + } + + /// 12-bit samples as an MSB-first bit stream inside 32-bit words in `order` (rows are whole words). + fn words32(px: &[u16], w: usize, order: ByteOrder) -> Vec { + let mut out = Vec::new(); + for row in px.chunks(w) { + let stream = row.iter().fold(Vec::new(), |mut bits: Vec, v| { + bits.extend((0..12).rev().map(|b| v >> b & 1 == 1)); + bits + }); + for word in stream.chunks(32) { + let v = word.iter().enumerate().fold(0u32, |a, (i, b)| a | (*b as u32) << (31 - i)); + out.extend_from_slice(&match order { + ByteOrder::Little => v.to_le_bytes(), + ByteOrder::Big => v.to_be_bytes(), + }); + } + } + out + } + + #[test] + fn nrw_block_selects_32_bit_word_packing_and_supplies_the_black_level() { + let (w, h) = (16usize, 6usize); + let px: Vec = (0..w * h).map(|i| (200 + i * 131 % 3800) as u16).collect(); + let packed = words32(&px, w, ByteOrder::Little); + assert_eq!(packed.len(), w * h * 3 / 2); + let decode = |note: Option| { + crate::decode(&nef_in(ByteOrder::Little, 1, 12, vec![packed.clone()], w as u32, h as u32, h as u32, note)).unwrap() + }; + let r = decode(Some(nrw_note(200))); + assert_eq!(r.data, RawData::U16(px.clone())); + assert_eq!(r.black.values, vec![200.0]); + assert_eq!(decode(Some(nrw_note(0))).black.values, vec![0.0]); + // an implausible field is not a black level + assert_eq!(decode(Some(nrw_note(60000))).black.values, vec![0.0]); + // without the block the same bytes are an MSB-first byte stream + let plain = decode(None); + assert_ne!(plain.data, RawData::U16(px)); + assert_eq!(plain.black.values, vec![0.0]); + // the maker-note black level (14-bit units) wins when present + let mut both = nrw_note(200); + both.set(BLACK_LEVEL, Value::Short(vec![400, 404, 408, 412])); + assert_eq!(decode(Some(both)).black.values, vec![100.0, 101.0, 102.0, 103.0]); + } + + #[test] + fn compressed_without_table_is_unsupported() { + let bytes = nef(34713, 14, vec![vec![0; 64]], 8, 8, 8); + let Err(RawError::Unsupported(why)) = crate::decode(&bytes) else { panic!("expected unsupported") }; + assert!(why.contains("linearization table"), "{why}"); + } + + /// storytold/lightcraft#193: a High Efficiency NEF (Z f, Z 8: compression 34713, no `0x0096` table, a JPEG XS codestream) is + /// named as such instead of being blamed on the missing table, and the header probe reports the same reason. + #[test] + fn high_efficiency_nef_is_named() { + let mut strip = vec![0xff, 0x10, 0xff, 0x50, 0x00, 0x22]; + strip.resize(64, 0); + let bytes = nef(34713, 14, vec![strip], 8, 8, 8); + for result in [crate::decode(&bytes).map(|_| ()), crate::probe_info(&bytes).map(|_| ())] { + let Err(RawError::Unsupported(why)) = result else { panic!("expected unsupported") }; + assert_eq!(why, HIGH_EFFICIENCY); + assert!(why.starts_with("Nikon High Efficiency NEF ("), "{why}"); + } + // a strip too short to hold the markers is still the missing-table case, not a panic + let short = nef(34713, 14, vec![vec![0xff, 0x10]], 1, 1, 1); + assert!(matches!(crate::decode(&short), Err(RawError::Unsupported(w)) if w.contains("linearization table"))); + } + + fn samples(n: usize, bits: u32) -> Vec { + (0..n).map(|i| (((i * 2654435761usize) >> 7) as u32 % (1 << bits)) as u16).collect() + } + + fn pack_lsb(row: &[u16], bits: u32) -> Vec { + let (mut out, mut acc, mut n) = (Vec::new(), 0u64, 0u32); + for &v in row { + acc |= (v as u64) << n; + n += bits; + while n >= 8 { + out.push(acc as u8); + acc >>= 8; + n -= 8; + } + } + if n > 0 { + out.push(acc as u8); + } + out + } + + fn pack_msb12(row: &[u16]) -> Vec { + let mut out = Vec::new(); + for pair in row.chunks(2) { + let (a, b) = (pair[0], pair.get(1).copied().unwrap_or(0)); + out.extend([(a >> 4) as u8, ((a & 15) << 4) as u8 | (b >> 8) as u8, b as u8]); + } + out + } + + /// Rows of `samples` packed by `row`, each padded with `pad` bytes of 0xA5 (never decoded). + fn strip(data: &[u16], w: usize, pad: usize, row: impl Fn(&[u16]) -> Vec) -> Vec { + data.chunks(w).flat_map(|r| row(r).into_iter().chain(std::iter::repeat_n(0xA5, pad))).collect() + } + + fn decoded(strip: Vec, bits: u16, w: u32, h: u32) -> Result> { + let bytes = nef(34713, bits, vec![strip], w, h, h); + match crate::decode(&bytes)?.data { + RawData::U16(d) => Ok(d), + _ => Err(RawError::Unsupported("float".into())), + } + } + + #[test] + fn packed_34713_layouts_round_trip() { + let (w, h) = (40usize, 6usize); + let d12 = samples(w * h, 12); + let d14 = samples(w * h, 14); + // group A: 12-bit, 3w/2 bytes per row + assert_eq!(decoded(strip(&d12, w, 0, |r| pack_lsb(r, 12)), 12, 40, 6).unwrap(), d12); + // group B: 12-bit, rows padded to 16 bytes (60 -> 64) + assert_eq!(decoded(strip(&d12, w, 4, |r| pack_lsb(r, 12)), 12, 40, 6).unwrap(), d12); + // group C: 14-bit, 70 bytes -> 80 + assert_eq!(decoded(strip(&d14, w, 10, |r| pack_lsb(r, 14)), 14, 40, 6).unwrap(), d14); + // group E: 16-byte chunks, 15 data bytes (ten samples) + a zero byte; 16 chunks = 160 samples (at 40 wide the + // 64-byte stride would be group B's) + let (w, h) = (160usize, 3usize); + let d12 = samples(w * h, 12); + let e = strip(&d12, w, 0, |r| r.chunks(10).flat_map(|c| pack_msb12(c).into_iter().chain([0u8])).collect()); + assert_eq!(e.len(), 256 * h); + assert_eq!(decoded(e, 12, 160, 3).unwrap(), d12); + } + + /// Z 6II "Packed 14 bits" (maker note 0x93 = 10, storytold/lightcraft#769): 6064x4040, 14-bit, one strip of 4040 rows of 10624 + /// bytes (6064 * 14 / 8 = 10612, padded to a multiple of 16). Four rows keep the real stride. + #[test] + fn packed_14_bit_z6ii_geometry_with_row_padding() { + let (w, h) = (6064usize, 4usize); + let d14 = samples(w * h, 14); + let s = strip(&d14, w, 12, |r| pack_lsb(r, 14)); + assert_eq!(s.len(), 10624 * h); + assert_eq!(decoded(s, 14, w as u32, h as u32).unwrap(), d14); + assert_eq!(packed_layout(6064, 4040, 14, 42_920_960, false), Some(PackedLayout::Lsb { stride: 10624 })); + } + + /// The maker note's compression mode routes the strip: 6 and 10 are packed samples even when a `0x0096` + /// table is present (which the Huffman reader would otherwise take), in the old tag and in the `0x0051` + /// record, in either byte order; a depth that contradicts the mode, and High Efficiency (13, 14), are refused. + #[test] + fn maker_note_compression_mode_routes_packed_and_high_efficiency() { + let (w, h) = (40usize, 6usize); + let note = |order: ByteOrder, modern: bool, mode: u16| { + let mut n = IfdBuilder::new(); + n.set(LINEARIZATION_TABLE, Value::Undefined(vec![0x44, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])); + if modern { + let mut record = b"01000400\0\0".to_vec(); + record.extend(if order == ByteOrder::Big { mode.to_be_bytes() } else { mode.to_le_bytes() }); + n.set(NEF_COMPRESSION_RECORD, Value::Undefined(record)); + } else { + n.set(NEF_COMPRESSION, Value::Short(vec![mode])); + } + n + }; + let file = |order, modern, mode, bits: u16, image: Vec| { + nef_in(order, 34713, bits, vec![image], w as u32, h as u32, h as u32, Some(note(order, modern, mode))) + }; + for (bits, mode) in [(12u32, 6u16), (14, 10)] { + let data = samples(w * h, bits); + // rows padded to 16 bytes, and tight rows (14-bit tight rows only when the mode is stated) + for pad in [(16 - (w * bits as usize / 8) % 16) % 16, 0] { + let image = strip(&data, w, pad, |r| pack_lsb(r, bits)); + for order in [ByteOrder::Big, ByteOrder::Little] { + for modern in [false, true] { + let RawData::U16(got) = crate::decode(&file(order, modern, mode, bits as u16, image.clone())).unwrap().data else { + panic!("integer samples") + }; + assert_eq!(got, data, "{bits}-bit, pad {pad}, {order:?}, modern {modern}"); + } + } + } + } + let d14 = samples(w * h, 14); + let image = strip(&d14, w, 10, |r| pack_lsb(r, 14)); + // tight 14-bit rows without a stated mode stay refused + assert!(decoded(strip(&d14, w, 0, |r| pack_lsb(r, 14)), 14, w as u32, h as u32).is_err()); + // the mode says 12-bit packed, the IFD 14 bits + assert!(matches!(crate::decode(&file(ByteOrder::Big, false, 6, 14, image.clone())), Err(RawError::Unsupported(_)))); + for mode in [13, 14] { + for modern in [false, true] { + let err = crate::decode(&file(ByteOrder::Big, modern, mode, 14, image.clone())).unwrap_err(); + assert!(matches!(&err, RawError::Unsupported(m) if m == HIGH_EFFICIENCY), "{err:?}"); + } + } + } + + #[test] + fn packed_layout_refuses_overflowing_dimensions() { + for bits in [12, 14] { + assert!(packed_layout(usize::MAX - 3, 1, bits, u64::MAX, false).is_none()); + } + assert!(d100_layout(usize::MAX, 1, 12, u64::MAX).is_none()); + } + + /// Header probes inspect only the same top rows that the masked-column test uses. Routing + /// still validates the complete strip's layout, so header and full decode keep the same crop. + #[test] + fn packed_header_probe_reads_only_the_mask_rows() { + let (w, h) = (160usize, MASK_ROWS + 7); + let data = samples(w * h, 12); + for image in [strip(&data, w, 0, |r| pack_lsb(r, 12)), be16(&data)] { + let bytes = nef(34713, 12, vec![image], w as u32, h as u32, h as u32); + let tiff = Tiff::parse(&bytes).unwrap(); + let info = raw_ifd(&tiff).unwrap().image().unwrap(); + let RawData::U16(top) = packed(&bytes, &info, MASK_ROWS, false, &Limits::default()).unwrap() else { panic!("integer samples") }; + assert_eq!(top, data[..w * MASK_ROWS]); + let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap(); + let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap(); + assert_eq!((header.active_area, header.crop), (full.active_area, full.crop)); + assert!(matches!(header.data, RawData::U16(ref samples) if samples.is_empty())); + } + } + + #[test] + fn packed_d100_rows_carry_extra_columns() { + // IFD width 154, rows hold 160 samples (16 chunks); the last 6 are dropped + let (w, wide, h) = (154usize, 160usize, 3usize); + let wide_data = samples(wide * h, 12); + let e = strip(&wide_data, wide, 0, |r| r.chunks(10).flat_map(|c| pack_msb12(c).into_iter().chain([0u8])).collect()); + let got = decoded(e, 12, w as u32, h as u32).unwrap(); + let want: Vec = wide_data.chunks(wide).flat_map(|r| r[..w].iter().copied()).collect(); + assert_eq!(got, want); + } + + #[test] + fn packed_34713_wrong_sizes_stay_refused() { + let d12 = samples(40 * 6, 12); + let good = strip(&d12, 40, 0, |r| pack_lsb(r, 12)); + // one byte more or fewer, a stride that is neither tight nor 16-aligned, a 13-bit file + for len in [good.len() - 1, good.len() + 1, good.len() + 6] { + let mut s = good.clone(); + s.resize(len, 0); + assert!(matches!(decoded(s, 12, 40, 6), Err(RawError::Unsupported(_))), "len {len}"); + } + assert!(matches!(decoded(good.clone(), 13, 40, 6), Err(RawError::Unsupported(_)))); + // 14-bit rows must be padded to 16 bytes + let d14 = samples(40 * 6, 14); + assert!(matches!(decoded(strip(&d14, 40, 0, |r| pack_lsb(r, 14)), 14, 40, 6), Err(RawError::Unsupported(_)))); + } + + /// A small raw strip of `[Y0 Y1 Cb Cr]` groups, rows of `3 * w` bytes, LSB-first 12-bit. + fn small_strip(groups: &[[u16; 4]], w: usize) -> Vec { + let flat: Vec = groups.iter().flatten().copied().collect(); + assert_eq!(flat.len() % (2 * w), 0); + flat.chunks(2 * w).flat_map(|r| pack_lsb(r, 12)).collect() + } + + fn small(strip: Vec, bits: u16, w: u32, h: u32) -> Result { + crate::decode(&nef(34713, bits, vec![strip], w, h, h)) + } + + #[test] + fn small_raw_422_round_trips() { + let (w, h) = (8usize, 3usize); + // flat chroma per row so the interpolation is exact; distinct luma everywhere + let mut groups = Vec::new(); + for y in 0..h { + for k in 0..w / 2 { + let (cb, cr) = (2048 + 10 * y as u16, 2048 - 5 * y as u16); + groups.push([100 + (y * w + 2 * k) as u16, 100 + (y * w + 2 * k + 1) as u16, cb, cr]); + } + } + let raw = small(small_strip(&groups, w), 12, w as u32, h as u32).unwrap(); + assert_eq!((raw.width, raw.height, raw.cpp, raw.cfa.is_none()), (w, h, 3, true)); + assert_eq!(raw.wb_multipliers, Some([1.0; 3])); + let RawData::U16(d) = &raw.data else { panic!() }; + for y in 0..h { + for x in 0..w { + let luma = (100 + y * w + x) as f32; + let (u, v) = (10.0 * y as f32, -5.0 * y as f32); + let (r, b) = (luma + SMALL_RAW_R_GAIN * v, luma + SMALL_RAW_B_GAIN * u); + let g = (luma - 0.299 * r - 0.114 * b) / 0.587; + for (i, want) in [r, g, b].into_iter().enumerate() { + let got = f32::from(d[(y * w + x) * 3 + i]) / SMALL_RAW_SCALE; + assert!((got - want).abs() < 0.1, "({x},{y}) channel {i}: {got} vs {want}"); + } + } + } + } + + #[test] + fn small_raw_neutral_chroma_is_grey_and_chroma_is_interpolated() { + let w = 8usize; + let groups: Vec<[u16; 4]> = (0..w / 2).map(|k| [200 + 40 * k as u16, 200 + 40 * k as u16, 2048, 2048]).collect(); + let raw = small(small_strip(&groups, w), 12, w as u32, 1).unwrap(); + let RawData::U16(d) = &raw.data else { panic!() }; + for x in 0..w { + let px = &d[x * 3..x * 3 + 3]; + assert!(px.iter().all(|&v| v == px[0]), "{px:?}"); + assert_eq!(f32::from(px[0]), (200 + 40 * (x / 2)) as f32 * SMALL_RAW_SCALE); + } + // a step in Cb between the first two pairs is shared 3:1 by the pixels next to the boundary + let mut groups = groups; + groups[1][2] = 2048 + 100; + let raw = small(small_strip(&groups, w), 12, w as u32, 1).unwrap(); + let RawData::U16(d) = &raw.data else { panic!() }; + let blue_minus_luma = |x: usize, luma: f32| f32::from(d[x * 3 + 2]) / SMALL_RAW_SCALE - luma; + assert!((blue_minus_luma(1, 200.0) - SMALL_RAW_B_GAIN * 25.0).abs() < 0.1); + assert!((blue_minus_luma(2, 240.0) - SMALL_RAW_B_GAIN * 75.0).abs() < 0.1); + } + + #[test] + fn small_raw_wrong_sizes_and_depths_are_refused() { + let (w, h) = (8usize, 3usize); + let good = small_strip(&vec![[100, 100, 2048, 2048]; w / 2 * h], w); + assert!(small(good.clone(), 12, w as u32, h as u32).is_ok()); + for len in [good.len() - 1, good.len() + 1] { + let mut s = good.clone(); + s.resize(len, 0); + assert!(matches!(small(s, 12, w as u32, h as u32), Err(RawError::Unsupported(_))), "len {len}"); + } + assert!(matches!(small(good.clone(), 14, w as u32, h as u32), Err(RawError::Unsupported(_)))); + // odd width: 3 bytes per pixel does not split into pixel pairs + assert!(matches!(small(vec![0; 3 * 7 * 3], 12, 7, 3), Err(RawError::Unsupported(_)))); + } + + #[test] + fn small_raw_short_input_does_not_panic() { + for len in [0, 1, 2, 3, 5, 11, 12, 13] { + let mut out = vec![0u16; 12]; + small_raw_row(&vec![0xFF; len], 4, &mut out); + } + let mut bytes = nef(34713, 12, vec![vec![0x55; 3 * 8 * 3]], 8, 3, 3); + bytes.truncate(bytes.len() - 40); + assert!(crate::decode(&bytes).is_err(), "a truncated small-raw strip is corrupt"); + } + + #[test] + fn small_raw_header_has_full_geometry_without_allocating_pixels() { + let (w, h) = (8usize, 3usize); + let bytes = nef(34713, 12, vec![small_strip(&vec![[100, 110, 2048, 2048]; w / 2 * h], w)], w as u32, h as u32, h as u32); + let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap(); + let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap(); + assert_eq!(header.info(), full.info()); + assert!(matches!(header.data, RawData::U16(ref samples) if samples.is_empty())); + assert!(matches!(full.data, RawData::U16(ref samples) if samples.len() == 3 * w * h)); + } + + #[test] + fn small_raw_rejects_excessive_dimensions_before_allocation() { + let bytes = nef(34713, 12, vec![vec![0; 72]], 8, 3, 3); + let tiff = Tiff::parse(&bytes).unwrap(); + let mut info = raw_ifd(&tiff).unwrap().image().unwrap(); + let strip = info.chunks(bytes.len() as u64)[0]; + info.width = u32::MAX - 1; + info.height = u32::MAX; + assert!(matches!(small_raw(&bytes, &tiff, &info, strip, Mode::Header, &Limits::default()), Err(RawError::Limit(_)))); + let mut out = [0; 12]; + small_raw_row(&[], usize::MAX, &mut out); + small_raw_row(&[], 3, &mut out); + assert_eq!(out, [0; 12]); + } + + /// Big-endian 16-bit words holding the 12-bit samples in their upper bits. + fn be16(data: &[u16]) -> Vec { + data.iter().flat_map(|v| (v << 4).to_be_bytes()).collect() + } + + #[test] + fn coolpix_be16_words_decode_and_wrong_sizes_are_refused() { + let (w, h) = (40usize, 6usize); + let d12 = samples(w * h, 12); + let good = be16(&d12); + assert_eq!(good.len(), 2 * w * h); + assert_eq!(decoded(good.clone(), 12, 40, 6).unwrap(), d12); + // a half-LSB in the low nibble does not change the sample + let mut half = good.clone(); + for b in half.iter_mut().skip(1).step_by(2) { + *b |= 8; + } + assert_eq!(decoded(half, 12, 40, 6).unwrap(), d12); + // any other strip size, or another bit depth, is not this layout + for len in [good.len() - 2, good.len() - 1, good.len() + 1, good.len() + 2, good.len() + 16] { + let mut s = good.clone(); + s.resize(len, 0); + assert!(matches!(decoded(s, 12, 40, 6), Err(RawError::Unsupported(_))), "len {len}"); + } + assert!(matches!(decoded(good.clone(), 13, 40, 6), Err(RawError::Unsupported(_)))); + // the black level is 200, but never above the darkest sample + let black = |data: &[u16]| crate::decode(&nef(34713, 12, vec![be16(data)], 40, 6, 6)).unwrap().black.values[0]; + let mut d = d12.clone(); + for v in &mut d { + *v = (*v).max(300); + } + assert_eq!(black(&d), 200.0); + d[5] = 150; + assert_eq!(black(&d), 150.0); + } + + #[test] + fn coolpix_be16_does_not_take_other_layouts() { + // every width where another group's stride equals 2w must still go to that group (the narrow rows where they + // coincide are below the real-world widths; the other groups are tried first) + for w in (2..=400usize).step_by(2) { + let strip = 2 * w as u64 * 6; + let other = packed_layout(w, 6, 12, strip, false).or_else(|| d100_layout(w, 6, 12, strip)); + if other.is_some() { + assert_eq!( + packed_layout(w, 6, 12, strip, false).or_else(|| d100_layout(w, 6, 12, strip)).or_else(|| be16_layout(w, 6, 12, strip)), + other + ); + } + } + assert_eq!(be16_layout(4032, 3024, 12, 24385536), Some(PackedLayout::Be16)); + assert!(packed_layout(4032, 3024, 12, 24385536, false).is_none() && d100_layout(4032, 3024, 12, 24385536).is_none()); + assert!(be16_layout(4032, 3024, 14, 24385536).is_none()); + assert!(be16_layout(usize::MAX, usize::MAX, 12, u64::MAX).is_none()); + } + + #[test] + fn packed_34713_short_input_does_not_panic() { + let (w, h) = (40usize, 6usize); + for layout in [PackedLayout::Lsb { stride: 64 }, PackedLayout::Lsb { stride: 80 }, PackedLayout::Chunked16 { stride: 64 }, PackedLayout::Be16] + { + for len in [0, 1, 15, 16, 17, 63, 64, 65, 200] { + let bits = if matches!(layout, PackedLayout::Lsb { stride: 80 }) { 14 } else { 12 }; + assert_eq!(unpack_packed(&vec![0xFF; len], w, h, bits, layout).len(), w * h); + } + } + // a strip that claims more bytes than the file has + let mut bytes = nef(34713, 12, vec![vec![0x55; 60 * 6]], 40, 6, 6); + bytes.truncate(bytes.len() - 100); + assert!(crate::decode(&bytes).is_err(), "a truncated packed strip is corrupt"); + } + /// Keep every directory intact before a relocated tail strip, then remove its last rows. + fn truncated_tail_strip(mut bytes: Vec, keep: usize) -> Vec { + let tiff = Tiff::parse(&bytes).unwrap(); + let raw = raw_ifd(&tiff).unwrap(); + let info = raw.image().unwrap(); + let chunk = info.chunks(bytes.len() as u64)[0]; + let data = craft_tiff::image::chunk_bytes(&bytes, &chunk).unwrap().to_vec(); + let offset_entry = raw.get(t::STRIP_OFFSETS).unwrap().offset as usize; + let start = bytes.len() as u32; + bytes[offset_entry..offset_entry + 4].copy_from_slice(&start.to_be_bytes()); + bytes.extend_from_slice(&data[..keep]); + bytes + } + + #[test] + fn packed_rejects_tail_truncation_with_intact_directories() { + let bytes = nef(34713, 12, vec![vec![0x55; 360]], 40, 6, 6); + for keep in [0, 60, 359] { + let short = truncated_tail_strip(bytes.clone(), keep); + for mode in [Mode::Header, Mode::Full] { + assert!(matches!(decode(&short, mode, &Limits::default()), Err(RawError::Corrupt(_)))); + } + } + } + + #[test] + fn small_raw_rejects_tail_truncation_with_intact_directories() { + let bytes = nef(34713, 12, vec![vec![0x55; 72]], 8, 3, 3); + for keep in [0, 24, 71] { + let short = truncated_tail_strip(bytes.clone(), keep); + for mode in [Mode::Header, Mode::Full] { + assert!(matches!(decode(&short, mode, &Limits::default()), Err(RawError::Corrupt(_)))); + } + } + } +} diff --git a/crates/raw/src/nef_lens.rs b/crates/raw/src/nef_lens.rs new file mode 100644 index 000000000..9f8dafb0e --- /dev/null +++ b/crates/raw/src/nef_lens.rs @@ -0,0 +1,259 @@ +//! Nikon's in-file distortion correction (raw SubIFD tag `0xc7d5`, NikonNEFInfo), expressed as a DNG warp for +//! the shared optics pipeline. +//! +//! Tag identity: the ExifTool Nikon tag-name documentation (NikonNEFInfo `0xc7d5`; its DistortionInfo `0x0005` +//! with DistortionCorrection and RadialDistortionCoefficient1..3). The model below (which stored value multiplies +//! which power, the normalisation, the fill-frame zoom and the optical-centre offset in entry `0x0008`) was inferred +//! independently by matching the raw against the camera's own corrected JPEG; see docs/nikon-lens-corrections.md. +//! No lens database or external decoder code. + +use crate::{Opcode, Rect}; +use craft_tiff::{ByteOrder, Ifd, ParseOptions, parse_ifd_at}; + +const NEF_INFO: u16 = 0xc7d5; +const DISTORTION_INFO: u16 = 0x0005; +const CENTRE_OFFSET: u16 = 0x0008; +/// Bodies whose files were checked against the camera's JPEG (docs/nikon-lens-corrections.md). Matching tags on +/// another body do not establish its pixel pitch or image area; add a body only after checking its files. +const VALIDATED: &[&str] = &["NIKON Z 6_2", "NIKON Z 8", "NIKON Z f"]; +/// The normalisation radius every checked file stores: the FX half-diagonal in millimetres. +const FX_HALF_DIAGONAL_MM: f64 = 21.6; +const SAMPLES: usize = 256; + +/// `r_src = r_out * (1 + k3 rho^2 + k2 rho^4 + k1 rho^6) / (1 + zoom)` around the optical centre, `rho` the radius +/// over the half-diagonal of the image area (`k1..k3` as ExifTool names them, in the order they are stored). +#[derive(Clone, Copy, Debug, PartialEq)] +struct Distortion { + k: [f64; 3], + zoom: f64, +} + +/// A signed rational at `at` (`i32 / i32`), when it is one. +fn rational(b: &[u8], at: usize, order: ByteOrder) -> Option { + let n = order.u32(b.get(at..at + 4)?.try_into().ok()?) as i32; + let d = order.u32(b.get(at + 4..at + 8)?.try_into().ok()?) as i32; + (d > 0).then(|| f64::from(n) / f64::from(d)).filter(|v| v.is_finite()) +} + +/// The NikonNEFInfo entries: the blob is `"Nikon\0"`, a 4-byte version, then a little TIFF stream. +fn nef_info(raw: &Ifd) -> Option<(Ifd, ByteOrder)> { + let tiff = raw.bytes(NEF_INFO)?.strip_prefix(b"Nikon\0")?.get(4..)?; + let (order, false) = craft_tiff::Tiff::sniff(tiff)? else { return None }; + let first = order.u32(tiff.get(4..8)?.try_into().ok()?); + let opts = ParseOptions { follow_children: false, ..Default::default() }; + let (ifd, _) = parse_ifd_at(tiff, u64::from(first), order, 0, false, &opts).ok()?; + Some((ifd, order)) +} + +/// DistortionInfo: `"0100"`, the correction mode (0 none), the normalisation radius (mm) at 8, the term count (4) +/// at 16, the coefficients at 20 / 28 / 36 and the zoom at 64, each a signed rational. +fn distortion_info(b: &[u8], order: ByteOrder) -> Option { + if b.get(0..4)? != b"0100" || *b.get(4)? == 0 { + return None; + } + let norm = rational(b, 8, order)?; + let terms = order.u32(b.get(16..20)?.try_into().ok()?); + if (norm - FX_HALF_DIAGONAL_MM).abs() > 0.05 || terms != 4 { + return None; + } + let k = [rational(b, 20, order)?, rational(b, 28, order)?, rational(b, 36, order)?]; + let zoom = rational(b, 64, order)?; + // real lenses stay far inside these (the checked files: |k| < 0.13, |zoom| < 0.09) + (k.iter().all(|v| v.abs() < 1.0) && zoom.abs() < 0.5).then_some(Distortion { k, zoom }) +} + +/// Entry `0x0008`: `"0100"`, then the optical centre's offset from the image-area centre in pixels (x right, +/// y down), two signed rationals. Zero when the lens stabilises (the sensor stays centred). +fn centre_offset(b: &[u8], order: ByteOrder) -> Option<(f64, f64)> { + (b.get(0..4)? == b"0100").then_some(())?; + Some((rational(b, 4, order)?, rational(b, 12, order)?)) +} + +/// The correction of a `model` NEF whose raw IFD is `raw`, for the image area `crop` inside `active` (both in +/// sensor pixels), as an `OpcodeList3` warp in active-area coordinates. `None` for unvalidated bodies, other +/// image areas, missing or implausible data, and maps that fold. +pub(super) fn distortion(model: &str, raw: &Ifd, active: Rect, crop: Rect) -> Option { + if !VALIDATED.contains(&model.trim()) { + return None; + } + let crop = crop.clipped(active.width, active.height); + if active.width < 2 || active.height < 2 || crop.width < 2 || crop.height < 2 { + return None; + } + // the stored radius is the FX half-diagonal: only the full 3:2 image area (not a DX or other crop of the + // sensor) is normalised by it + if (crop.width as f64 / crop.height as f64 - 1.5).abs() > 0.005 || (crop.width as f64) < active.width as f64 * 0.95 { + return None; + } + let (info, order) = nef_info(raw)?; + let d = distortion_info(info.bytes(DISTORTION_INFO)?, order)?; + let (ox, oy) = info.bytes(CENTRE_OFFSET).and_then(|b| centre_offset(b, order)).unwrap_or((0.0, 0.0)); + let (cw, ch) = (crop.width as f64 - 1.0, crop.height as f64 - 1.0); + if ox.abs() > cw * 0.05 || oy.abs() > ch * 0.05 { + return None; + } + let half_diagonal = cw.hypot(ch) / 2.0; + let poly = |rho: f64| { + let x = rho * rho; + 1.0 + x * (d.k[2] + x * (d.k[1] + x * d.k[0])) + }; + // DNG normalises by the farthest active-area corner from the centre; the image area reaches at most there + let (ax, ay) = (active.width as f64 - 1.0, active.height as f64 - 1.0); + let (cx, cy) = (crop.x as f64 + cw / 2.0 + ox, crop.y as f64 + ch / 2.0 + oy); + let farthest = cx.max(ax - cx).hypot(cy.max(ay - cy)); + let a = farthest / half_diagonal; + // the source radius must grow with the output radius out to the farthest corner, or the map folds + let mut previous = -1.0; + for j in 0..=SAMPLES { + let rho = a * j as f64 / SAMPLES as f64; + let source = rho * poly(rho) / (1.0 + d.zoom); + if !source.is_finite() || source <= previous { + return None; + } + previous = source; + } + let s = 1.0 / (1.0 + d.zoom); + let a2 = a * a; + let planes = vec![[s, d.k[2] * a2 * s, d.k[1] * a2 * a2 * s, d.k[0] * a2 * a2 * a2 * s, 0.0, 0.0]]; + Some(Opcode::WarpRectilinear { planes, center: [cx / ax, cy / ay] }) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use craft_tiff::{Entry, IfdBuilder, TiffWriter, Value}; + + const AREA: Rect = Rect { x: 0, y: 0, width: 6064, height: 4040 }; + const CROP: Rect = Rect { x: 8, y: 8, width: 6048, height: 4024 }; + + fn rat(v: f64, den: i32) -> [u8; 8] { + let mut b = [0u8; 8]; + b[..4].copy_from_slice(&((v * f64::from(den)).round() as i32).to_le_bytes()); + b[4..].copy_from_slice(&den.to_le_bytes()); + b + } + + /// DistortionInfo as the checked files lay it out (84 bytes, little-endian). + pub(crate) fn distortion_block(mode: u8, norm: f64, terms: u32, k: [f64; 3], zoom: f64) -> Vec { + let mut b = vec![0u8; 84]; + b[..4].copy_from_slice(b"0100"); + b[4] = mode; + b[5] = 1; + b[8..16].copy_from_slice(&rat(norm, 4096)); + b[16..20].copy_from_slice(&terms.to_le_bytes()); + for (i, v) in k.iter().enumerate() { + b[20 + 8 * i..28 + 8 * i].copy_from_slice(&rat(*v, 1 << 20)); + } + b[44..52].copy_from_slice(&rat(0.0, 1)); + b[52..60].copy_from_slice(&rat(0.0, 1)); + b[64..72].copy_from_slice(&rat(zoom, 1 << 20)); + b + } + + pub(crate) fn offset_block(x: f64, y: f64) -> Vec { + let mut b = b"0100".to_vec(); + b.extend_from_slice(&rat(x, 256)); + b.extend_from_slice(&rat(y, 256)); + b.extend_from_slice(&[0, 0, 0, 0]); + b + } + + /// A NikonNEFInfo (`0xc7d5`) blob with the given entries. + pub(crate) fn nef_info_blob(entries: &[(u16, Vec)]) -> Vec { + let mut info = IfdBuilder::new(); + for (tag, data) in entries { + info.set(*tag, Value::Undefined(data.clone())); + } + let tiff = TiffWriter::new(ByteOrder::Little, false).write(&[info]).unwrap(); + let mut blob = b"Nikon\0\x01\x03\0\0".to_vec(); + blob.extend_from_slice(&tiff); + blob + } + + /// A raw IFD carrying a NikonNEFInfo blob with the given entries. + pub(super) fn raw_with(entries: &[(u16, Vec)]) -> Ifd { + Ifd { entries: vec![Entry { tag: NEF_INFO, value: Value::Undefined(nef_info_blob(entries)), offset: 0 }], ..Default::default() } + } + + // a Z 24-70 f/4 S at 24 mm and at 70 mm, as stored in two of the checked files + const WIDE: ([f64; 3], f64) = ([0.039895, 0.019211, -0.127218], -0.025043); + const TELE: ([f64; 3], f64) = ([0.01681, -0.010601, 0.056758], 0.088301); + + fn warp(model: &str, raw: &Ifd) -> Option<([f64; 6], [f64; 2])> { + match distortion(model, raw, AREA, CROP)? { + Opcode::WarpRectilinear { planes, center } => Some((planes[0], center)), + _ => None, + } + } + + /// The source radius the warp gives for an output radius `rho` (in image-area half-diagonals) about the centre. + fn source_rho(p: [f64; 6], centre: [f64; 2], rho: f64) -> f64 { + let (ax, ay) = (AREA.width as f64 - 1.0, AREA.height as f64 - 1.0); + let (cx, cy) = (centre[0] * ax, centre[1] * ay); + let m = cx.max(ax - cx).hypot(cy.max(ay - cy)); + let hd = (CROP.width as f64 - 1.0).hypot(CROP.height as f64 - 1.0) / 2.0; + let t2 = (rho * hd / m).powi(2); + rho * (p[0] + t2 * (p[1] + t2 * (p[2] + t2 * p[3]))) + } + + #[test] + fn the_warp_is_the_stored_polynomial_over_the_zoom() { + for (k, zoom) in [WIDE, TELE] { + let raw = raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, k, zoom))]); + let (p, c) = warp("NIKON Z 6_2", &raw).unwrap(); + for rho in [0.0, 0.25, 0.5, 0.75, 1.0] { + let x = rho * rho; + let want = rho * (1.0 + k[2] * x + k[1] * x * x + k[0] * x * x * x) / (1.0 + zoom); + assert!((source_rho(p, c, rho) - want).abs() < 2e-6, "{k:?} at {rho}"); + } + // centred on the image area, which sits 8 px in from the sensor's top-left + assert!((c[0] - (8.0 + 6047.0 / 2.0) / 6063.0).abs() < 1e-12 && (c[1] - (8.0 + 4023.0 / 2.0) / 4039.0).abs() < 1e-12); + } + } + + #[test] + fn the_optical_centre_follows_the_sensor_shift() { + let raw = raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, TELE.0, TELE.1)), (CENTRE_OFFSET, offset_block(-90.1, 18.59))]); + let (_, c) = warp("NIKON Z 6_2", &raw).unwrap(); + assert!((c[0] * 6063.0 - (8.0 + 6047.0 / 2.0 - 90.1)).abs() < 0.01, "{c:?}"); + assert!((c[1] * 4039.0 - (8.0 + 4023.0 / 2.0 + 18.59)).abs() < 0.01, "{c:?}"); + } + + #[test] + fn refuses_unvalidated_bodies_and_unknown_layouts() { + let good = distortion_block(3, 21.6, 4, WIDE.0, WIDE.1); + assert!(warp("NIKON Z 6_2", &raw_with(&[(DISTORTION_INFO, good.clone())])).is_some()); + assert!(warp("NIKON Z f", &raw_with(&[(DISTORTION_INFO, good.clone())])).is_some()); + assert!(warp("NIKON Z 8", &raw_with(&[(DISTORTION_INFO, good.clone())])).is_some()); + for model in ["", "NIKON Z 6", "NIKON Z 6_3", "NIKON Z 9", "NIKON D850", "NIKON Z 6_2 x"] { + assert!(warp(model, &raw_with(&[(DISTORTION_INFO, good.clone())])).is_none(), "{model}"); + } + // correction off, a DX radius, another term count, a wild coefficient or zoom, no NEFInfo at all + for bad in [ + distortion_block(0, 21.6, 4, WIDE.0, WIDE.1), + distortion_block(3, 14.2, 4, WIDE.0, WIDE.1), + distortion_block(3, 21.6, 3, WIDE.0, WIDE.1), + distortion_block(3, 21.6, 4, [0.0, 0.0, 1.5], 0.0), + distortion_block(3, 21.6, 4, WIDE.0, 0.9), + b"0200".iter().copied().chain([0u8; 80]).collect(), + vec![0u8; 12], + ] { + assert!(warp("NIKON Z 6_2", &raw_with(&[(DISTORTION_INFO, bad)])).is_none()); + } + assert!(warp("NIKON Z 6_2", &Ifd::default()).is_none()); + // a map that folds (the source radius turns back) is refused + assert!(warp("NIKON Z 6_2", &raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, [-0.9, 0.0, 0.0], 0.0))])).is_none()); + } + + #[test] + fn only_the_full_3_2_image_area_is_corrected() { + let raw = raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, WIDE.0, WIDE.1))]); + // a DX or 1:1 crop of the sensor would need another normalisation + assert!(distortion("NIKON Z 6_2", &raw, AREA, Rect::new(1016, 676, 4032, 2688)).is_none(), "DX"); + assert!(distortion("NIKON Z 6_2", &raw, AREA, Rect::new(1016, 8, 4024, 4024)).is_none()); + assert!(distortion("NIKON Z 6_2", &raw, AREA, Rect::new(0, 0, 1, 1)).is_none()); + // a sensor-shift offset beyond 5 % of the frame is not trusted + let far = raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, WIDE.0, WIDE.1)), (CENTRE_OFFSET, offset_block(900.0, 0.0))]); + assert!(warp("NIKON Z 6_2", &far).is_none()); + } +} diff --git a/crates/raw/src/nefc.rs b/crates/raw/src/nefc.rs new file mode 100644 index 000000000..9904f7cca --- /dev/null +++ b/crates/raw/src/nefc.rs @@ -0,0 +1,794 @@ +//! Nikon Huffman-compressed NEF image data (TIFF compression 34713): "lossless compressed" and "lossy compressed" +//! (type 1 and type 2) at 12 and 14 bits. +//! +//! **Clean-room.** No raw-decoder source code (dcraw, LibRaw, rawspeed, rawloader/rawler, darktable, RawTherapee, +//! libopenraw, ExifTool's Perl code, …) was read or consulted. What we used: +//! +//! - Prose format descriptions: Laurent Clévy, "Nikon Electronic File (NEF) format" (), +//! the maker-note tag table and the "0x96 (linearization table) tag format" table (version bytes, four `u16` +//! predictor seeds at offset 2, curve size at 10, curve at 12, split value at 562 for lossy type 2, "lossy type +//! 2 reads an incomplete table and interpolation is required"); Bill Claff, "NEF Compression" +//! (): a lossy encoding curve followed by a +//! non-adaptive (fixed-table) Huffman stage, and the number of encoded values per camera (683, 769, 1025, 2753, +//! 3073, 4097 …), which confirmed our curve interpolation; the libopenraw format notes +//! (, prose only); ExifTool's Nikon tag-name documentation +//! (`0x0093` NEFCompression, `0x0096` NEFLinearizationTable); ITU-T T.81 Annex F/H for the "difference category +//! + additional bits" coding and canonical (BITS/HUFFVAL) Huffman codes. +//! - Our own black-box analysis of CC0 samples from raw.pixls.us. Nikon's Huffman tables are not stored in the +//! files and no prose source lists them, so they were recovered from the data: several cameras (D5100, D300, +//! D700) were shot with the same scene in compressed and uncompressed modes; a beam search assigned code words +//! to difference categories so that the decoded compressed image matched the uncompressed one (likelihood of +//! the decoded values given the reference, minus the bits consumed, plus a description-length penalty per new +//! code word). Each table was then verified on ~45 files from ~35 bodies (D40 … D850, Df, Z 6, Z 50, 1 J1): with +//! the right table the whole strip decodes and ends within the last byte of the data, with every value inside +//! the encoded range. The predictor and difference coding are the same as Pentax PEF (see `pef.rs`). +//! +//! Format, as established: +//! - One strip, MSB-first bit stream, no byte stuffing, continuous across rows. Per pixel a canonical Huffman code +//! gives the category `n`, followed by `n` additional bits (T.81: a leading 0 means a negative difference). +//! - Each pixel is predicted from the same-colour pixel two to the left; the first two pixels of a row from the +//! first two pixels of the previous row of the same parity, starting at the seeds of maker note `0x0096`. +//! - Lossless (`0x0096` version `0x46`): the decoded values are the samples. Lossy (version `0x44`): they index a +//! curve. Type 1 (`0x44 0x10`) stores the full curve; type 2 (`0x44 0x20`, `0x44 0x40`) stores 257 points, +//! one every `2^bits / 256` codes (`0x20`) or `2^bits / 1024` codes (`0x40`), linearly interpolated. The span +//! was established from the files: in every `0x20` file the first interval of the 257 points rises by +//! `2^bits / 256` (16 at 12 bit, 64 at 14 bit) and in every `0x40` file by `2^bits / 1024` (4 and 16), i.e. +//! the curve is the identity in the dark part, one code per sample. Read with a point every `2^bits / 256` +//! codes instead, the 12 `0x40` files that are not split decoded to values that never exceeded 255 (12 bit) or +//! 1023 (14 bit) and whose darkest 0.01 % sat at a quarter of the black level of maker note `0x003d` (63 against +//! 252 in 12-bit units); with the first-interval step they sit at that black level. +//! - Values in the maker note are in the maker note's byte order (newer bodies write little-endian notes). +//! +//! "Lossy after split" (non-zero split row at `0x0096` offset 562, type 2 tables only, found in 11 of the CC0 +//! samples): derived by black-box analysis of those files. Rows below the split row use the lossy Huffman path +//! above unchanged. Row `split` starts at the very next bit (no alignment); from there every pixel is a word of +//! `bits - 4` bits (8 at 12 bit, 10 at 14 bit), MSB first, with no row padding. A word is a complete prefix code +//! for a difference category `n` followed by the top `W - prefix length` bits of the T.81 additional bits (the low +//! bits are dropped); the difference is the centre of the magnitude bin that was kept. The predictor is the same +//! as above and the running value is not clamped (only the sample index is, before the curve lookup). The camera +//! ends the strip with 0 to 7 one-bits after the last full row, which the decoder checks. +//! +//! Not supported (returned as [`RawError::Unsupported`], the embedded preview is used instead): +//! - "Lossy after split" files whose fixed-rate section does not end within the last byte of the strip (see +//! below): refused rather than decoded wrongly. +//! - Other `0x0096` versions, bit depths other than 12/14. +//! - Code words that never occurred in any sample (the lossy 12-bit 8-bit code word `11111110`, and code words +//! longer than 9 bits in the lossy tables) are rejected as corrupt; they cannot code a difference within the +//! lossy value range anyway. + +use crate::huffman::{Bits, Huffman, diff}; +use crate::{MAX_SAMPLES, RawError, Result}; +use craft_tiff::ByteOrder; + +/// A table code word whose symbol was never observed (see the module docs). +const UNSEEN: u8 = u8::MAX; + +/// Canonical Huffman table: number of codes of length 1..=16 and the symbols (difference categories) in code order. +struct Table { + counts: &'static [u8], + symbols: &'static [u8], +} + +const LOSSLESS_12: Table = Table { counts: &[0, 1, 4, 2, 3, 1, 2], symbols: &[5, 4, 6, 3, 7, 2, 8, 1, 9, 0, 10, 11, 12] }; +const LOSSLESS_14: Table = Table { counts: &[0, 1, 4, 2, 2, 3, 1, 2], symbols: &[7, 6, 8, 5, 9, 4, 10, 3, 11, 12, 2, 0, 1, 13, 14] }; +const LOSSY_12: Table = Table { counts: &[0, 1, 5, 1, 1, 1, 1, 1, 1], symbols: &[5, 4, 3, 6, 2, 7, 1, 0, 8, 9, UNSEEN, 10] }; +const LOSSY_14: Table = Table { counts: &[0, 1, 4, 3, 1, 1, 1, 1, 1], symbols: &[5, 6, 4, 7, 8, 3, 9, 2, 1, 0, 10, 11, 12] }; + +impl Table { + /// `(left-aligned 12-bit code, length)` per symbol, for [`Huffman::new`]. + fn codes(&self) -> Vec<(u16, u8)> { + let mut out = vec![(0u16, 0u8); 17]; + let mut code = 0u32; + let mut symbols = self.symbols.iter(); + for (i, &n) in self.counts.iter().enumerate() { + let len = i as u32 + 1; + for _ in 0..n { + if let Some(&s) = symbols.next() + && let Some(slot) = out.get_mut(s as usize) + { + *slot = ((code << (12 - len)) as u16, len as u8); + } + code += 1; + } + code <<= 1; + } + out + } +} + +/// How the decoded values map to samples. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Encoding { + /// The decoded values are the samples (`0..2^bits`). + Lossless, + /// The decoded values index this curve. + Lossy(Vec), +} + +/// Maker note `0x0096` (NEF linearization table), parsed. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct DecodeTable { + pub encoding: Encoding, + /// Predictor seeds `[row parity][column]`. + pub seeds: [[i32; 2]; 2], + /// First row of the fixed-rate section ("lossy after split"), 0 when the whole strip is Huffman coded. + pub split: usize, +} + +/// Parse maker note `0x0096` for a `bits`-bit image. `order` is the maker note's byte order. +pub(crate) fn parse_table(t: &[u8], order: ByteOrder, bits: u32) -> Result { + let u16_at = |o: usize| t.get(o..o + 2).and_then(|s| <[u8; 2]>::try_from(s).ok()).map(|b| order.u16(b)); + let short = || RawError::Corrupt(format!("NEF: linearization table too short ({} bytes)", t.len())); + if bits != 12 && bits != 14 { + return Err(RawError::Unsupported(format!("Nikon compressed NEF with {bits}-bit samples"))); + } + let (v0, v1) = (t.first().copied().ok_or_else(short)?, t.get(1).copied().ok_or_else(short)?); + let mut seeds = [[0i32; 2]; 2]; + for (i, s) in seeds.iter_mut().flatten().enumerate() { + *s = u16_at(2 + 2 * i).ok_or_else(short)? as i32; + } + let mut split = 0; + let encoding = match (v0, v1) { + (0x46, _) => Encoding::Lossless, + (0x44, 0x10 | 0x20 | 0x40) => { + let n = u16_at(10).ok_or_else(short)? as usize; + let points: Vec = (0..n).map(|i| u16_at(12 + 2 * i)).collect::>().ok_or_else(short)?; + if n < 2 { + return Err(RawError::Corrupt(format!("NEF: linearization curve of {n} points"))); + } + if v1 == 0x10 { + Encoding::Lossy(points) + } else { + split = u16_at(562).unwrap_or(0) as usize; + // type 2: `n` points covering `range` codes, one every `range / (n - 1)`; the curve's first interval + // rises by exactly that step (identity in the dark part) and the table version sets the span: + // `0x20` covers all 2^bits codes, `0x40` the first quarter (see the module docs) + let range = if v1 == 0x40 { 1usize << (bits - 2) } else { 1usize << bits }; + if n - 1 > range || !range.is_multiple_of(n - 1) { + return Err(RawError::Unsupported(format!("NEF: {n}-point lossy curve for {bits}-bit data"))); + } + let step = range / (n - 1); + let mut curve = Vec::with_capacity(range + 1); + for pair in points.windows(2) { + let &[a, b] = pair else { continue }; + let (a, b) = (a as u32, b as u32); + for k in 0..step as u32 { + let s = step as u32; + curve.push(((a * (s - k) + b * k) / s) as u16); + } + } + curve.extend(points.last().copied()); + Encoding::Lossy(curve) + } + } + _ => return Err(RawError::Unsupported(format!("Nikon compressed NEF version {v0:#04x} {v1:#04x}"))), + }; + Ok(DecodeTable { encoding, seeds, split }) +} + +/// Difference-category prefix codes of the fixed-rate section: `(prefix bits, prefix length, category n)`. +/// A word of `W` bits is a prefix followed by `W - length` bits of the T.81 additional bits of category `n`. +/// (The 14-bit `010` = category 12 never occurs in the samples; its category is deduced.) +const FIXED_12: &[(u32, u32, u32)] = &[ + (0b00, 2, 9), + (0b010, 3, 10), + (0b011, 3, 8), + (0b100, 3, 7), + (0b101, 3, 6), + (0b110, 3, 5), + (0b1110, 4, 4), + (0b11110, 5, 3), + (0b111110, 6, 2), + (0b1111110, 7, 1), + (0b11111110, 8, 0), +]; +const FIXED_14: &[(u32, u32, u32)] = &[ + (0b00, 2, 8), + (0b010, 3, 12), + (0b011, 3, 11), + (0b100, 3, 10), + (0b101, 3, 9), + (0b110, 3, 7), + (0b1110, 4, 6), + (0b11110, 5, 5), + (0b111110, 6, 4), + (0b1111110, 7, 3), + (0b11111110, 8, 2), + (0b111111110, 9, 1), + (0b1111111110, 10, 0), +]; + +/// Word (`bits - 4` bits) to dequantised difference for the fixed-rate section; `None` marks the unused all-ones word. +fn fixed_word_table(bits: u32) -> Vec> { + let (w, codes) = if bits == 12 { (8, FIXED_12) } else { (10, FIXED_14) }; + let mut lut = vec![None; 1usize << w]; + for &(prefix, len, n) in codes { + let kept = w - len; + for m in 0..1u32 << kept { + let e = n.saturating_sub(kept); + let ex = (m << e) as i32; + let half = (1i32 << e) >> 1; + let d = if n == 0 { + 0 + } else if ex >= 1 << (n - 1) { + ex + half + } else if e > 0 { + ex + half - (1 << n) + } else { + ex - ((1 << n) - 1) + }; + if let Some(slot) = lut.get_mut(((prefix << kept) | m) as usize) { + *slot = Some(d); + } + } + } + lut +} + +/// Decode a whole `w × h` Nikon Huffman-compressed strip `src` with `bits`-bit samples. +#[cfg(test)] +pub(crate) fn decode(src: &[u8], w: usize, h: usize, bits: u32, table: &DecodeTable) -> Result> { + decode_rows(src, w, h, h, bits, table) +} + +/// The first `rows` rows (`w × rows` samples) of a `w × h` strip. The stream is sequential, so a header-only probe +/// decodes just the rows its masked-column test looks at (storytold/lightcraft#708) instead of the whole image. The size and +/// truncation checks still cover the whole image; the "lossy after split" end-of-strip rule (and corruption +/// further down) can only be seen when the whole strip is decoded. +pub(crate) fn decode_rows(src: &[u8], w: usize, h: usize, rows: usize, bits: u32, table: &DecodeTable) -> Result> { + if bits != 12 && bits != 14 { + return Err(RawError::Unsupported(format!("Nikon compressed NEF with {bits}-bit samples"))); + } + let whole = w.checked_mul(h).filter(|&n| n > 0 && n <= MAX_SAMPLES).ok_or(RawError::Limit("NEF image size"))?; + // every code word is at least 2 bits long: a strip that can't hold the image is truncated (also bounds the + // allocation by the file size) + if whole / 4 > src.len() { + return Err(RawError::Corrupt(format!("NEF: {} bytes of compressed data for {w}x{h} pixels", src.len()))); + } + let rows = rows.min(h); + let n = w.saturating_mul(rows); + let huff = Huffman::new( + &match (&table.encoding, bits) { + (Encoding::Lossless, 12) => LOSSLESS_12, + (Encoding::Lossless, _) => LOSSLESS_14, + (Encoding::Lossy(_), 12) => LOSSY_12, + (Encoding::Lossy(_), _) => LOSSY_14, + } + .codes(), + )?; + let (lut, max): (&[u16], i32) = match &table.encoding { + Encoding::Lossless => (&[], (1i32 << bits) - 1), + Encoding::Lossy(curve) => (curve, curve.len() as i32 - 1), + }; + if table.split != 0 && table.split >= h { + return Err(RawError::Corrupt(format!("NEF: split row {} outside the {h} rows", table.split))); + } + let fixed = if table.split != 0 { fixed_word_table(bits) } else { Vec::new() }; + let word = bits - 4; + let mut out = vec![0u16; n]; + let mut stream = Bits::new(src); + let mut vpred = table.seeds; + let mut clipped = 0usize; + for (y, row) in out.chunks_exact_mut(w).enumerate() { + let seeds = vpred.get_mut(y & 1).ok_or(RawError::Limit("NEF row"))?; + let mut hpred = [0i32; 2]; + for (x, o) in row.iter_mut().enumerate() { + let d = if table.split != 0 && y >= table.split { + let code = stream.get(word) as usize; + fixed.get(code).copied().flatten().ok_or_else(|| RawError::Corrupt(format!("NEF: invalid fixed-rate word at row {y}")))? + } else { + diff(&mut stream, &huff).ok_or_else(|| RawError::Corrupt(format!("NEF: invalid Huffman code at row {y}")))? + }; + // a run of large differences in a corrupt stream can drive a predictor past i32 (storytold/lightcraft#1515) + let overflow = || RawError::Corrupt(format!("NEF: predictor out of range at row {y}")); + let p = if x < 2 { + let s = seeds.get_mut(x).ok_or(RawError::Limit("NEF column"))?; + *s = s.checked_add(d).ok_or_else(overflow)?; + hpred[x & 1] = *s; + *s + } else { + let p = hpred[x & 1].checked_add(d).ok_or_else(overflow)?; + hpred[x & 1] = p; + p + }; + let v = p.clamp(0, max); + clipped += (v != p && (table.split == 0 || y < table.split)) as usize; + *o = if lut.is_empty() { v as u16 } else { lut.get(v as usize).copied().unwrap_or(0) }; + } + // the camera pads the strip; reading more than a few bytes past its end means truncated or corrupt data + if stream.consumed_bits() > src.len() * 8 + 64 { + return Err(RawError::Corrupt(format!("NEF: compressed data ends at row {y} of {h}"))); + } + } + if table.split != 0 && rows == h { + // the fixed-rate section must fill the rest of the strip: whole rows of w words and 0..7 bits of padding. + // A strip may hold a few more rows than the image (one body stores 4022 rows for 4020); anything else means + // the file does not follow the rule, so refuse it instead of decoding it wrongly. + let left = (src.len() * 8).checked_sub(stream.consumed_bits()); + let row_bits = w * word as usize; + if !left.is_some_and(|l| l % row_bits < 8 && l / row_bits <= 4) { + return Err(RawError::Unsupported(format!( + "Nikon \"lossy after split\" NEF whose fixed-rate section (from row {}) does not end within the last byte", + table.split + ))); + } + } + // a wrong table / corrupt stream drifts out of range quickly; real files stay inside (a handful of edge pixels) + if clipped > n / 1000 + 16 { + return Err(RawError::Corrupt(format!("NEF: {clipped} decoded values out of range"))); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// MSB-first bit writer for the test encoder. + #[derive(Default)] + struct Writer { + out: Vec, + acc: u64, + n: u32, + } + + impl Writer { + fn put(&mut self, v: u32, len: u32) { + for i in (0..len).rev() { + self.acc = (self.acc << 1) | ((v >> i) & 1) as u64; + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push((self.acc << (8 - self.n)) as u8); + } + self.out.extend_from_slice(&[0; 4]); + self.out + } + } + + fn category(d: i32) -> u32 { + 32 - d.unsigned_abs().leading_zeros() + } + + /// Encode `values` (decoded domain: samples or curve indices) the way the camera does. + fn encode(values: &[i32], w: usize, table: &Table, seeds: [[i32; 2]; 2]) -> Vec { + let codes = table.codes(); + let mut wr = Writer::default(); + let mut vpred = seeds; + for (y, row) in values.chunks(w).enumerate() { + for (x, &v) in row.iter().enumerate() { + let pred = if x < 2 { vpred[y & 1][x] } else { row[x - 2] }; + if x < 2 { + vpred[y & 1][x] = v; + } + let d = v - pred; + let k = category(d); + let (code, len) = codes[k as usize]; + assert!(len > 0, "no code for category {k}"); + wr.put(code as u32 >> (12 - len), len as u32); + let extra = if d < 0 { d + (1 << k) - 1 } else { d }; + wr.put(extra as u32, k); + } + } + wr.finish() + } + + /// Deterministic test image: smooth gradient + texture + hard edges, within `0..=max`. + fn image(w: usize, h: usize, max: i32, seed: u32) -> Vec { + let mut s = seed; + (0..w * h) + .map(|i| { + let (x, y) = ((i % w) as i32, (i / w) as i32); + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let noise = (s >> 24) as i32 % 9 - 4; + let base = (x * 37 + y * 11) % (max / 2) + if (x / 7 + y / 5) % 3 == 0 { max / 3 } else { 0 }; + (base + noise).clamp(0, max) + }) + .collect() + } + + fn lossless(bits: u32) -> DecodeTable { + let s = 1 << (bits - 3); + DecodeTable { encoding: Encoding::Lossless, seeds: [[s, s], [s, s]], split: 0 } + } + + #[test] + fn tables_are_complete_prefix_codes() { + for t in [LOSSLESS_12, LOSSLESS_14, LOSSY_12, LOSSY_14] { + assert_eq!(t.counts.iter().map(|&c| c as usize).sum::(), t.symbols.len()); + let kraft: f64 = t.counts.iter().enumerate().map(|(i, &c)| c as f64 / (1u64 << (i + 1)) as f64).sum(); + assert!(kraft <= 1.0); + assert!(Huffman::new(&t.codes()).is_ok()); + } + assert_eq!(LOSSLESS_14.codes()[7], (0b00 << 10, 2)); + assert_eq!(LOSSLESS_14.codes()[0], (0b111110 << 6, 6)); + assert_eq!(LOSSY_12.codes()[10], (0b111111110 << 3, 9)); + } + + #[test] + fn lossless_roundtrip_12_and_14() { + for (bits, table) in [(12, LOSSLESS_12), (14, LOSSLESS_14)] { + let (w, h) = (64, 9); + let max = (1 << bits) - 1; + let mut img = image(w, h, max, bits); + // extreme jumps exercise the longest categories + img[5] = 0; + img[7] = max; + img[w + 9] = max; + let t = lossless(bits); + let src = encode(&img, w, &table, t.seeds); + let out = decode(&src, w, h, bits, &t).unwrap(); + assert_eq!(out, img.iter().map(|&v| v as u16).collect::>(), "{bits}-bit"); + } + } + + fn lossy_table(bits: u32, order: ByteOrder) -> Vec { + // version 0x44 0x20, seeds, 257 points of a concave curve, padding up to the split value (0) + let mut t = vec![0x44, 0x20]; + let put = |t: &mut Vec, v: u16| t.extend_from_slice(&if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() }); + for _ in 0..4 { + put(&mut t, 300); + } + put(&mut t, 257); + let max = ((1u32 << bits) - 1) as f64; + for i in 0..257u32 { + put(&mut t, (max * (i as f64 / 64.0).min(1.0).sqrt()) as u16); + } + t.resize(624, 0); + t + } + + #[test] + fn lossy_type2_curve_and_roundtrip() { + for (bits, table) in [(12u32, LOSSY_12), (14, LOSSY_14)] { + for order in [ByteOrder::Big, ByteOrder::Little] { + let t = parse_table(&lossy_table(bits, order), order, bits).unwrap(); + assert_eq!(t.seeds, [[300, 300], [300, 300]]); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + let step = (1usize << bits) / 256; + assert_eq!(curve.len(), (1 << bits) + 1); + assert_eq!(curve[64 * step], (1 << bits) - 1); + assert!(curve.windows(2).all(|p| p[0] <= p[1])); + // the encoded range is the curve's rising part + let (w, h) = (48, 7); + let img = image(w, h, 64 * step as i32 - 1, 7); + let src = encode(&img, w, &table, t.seeds); + let out = decode(&src, w, h, bits, &t).unwrap(); + assert_eq!(out, img.iter().map(|&v| curve[v as usize]).collect::>()); + } + } + } + + /// Maker note `0x0096` as bytes, written out by hand: version `0x44 `, four seeds, 257 points (little-endian + /// as newer bodies write it) of a curve whose first 256 points rise by `step` and whose last is the white + /// level, and a zero split value at offset 562. + fn hand_table(v1: u8, seed: u16, step: u16, white: u16) -> Vec { + let mut t = vec![0x44, v1]; + for _ in 0..4 { + t.extend_from_slice(&seed.to_le_bytes()); + } + t.extend_from_slice(&257u16.to_le_bytes()); + for k in 0..256u16 { + t.extend_from_slice(&(k * step).to_le_bytes()); + } + t.extend_from_slice(&white.to_le_bytes()); + t.resize(624, 0); + t + } + + /// Known answers for the `0x44 0x40` curve, from bytes and bit strings written by hand (nothing here uses the + /// decoder's span rule or the test encoder): the first interval of the points rises by `2^bits / 1024`, so + /// the curve is the identity for one code per sample and spans `2^bits / 4` codes. The `0x20` curve of the + /// same sizes keeps its span of all `2^bits` codes. + #[test] + fn lossy_type2_version_0x40_spans_a_quarter_of_the_range() { + // 12 bit: points 0, 4, 8 … 1020, 4095 + let t = parse_table(&hand_table(0x40, 252, 4, 4095), ByteOrder::Little, 12).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + assert_eq!(curve.len(), 1025); + assert_eq!(curve[0..6], [0, 1, 2, 3, 4, 5]); + assert_eq!((curve[252], curve[255], curve[1020]), (252, 255, 1020)); + assert_eq!((curve[1021], curve[1022], curve[1023], curve[1024]), (1788, 2557, 3326, 4095)); + // the pixels p0 = 252 + 4, p1 = 252, p2 = p0 - 1, p3 = p1, coded as 011 100 | 11110 | 1110 0 | 11110 + // (LOSSY_12: category 3 = 011, 0 = 11110, 1 = 1110; extra bits 100 = +4, 0 = -1) + let out = decode(&[0x73, 0xdc, 0xf0], 4, 1, 12, &t).unwrap(); + assert_eq!(out, [256, 252, 255, 252]); + // 14 bit: points 0, 16, 32 … 4080, 16383 + let t = parse_table(&hand_table(0x40, 1008, 16, 16383), ByteOrder::Little, 14).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + assert_eq!(curve.len(), 4097); + assert_eq!(curve[0..6], [0, 1, 2, 3, 4, 5]); + assert_eq!((curve[1008], curve[4080], curve[4096]), (1008, 4080, 16383)); + // the same pixels coded as 1100 100 | 111110 | 11110 0 | 111110 + // (LOSSY_14: category 3 = 1100, 0 = 111110, 1 = 11110) + let out = decode(&[0xc9, 0xf7, 0x9f, 0x00], 4, 1, 14, &t).unwrap(); + assert_eq!(out, [1012, 1008, 1011, 1008]); + // version 0x20 with the same points is a curve over all 2^bits codes: the points are 16 codes apart + let t = parse_table(&hand_table(0x20, 252, 16, 4095), ByteOrder::Little, 12).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + assert_eq!((curve.len(), curve[16], curve[17], curve[4095]), (4097, 16, 17, 4094)); + // 200 points can't be spaced evenly over the 1024 codes of a 12-bit 0x40 table + let mut odd = hand_table(0x40, 252, 4, 4095); + odd[10..12].copy_from_slice(&200u16.to_le_bytes()); + assert!(parse_table(&odd, ByteOrder::Little, 12).is_err()); + } + + /// Pack a string of `0`/`1` (spaces ignored) MSB first, padding the last byte with one-bits like the camera. + fn pack(bits: &str) -> Vec { + let b: Vec = bits.chars().filter(|c| !c.is_whitespace()).map(|c| c == '1').collect(); + b.chunks(8).map(|c| (0..8).fold(0u8, |a, i| (a << 1) | c.get(i).copied().unwrap_or(true) as u8)).collect() + } + + fn split_table(v1: u8, seed: u16, step: u16, white: u16, split: u16) -> DecodeTable { + let mut t = hand_table(v1, seed, step, white); + t[562..564].copy_from_slice(&split.to_le_bytes()); + parse_table(&t, ByteOrder::Little, if step == 4 { 12 } else { 14 }).unwrap() + } + + /// 12-bit "lossy after split": row 0 Huffman coded, rows 1 and 2 as 8-bit words that follow the last Huffman + /// bit directly. Bit strings written by hand from the rule (prefix, then the kept top bits of the extra bits). + const SPLIT_12: &str = "011 100 11110 1110 0 11110 11111110 11110100 11100111 01110000 00000001 11111110 01111111 11111110"; + + #[test] + fn lossy_after_split_known_answers_12_bit() { + let t = split_table(0x40, 252, 4, 4095, 1); + assert_eq!(t.split, 1); + let out = decode(&pack(SPLIT_12), 4, 3, 12, &t).unwrap(); + // row 1: d = 0 (n0), +4 (n3, exact), -8 (n4, exact), +132 (n8 keeps 5 bits: 16 << 3 plus half a bin, 4); + // row 2: -500 (n9 keeps 6 bits: bin 1 of the negative side, centre -508 + 8), then +252 on the running + // value -244 which is not clamped (8, not 252) + assert_eq!(out, [256, 252, 255, 252, 252, 256, 244, 388, 0, 252, 8, 252]); + } + + #[test] + fn lossy_after_split_known_answers_14_bit() { + let t = split_table(0x40, 1008, 16, 16383, 1); + // row 0 as in the 0x40 test above; row 1 in 10-bit words: 0 (n0), +32 (n6, exact), -254 (n8, exact), + // +510 (n9 keeps 7 bits: 127 << 2 plus 2) + let bits = "1100 100 111110 11110 0 111110 1111111110 1110100000 0000000001 1011111111"; + let out = decode(&pack(bits), 4, 2, 14, &t); + // word 3 is `00` + 8 bits: 00 00000001 -> n8 extra 00000001 = -254 + assert_eq!(out.unwrap(), [1012, 1008, 1011, 1008, 1008, 1040, 754, 1550]); + } + + #[test] + fn lossy_after_split_dequantisation_is_symmetric() { + let lut = fixed_word_table(12); + assert_eq!((lut[0b00_000000], lut[0b00_111111]), (Some(-508), Some(508))); + assert_eq!((lut[0b011_00000], lut[0b011_11111]), (Some(-252), Some(252))); + assert_eq!(lut[0b11111110], Some(0)); + assert_eq!(lut[0xff], None); + let lut = fixed_word_table(14); + assert_eq!((lut[0b00_00000000], lut[0b00_11111111]), (Some(-255), Some(255))); + assert_eq!(lut[0b101_0000000], Some(-510)); + assert_eq!(lut[0b1111111110], Some(0)); + assert_eq!(lut[0x3ff], None); + for bits in [12, 14] { + let lut = fixed_word_table(bits); + assert_eq!(lut.iter().filter(|d| d.is_none()).count(), 1, "{bits}-bit: only the all-ones word is unused"); + } + } + + #[test] + fn lossy_after_split_refuses_strips_that_do_not_follow_the_rule() { + let t = split_table(0x40, 252, 4, 4095, 1); + let mut longer = pack(SPLIT_12); + longer.extend_from_slice(&[0xff; 3]); + assert!(matches!(decode(&longer, 4, 3, 12, &t), Err(RawError::Unsupported(_)))); + let shorter = &pack(SPLIT_12)[..10]; + assert!(decode(shorter, 4, 3, 12, &t).is_err()); + // the unused all-ones word + let bad = pack("011 100 11110 1110 0 11110 11111111 11111110 11111110 11111110"); + assert!(matches!(decode(&bad, 4, 2, 12, &t), Err(RawError::Corrupt(_)))); + // a split row beyond the image + assert!(matches!(decode(&pack(SPLIT_12), 4, 1, 12, &t), Err(RawError::Corrupt(_)))); + // through the container + let mut table = hand_table(0x40, 252, 4, 4095); + table[562..564].copy_from_slice(&1u16.to_le_bytes()); + let ok = crate::decode(&nef_file(pack(SPLIT_12), 4, 3, 12, table.clone(), ByteOrder::Little)).unwrap(); + assert_eq!(ok.data, crate::RawData::U16(vec![256, 252, 255, 252, 252, 256, 244, 388, 0, 252, 8, 252])); + let e = crate::decode(&nef_file(longer, 4, 3, 12, table, ByteOrder::Little)); + assert!(matches!(e, Err(RawError::Unsupported(_))), "{e:?}"); + } + + #[test] + fn lossy_type1_full_curve() { + let mut t = vec![0x44, 0x10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 5]; + for v in [0u16, 100, 400, 900, 1600] { + t.extend_from_slice(&v.to_be_bytes()); + } + let parsed = parse_table(&t, ByteOrder::Big, 12).unwrap(); + assert_eq!(parsed.encoding, Encoding::Lossy(vec![0, 100, 400, 900, 1600])); + let img = vec![0, 4, 1, 3, 2, 2, 4, 0]; + let src = encode(&img, 4, &LOSSY_12, parsed.seeds); + assert_eq!(decode(&src, 4, 2, 12, &parsed).unwrap(), vec![0, 1600, 100, 900, 400, 400, 1600, 0]); + } + + #[test] + fn rejects_unsupported_and_malformed_tables() { + let unsupported = |r: Result| matches!(r, Err(RawError::Unsupported(_))); + let corrupt = |r: Result| matches!(r, Err(RawError::Corrupt(_))); + let mut split = lossy_table(12, ByteOrder::Big); + split[562..564].copy_from_slice(&345u16.to_be_bytes()); + assert_eq!(parse_table(&split, ByteOrder::Big, 12).unwrap().split, 345); + assert!(unsupported(parse_table(&[0x49, 0x30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], ByteOrder::Big, 12))); + assert!(unsupported(parse_table(&lossy_table(12, ByteOrder::Big), ByteOrder::Big, 16))); + assert!(corrupt(parse_table(&[], ByteOrder::Big, 12))); + assert!(corrupt(parse_table(&[0x46, 0x30, 8], ByteOrder::Big, 14))); + assert!(corrupt(parse_table(&lossy_table(12, ByteOrder::Big)[..100], ByteOrder::Big, 12))); + // 300 points don't divide the 12-bit range + let mut odd = lossy_table(12, ByteOrder::Big); + odd[10..12].copy_from_slice(&300u16.to_be_bytes()); + assert!(parse_table(&odd, ByteOrder::Big, 12).is_err()); + assert!(parse_table(&[0x44, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], ByteOrder::Big, 12).is_err()); + } + + /// A minimal NEF: CFA SubIFD with one compressed strip, Exif maker note (`Nikon\0` v2 header + embedded TIFF in + /// `mn_order`) holding the linearization table. + fn nef_file(strip: Vec, w: u32, h: u32, bits: u16, table: Vec, mn_order: ByteOrder) -> Vec { + use craft_tiff::tags::{self as t, photometric}; + use craft_tiff::{IfdBuilder, ImageData, TiffWriter, Value}; + let mut mn = IfdBuilder::new(); + mn.set(0x0096, Value::Undefined(table)); + let mut note = b"Nikon\0\x02\x10\0\0".to_vec(); + note.extend(TiffWriter::new(mn_order, false).write(&[mn]).unwrap()); + let mut exif = IfdBuilder::new(); + exif.set(t::MAKER_NOTE, Value::Undefined(note)); + let mut raw = IfdBuilder::new(); + raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0])); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![w])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![h])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![bits])); + raw.set(t::COMPRESSION, Value::Short(vec![t::compression::NIKON])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])); + raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2])); + raw.set(t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2])); + raw.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![strip] }); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("NIKON CORPORATION".into())); + ifd0.set(t::MODEL, Value::Ascii("NIKON TEST".into())); + ifd0.set_child(t::EXIF_IFD, exif); + ifd0.add_sub_ifd(raw); + TiffWriter::new(ByteOrder::Big, false).write(&[ifd0]).unwrap() + } + + #[test] + fn decodes_through_the_nef_container() { + // lossless 14-bit, big-endian maker note + let (w, h) = (40usize, 6usize); + let img = image(w, h, 16383, 11); + let mut table = vec![0x46, 0x30]; + for _ in 0..4 { + table.extend_from_slice(&2048u16.to_be_bytes()); + } + table.resize(46, 0); + let src = encode(&img, w, &LOSSLESS_14, [[2048; 2]; 2]); + let bytes = nef_file(src, w as u32, h as u32, 14, table, ByteOrder::Big); + let r = crate::decode(&bytes).unwrap(); + assert_eq!(r.data, crate::RawData::U16(img.iter().map(|&v| v as u16).collect())); + assert_eq!(r.cfa.as_ref().unwrap().name(), "RGGB"); + assert_eq!(crate::probe_info(&bytes).unwrap(), r.info()); + // lossy 12-bit type 2, little-endian maker note (newer bodies) + let table = lossy_table(12, ByteOrder::Little); + let t = parse_table(&table, ByteOrder::Little, 12).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + let img = image(w, h, 1023, 5); + let src = encode(&img, w, &LOSSY_12, t.seeds); + let r = crate::decode(&nef_file(src.clone(), w as u32, h as u32, 12, table.clone(), ByteOrder::Little)).unwrap(); + assert_eq!(r.data, crate::RawData::U16(img.iter().map(|&v| curve[v as usize]).collect())); + // a split file whose strip does not follow the fixed-rate rule is refused, never decoded wrongly + let mut split = table; + split[562..564].copy_from_slice(&3u16.to_le_bytes()); + let e = crate::decode(&nef_file(src, w as u32, h as u32, 12, split, ByteOrder::Little)); + assert!(e.is_err(), "{e:?}"); + } + + /// storytold/lightcraft#708: the header probe of a compressed NEF decodes only the first [`crate::nef::MASK_ROWS`] + /// rows (the ones the masked-column test reads) instead of the whole Huffman stream, and still finds the masked + /// columns the full decode finds. A strip cut off after those rows proves it: the probe succeeds with the same + /// description while the full decode reports the truncation. + #[test] + fn header_probe_stops_after_the_masked_column_rows() { + use crate::nef::MASK_ROWS; + let (w, h) = (160usize, MASK_ROWS * 3); + let mut img = image(w, h, 16383, 7); + // eight optically masked columns at 0, as the D5100 / D7000 write them + for row in img.chunks_mut(w) { + row[w - 8..].fill(0); + } + let t = lossless(14); + let table = { + let mut v = vec![0x46, 0x30]; + for s in t.seeds.iter().flatten() { + v.extend_from_slice(&(*s as u16).to_be_bytes()); + } + v.resize(46, 0); + v + }; + let src = encode(&img, w, &LOSSLESS_14, t.seeds); + let whole = nef_file(src.clone(), w as u32, h as u32, 14, table.clone(), ByteOrder::Big); + let full = crate::decode(&whole).unwrap(); + assert_eq!(full.active_area, crate::Rect::new(0, 0, w - 8, h), "the masked columns are found in a compressed file"); + assert_eq!(crate::probe_info(&whole).unwrap(), full.info()); + // the same stream cut off after two thirds of the rows (the first rows' codes are a prefix of the whole) + let cut = encode(&img[..w * (h * 2 / 3)], w, &LOSSLESS_14, t.seeds).len(); + assert!(cut > w * h / 4, "enough bytes to pass the size check"); + let truncated = nef_file(src[..cut].to_vec(), w as u32, h as u32, 14, table, ByteOrder::Big); + assert!(matches!(crate::decode(&truncated), Err(RawError::Corrupt(_))), "the full decode sees the truncation"); + assert_eq!(crate::probe_info(&truncated).unwrap(), full.info(), "the header probe never reads that far"); + // decode_rows itself: a prefix of the whole decode, and the whole image's size check still applies + let top = decode_rows(&src, w, h, MASK_ROWS, 14, &t).unwrap(); + assert_eq!(top.len(), w * MASK_ROWS); + assert_eq!(top[..], decode(&src, w, h, 14, &t).unwrap()[..w * MASK_ROWS]); + assert_eq!(decode_rows(&src, w, h, h + 5, 14, &t).unwrap().len(), w * h, "rows are clamped to the height"); + assert!(decode_rows(&src[..w * h / 4 - 1], w, h, 1, 14, &t).is_err(), "a strip too short for the image is corrupt in any mode"); + } + + /// A long run of the largest difference drove a predictor past i32: a panic in builds with overflow checks, + /// wrapped values otherwise (storytold/lightcraft#1515). Both the row and the column predictors now stop with an error. + #[test] + fn predictor_overflow_is_an_error_not_a_panic() { + let codes = LOSSLESS_14.codes(); + let k = (0..codes.len()).rev().find(|&k| codes[k].1 > 0).unwrap(); + let d = (1u32 << k) - 1; + let stream = |n: usize| { + let mut wr = Writer::default(); + let (code, len) = codes[k]; + for _ in 0..n { + wr.put(code as u32 >> (12 - len), len as u32); + wr.put(d, k as u32); + } + wr.finish() + }; + // differences one predictor needs to pass i32::MAX + let steps = (i32::MAX as u32 / d) as usize + 2; + let table = DecodeTable { encoding: Encoding::Lossless, seeds: [[0; 2]; 2], split: 0 }; + // along a row: every other sample adds to the same predictor + let w = 2 * steps + 2; + let err = decode(&stream(w), w, 1, 14, &table).unwrap_err(); + assert!(matches!(err, RawError::Corrupt(ref m) if m.contains("predictor")), "{err}"); + // down two columns: each row adds to the seed of its parity + let h = 2 * steps + 2; + let err = decode(&stream(2 * h), 2, h, 14, &table).unwrap_err(); + assert!(matches!(err, RawError::Corrupt(ref m) if m.contains("predictor")), "{err}"); + } + + #[test] + fn truncated_and_corrupted_streams_error_never_panic() { + let (w, h) = (64, 16); + let t = lossless(14); + let img = image(w, h, 16383, 3); + let src = encode(&img, w, &LOSSLESS_14, t.seeds); + // truncation: too short for the image, or ends early + for cut in [0, 1, 10, src.len() / 2, src.len() - 40] { + assert!(decode(&src[..cut], w, h, 14, &t).is_err(), "cut {cut}"); + } + // garbage and bit flips: an error or (rarely) some image, never a panic + let mut s = 12345u32; + for i in 0..200 { + let mut bad = src.clone(); + for _ in 0..1 + i % 8 { + s = s.wrapping_mul(1_103_515_245).wrapping_add(12345); + let at = (s >> 8) as usize % bad.len(); + bad[at] ^= 1 << (s % 8); + } + let _ = decode(&bad, w, h, 14, &t); + let noise: Vec = (0..src.len()).map(|k| ((k as u32).wrapping_mul(2_654_435_761) >> (i % 24)) as u8).collect(); + let _ = decode(&noise, w, h, 14, &t); + let _ = decode(&noise, w, h, 12, &lossless(12)); + } + // all-ones never forms a valid lossy 12-bit code + assert!(decode(&[0xff; 64], 8, 8, 12, &parse_table(&lossy_table(12, ByteOrder::Big), ByteOrder::Big, 12).unwrap()).is_err()); + // absurd dimensions are limited before allocating + assert!(decode(&src, usize::MAX, 2, 14, &t).is_err()); + assert!(decode(&src, 1 << 20, 1 << 20, 14, &t).is_err()); + assert!(decode(&src, 0, 2, 14, &t).is_err()); + } +} diff --git a/crates/raw/tests/it/vendor_robust.rs b/crates/raw/tests/it/vendor_robust.rs index 68eccf554..65fb139fb 100644 --- a/crates/raw/tests/it/vendor_robust.rs +++ b/crates/raw/tests/it/vendor_robust.rs @@ -61,6 +61,8 @@ fn cr2() -> Vec { fn samples() -> Vec> { vec![ cr2(), + cfa_tiff("NIKON CORPORATION", 1, 12, ByteOrder::Big), + cfa_tiff("NIKON CORPORATION", 34713, 14, ByteOrder::Big), pef(1), pef(65535), cfa_tiff("SAMSUNG", 1, 12, ByteOrder::Little),