raw: dng_lines reuses the DNG reader's unpackers and JPEG decoder, drops the image dependency

- uncompressed rows go through crate::unpack (read_u16s / unpack_msb), as in dng.rs
- lossy JPEG tiles (34892) use tiffraw::lossy_jpeg_with_tile (zune-jpeg) with its tile-size checks
- line-dng now implies dng instead of pulling in image
- tests: every layout and compression read against the full reader, refusals, unpack depths
This commit is contained in:
FredApps committed 2026-11-16 08:27:55 +01:00
1 parent 3fd0383c81
commit 59d28093e6
7 files changed
+285 -65

No files matched your search

Generated
-1
View File
@@ -2609,7 +2609,6 @@ dependencies = [
"craft-crw",
"craft-metadata",
"craft-tiff",
"image",
"jpeg-encoder 0.7.1",
"jxl-oxide",
"miniz_oxide 0.9.1",
+2 -3
View File
@@ -11,8 +11,8 @@ repository.workspace = true
workspace = true
[features]
# Bounded DNG strip/tile reader, with FilmCraft-compatible baseline JPEG decoding.
line-dng = ["dep:image"]
# Bounded DNG strip/tile reader (`dng_lines`): the `dng` reader's unpackers and JPEG decoder.
line-dng = ["dng"]
# Decode DNG tile bands in parallel.
line-dng-threads = ["line-dng"]
default = ["dng", "jxl", "crw", "hasselblad", "mrw", "erf", "mef", "kodak", "pef", "srw", "cr2", "nef", "rw2", "raf", "orf", "arw", "cr3"]
@@ -52,7 +52,6 @@ cr3 = ["dep:craft-metadata"]
crw = ["dep:craft-crw", "dep:chrono"]
[dependencies]
image = { version = "0.25", default-features = false, features = ["jpeg"], optional = true }
craft-tiff = { workspace = true }
craft-crw = { workspace = true, optional = true }
# The CR3 container (ISO base media boxes, CMT1-4, tracks, CMP1 / CDI1): the one CR3 parser.
+30 -52
View File
@@ -229,25 +229,14 @@ pub fn unpack(bytes: &[u8], w: usize, h: usize, spp: usize, bits: u32, big_endia
if bytes.len() < need {
return Err(Error::Truncated);
}
let mut out = Vec::with_capacity(per_row * h);
for row in bytes.chunks_exact(row_bytes.max(1)).take(h) {
let mut out = vec![0u16; per_row * h];
let order = if big_endian { craft_tiff::ByteOrder::Big } else { craft_tiff::ByteOrder::Little };
// the same unpackers the full DNG reader uses (`crate::unpack`)
for (row, dst) in bytes.chunks_exact(row_bytes.max(1)).zip(out.chunks_exact_mut(per_row.max(1))) {
match bits {
8 => out.extend(row.iter().map(|&b| u16::from(b))),
16 => out.extend(row.as_chunks::<2>().0.iter().map(|&b| if big_endian { u16::from_be_bytes(b) } else { u16::from_le_bytes(b) })),
_ => {
let mut acc: u32 = 0;
let mut n: u32 = 0;
let mut it = row.iter();
for _ in 0..per_row {
while n < bits {
acc = (acc << 8) | u32::from(*it.next().ok_or(Error::Truncated)?);
n += 8;
}
n -= bits;
out.push(((acc >> n) & ((1 << bits) - 1)) as u16);
acc &= (1 << n) - 1;
}
}
8 => dst.iter_mut().zip(row).for_each(|(o, &b)| *o = u16::from(b)),
16 => crate::unpack::read_u16s(row, order, dst),
_ => crate::unpack::unpack_msb(row, bits, dst),
}
}
Ok(out)
@@ -258,7 +247,7 @@ fn decode_chunk(c: &Chunk, l: &RawLayout, big_endian: bool) -> Result<Vec<u16>>
let need = c.w * c.h * l.samples_per_pixel;
match l.compression {
1 => unpack(c.bytes, c.w, c.h, l.samples_per_pixel, l.bits, big_endian),
34892 => lossy_jpeg(c.bytes, need, l.samples_per_pixel),
34892 => lossy_jpeg(c, l),
_ => {
// the JPEG's samples fill the chunk in raster order (its own width × components may
// differ from the chunk's, e.g. two components of half the width)
@@ -275,39 +264,28 @@ fn decode_chunk(c: &Chunk, l: &RawLayout, big_endian: bool) -> Result<Vec<u16>>
}
}
/// Decode a baseline (lossy) JPEG tile or strip (Compression 34892) into `need` 8-bit samples:
/// one component (greyscale) or three (colour, converted from YCbCr by the JPEG decoder as the
/// JPEG says), in raster order.
fn lossy_jpeg(bytes: &[u8], need: usize, spp: usize) -> Result<Vec<u16>> {
use image::ImageDecoder;
let bad = |_| Error::Malformed("lossy JPEG tile");
let decode = || -> Result<Vec<u16>> {
let mut d = image::codecs::jpeg::JpegDecoder::new(std::io::Cursor::new(bytes)).map_err(bad)?;
let channels = match d.color_type() {
image::ColorType::L8 => 1usize,
image::ColorType::Rgb8 => 3,
c => return Err(Error::Unsupported(format!("lossy JPEG DNG tile of colour type {c:?}"))),
};
if channels != spp {
return Err(Error::Malformed("lossy JPEG components differ from SamplesPerPixel"));
}
let (w, h) = d.dimensions();
let total = (w as usize).checked_mul(h as usize).and_then(|v| v.checked_mul(channels)).ok_or(Error::TooLarge)?;
if total < need {
return Err(Error::Malformed("lossy JPEG smaller than its tile or strip"));
}
if total > need.saturating_mul(4).saturating_add(65_536) || total > MAX_SAMPLES {
return Err(Error::TooLarge);
}
let mut limits = image::Limits::default();
limits.max_alloc = Some(total as u64 * 4 + (1 << 20));
d.set_limits(limits).map_err(bad)?;
let mut buf = vec![0u8; total];
d.read_image(&mut buf).map_err(bad)?;
Ok(buf.iter().take(need).map(|&v| u16::from(v)).collect())
};
// a third-party decoder: a panic in it is an error for this file, not a crash
std::panic::catch_unwind(std::panic::AssertUnwindSafe(decode)).unwrap_or(Err(Error::Malformed("lossy JPEG decoder failed")))
/// Decode a baseline (lossy) JPEG tile or strip (Compression 34892) into the chunk's 8-bit
/// samples: one component (greyscale) or three (colour, converted from YCbCr as the JPEG says), in
/// raster order. It is the full DNG reader's decoder (`tiffraw`, zune-jpeg), with the same check
/// that the JPEG covers the chunk's visible part and is at most MCU-padded beyond it.
fn lossy_jpeg(c: &Chunk, l: &RawLayout) -> Result<Vec<u16>> {
let spp = l.samples_per_pixel;
let need = c.w * c.h * spp;
let visible = (c.w.min(l.width.saturating_sub(c.x)), c.h.min(l.height.saturating_sub(c.y)));
let max = need.saturating_mul(4).saturating_add(65_536).min(MAX_SAMPLES);
let (px, w, h, _) = crate::tiffraw::lossy_jpeg_with_tile(c.bytes, spp, max, Some((c.w, c.h, visible.0, visible.1))).map_err(|e| match e {
crate::RawError::Unsupported(m) => Error::Unsupported(m),
_ => Error::Malformed("lossy JPEG tile"),
})?;
// the JPEG's rows are its own width; the chunk keeps its `c.w` x `c.h` top-left corner, and
// the part of a coded chunk past a smaller JPEG (outside the image) stays zero
let mut out = vec![0u16; need];
let n = c.w.min(w) * spp;
for (y, dst) in out.chunks_exact_mut(c.w * spp).take(h).enumerate() {
let src = px.get(y * w * spp..y * w * spp + n).ok_or(Error::Truncated)?;
dst[..n].iter_mut().zip(src).for_each(|(o, &v)| *o = u16::from(v));
}
Ok(out)
}
/// Read the raw image of `ifd`.
+1 -1
View File
@@ -461,7 +461,7 @@ pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<(
/// Validate the actual frame before decode allocates its output. Edge tiles may store
/// a smaller frame covering all visible samples or bounded whole-MCU padding.
#[cfg(feature = "dng")]
fn lossy_jpeg_with_tile(
pub(crate) fn lossy_jpeg_with_tile(
src: &[u8],
cpp: usize,
max_samples: usize,
+8 -8
View File
@@ -15,7 +15,7 @@ fn scene_value(x: usize, y: usize, c: u8, max: f32) -> f32 {
v * max
}
fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
pub(super) fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
let black = 256.0;
let white = 16000.0;
let mut data = Vec::with_capacity(w * h * cpp);
@@ -194,7 +194,7 @@ fn deflate_roundtrips_integer_and_float() {
// ---------------------------------------------------------------- hand-built variants
fn base_ifd(w: usize, h: usize, bits: u16, cpp: usize, cfa: bool) -> IfdBuilder {
pub(super) fn base_ifd(w: usize, h: usize, bits: u16, cpp: usize, cfa: bool) -> IfdBuilder {
let mut ifd = IfdBuilder::new();
ifd.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
ifd.set(t::IMAGE_WIDTH, Value::Long(vec![w as u32]));
@@ -209,7 +209,7 @@ fn base_ifd(w: usize, h: usize, bits: u16, cpp: usize, cfa: bool) -> IfdBuilder
ifd
}
fn dng(raw: IfdBuilder, order: ByteOrder) -> Vec<u8> {
pub(super) fn dng(raw: IfdBuilder, order: ByteOrder) -> Vec<u8> {
dng_with(raw, order, |_| {})
}
@@ -237,12 +237,12 @@ fn dng_with(raw: IfdBuilder, order: ByteOrder, extra: impl FnOnce(&mut IfdBuilde
TiffWriter::new(order, false).write(&[ifd0]).unwrap()
}
fn pattern(w: usize, h: usize, cpp: usize, bits: u32) -> Vec<u16> {
pub(super) fn pattern(w: usize, h: usize, cpp: usize, bits: u32) -> Vec<u16> {
let max = (1u32 << bits) - 1;
(0..w * h * cpp).map(|i| ((i as u32).wrapping_mul(2654435761u32) >> 7) % (max + 1)).map(|v| v as u16).collect()
}
fn pack_msb(v: &[u16], bits: u32, w: usize) -> Vec<u8> {
pub(super) fn pack_msb(v: &[u16], bits: u32, w: usize) -> Vec<u8> {
let mut out = Vec::new();
for row in v.chunks(w) {
let mut acc = 0u64;
@@ -262,7 +262,7 @@ fn pack_msb(v: &[u16], bits: u32, w: usize) -> Vec<u8> {
out
}
fn u16_bytes(v: &[u16], order: ByteOrder) -> Vec<u8> {
pub(super) fn u16_bytes(v: &[u16], order: ByteOrder) -> Vec<u8> {
let mut b = Vec::new();
v.iter().for_each(|&x| order.put_u16(&mut b, x));
b
@@ -407,7 +407,7 @@ fn floating_point_dngs() {
/// The chunks (tiles of `tw` x `th`, or strips when `tw == w`) of an image of `cpp` interleaved samples per pixel, each
/// nominal-sized (strips: clipped to the image height) with zero padding past the image edge: one chunk per tile, and for `planar` one set of chunks per
/// plane (one sample per pixel), plane after plane.
fn chunk_samples<T: Copy + Default>(px: &[T], w: usize, h: usize, cpp: usize, tw: usize, th: usize, planar: bool) -> Vec<Vec<T>> {
pub(super) fn chunk_samples<T: Copy + Default>(px: &[T], w: usize, h: usize, cpp: usize, tw: usize, th: usize, planar: bool) -> Vec<Vec<T>> {
let planes = if planar { cpp } else { 1 };
let ccpp = if planar { 1 } else { cpp };
let mut out = Vec::new();
@@ -432,7 +432,7 @@ fn chunk_samples<T: Copy + Default>(px: &[T], w: usize, h: usize, cpp: usize, tw
out
}
fn layout_image(raw: &mut IfdBuilder, chunks: Vec<Vec<u8>>, w: usize, tile: (usize, usize)) {
pub(super) fn layout_image(raw: &mut IfdBuilder, chunks: Vec<Vec<u8>>, w: usize, tile: (usize, usize)) {
raw.set_image(if tile.0 == w {
ImageData::Strips { rows_per_strip: tile.1 as u32, strips: chunks }
} else {
+242
View File
@@ -0,0 +1,242 @@
//! The bounded strip/tile reader (`dng_lines`) against the full DNG reader: every layout the writer
//! and the hand-built variants produce reads to the same samples, and the layouts the bounded reader
//! does not take are refused with a reason.
use super::dng::{base_ifd, chunk_samples, dng, layout_image, pack_msb, pattern, synthetic, u16_bytes};
use craft_raw::dng_lines::{self, Error};
use craft_raw::dngwrite::{DngImage, write_lossy_dng};
use craft_raw::{Cfa, DngCompression, DngWriteOptions, RawData, RawError, decode, write_dng};
use craft_tiff::classic::Tiff;
use craft_tiff::tags as t;
use craft_tiff::{ByteOrder, ImageData, Value};
fn lines(bytes: &[u8]) -> Result<dng_lines::RawImage, Error> {
let (tiff, first) = Tiff::new(bytes)?;
let ifds = tiff.all_ifds(first)?;
let ifd = dng_lines::find_raw_ifd(&tiff, &ifds)?;
dng_lines::read(&tiff, ifd)
}
fn full(bytes: &[u8]) -> Vec<u16> {
match decode(bytes).unwrap().data {
RawData::U16(v) => v,
other => panic!("integer samples expected, got {other:?}"),
}
}
#[test]
fn uncompressed_layouts_match_the_full_reader() {
let (w, h) = (37, 23);
for cpp in [1usize, 3] {
for bits in [8u32, 12, 16] {
let px = pattern(w, h, cpp, bits);
for tile in [(16, 16), (w, 5), (w, h)] {
for order in [ByteOrder::Little, ByteOrder::Big] {
let chunks = chunk_samples(&px, w, h, cpp, tile.0, tile.1, false)
.iter()
.map(|c| match bits {
8 => c.iter().map(|&v| v as u8).collect(),
16 => u16_bytes(c, order),
_ => pack_msb(c, bits, tile.0 * cpp),
})
.collect();
let mut raw = base_ifd(w, h, bits as u16, cpp, cpp == 1);
raw.set(t::COMPRESSION, Value::Short(vec![1]));
layout_image(&mut raw, chunks, w, tile);
let bytes = dng(raw, order);
let img = lines(&bytes).unwrap();
let what = format!("cpp {cpp} bits {bits} tile {tile:?} {order:?}");
assert_eq!((img.width, img.height, img.samples_per_pixel, img.bits), (w, h, cpp, bits), "{what}");
assert_eq!(img.data, px, "{what}");
assert_eq!(img.data, full(&bytes), "{what}");
}
}
}
}
}
#[test]
fn written_dngs_read_back_in_every_compression() {
for (w, h) in [(64, 48), (33, 17)] {
for cfa in [Some(Cfa::bayer("RGGB").unwrap()), None] {
let cpp = if cfa.is_some() { 1 } else { 3 };
let raw = synthetic(w, h, cfa, cpp);
for comp in [DngCompression::Uncompressed, DngCompression::Lj92 { tile: 16 }, DngCompression::Lj92 { tile: 256 }] {
for order in [ByteOrder::Little, ByteOrder::Big] {
let bytes = write_dng(&raw, &DngWriteOptions { compression: comp, order, xmp: None, ..Default::default() }).unwrap();
let img = lines(&bytes).unwrap();
assert_eq!(RawData::U16(img.data.clone()), raw.data, "{w}x{h} cpp {cpp} {comp:?} {order:?}");
assert_eq!(img.data, full(&bytes));
}
}
}
}
}
#[test]
fn lossy_jpeg_tiles_read_like_the_full_reader() {
let (w, h) = (40, 21);
let mut raw = synthetic(w, h, None, 3);
raw.bits = 8;
raw.data = RawData::U16(pattern(w, h, 3, 8));
raw.white = vec![255.0];
raw.black = craft_raw::BlackLevel::uniform(0.0);
let linearization: Vec<u16> = (0..256).map(|v| v * 257).collect();
let encode = |pixels: Vec<u8>, tile: u32, quality: u8| {
let mut out = Vec::new();
jpeg_encoder::Encoder::new(&mut out, quality)
.encode(&pixels, tile as u16, tile as u16, jpeg_encoder::ColorType::Rgb)
.map_err(|e| RawError::Corrupt(e.to_string()))?;
Ok(out)
};
for tile in [16, 64] {
let bytes = write_lossy_dng(&DngImage::from(&raw), &DngWriteOptions::default(), tile, 90, &linearization, &encode).unwrap();
let img = lines(&bytes).unwrap();
assert_eq!((img.width, img.height, img.samples_per_pixel, img.bits), (w, h, 3, 8));
// the same decoder as the full reader, so the same 8-bit codes; the full reader also
// applies the LinearizationTable, the bounded reader leaves that to its caller
let codes: Vec<u16> = img.data.iter().map(|&v| linearization[usize::from(v)]).collect();
assert_eq!(codes, full(&bytes), "tile {tile}");
}
// a lossy tile that is not a JPEG
let mut bad = base_ifd(16, 16, 8, 3, false);
bad.set(t::COMPRESSION, Value::Short(vec![34892]));
bad.set_image(ImageData::Tiles { tile_width: 16, tile_height: 16, tiles: vec![vec![0x55; 512]] });
assert!(matches!(lines(&dng(bad, ByteOrder::Little)), Err(Error::Malformed(_) | Error::Unsupported(_))));
}
fn refused(edit: impl FnOnce(&mut craft_tiff::IfdBuilder), cpp: usize, cfa: bool) -> Error {
let (w, h) = (8, 4);
let mut raw = base_ifd(w, h, 16, cpp, cfa);
raw.set(t::COMPRESSION, Value::Short(vec![1]));
raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![vec![0; w * h * cpp * 2]] });
edit(&mut raw);
lines(&dng(raw, ByteOrder::Little)).unwrap_err()
}
#[test]
fn unsupported_and_damaged_layouts_are_refused() {
let unsupported = |e: Error| matches!(e, Error::Unsupported(_));
assert!(unsupported(refused(
|r| {
r.set(t::COMPRESSION, Value::Short(vec![8]));
},
1,
true
)));
assert!(unsupported(refused(
|r| {
r.set(t::COMPRESSION, Value::Short(vec![52546]));
},
1,
true
)));
assert!(unsupported(refused(
|r| {
r.set(t::COMPRESSION, Value::Short(vec![5]));
},
1,
true
)));
assert!(unsupported(refused(
|r| {
r.set(t::COMPRESSION, Value::Short(vec![34892]));
},
1,
true
)));
assert!(unsupported(refused(
|r| {
r.set(t::COMPRESSION, Value::Short(vec![34892]));
},
3,
false
)));
assert!(unsupported(refused(
|r| {
r.set(t::PREDICTOR, Value::Short(vec![2]));
},
1,
true
)));
assert!(unsupported(refused(
|r| {
r.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
},
3,
false
)));
assert!(unsupported(refused(
|r| {
r.set(t::SAMPLE_FORMAT, Value::Short(vec![3; 3]));
},
3,
false
)));
assert!(unsupported(refused(
|r| {
r.set(t::BITS_PER_SAMPLE, Value::Short(vec![16, 12, 16]));
},
3,
false
)));
assert!(unsupported(refused(
|r| {
r.set(t::BITS_PER_SAMPLE, Value::Short(vec![32]));
},
1,
true
)));
assert!(unsupported(refused(
|r| {
r.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![5]));
},
1,
false
)));
assert!(matches!(
refused(
|r| {
r.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![3]));
},
1,
true
),
Error::Malformed(_)
));
assert!(matches!(
refused(
|r| {
r.set(t::IMAGE_WIDTH, Value::Long(vec![0]));
},
1,
true
),
Error::Malformed(_)
));
// a strip shorter than its rows, and fewer strips than the image needs
let short = |r: &mut craft_tiff::IfdBuilder| {
r.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0; 10]] });
};
assert_eq!(refused(short, 1, true), Error::Truncated);
let few = |r: &mut craft_tiff::IfdBuilder| {
r.set_image(ImageData::Strips { rows_per_strip: 1, strips: vec![vec![0; 16]] });
};
assert!(matches!(refused(few, 1, true), Error::Malformed(_)));
// no full-resolution CFA or LinearRaw IFD at all
let none = |r: &mut craft_tiff::IfdBuilder| {
r.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![1]));
};
assert!(unsupported(refused(none, 1, true)));
assert_eq!(lines(b"not a tiff at all").unwrap_err(), Error::NotTiff);
}
#[test]
fn unpack_reads_every_depth_in_both_byte_orders() {
assert_eq!(dng_lines::unpack(&[1, 2, 3, 4], 2, 1, 1, 16, false).unwrap(), [0x0201, 0x0403]);
assert_eq!(dng_lines::unpack(&[1, 2, 3, 4], 2, 1, 1, 16, true).unwrap(), [0x0102, 0x0304]);
assert_eq!(dng_lines::unpack(&[7, 9], 1, 2, 1, 8, false).unwrap(), [7, 9]);
// rows start on a byte boundary: 3 x 4 bits per row is two bytes
assert_eq!(dng_lines::unpack(&[0x12, 0x30, 0x45, 0x60], 3, 2, 1, 4, true).unwrap(), [1, 2, 3, 4, 5, 6]);
assert_eq!(dng_lines::unpack(&[0x12], 3, 1, 1, 4, true).unwrap_err(), Error::Truncated);
}
+2
View File
@@ -2,6 +2,8 @@
#[cfg(feature = "dng")]
mod dng;
#[cfg(feature = "line-dng")]
mod dng_lines;
#[cfg(feature = "dng")]
mod robust;
mod vendor_robust;