From c004e38e759a2be63fa1b7b39695ec4e32b9cc3e Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 3 Nov 2026 20:37:28 -0500 Subject: [PATCH] raw: decode a tile range once when several tiles share it (#11041) read_plane decoded every segment separately, even when a file pointed every tile at the same bytes. Each lossless JPEG decode rebuilds its Huffman tables and each result is retained until assembly, so one small range repeated across many tiles multiplied header work and memory by the tile count. Decode each distinct (offset, length, geometry) once and scatter the shared samples into every tile that names it. --- crates/raw/src/sensor.rs | 29 +++++++++++++++++++++++------ crates/raw/tests/it/hostile.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/crates/raw/src/sensor.rs b/crates/raw/src/sensor.rs index bae018c244..5c83066c21 100644 --- a/crates/raw/src/sensor.rs +++ b/crates/raw/src/sensor.rs @@ -279,8 +279,24 @@ pub(crate) fn read_plane(t: &Tiff, ifd: &Ifd, limits: &Limits, layout: JpegLayou retained_segment_bytes(&segments, samples, limits)?; let le = t.le; let tiled = t.tag_uint(ifd, tag::TILE_WIDTH).is_some(); - let decoded: Vec>> = par::map(segments.len(), |i| { - let s = &segments[i]; + // Segments naming the same bytes at the same geometry decode to the same samples, so each + // distinct one is decoded once and shared. A file can point every tile at one small range; + // decoding it per tile multiplied header work (a lossless JPEG rebuilds its Huffman tables on + // every decode) and retained buffers by the tile count. + let mut first_of = std::collections::HashMap::new(); + let mut unique: Vec = Vec::new(); + let slot: Vec = segments + .iter() + .enumerate() + .map(|(i, s)| { + *first_of.entry((s.offset, s.len, s.w, s.h, s.h.min(height - s.y))).or_insert_with(|| { + unique.push(i); + unique.len() - 1 + }) + }) + .collect(); + let decoded: Vec>> = par::map(unique.len(), |u| { + let s = &segments[unique[u]]; let rows = s.h.min(height - s.y); let src = t.bytes(s.offset, s.len).ok_or_else(|| RawError::malformed("raw data lies outside the file"))?; // Every branch below sizes at least one buffer from the segment's own geometry, which @@ -322,9 +338,9 @@ pub(crate) fn read_plane(t: &Tiff, ifd: &Ifd, limits: &Limits, layout: JpegLayou // One segment holding the whole image row for row (a single strip): its buffer is the plane, no // second full-size buffer and no copy. The scatter below then has no segment left to place. let whole = matches!(segments.as_slice(), [s] if s.x == 0 && s.y == 0 && s.w == width && s.h >= height); - let mut decoded = decoded.into_iter(); + let mut decoded: Vec> = decoded.into_iter().collect::>()?; let taken = if whole { - let mut d = decoded.next().ok_or_else(|| RawError::malformed("raw segment missing"))??; + let mut d = decoded.pop().ok_or_else(|| RawError::malformed("raw segment missing"))?; if d.len() < width * height * samples { return Err(RawError::malformed("raw segment is smaller than the image")); } @@ -334,8 +350,9 @@ pub(crate) fn read_plane(t: &Tiff, ifd: &Ifd, limits: &Limits, layout: JpegLayou None }; let mut data = taken.unwrap_or_else(|| vec![0u16; width * height * samples]); - for (s, d) in segments.iter().zip(decoded) { - let d = d?; + // The whole-image segment above is the plane itself: nothing is left to scatter. + for (s, &k) in segments.iter().zip(&slot).filter(|_| !whole) { + let d = &decoded[k]; let rows = s.h.min(height - s.y); let cols = s.w.min(width - s.x); for r in 0..rows { diff --git a/crates/raw/tests/it/hostile.rs b/crates/raw/tests/it/hostile.rs index 6fdd0e7434..a60025be69 100644 --- a/crates/raw/tests/it/hostile.rs +++ b/crates/raw/tests/it/hostile.rs @@ -232,6 +232,37 @@ fn patch_long(b: &mut [u8], tag: u16, to: u32) -> bool { patched == 1 } +/// Tiles that all point at the same bytes decode once and fill every tile (the range is shared, +/// not decoded per tile), and give the image a lossless JPEG of that one tile repeated. +#[test] +fn tiles_sharing_one_range_fill_every_tile() { + let mut d = DngSpec::cfa(16, 16, mosaic(&scene(16, 16), 16, [0, 1, 1, 2], 64, 4000)); + d.bits = 12; + d.white = 4000; + d.black = vec![64]; + d.storage = DngStorage::Lj92Tiles { width: 8, height: 8 }; + let mut b = d.build(); + // TileOffsets (324) and TileByteCounts (325): LONG x 4, stored out of line. + for tag in [324u16, 325] { + let e = (8..b.len() - 12) + .find(|&i| b[i..i + 2] == tag.to_le_bytes() && b[i + 2..i + 4] == 4u16.to_le_bytes() && b[i + 4..i + 8] == 4u32.to_le_bytes()) + .expect("tile tag"); + let at = u32::from_le_bytes(b[e + 8..e + 12].try_into().unwrap()) as usize; + let first: [u8; 4] = b[at..at + 4].try_into().unwrap(); + for k in 1..4 { + b[at + 4 * k..at + 4 * k + 4].copy_from_slice(&first); + } + } + let shared = decode(&b, &Limits::default()).expect("a repeated tile range decodes"); + let plane = &shared.data; + // Every 8x8 tile holds the first tile's samples. + for (ty, tx) in [(0, 8), (8, 0), (8, 8)] { + for y in 0..8 { + assert_eq!(plane[(ty + y) * 16 + tx..(ty + y) * 16 + tx + 8], plane[y * 16..y * 16 + 8]); + } + } +} + /// The lossless-JPEG intermediate is sized from the segment, so the plane check alone /// does not bound it: it has to be charged against `max_alloc` too (#2925). #[test]