From f25ce3949366a04f63d30b93def0a43b32ffba99 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 17:48:59 +0000 Subject: [PATCH 01/11] Share per-face shaping plan cache --- crates/font-catalog/src/lib.rs | 2 ++ crates/font-catalog/src/plan_cache.rs | 52 +++++++++++++++++++++++++++ crates/font-catalog/src/tests.rs | 19 ++++++++++ 3 files changed, 73 insertions(+) create mode 100644 crates/font-catalog/src/plan_cache.rs diff --git a/crates/font-catalog/src/lib.rs b/crates/font-catalog/src/lib.rs index 992b75080..b4271db7e 100644 --- a/crates/font-catalog/src/lib.rs +++ b/crates/font-catalog/src/lib.rs @@ -27,6 +27,7 @@ mod fontconfig; mod ligature; mod line_break; mod outline; +mod plan_cache; mod registry; mod scan; mod segments; @@ -42,6 +43,7 @@ pub use fontconfig::fontconfig_dirs; pub use ligature::ligature_caret_stops; pub use line_break::settle_line_break; pub use outline::{FontOutline, GlyphPen, OutlineError}; +pub use plan_cache::ShapePlanCache; pub use scan::{FaceInfo, LocalizedName, Source, is_font_file, style_italic, style_weight, style_width}; pub use segments::SegmentBoundary; pub use shape::{RunGlyph, RunInput, ShapeRunError, shape_run}; diff --git a/crates/font-catalog/src/plan_cache.rs b/crates/font-catalog/src/plan_cache.rs new file mode 100644 index 000000000..d428de4cd --- /dev/null +++ b/crates/font-catalog/src/plan_cache.rs @@ -0,0 +1,52 @@ +//! Per-face bounded shaping plan reuse. + +use std::sync::{Arc, Mutex}; + +/// A cache for one font face and variation instance. Do not share it across faces or instances. +/// Plans are created outside the lock, then checked again to avoid retaining duplicates. +#[derive(Default)] +pub struct ShapePlanCache { + plans: Mutex>>, +} + +impl ShapePlanCache { + /// Maximum number of plans retained; the oldest is evicted when this is reached. + pub const CAPACITY: usize = 32; + + /// Returns a matching plan or builds and retains one for this face. + pub fn get_or_create( + &self, + shaper: &harfrust::Shaper<'_>, + instance: Option<&harfrust::ShaperInstance>, + direction: harfrust::Direction, + script: Option, + language: Option<&harfrust::Language>, + features: &[harfrust::Feature], + ) -> Arc { + let key = harfrust::ShapePlanKey::new(script, direction).language(language).instance(instance).features(features); + let kept = |plans: &[Arc]| plans.iter().rev().find(|p| key.matches(p)).cloned(); + if let Some(plan) = kept(&self.plans.lock().unwrap_or_else(|e| e.into_inner())) { + return plan; + } + let plan = Arc::new(harfrust::ShapePlan::new(shaper, direction, script, language, features)); + let mut plans = self.plans.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(made) = kept(&plans) { + return made; + } + if plans.len() >= Self::CAPACITY { + plans.remove(0); + } + plans.push(plan.clone()); + plan + } + + /// Number of retained plans. + pub fn len(&self) -> usize { + self.plans.lock().unwrap_or_else(|e| e.into_inner()).len() + } + + /// Whether this cache contains no plans. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} diff --git a/crates/font-catalog/src/tests.rs b/crates/font-catalog/src/tests.rs index 5137a4295..c7623da43 100644 --- a/crates/font-catalog/src/tests.rs +++ b/crates/font-catalog/src/tests.rs @@ -568,3 +568,22 @@ fn a_read_is_capped_where_the_file_has_grown() { assert!(matches!(read_capped(&path.with_extension("missing"), 9), Err(FontFileError::Io(_)))); let _ = std::fs::remove_file(&path); } + +#[test] +fn shape_plan_cache_reuses_and_evicts_plans() { + let data = font(&spec("Test", "Regular", 400)); + let face = harfrust::FontRef::from_index(&data, 0).unwrap(); + let shaper_data = harfrust::ShaperData::new(&face); + let shaper = shaper_data.shaper(&face).build(); + let cache = ShapePlanCache::default(); + let make = |language: &harfrust::Language| cache.get_or_create(&shaper, None, harfrust::Direction::LeftToRight, None, Some(language), &[]); + let languages: Vec = ('a'..='z').flat_map(|a| ['a', 'b'].map(move |b| format!("{a}{b}").parse().unwrap())).collect(); + let first = make(&languages[0]); + assert!(Arc::ptr_eq(&first, &make(&languages[0]))); + for language in languages.iter().skip(1) { + make(language); + } + assert_eq!(cache.len(), ShapePlanCache::CAPACITY); + assert!(!Arc::ptr_eq(&first, &make(&languages[0]))); + assert_eq!(cache.len(), ShapePlanCache::CAPACITY); +} From 83b240500fb113b484df2812b6b708566a9c72c0 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 18:16:45 +0000 Subject: [PATCH 02/11] Share font face data ownership --- crates/font-catalog/src/face_data.rs | 89 ++++++++++++++++++++++++++++ crates/font-catalog/src/lib.rs | 2 + 2 files changed, 91 insertions(+) create mode 100644 crates/font-catalog/src/face_data.rs diff --git a/crates/font-catalog/src/face_data.rs b/crates/font-catalog/src/face_data.rs new file mode 100644 index 000000000..655c2f408 --- /dev/null +++ b/crates/font-catalog/src/face_data.rs @@ -0,0 +1,89 @@ +//! Shared ownership of a font file and a face within it. + +use std::sync::Arc; + +/// Font file bytes and the index of a face in that file. +/// +/// Clones share owned bytes, so collection faces and variable instances do not copy the file. +/// Static bytes let applications use embedded fonts without allocating a copy. +#[derive(Clone, Debug)] +pub struct FaceData { + bytes: Bytes, + index: u32, +} + +#[derive(Clone, Debug)] +enum Bytes { + Static(&'static [u8]), + Owned(Arc>), + Shared(Arc<[u8]>), +} + +impl FaceData { + /// A face in statically embedded font bytes. + pub fn from_static(bytes: &'static [u8], index: u32) -> Self { + Self { bytes: Bytes::Static(bytes), index } + } + + /// A face in a shared vector, without copying its allocation. + pub fn from_owned(bytes: Arc>, index: u32) -> Self { + Self { bytes: Bytes::Owned(bytes), index } + } + + /// A face in bytes shared by a catalog. + pub fn from_shared(bytes: Arc<[u8]>, index: u32) -> Self { + Self { bytes: Bytes::Shared(bytes), index } + } + + /// The entire font file. + pub fn bytes(&self) -> &[u8] { + match &self.bytes { + Bytes::Static(bytes) => bytes, + Bytes::Owned(bytes) => bytes.as_slice(), + Bytes::Shared(bytes) => bytes, + } + } + + /// The index of this face in the file. + pub fn index(&self) -> u32 { + self.index + } + + /// Parse this face for font metadata and outlines. + pub fn skrifa(&self) -> Option> { + skrifa::FontRef::from_index(self.bytes(), self.index).ok() + } + + /// Parse this face for shaping. + pub fn harfrust(&self) -> Option> { + harfrust::FontRef::from_index(self.bytes(), self.index).ok() + } +} + +impl From for FaceData { + fn from(data: crate::FontData) -> Self { + Self::from_shared(data.bytes, data.index) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sources_keep_bytes_and_face_index_without_copying() { + static DATA: &[u8] = b"not a font"; + let owned = Arc::new(DATA.to_vec()); + let shared: Arc<[u8]> = Arc::from(DATA); + for data in [FaceData::from_static(DATA, 2), FaceData::from_owned(owned.clone(), 2), FaceData::from_shared(shared.clone(), 2)] { + let clone = data.clone(); + assert_eq!(clone.bytes(), DATA); + assert_eq!(clone.bytes().as_ptr(), data.bytes().as_ptr()); + assert_eq!(clone.index(), 2); + assert!(clone.skrifa().is_none()); + assert!(clone.harfrust().is_none()); + } + assert_eq!(Arc::strong_count(&owned), 1); + assert_eq!(Arc::strong_count(&shared), 1); + } +} diff --git a/crates/font-catalog/src/lib.rs b/crates/font-catalog/src/lib.rs index b4271db7e..909b9de9a 100644 --- a/crates/font-catalog/src/lib.rs +++ b/crates/font-catalog/src/lib.rs @@ -22,6 +22,7 @@ mod catalog; mod env; +mod face_data; mod fallback; mod fontconfig; mod ligature; @@ -38,6 +39,7 @@ mod visual_order; pub use catalog::{Catalog, FontData, FontFileError, Limits, Lister, LoadedFace, Match, Watch}; pub use env::{FontEnv, Os, font_dirs, font_store_dir, office_font_dirs}; +pub use face_data::FaceData; pub use fallback::{cluster_face, drawable_characters}; pub use fontconfig::fontconfig_dirs; pub use ligature::ligature_caret_stops; From 1864bb7d1de5001c99605b445537803a172a4cd7 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 17:44:43 -0500 Subject: [PATCH 03/11] raw: let the DNG writer borrow application image buffers Reuse existing writer and image layout validation rather than cloning RawImage samples or importing another wire writer. Metadata matrices are value copies; profile/opcode tables and pixel data remain borrowed. --- crates/raw/src/dngwrite.rs | 125 +++++++++++++++++++++++++++++++++---- crates/raw/src/lib.rs | 69 +++++++++++--------- 2 files changed, 154 insertions(+), 40 deletions(-) diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index ace0c07f6..b3424087f 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -119,8 +119,111 @@ fn rat_vec(v: &[f64]) -> Value { Value::Rational(v.iter().map(|&x| rational(x)).collect()) } +/// Colour tags copied by value, with potentially large profile tables borrowed. +pub struct DngColor<'a> { + pub illuminant: [u16; 2], + pub color_matrix: [Option; 2], + pub forward_matrix: [Option; 2], + pub camera_calibration: [Option; 2], + pub analog_balance: Option<[f64; 3]>, + pub as_shot_neutral: Option<[f64; 3]>, + pub as_shot_white_xy: Option<[f64; 2]>, + pub baseline_exposure: f64, + pub baseline_sharpness: Option, + pub profile: &'a crate::profile::ProfileLook, +} + +impl<'a> From<&'a crate::ColorData> for DngColor<'a> { + fn from(color: &'a crate::ColorData) -> Self { + Self { + illuminant: color.illuminant, + color_matrix: color.color_matrix, + forward_matrix: color.forward_matrix, + camera_calibration: color.camera_calibration, + analog_balance: color.analog_balance, + as_shot_neutral: color.as_shot_neutral, + as_shot_white_xy: color.as_shot_white_xy, + baseline_exposure: color.baseline_exposure, + baseline_sharpness: color.baseline_sharpness, + profile: &color.profile, + } + } +} + +/// Borrowed wire-format input for callers with their own application image type. +/// Pixel buffers and large profile/opcode tables are not copied. +pub struct DngImage<'a> { + pub width: usize, + pub height: usize, + pub cpp: usize, + pub data: &'a RawData, + pub cfa: Option<&'a crate::Cfa>, + pub bits: u32, + pub black: &'a crate::BlackLevel, + pub white: &'a [f32], + pub active_area: crate::Rect, + pub crop: crate::Rect, + pub orientation: crate::Orientation, + pub color: DngColor<'a>, + pub wb_multipliers: Option<[f32; 3]>, + pub opcodes: &'a crate::OpcodeLists, + pub make: Option<&'a str>, + pub model: Option<&'a str>, + pub cinema: &'a crate::CinemaTags, + /// Optional application-supplied UniqueCameraModel fallback when make/model are absent. + pub unknown_camera_model: Option<&'a str>, +} + +impl<'a> From<&'a RawImage> for DngImage<'a> { + fn from(raw: &'a RawImage) -> Self { + Self { + width: raw.width, + height: raw.height, + cpp: raw.cpp, + data: &raw.data, + cfa: raw.cfa.as_ref(), + bits: raw.bits, + black: &raw.black, + white: &raw.white, + active_area: raw.active_area, + crop: raw.crop, + orientation: raw.orientation, + color: DngColor::from(&raw.color), + wb_multipliers: raw.wb_multipliers, + opcodes: &raw.opcodes, + make: raw.camera.make.as_deref(), + model: raw.camera.model.as_deref(), + cinema: &raw.cinema, + unknown_camera_model: None, + } + } +} + +impl DngImage<'_> { + fn validate(&self) -> Result<()> { + crate::validate_image_layout(self.width, self.height, self.cpp, Some(self.data.len()), self.cfa, self.active_area)?; + if self.width > u32::MAX as usize || self.height > u32::MAX as usize { + return Err(RawError::Limit("DNG dimensions exceed TIFF fields")); + } + Ok(()) + } + + fn white_at(&self, sample: usize) -> f32 { + self.white.get(sample).or_else(|| self.white.first()).copied().unwrap_or(65535.0) + } +} + +/// Write an image without taking ownership of its pixels or application metadata. +pub fn write_dng_view(raw: &DngImage<'_>, opts: &DngWriteOptions) -> Result> { + write_image(raw, opts) +} + /// Serialise `raw` as a DNG. pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { + write_dng_view(&DngImage::from(raw), opts) +} + +fn write_image(raw: &DngImage<'_>, opts: &DngWriteOptions) -> Result> { raw.validate()?; if opts.software.trim() == "LightCraft" { return Err(RawError::Unsupported("the Software tag `LightCraft` marks files with a sensor-anchored CFAPattern".into())); @@ -137,7 +240,7 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { ifd.set(t::DNG_BACKWARD_VERSION, Value::Byte(vec![1, 3, 0, 0])); write_identity_tags(&mut ifd, raw, opts); - if let Some(cfa) = &raw.cfa { + if let Some(cfa) = raw.cfa { ifd.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![cfa.height as u16, cfa.width as u16])); // the tag is anchored at the ActiveArea's top-left, `raw.cfa` at the sensor origin let a = raw.active_area; @@ -145,7 +248,7 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { ifd.set(t::CFA_PLANE_COLOR, Value::Byte(vec![0, 1, 2])); ifd.set(t::CFA_LAYOUT, Value::Short(vec![1])); } - let b = &raw.black; + let b = raw.black; ifd.set(t::BLACK_LEVEL_REPEAT_DIM, Value::Short(vec![b.repeat_rows as u16, b.repeat_cols as u16])); let mut bl: Vec = b.values.iter().map(|&v| v as f64).collect(); if bl.len() == 1 && cpp > 1 { @@ -174,7 +277,7 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { write_profile_tags(&mut ifd, raw, opts.order); write_opcode_tags(&mut ifd, raw); - match (&raw.data, opts.compression) { + match (raw.data, opts.compression) { (RawData::F32(v), DngCompression::Deflate { tile, half }) => { let tile = deflate_tile(tile); let bp = if half { 2 } else { 4 }; @@ -311,9 +414,9 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { } /// Make, Model, UniqueCameraModel, Orientation, Software and the XMP packet. -fn write_identity_tags(ifd: &mut IfdBuilder, raw: &RawImage, opts: &DngWriteOptions) { - let make = raw.camera.make.as_deref().unwrap_or_default(); - let model = raw.camera.model.as_deref().unwrap_or_default(); +fn write_identity_tags(ifd: &mut IfdBuilder, raw: &DngImage<'_>, opts: &DngWriteOptions) { + let make = raw.make.unwrap_or_default(); + let model = raw.model.unwrap_or_default(); if !make.is_empty() { ifd.set(t::MAKE, Value::Ascii(make.to_owned())); } @@ -332,7 +435,7 @@ fn write_identity_tags(ifd: &mut IfdBuilder, raw: &RawImage, opts: &DngWriteOpti } else { format!("{make} {model}") }; - ifd.set(t::UNIQUE_CAMERA_MODEL, Value::Ascii(if unique.is_empty() { "Unknown".into() } else { unique })); + ifd.set(t::UNIQUE_CAMERA_MODEL, Value::Ascii(if unique.is_empty() { raw.unknown_camera_model.unwrap_or("Unknown").into() } else { unique })); ifd.set(t::ORIENTATION, Value::Short(vec![raw.orientation.to_exif()])); ifd.set(t::SOFTWARE, Value::Ascii(opts.software.clone())); if let Some(xmp) = &opts.xmp { @@ -341,7 +444,7 @@ fn write_identity_tags(ifd: &mut IfdBuilder, raw: &RawImage, opts: &DngWriteOpti } /// The colour tags (matrices, white balance, baseline exposure and sharpness) and the CinemaDNG frame tags. -fn write_color_tags(ifd: &mut IfdBuilder, raw: &RawImage) { +fn write_color_tags(ifd: &mut IfdBuilder, raw: &DngImage<'_>) { let c = &raw.color; for (i, tag) in [t::CALIBRATION_ILLUMINANT_1, t::CALIBRATION_ILLUMINANT_2].into_iter().enumerate() { if c.color_matrix[i].is_some() { @@ -385,9 +488,9 @@ fn write_color_tags(ifd: &mut IfdBuilder, raw: &RawImage) { } } -fn write_profile_tags(ifd: &mut IfdBuilder, raw: &RawImage, order: ByteOrder) { +fn write_profile_tags(ifd: &mut IfdBuilder, raw: &DngImage<'_>, order: ByteOrder) { // The source's own profile look travels with its data (conversions, smart previews, merges). - let profile = &raw.color.profile; + let profile = raw.color.profile; let table_tags = |ifd: &mut IfdBuilder, table: &crate::profile::HsvTable, dims: u16, enc: u16| { ifd.set(dims, Value::Long(vec![table.hue_divisions as u32, table.sat_divisions as u32, table.val_divisions as u32])); if table.srgb_value { @@ -425,7 +528,7 @@ fn write_profile_tags(ifd: &mut IfdBuilder, raw: &RawImage, order: ByteOrder) { } } -fn write_opcode_tags(ifd: &mut IfdBuilder, raw: &RawImage) { +fn write_opcode_tags(ifd: &mut IfdBuilder, raw: &DngImage<'_>) { for (list, tag) in [(&raw.opcodes.list1, t::OPCODE_LIST_1), (&raw.opcodes.list2, t::OPCODE_LIST_2), (&raw.opcodes.list3, t::OPCODE_LIST_3)] { if !list.is_empty() { ifd.set(tag, Value::Undefined(opcodes::write_list(list))); diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs index 4fa225364..c9f829e14 100644 --- a/crates/raw/src/lib.rs +++ b/crates/raw/src/lib.rs @@ -1063,6 +1063,45 @@ pub struct RawImage { pub notices: RawNotices, } +pub(crate) fn validate_image_layout( + width: usize, + height: usize, + cpp: usize, + data_len: Option, + cfa: Option<&Cfa>, + active_area: Rect, +) -> Result<()> { + if width == 0 || height == 0 || !(1..=4).contains(&cpp) { + return Err(RawError::Corrupt("bad dimensions".into())); + } + // two samples a pixel are neither one colour nor RGB: nothing downstream can develop them (storytold/lightcraft#1511) + if cpp == 2 { + return Err(RawError::Unsupported("2 samples per pixel".into())); + } + if let Some(data_len) = data_len { + let expected = crate::checked_pixel_count(width as u64, height as u64) + .and_then(|n| n.checked_mul(cpp as u64)) + .ok_or_else(|| RawError::Corrupt("data length mismatch".into()))?; + let actual = u64::try_from(data_len).map_err(|_| RawError::Corrupt("data length mismatch".into()))?; + if actual != expected { + return Err(RawError::Corrupt("data length mismatch".into())); + } + } + if let Some(c) = cfa + && (!c.valid() || c.pattern.iter().any(|&p| p > 2)) + { + return Err(RawError::Unsupported(format!("CFA pattern {}", c.name()))); + } + let a = active_area; + // checked: a crop origin from a maker note can be `u64::MAX`, and `x + width` would wrap + // back inside the image here and again in `normalized`'s index arithmetic. + let fits = |x: usize, len: usize, limit: usize| x.checked_add(len).is_some_and(|end| end <= limit); + if a.width == 0 || a.height == 0 || !fits(a.x, a.width, width) || !fits(a.y, a.height, height) { + return Err(RawError::Corrupt("active area outside image".into())); + } + Ok(()) +} + impl RawImage { /// The description without the samples (what [`probe_info`] returns for the same file). pub fn info(&self) -> RawInfo { @@ -1120,35 +1159,7 @@ impl RawImage { } fn check(&self, samples: bool) -> Result<()> { - if self.width == 0 || self.height == 0 || !(1..=4).contains(&self.cpp) { - return Err(RawError::Corrupt("bad dimensions".into())); - } - // two samples a pixel are neither one colour nor RGB: nothing downstream can develop them (storytold/lightcraft#1511) - if self.cpp == 2 { - return Err(RawError::Unsupported("2 samples per pixel".into())); - } - if samples { - let expected = crate::checked_pixel_count(self.width as u64, self.height as u64) - .and_then(|n| n.checked_mul(self.cpp as u64)) - .ok_or_else(|| RawError::Corrupt("data length mismatch".into()))?; - let actual = u64::try_from(self.data.len()).map_err(|_| RawError::Corrupt("data length mismatch".into()))?; - if actual != expected { - return Err(RawError::Corrupt("data length mismatch".into())); - } - } - if let Some(c) = &self.cfa - && (!c.valid() || c.pattern.iter().any(|&p| p > 2)) - { - return Err(RawError::Unsupported(format!("CFA pattern {}", c.name()))); - } - let a = self.active_area; - // checked: a crop origin from a maker note can be `u64::MAX`, and `x + width` would wrap - // back inside the image here and again in `normalized`'s index arithmetic. - let fits = |x: usize, len: usize, limit: usize| x.checked_add(len).is_some_and(|end| end <= limit); - if a.width == 0 || a.height == 0 || !fits(a.x, a.width, self.width) || !fits(a.y, a.height, self.height) { - return Err(RawError::Corrupt("active area outside image".into())); - } - Ok(()) + validate_image_layout(self.width, self.height, self.cpp, samples.then_some(self.data.len()), self.cfa.as_ref(), self.active_area) } /// White level for sample `s`. From 10be8769da666e90199893262448e5315398bc18 Mon Sep 17 00:00:00 2001 From: Andrew Frahn Date: Sun, 15 Nov 2026 17:44:43 -0500 Subject: [PATCH 04/11] raw: share prepared lossy DNG tile serialization Extract the format portion of LightCraft cc3eb1c8 (lossy DNG export) preserving tile/quality clamps and RGB edge replication. Application rendering/resize/transfer policy stays in the consumer. Callback ownership and fallible allocation integration by Dylan M. Taylor. --- crates/raw/src/dngwrite.rs | 314 ++++++++++++++++++++++--------------- 1 file changed, 191 insertions(+), 123 deletions(-) diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index b3424087f..30170fb28 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -215,7 +215,71 @@ impl DngImage<'_> { /// Write an image without taking ownership of its pixels or application metadata. pub fn write_dng_view(raw: &DngImage<'_>, opts: &DngWriteOptions) -> Result> { - write_image(raw, opts) + write_image(raw, opts, None) +} + +/// Encode prepared 8-bit RGB codes as lossy DNG JPEG tiles. Rendering, scaling and the +/// transfer curve are application policy; `linearization` supplies exactly 256 code values. +/// The callback receives one bounded RGB tile and must return its JPEG encoding or an error. +/// It can be invoked concurrently. The callback is a trusted encoder: it owns its output +/// allocation and JPEG-validity contract; this API does not impose an aggregate output budget. +/// Only the byte order, XMP and Software fields of `opts` +/// apply here; `tile` and `quality` select the compression instead of `opts.compression`. +pub fn write_lossy_dng( + raw: &DngImage<'_>, + opts: &DngWriteOptions, + tile: u32, + quality: u8, + linearization: &[u16], + encode: &(impl Fn(Vec, u32, u8) -> Result> + Sync), +) -> Result> { + write_image(raw, opts, Some(LossyTiles { tile, quality, linearization, encode })) +} + +type JpegTileEncoder<'a> = dyn Fn(Vec, u32, u8) -> Result> + Sync + 'a; +struct LossyTiles<'a> { + tile: u32, + quality: u8, + linearization: &'a [u16], + encode: &'a JpegTileEncoder<'a>, +} + +fn lossy_tile_edge(tile: u32) -> usize { + (tile.clamp(16, 4096) & !15) as usize +} + +fn set_lossy_tiles(ifd: &mut IfdBuilder, raw: &DngImage<'_>, options: LossyTiles<'_>) -> Result<()> { + let RawData::U16(samples) = raw.data else { return Err(RawError::Unsupported("lossy DNG requires prepared 8-bit RGB codes".into())) }; + if raw.cpp != 3 || raw.cfa.is_some() || options.linearization.len() != 256 || samples.iter().any(|&v| v > 255) { + return Err(RawError::Unsupported("lossy DNG requires prepared 8-bit RGB codes and a 256-entry linearization table".into())); + } + // Preserve the existing profile-export limit: <=4096² RGB bytes per worker, never + // a caller-supplied unbounded square. Existing codec output/worker budgets still apply. + let tile = lossy_tile_edge(options.tile); + let (across, down) = (raw.width.div_ceil(tile), raw.height.div_ceil(tile)); + let count = across.checked_mul(down).ok_or(RawError::Limit("too many DNG tiles"))?; + let mut tiles = Vec::new(); + tiles.try_reserve_exact(count).map_err(|_| RawError::Limit("DNG tile table allocation failed"))?; + tiles.resize_with(count, Vec::new); + tiles.par_iter_mut().enumerate().try_for_each(|(i, output)| -> Result<()> { + let (tx, ty) = (i % across * tile, i / across * tile); + let mut pixels = Vec::new(); + pixels.try_reserve_exact(tile * tile * 3).map_err(|_| RawError::Limit("DNG tile allocation failed"))?; + for y in 0..tile { + let sy = (ty + y).min(raw.height - 1); + for x in 0..tile { + let sx = (tx + x).min(raw.width - 1); + pixels.extend(samples[(sy * raw.width + sx) * 3..][..3].iter().map(|&v| v as u8)); + } + } + *output = (options.encode)(pixels, tile as u32, options.quality.clamp(1, 100))?; + Ok(()) + })?; + ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![8; 3])); + ifd.set(t::COMPRESSION, Value::Short(vec![compression::LOSSY_JPEG])); + ifd.set(t::LINEARIZATION_TABLE, Value::Short(options.linearization.to_vec())); + ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); + Ok(()) } /// Serialise `raw` as a DNG. @@ -223,7 +287,7 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { write_dng_view(&DngImage::from(raw), opts) } -fn write_image(raw: &DngImage<'_>, opts: &DngWriteOptions) -> Result> { +fn write_image(raw: &DngImage<'_>, opts: &DngWriteOptions, lossy: Option>) -> Result> { raw.validate()?; if opts.software.trim() == "LightCraft" { return Err(RawError::Unsupported("the Software tag `LightCraft` marks files with a sensor-anchored CFAPattern".into())); @@ -277,135 +341,139 @@ fn write_image(raw: &DngImage<'_>, opts: &DngWriteOptions) -> Result> { write_profile_tags(&mut ifd, raw, opts.order); write_opcode_tags(&mut ifd, raw); - match (raw.data, opts.compression) { - (RawData::F32(v), DngCompression::Deflate { tile, half }) => { - let tile = deflate_tile(tile); - let bp = if half { 2 } else { 4 }; - ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![(bp * 8) as u16; cpp])); - ifd.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp])); - ifd.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE])); - ifd.set(t::PREDICTOR, Value::Short(vec![3])); - let (ta, td) = (w.div_ceil(tile), h.div_ceil(tile)); - let tiles: Vec> = (0..ta * td) - .into_par_iter() - .map(|i| { - let (tx, ty) = (i % ta * tile, i / ta * tile); - let mut buf = Vec::with_capacity(tile * tile * cpp * bp); - let mut row = Vec::with_capacity(tile * cpp * bp); - for y in 0..tile { - let sy = (ty + y).min(h - 1); - row.clear(); - for x in 0..tile { - let sx = (tx + x).min(w - 1); - for s in 0..cpp { - let f = v[(sy * w + sx) * cpp + s]; - if half { - row.extend_from_slice(&f32_to_f16(f).to_be_bytes()); - } else { - row.extend_from_slice(&f.to_bits().to_be_bytes()); + if let Some(lossy) = lossy { + set_lossy_tiles(&mut ifd, raw, lossy)?; + } else { + match (raw.data, opts.compression) { + (RawData::F32(v), DngCompression::Deflate { tile, half }) => { + let tile = deflate_tile(tile); + let bp = if half { 2 } else { 4 }; + ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![(bp * 8) as u16; cpp])); + ifd.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp])); + ifd.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE])); + ifd.set(t::PREDICTOR, Value::Short(vec![3])); + let (ta, td) = (w.div_ceil(tile), h.div_ceil(tile)); + let tiles: Vec> = (0..ta * td) + .into_par_iter() + .map(|i| { + let (tx, ty) = (i % ta * tile, i / ta * tile); + let mut buf = Vec::with_capacity(tile * tile * cpp * bp); + let mut row = Vec::with_capacity(tile * cpp * bp); + for y in 0..tile { + let sy = (ty + y).min(h - 1); + row.clear(); + for x in 0..tile { + let sx = (tx + x).min(w - 1); + for s in 0..cpp { + let f = v[(sy * w + sx) * cpp + s]; + if half { + row.extend_from_slice(&f32_to_f16(f).to_be_bytes()); + } else { + row.extend_from_slice(&f.to_bits().to_be_bytes()); + } } } + buf.extend_from_slice(&float_predict(&row, tile * cpp, bp, cpp)); } - buf.extend_from_slice(&float_predict(&row, tile * cpp, bp, cpp)); - } - miniz_oxide::deflate::compress_to_vec_zlib(&buf, DEFLATE_LEVEL) - }) - .collect(); - ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); - } - (RawData::F32(v), _) => { - ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![32; cpp])); - ifd.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp])); - ifd.set(t::COMPRESSION, Value::Short(vec![compression::NONE])); - let mut bytes = Vec::with_capacity(v.len() * 4); - for &x in v { - let mut tmp = Vec::with_capacity(4); - opts.order.put_u32(&mut tmp, x.to_bits()); - bytes.extend_from_slice(&tmp); + miniz_oxide::deflate::compress_to_vec_zlib(&buf, DEFLATE_LEVEL) + }) + .collect(); + ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); } - ifd.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![bytes] }); - } - (RawData::U16(v), _) => { - // Dark frames may never reach the sensor's white level, which must still fit. - let maxv = u32::from(v.iter().copied().max().unwrap_or(0)).max(max_white); - let bits = (32 - maxv.leading_zeros()).max(raw.bits).clamp(2, 16) as u8; - ifd.set( - t::BITS_PER_SAMPLE, - Value::Short(vec![if matches!(opts.compression, DngCompression::Uncompressed) { 16 } else { bits as u16 }; cpp]), - ); - match opts.compression { - DngCompression::Uncompressed => { - ifd.set(t::COMPRESSION, Value::Short(vec![compression::NONE])); - let rows = (65536 / (w * cpp * 2)).clamp(1, h); - let strips = v - .chunks(rows * w * cpp) - .map(|s| { - let mut b = Vec::with_capacity(s.len() * 2); - s.iter().for_each(|&x| opts.order.put_u16(&mut b, x)); - b - }) - .collect(); - ifd.set_image(ImageData::Strips { rows_per_strip: rows as u32, strips }); + (RawData::F32(v), _) => { + ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![32; cpp])); + ifd.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp])); + ifd.set(t::COMPRESSION, Value::Short(vec![compression::NONE])); + let mut bytes = Vec::with_capacity(v.len() * 4); + for &x in v { + let mut tmp = Vec::with_capacity(4); + opts.order.put_u32(&mut tmp, x.to_bits()); + bytes.extend_from_slice(&tmp); } - DngCompression::Deflate { tile, .. } => { - let tile = deflate_tile(tile); - ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![16; cpp])); - ifd.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE])); - ifd.set(t::PREDICTOR, Value::Short(vec![2])); - let (ta, td) = (w.div_ceil(tile), h.div_ceil(tile)); - let tiles: Vec> = (0..ta * td) - .into_par_iter() - .map(|i| { - let (tx, ty) = (i % ta * tile, i / ta * tile); - let mut buf = Vec::with_capacity(tile * tile * cpp * 2); - let mut row = vec![0u16; tile * cpp]; - for y in 0..tile { - let sy = (ty + y).min(h - 1); - for x in 0..tile { - let sx = (tx + x).min(w - 1); - for s in 0..cpp { - row[x * cpp + s] = v[(sy * w + sx) * cpp + s]; - } - } - for i in (cpp..row.len()).rev() { - row[i] = row[i].wrapping_sub(row[i - cpp]); - } - row.iter().for_each(|&x| opts.order.put_u16(&mut buf, x)); - } - miniz_oxide::deflate::compress_to_vec_zlib(&buf, DEFLATE_LEVEL) - }) - .collect(); - ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); - } - DngCompression::Lj92 { tile } => { - let tile = (tile.max(2) & !1) as usize; - if tile > MAX_LJ92_TILE { - return Err(RawError::Limit("tile too large")); + ifd.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![bytes] }); + } + (RawData::U16(v), _) => { + // Dark frames may never reach the sensor's white level, which must still fit. + let maxv = u32::from(v.iter().copied().max().unwrap_or(0)).max(max_white); + let bits = (32 - maxv.leading_zeros()).max(raw.bits).clamp(2, 16) as u8; + ifd.set( + t::BITS_PER_SAMPLE, + Value::Short(vec![if matches!(opts.compression, DngCompression::Uncompressed) { 16 } else { bits as u16 }; cpp]), + ); + match opts.compression { + DngCompression::Uncompressed => { + ifd.set(t::COMPRESSION, Value::Short(vec![compression::NONE])); + let rows = (65536 / (w * cpp * 2)).clamp(1, h); + let strips = v + .chunks(rows * w * cpp) + .map(|s| { + let mut b = Vec::with_capacity(s.len() * 2); + s.iter().for_each(|&x| opts.order.put_u16(&mut b, x)); + b + }) + .collect(); + ifd.set_image(ImageData::Strips { rows_per_strip: rows as u32, strips }); } - ifd.set(t::COMPRESSION, Value::Short(vec![compression::JPEG])); - let (ta, td) = (w.div_ceil(tile), h.div_ceil(tile)); - let tiles: Vec> = (0..ta * td) - .into_par_iter() - .map(|i| { - let (tx, ty) = (i % ta * tile, i / ta * tile); - let mut buf = Vec::with_capacity(tile * tile * cpp); - for y in 0..tile { - let sy = (ty + y).min(h - 1); - for x in 0..tile { - let sx = (tx + x).min(w - 1); - for s in 0..cpp { - buf.push(v[(sy * w + sx) * cpp + s]); + DngCompression::Deflate { tile, .. } => { + let tile = deflate_tile(tile); + ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![16; cpp])); + ifd.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE])); + ifd.set(t::PREDICTOR, Value::Short(vec![2])); + let (ta, td) = (w.div_ceil(tile), h.div_ceil(tile)); + let tiles: Vec> = (0..ta * td) + .into_par_iter() + .map(|i| { + let (tx, ty) = (i % ta * tile, i / ta * tile); + let mut buf = Vec::with_capacity(tile * tile * cpp * 2); + let mut row = vec![0u16; tile * cpp]; + for y in 0..tile { + let sy = (ty + y).min(h - 1); + for x in 0..tile { + let sx = (tx + x).min(w - 1); + for s in 0..cpp { + row[x * cpp + s] = v[(sy * w + sx) * cpp + s]; + } + } + for i in (cpp..row.len()).rev() { + row[i] = row[i].wrapping_sub(row[i - cpp]); + } + row.iter().for_each(|&x| opts.order.put_u16(&mut buf, x)); + } + miniz_oxide::deflate::compress_to_vec_zlib(&buf, DEFLATE_LEVEL) + }) + .collect(); + ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); + } + DngCompression::Lj92 { tile } => { + let tile = (tile.max(2) & !1) as usize; + if tile > MAX_LJ92_TILE { + return Err(RawError::Limit("tile too large")); + } + ifd.set(t::COMPRESSION, Value::Short(vec![compression::JPEG])); + let (ta, td) = (w.div_ceil(tile), h.div_ceil(tile)); + let tiles: Vec> = (0..ta * td) + .into_par_iter() + .map(|i| { + let (tx, ty) = (i % ta * tile, i / ta * tile); + let mut buf = Vec::with_capacity(tile * tile * cpp); + for y in 0..tile { + let sy = (ty + y).min(h - 1); + for x in 0..tile { + let sx = (tx + x).min(w - 1); + for s in 0..cpp { + buf.push(v[(sy * w + sx) * cpp + s]); + } } } - } - if cpp == 1 { - ljpeg::encode(&buf, tile / 2, tile, 2, bits, 1, 0) - } else { - ljpeg::encode(&buf, tile, tile, cpp, bits, 1, 0) - } - }) - .collect(); - ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); + if cpp == 1 { + ljpeg::encode(&buf, tile / 2, tile, 2, bits, 1, 0) + } else { + ljpeg::encode(&buf, tile, tile, cpp, bits, 1, 0) + } + }) + .collect(); + ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); + } } } } From 61c1b28c5dc486dea1d1f24ff3ccf27acc417273 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 17:44:43 -0500 Subject: [PATCH 05/11] test(raw): compare borrowed DNG output with frozen writer and test lossy boundaries --- crates/raw/src/dngwrite.rs | 77 +++++ crates/raw/src/dngwrite_legacy_tests.rs | 435 ++++++++++++++++++++++++ 2 files changed, 512 insertions(+) create mode 100644 crates/raw/src/dngwrite_legacy_tests.rs diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index 30170fb28..c7caa0224 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -604,6 +604,10 @@ fn write_opcode_tags(ifd: &mut IfdBuilder, raw: &DngImage<'_>) { } } +#[cfg(test)] +#[path = "dngwrite_legacy_tests.rs"] +mod legacy; + #[cfg(test)] mod tests { use super::*; @@ -638,6 +642,79 @@ mod tests { } } + #[test] + fn borrowed_writer_preserves_owned_output() { + let mut raw = tiny_raw(); + for floating in [false, true] { + if floating { + raw.data = RawData::F32((0..64).map(|v| v as f32 / 64.0).collect()); + raw.white = vec![1.0]; + } + for order in [ByteOrder::Little, ByteOrder::Big] { + for (compression, old_compression) in [ + (DngCompression::Uncompressed, legacy::DngCompression::Uncompressed), + (DngCompression::Lj92 { tile: 16 }, legacy::DngCompression::Lj92 { tile: 16 }), + (DngCompression::Deflate { tile: 16, half: false }, legacy::DngCompression::Deflate { tile: 16, half: false }), + (DngCompression::Deflate { tile: 16, half: true }, legacy::DngCompression::Deflate { tile: 16, half: true }), + ] { + let xmp = Some("retained".to_string()); + let options = DngWriteOptions { compression, order, xmp: xmp.clone(), software: "Test writer".into() }; + let old_options = legacy::DngWriteOptions { compression: old_compression, order, xmp, software: "Test writer".into() }; + let expected = legacy::write_dng(&raw, &old_options).unwrap(); + assert_eq!(expected, write_dng_view(&DngImage::from(&raw), &options).unwrap()); + assert_eq!(expected, write_dng(&raw, &options).unwrap()); + } + } + } + } + + #[test] + fn lossy_tiles_preserve_prepared_codes_padding_and_tags() { + let mut raw = tiny_raw(); + raw.width = 2; + raw.height = 1; + raw.cpp = 3; + raw.bits = 8; + raw.data = RawData::U16(vec![1, 2, 3, 250, 251, 252]); + raw.active_area = Rect::new(0, 0, 2, 1); + raw.crop = raw.active_area; + let linearization: Vec = (0..256).map(|v| v * 257).collect(); + let called = std::sync::atomic::AtomicUsize::new(0); + let encoder = |pixels: Vec, tile: u32, quality: u8| { + called.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + assert_eq!((tile, quality), (16, 100)); + for row in pixels.chunks_exact(16 * 3) { + assert_eq!(&row[..3], &[1, 2, 3]); + assert!(row[3..].chunks_exact(3).all(|p| p == [250, 251, 252])); + } + let mut output = Vec::new(); + jpeg_encoder::Encoder::new(&mut output, quality) + .encode(&pixels, tile as u16, tile as u16, jpeg_encoder::ColorType::Rgb) + .map_err(|e| RawError::Corrupt(e.to_string()))?; + Ok(output) + }; + let bytes = write_lossy_dng(&DngImage::from(&raw), &DngWriteOptions::default(), 17, 255, &linearization, &encoder).unwrap(); + assert_eq!(called.load(std::sync::atomic::Ordering::Relaxed), 1); + let tiff = craft_tiff::Tiff::parse(&bytes).unwrap(); + let ifd = tiff.all_ifds()[0]; + assert_eq!(ifd.u16(t::COMPRESSION), Some(compression::LOSSY_JPEG)); + assert_eq!(ifd.u64s(t::LINEARIZATION_TABLE).unwrap(), linearization.iter().map(|&v| u64::from(v)).collect::>()); + assert_eq!((crate::decode(&bytes).unwrap().width, crate::decode(&bytes).unwrap().height), (2, 1)); + assert_eq!(lossy_tile_edge(u32::MAX), 4096); + assert_eq!(lossy_tile_edge(0), 16); + let failing = |_: Vec, _: u32, _: u8| Err(RawError::Limit("test encoder failure")); + assert!(matches!( + write_lossy_dng(&DngImage::from(&raw), &DngWriteOptions::default(), 16, 90, &linearization, &failing), + Err(RawError::Limit("test encoder failure")) + )); + let never = |_: Vec, _: u32, _: u8| -> Result> { panic!("malformed input reached encoder") }; + assert!(write_lossy_dng(&DngImage::from(&raw), &DngWriteOptions::default(), 16, 90, &linearization[..255], &never).is_err()); + raw.data = RawData::U16(vec![256; 6]); + assert!(write_lossy_dng(&DngImage::from(&raw), &DngWriteOptions::default(), 16, 90, &linearization, &never).is_err()); + raw.data = RawData::U16(vec![1; 5]); + assert!(write_lossy_dng(&DngImage::from(&raw), &DngWriteOptions::default(), 16, 90, &linearization, &never).is_err()); + } + #[test] fn oversized_lj92_padding_is_rejected() { let raw = tiny_raw(); diff --git a/crates/raw/src/dngwrite_legacy_tests.rs b/crates/raw/src/dngwrite_legacy_tests.rs new file mode 100644 index 000000000..f0eda677a --- /dev/null +++ b/crates/raw/src/dngwrite_legacy_tests.rs @@ -0,0 +1,435 @@ +// Frozen pre-extraction writer: test-only byte-parity oracle. +//! Writing DNG files from a [`RawImage`] (uncompressed strips, lossless-JPEG tiles or Deflate tiles), used for DNG +//! export and for bit-exact decoder round-trip tests. Lossy DNG and DNGs of rendered pictures start from a developed +//! image, so the develop pipeline builds those on top of this writer. The raw image is stored in IFD0 (allowed by DNG 1.7 §"File Structure"). + +use crate::{Matrix3, RawData, RawError, RawImage, Result, ljpeg, opcodes}; +use craft_tiff::tags::{self as t, compression, photometric}; +use craft_tiff::writer::{rational, srational}; +use craft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter, Value}; +use rayon::prelude::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DngCompression { + Uncompressed, + /// Lossless JPEG tiles of `tile × tile` pixels (`tile` even). CFA tiles are encoded as two interleaved + /// components of half the tile width (the common DNG layout), linear data with one component per sample. + /// Edges above 2048 (after even alignment) are rejected before padding the tile. + Lj92 { + tile: u32, + }, + /// Deflate (zlib) tiles of `tile × tile` pixels. Floating-point data is stored as 16-bit (`half`) or + /// 32-bit IEEE floats with the floating-point predictor (TIFF predictor 3, Adobe TIFF Technical Note 3); + /// integer data with horizontal differencing (predictor 2). 16-bit float is what HDR merges usually use. + Deflate { + tile: u32, + half: bool, + }, +} + +/// zlib level of Deflate tiles. On a 67 MP focus-stack merge (half floats) level 6 took 7× as long as level 1 +/// (3.9 s vs 0.55 s on 4 cores) to save 3 % of the file (266 vs 275 MB); 5–12 % on raw and rendered DNGs. +const DEFLATE_LEVEL: u8 = 1; + +/// Largest Deflate tile edge the writer pads to (`tile` is caller-supplied in [`DngCompression::Deflate`]): +/// square-tile buffers grow with the square, so an unclamped edge lets a request pad a tiny valid image into +/// a multi-gigabyte buffer and billions of edge-replicate iterations. 2048² samples (≤ 24 MB of f32) is far +/// above the 256 default and every real tile this writer makes. +const MAX_DEFLATE_TILE: u32 = 2048; + +// Match the existing padded Deflate edge budget. JPEG's 16-bit dimension limit alone would permit +// a 65534²-sample allocation even for a tiny source image. At most four validated channels make +// this bound at most 32 MiB of u16 padding per tile, before encoder scratch/output. +const MAX_LJ92_TILE: usize = MAX_DEFLATE_TILE as usize; + +/// `tile` aligned down to a multiple of 16 and clamped to [`MAX_DEFLATE_TILE`]. +fn deflate_tile(tile: u32) -> usize { + (tile.max(16) & !15).min(MAX_DEFLATE_TILE) as usize +} + +/// IEEE 754 binary32 → binary16 (round to nearest even; overflow → ±inf, NaN kept). +pub fn f32_to_f16(v: f32) -> u16 { + let b = v.to_bits(); + let sign = ((b >> 16) & 0x8000) as u16; + let exp = ((b >> 23) & 0xff) as i32; + let man = b & 0x7f_ffff; + if exp == 0xff { + return sign | 0x7c00 | if man != 0 { 0x200 } else { 0 }; + } + let e = exp - 127 + 15; + if e >= 0x1f { + return sign | 0x7c00; + } + if e <= 0 { + if e < -10 { + return sign; + } + // subnormal half + let m = man | 0x80_0000; + let shift = (14 - e) as u32; + let half = m >> shift; + let rem = m & ((1 << shift) - 1); + let mid = 1 << (shift - 1); + let r = if rem > mid || (rem == mid && half & 1 == 1) { half + 1 } else { half }; + return sign | r as u16; + } + let half = ((e as u32) << 10) | (man >> 13); + let rem = man & 0x1fff; + let r = if rem > 0x1000 || (rem == 0x1000 && half & 1 == 1) { half + 1 } else { half }; + sign | r as u16 +} + +/// Apply the floating-point predictor (inverse of `unpack::undo_float_predictor`) to one row of +/// `n` big-endian samples of `bytes_per` bytes: split into byte planes (most significant first), then +/// byte-difference with `stride` samples. +fn float_predict(row: &[u8], n: usize, bytes_per: usize, stride: usize) -> Vec { + let mut planes = vec![0u8; row.len()]; + for i in 0..n { + for b in 0..bytes_per { + planes[b * n + i] = row[i * bytes_per + b]; + } + } + for i in (stride..planes.len()).rev() { + planes[i] = planes[i].wrapping_sub(planes[i - stride]); + } + planes +} + +#[derive(Clone, Debug, PartialEq)] +pub struct DngWriteOptions { + pub compression: DngCompression, + pub order: ByteOrder, + /// XMP packet to embed (tag 700); none when `None`. + pub xmp: Option, + /// The `Software` tag: the writing application. It must not be the bare string `LightCraft`, which the reader + /// takes for that app's early files (their `CFAPattern` is anchored at the sensor origin, not the ActiveArea). + pub software: String, +} + +impl Default for DngWriteOptions { + fn default() -> Self { + Self { compression: DngCompression::Lj92 { tile: 256 }, order: ByteOrder::Little, xmp: None, software: "craft-raw".into() } + } +} + +fn srat_mat(m: &Matrix3) -> Value { + Value::SRational(m.iter().flatten().map(|&v| srational(v)).collect()) +} + +fn rat_vec(v: &[f64]) -> Value { + Value::Rational(v.iter().map(|&x| rational(x)).collect()) +} + +/// Serialise `raw` as a DNG. +pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { + raw.validate()?; + if opts.software.trim() == "LightCraft" { + return Err(RawError::Unsupported("the Software tag `LightCraft` marks files with a sensor-anchored CFAPattern".into())); + } + let (w, h, cpp) = (raw.width, raw.height, raw.cpp); + let mut ifd = IfdBuilder::new(); + ifd.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0])); + ifd.set(t::IMAGE_WIDTH, Value::Long(vec![w as u32])); + ifd.set(t::IMAGE_LENGTH, Value::Long(vec![h as u32])); + ifd.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![cpp as u16])); + ifd.set(t::PLANAR_CONFIGURATION, Value::Short(vec![1])); + ifd.set(t::PHOTOMETRIC, Value::Short(vec![if raw.cfa.is_some() { photometric::CFA } else { photometric::LINEAR_RAW }])); + ifd.set(t::DNG_VERSION, Value::Byte(vec![1, 4, 0, 0])); + ifd.set(t::DNG_BACKWARD_VERSION, Value::Byte(vec![1, 3, 0, 0])); + write_identity_tags(&mut ifd, raw, opts); + + if let Some(cfa) = &raw.cfa { + ifd.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![cfa.height as u16, cfa.width as u16])); + // the tag is anchored at the ActiveArea's top-left, `raw.cfa` at the sensor origin + let a = raw.active_area; + ifd.set(t::CFA_PATTERN_EP, Value::Byte(cfa.shifted(a.x, a.y).pattern)); + ifd.set(t::CFA_PLANE_COLOR, Value::Byte(vec![0, 1, 2])); + ifd.set(t::CFA_LAYOUT, Value::Short(vec![1])); + } + let b = &raw.black; + ifd.set(t::BLACK_LEVEL_REPEAT_DIM, Value::Short(vec![b.repeat_rows as u16, b.repeat_cols as u16])); + let mut bl: Vec = b.values.iter().map(|&v| v as f64).collect(); + if bl.len() == 1 && cpp > 1 { + bl = vec![bl[0]; cpp]; + } + ifd.set(t::BLACK_LEVEL, rat_vec(&bl)); + if !b.delta_h.is_empty() { + ifd.set(t::BLACK_LEVEL_DELTA_H, Value::SRational(b.delta_h.iter().map(|&v| srational(v as f64)).collect())); + } + if !b.delta_v.is_empty() { + ifd.set(t::BLACK_LEVEL_DELTA_V, Value::SRational(b.delta_v.iter().map(|&v| srational(v as f64)).collect())); + } + let white: Vec = (0..cpp).map(|s| raw.white_at(s).round().clamp(0.0, u32::MAX as f32) as u32).collect(); + let max_white = white.iter().copied().max().unwrap_or(0); + if matches!(raw.data, RawData::F32(_)) { + ifd.set(t::WHITE_LEVEL, rat_vec(&(0..cpp).map(|s| raw.white_at(s) as f64).collect::>())); + } else { + ifd.set(t::WHITE_LEVEL, Value::Long(white)); + } + let a = raw.active_area; + ifd.set(t::ACTIVE_AREA, Value::Long(vec![a.y as u32, a.x as u32, (a.y + a.height) as u32, (a.x + a.width) as u32])); + ifd.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![raw.crop.x as u32, raw.crop.y as u32])); + ifd.set(t::DEFAULT_CROP_SIZE, Value::Long(vec![raw.crop.width as u32, raw.crop.height as u32])); + + write_color_tags(&mut ifd, raw); + write_profile_tags(&mut ifd, raw, opts.order); + write_opcode_tags(&mut ifd, raw); + + match (&raw.data, opts.compression) { + (RawData::F32(v), DngCompression::Deflate { tile, half }) => { + let tile = deflate_tile(tile); + let bp = if half { 2 } else { 4 }; + ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![(bp * 8) as u16; cpp])); + ifd.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp])); + ifd.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE])); + ifd.set(t::PREDICTOR, Value::Short(vec![3])); + let (ta, td) = (w.div_ceil(tile), h.div_ceil(tile)); + let tiles: Vec> = (0..ta * td) + .into_par_iter() + .map(|i| { + let (tx, ty) = (i % ta * tile, i / ta * tile); + let mut buf = Vec::with_capacity(tile * tile * cpp * bp); + let mut row = Vec::with_capacity(tile * cpp * bp); + for y in 0..tile { + let sy = (ty + y).min(h - 1); + row.clear(); + for x in 0..tile { + let sx = (tx + x).min(w - 1); + for s in 0..cpp { + let f = v[(sy * w + sx) * cpp + s]; + if half { + row.extend_from_slice(&f32_to_f16(f).to_be_bytes()); + } else { + row.extend_from_slice(&f.to_bits().to_be_bytes()); + } + } + } + buf.extend_from_slice(&float_predict(&row, tile * cpp, bp, cpp)); + } + miniz_oxide::deflate::compress_to_vec_zlib(&buf, DEFLATE_LEVEL) + }) + .collect(); + ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); + } + (RawData::F32(v), _) => { + ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![32; cpp])); + ifd.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp])); + ifd.set(t::COMPRESSION, Value::Short(vec![compression::NONE])); + let mut bytes = Vec::with_capacity(v.len() * 4); + for &x in v { + let mut tmp = Vec::with_capacity(4); + opts.order.put_u32(&mut tmp, x.to_bits()); + bytes.extend_from_slice(&tmp); + } + ifd.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![bytes] }); + } + (RawData::U16(v), _) => { + // Dark frames may never reach the sensor's white level, which must still fit. + let maxv = u32::from(v.iter().copied().max().unwrap_or(0)).max(max_white); + let bits = (32 - maxv.leading_zeros()).max(raw.bits).clamp(2, 16) as u8; + ifd.set( + t::BITS_PER_SAMPLE, + Value::Short(vec![if matches!(opts.compression, DngCompression::Uncompressed) { 16 } else { bits as u16 }; cpp]), + ); + match opts.compression { + DngCompression::Uncompressed => { + ifd.set(t::COMPRESSION, Value::Short(vec![compression::NONE])); + let rows = (65536 / (w * cpp * 2)).clamp(1, h); + let strips = v + .chunks(rows * w * cpp) + .map(|s| { + let mut b = Vec::with_capacity(s.len() * 2); + s.iter().for_each(|&x| opts.order.put_u16(&mut b, x)); + b + }) + .collect(); + ifd.set_image(ImageData::Strips { rows_per_strip: rows as u32, strips }); + } + DngCompression::Deflate { tile, .. } => { + let tile = deflate_tile(tile); + ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![16; cpp])); + ifd.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE])); + ifd.set(t::PREDICTOR, Value::Short(vec![2])); + let (ta, td) = (w.div_ceil(tile), h.div_ceil(tile)); + let tiles: Vec> = (0..ta * td) + .into_par_iter() + .map(|i| { + let (tx, ty) = (i % ta * tile, i / ta * tile); + let mut buf = Vec::with_capacity(tile * tile * cpp * 2); + let mut row = vec![0u16; tile * cpp]; + for y in 0..tile { + let sy = (ty + y).min(h - 1); + for x in 0..tile { + let sx = (tx + x).min(w - 1); + for s in 0..cpp { + row[x * cpp + s] = v[(sy * w + sx) * cpp + s]; + } + } + for i in (cpp..row.len()).rev() { + row[i] = row[i].wrapping_sub(row[i - cpp]); + } + row.iter().for_each(|&x| opts.order.put_u16(&mut buf, x)); + } + miniz_oxide::deflate::compress_to_vec_zlib(&buf, DEFLATE_LEVEL) + }) + .collect(); + ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); + } + DngCompression::Lj92 { tile } => { + let tile = (tile.max(2) & !1) as usize; + if tile > MAX_LJ92_TILE { + return Err(RawError::Limit("tile too large")); + } + ifd.set(t::COMPRESSION, Value::Short(vec![compression::JPEG])); + let (ta, td) = (w.div_ceil(tile), h.div_ceil(tile)); + let tiles: Vec> = (0..ta * td) + .into_par_iter() + .map(|i| { + let (tx, ty) = (i % ta * tile, i / ta * tile); + let mut buf = Vec::with_capacity(tile * tile * cpp); + for y in 0..tile { + let sy = (ty + y).min(h - 1); + for x in 0..tile { + let sx = (tx + x).min(w - 1); + for s in 0..cpp { + buf.push(v[(sy * w + sx) * cpp + s]); + } + } + } + if cpp == 1 { + ljpeg::encode(&buf, tile / 2, tile, 2, bits, 1, 0) + } else { + ljpeg::encode(&buf, tile, tile, cpp, bits, 1, 0) + } + }) + .collect(); + ifd.set_image(ImageData::Tiles { tile_width: tile as u32, tile_height: tile as u32, tiles }); + } + } + } + } + Ok(TiffWriter::new(opts.order, false).write(&[ifd])?) +} + +/// Make, Model, UniqueCameraModel, Orientation, Software and the XMP packet. +fn write_identity_tags(ifd: &mut IfdBuilder, raw: &RawImage, opts: &DngWriteOptions) { + let make = raw.camera.make.as_deref().unwrap_or_default(); + let model = raw.camera.model.as_deref().unwrap_or_default(); + if !make.is_empty() { + ifd.set(t::MAKE, Value::Ascii(make.to_owned())); + } + if !model.is_empty() { + ifd.set(t::MODEL, Value::Ascii(model.to_owned())); + } + let model_includes_make = !make.is_empty() + && model.get(..make.len()).is_some_and(|prefix| prefix.eq_ignore_ascii_case(make)) + && model.get(make.len()..).is_some_and(|suffix| suffix.chars().next().is_none_or(|c| !c.is_alphanumeric() && c != '_')); + let unique = if let Some(unique) = raw.camera.unique_model.as_deref().filter(|u| !u.is_empty()) { + unique.to_owned() + } else if model_includes_make || make.is_empty() { + model.to_owned() + } else if model.is_empty() { + make.to_owned() + } else { + format!("{make} {model}") + }; + ifd.set(t::UNIQUE_CAMERA_MODEL, Value::Ascii(if unique.is_empty() { "Unknown".into() } else { unique })); + ifd.set(t::ORIENTATION, Value::Short(vec![raw.orientation.to_exif()])); + ifd.set(t::SOFTWARE, Value::Ascii(opts.software.clone())); + if let Some(xmp) = &opts.xmp { + ifd.set(t::XMP, Value::Byte(xmp.clone().into_bytes())); + } +} + +/// The colour tags (matrices, white balance, baseline exposure and sharpness) and the CinemaDNG frame tags. +fn write_color_tags(ifd: &mut IfdBuilder, raw: &RawImage) { + let c = &raw.color; + for (i, tag) in [t::CALIBRATION_ILLUMINANT_1, t::CALIBRATION_ILLUMINANT_2].into_iter().enumerate() { + if c.color_matrix[i].is_some() { + ifd.set(tag, Value::Short(vec![c.illuminant[i]])); + } + } + for (pair, tags) in [ + (&c.color_matrix, [t::COLOR_MATRIX_1, t::COLOR_MATRIX_2]), + (&c.forward_matrix, [t::FORWARD_MATRIX_1, t::FORWARD_MATRIX_2]), + (&c.camera_calibration, [t::CAMERA_CALIBRATION_1, t::CAMERA_CALIBRATION_2]), + ] { + for i in 0..2 { + if let Some(m) = &pair[i] { + ifd.set(tags[i], srat_mat(m)); + } + } + } + if let Some(v) = c.analog_balance { + ifd.set(t::ANALOG_BALANCE, rat_vec(&v)); + } + if let Some(v) = c.as_shot_neutral { + ifd.set(t::AS_SHOT_NEUTRAL, rat_vec(&v)); + } else if let Some(xy) = c.as_shot_white_xy { + ifd.set(t::AS_SHOT_WHITE_XY, rat_vec(&xy)); + } else if let Some(m) = raw.wb_multipliers.filter(|m| m.iter().all(|v| *v > 0.0 && v.is_finite())) { + // vendor raws (NEF, ARW…) carry white balance as multipliers: the DNG neutral is their reciprocal + ifd.set(t::AS_SHOT_NEUTRAL, rat_vec(&m.map(|v| 1.0 / v as f64))); + } + ifd.set(t::BASELINE_EXPOSURE, Value::SRational(vec![srational(c.baseline_exposure)])); + if let Some(v) = c.baseline_sharpness { + ifd.set(t::BASELINE_SHARPNESS, rat_vec(&[v])); + } + if let Some((n, d)) = raw.cinema.frame_rate.and_then(|(n, d)| Some((i32::try_from(n).ok()?, i32::try_from(d).ok()?))) { + ifd.set(51044, Value::SRational(vec![(n, d)])); + } + if let Some(tc) = raw.cinema.timecode { + ifd.set(51043, Value::Byte(tc.to_bytes().to_vec())); + } + if let Some(reel) = raw.cinema.reel_name.as_deref().filter(|r| !r.is_empty()) { + ifd.set(51081, Value::Ascii(reel.to_owned())); + } +} + +fn write_profile_tags(ifd: &mut IfdBuilder, raw: &RawImage, order: ByteOrder) { + // The source's own profile look travels with its data (conversions, smart previews, merges). + let profile = &raw.color.profile; + let table_tags = |ifd: &mut IfdBuilder, table: &crate::profile::HsvTable, dims: u16, enc: u16| { + ifd.set(dims, Value::Long(vec![table.hue_divisions as u32, table.sat_divisions as u32, table.val_divisions as u32])); + if table.srgb_value { + ifd.set(enc, Value::Long(vec![1])); + } + }; + let flat = |table: &crate::profile::HsvTable| Value::Float(table.data.iter().flatten().copied().collect()); + match &profile.hue_sat_map { + [Some(first), second] => { + table_tags(ifd, first, t::PROFILE_HUE_SAT_MAP_DIMS, t::PROFILE_HUE_SAT_MAP_ENCODING); + ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_1, flat(first)); + if let Some(second) = second.as_ref().filter(|second| { + (second.hue_divisions, second.sat_divisions, second.val_divisions) == (first.hue_divisions, first.sat_divisions, first.val_divisions) + }) { + ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_2, flat(second)); + } + } + [None, Some(table)] => { + table_tags(ifd, table, t::PROFILE_HUE_SAT_MAP_DIMS, t::PROFILE_HUE_SAT_MAP_ENCODING); + ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_1, flat(table)); + } + [None, None] => {} + } + if let Some(table) = &profile.look_table { + table_tags(ifd, table, t::PROFILE_LOOK_TABLE_DIMS, t::PROFILE_LOOK_TABLE_ENCODING); + ifd.set(t::PROFILE_LOOK_TABLE_DATA, flat(table)); + } + if let Some(curve) = &profile.tone_curve { + ifd.set(t::PROFILE_TONE_CURVE, Value::Float(curve.points.iter().flatten().copied().collect())); + } + if let Some(map) = &profile.gain_table_map { + // The raw IFD is IFD 0 here, where both tag versions are valid. + let (version2, bytes) = map.to_bytes(order); + ifd.set(if version2 { t::PROFILE_GAIN_TABLE_MAP_2 } else { t::PROFILE_GAIN_TABLE_MAP }, Value::Undefined(bytes)); + } +} + +fn write_opcode_tags(ifd: &mut IfdBuilder, raw: &RawImage) { + for (list, tag) in [(&raw.opcodes.list1, t::OPCODE_LIST_1), (&raw.opcodes.list2, t::OPCODE_LIST_2), (&raw.opcodes.list3, t::OPCODE_LIST_3)] { + if !list.is_empty() { + ifd.set(tag, Value::Undefined(opcodes::write_list(list))); + } + } +} From fa7d332cf4cd123728171706a36c5f0094cb0271 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 17:50:14 -0500 Subject: [PATCH 06/11] raw: preserve borrowed unique-camera-model identity --- crates/raw/src/dngwrite.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index c7caa0224..47c994b66 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -169,6 +169,7 @@ pub struct DngImage<'a> { pub opcodes: &'a crate::OpcodeLists, pub make: Option<&'a str>, pub model: Option<&'a str>, + pub unique_model: Option<&'a str>, pub cinema: &'a crate::CinemaTags, /// Optional application-supplied UniqueCameraModel fallback when make/model are absent. pub unknown_camera_model: Option<&'a str>, @@ -193,6 +194,7 @@ impl<'a> From<&'a RawImage> for DngImage<'a> { opcodes: &raw.opcodes, make: raw.camera.make.as_deref(), model: raw.camera.model.as_deref(), + unique_model: raw.camera.unique_model.as_deref(), cinema: &raw.cinema, unknown_camera_model: None, } @@ -494,7 +496,7 @@ fn write_identity_tags(ifd: &mut IfdBuilder, raw: &DngImage<'_>, opts: &DngWrite let model_includes_make = !make.is_empty() && model.get(..make.len()).is_some_and(|prefix| prefix.eq_ignore_ascii_case(make)) && model.get(make.len()..).is_some_and(|suffix| suffix.chars().next().is_none_or(|c| !c.is_alphanumeric() && c != '_')); - let unique = if let Some(unique) = raw.camera.unique_model.as_deref().filter(|u| !u.is_empty()) { + let unique = if let Some(unique) = raw.unique_model.filter(|u| !u.is_empty()) { unique.to_owned() } else if model_includes_make || make.is_empty() { model.to_owned() @@ -645,6 +647,7 @@ mod tests { #[test] fn borrowed_writer_preserves_owned_output() { let mut raw = tiny_raw(); + raw.camera.unique_model = Some("Exact camera model".into()); for floating in [false, true] { if floating { raw.data = RawData::F32((0..64).map(|v| v as f32 / 64.0).collect()); From 37bb0cff8d4d0424ac6a9d1999ca1f9b6d089193 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 18:06:34 -0500 Subject: [PATCH 07/11] test(raw): use fixed-size chunks in DNG padding assertions --- crates/raw/src/dngwrite.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index 47c994b66..4fef4ab43 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -686,9 +686,9 @@ mod tests { let encoder = |pixels: Vec, tile: u32, quality: u8| { called.fetch_add(1, std::sync::atomic::Ordering::Relaxed); assert_eq!((tile, quality), (16, 100)); - for row in pixels.chunks_exact(16 * 3) { + for row in pixels.as_chunks::<{ 16 * 3 }>().0 { assert_eq!(&row[..3], &[1, 2, 3]); - assert!(row[3..].chunks_exact(3).all(|p| p == [250, 251, 252])); + assert!(row[3..].as_chunks::<3>().0.iter().all(|p| *p == [250, 251, 252])); } let mut output = Vec::new(); jpeg_encoder::Encoder::new(&mut output, quality) From 159d7134d0c7cb45903864172122263353310401 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:13:19 +0100 Subject: [PATCH 08/11] fix(image-codecs): read the JPEGs zune-jpeg panics on with craft-jpeg zune-jpeg 0.5 asserts in its upsampler on some subsampled four-component frames whose first component isn't the largest. decode now catches that panic and reads the file with craft-jpeg, as it reads arithmetic-coded and 12-bit files. Refs #778. --- crates/image-codecs/src/codecs/jpeg.rs | 51 ++++++++++++++---------- crates/image-codecs/tests/it/png_jpeg.rs | 35 ++++++++++++++++ 2 files changed, 66 insertions(+), 20 deletions(-) diff --git a/crates/image-codecs/src/codecs/jpeg.rs b/crates/image-codecs/src/codecs/jpeg.rs index 15346ea56..7ec85e562 100644 --- a/crates/image-codecs/src/codecs/jpeg.rs +++ b/crates/image-codecs/src/codecs/jpeg.rs @@ -389,6 +389,30 @@ fn decode_craft_jpeg(bytes: &[u8], limits: &Limits, adobe_transform: Option, /// truncated or not, are far above this. const MAX_PIXELS_PER_BYTE: u64 = 2048; +/// Decode with craft-jpeg: the files [`uses_craft_jpeg`] routes away from `zune-jpeg`, and the +/// ones `zune-jpeg` panics on. +fn decode_whole(bytes: &[u8], limits: &Limits, meta: JpegMeta, end: DataEnd, scan_bytes: u64) -> Result { + // craft-jpeg sizes its buffers from the frame header before it reads a scan. + if scan_bytes == 0 { + return Err(err("the file has no scan data")); + } + if let Some((w, h, _)) = meta.frame + && end == DataEnd::Truncated + // Only truncated scans: arithmetic coding (SOF9–SOF11) can spend well under a bit on + // a flat block, so a complete scan may be shorter than this bound. + && u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes + { + return Err(err("the truncated scan is too short for the frame it declares")); + } + let (mut img, damaged) = decode_craft_jpeg(bytes, limits, meta.adobe_transform, meta.adobe_marker)?; + img.icc = meta.icc; + img.meta = Metadata { exif: meta.exif, xmp: meta.xmp, dpi: meta.dpi, photoshop_resources: meta.photoshop, ..Default::default() }; + if end == DataEnd::Truncated || damaged { + img.warnings.push(DecodeWarning::Truncated { format: F }); + } + Ok(img) +} + pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result { let (end, scan_bytes) = data_end(bytes); if end == DataEnd::Empty { @@ -399,25 +423,7 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result limits.check_bytes(w, h, u64::from(nc.max(1)))?; } if uses_craft_jpeg(bytes) { - // craft-jpeg sizes its buffers from the frame header before it reads a scan. - if scan_bytes == 0 { - return Err(err("the file has no scan data")); - } - if let Some((w, h, _)) = meta.frame - && end == DataEnd::Truncated - // Only truncated scans: arithmetic coding (SOF9–SOF11) can spend well under a bit on - // a flat block, so a complete scan may be shorter than this bound. - && u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes - { - return Err(err("the truncated scan is too short for the frame it declares")); - } - let (mut img, damaged) = decode_craft_jpeg(bytes, limits, meta.adobe_transform, meta.adobe_marker)?; - img.icc = meta.icc; - img.meta = Metadata { exif: meta.exif, xmp: meta.xmp, dpi: meta.dpi, photoshop_resources: meta.photoshop, ..Default::default() }; - if end == DataEnd::Truncated || damaged { - img.warnings.push(DecodeWarning::Truncated { format: F }); - } - return Ok(img); + return decode_whole(bytes, limits, meta, end, scan_bytes); } let options = DecoderOptions::default().set_strict_mode(false).set_max_width(65535).set_max_height(65535); let stripped = without_extended_xmp(bytes); @@ -443,7 +449,12 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result return Err(err(format!("{what} is too short for the frame it declares"))); } dec.set_options(dec.options().jpeg_set_out_colorspace(out_cs)); - let mut px = dec.decode().map_err(err)?; + // zune-jpeg 0.5 asserts on some subsampled four-component frames whose first component isn't + // the largest (its upsampler's scratch row is sized from that component): craft-jpeg reads them. + let Ok(decoded) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| dec.decode())) else { + return decode_whole(bytes, limits, meta, end, scan_bytes); + }; + let mut px = decoded.map_err(err)?; let expected = w as usize * h as usize * layout.channels(); if px.len() < expected { return Err(err("short pixel buffer")); diff --git a/crates/image-codecs/tests/it/png_jpeg.rs b/crates/image-codecs/tests/it/png_jpeg.rs index 07c57bf24..98a134121 100644 --- a/crates/image-codecs/tests/it/png_jpeg.rs +++ b/crates/image-codecs/tests/it/png_jpeg.rs @@ -759,3 +759,38 @@ fn bounded_png_stream_preserves_late_exif_across_metadata_yields() { assert_eq!(extras.orientation, 6); assert_eq!(extras.meta.text.len(), 150); } + +/// zune-jpeg 0.5 asserts on some subsampled CMYK frames whose first component isn't the largest +/// (jpeg-encoder writes C, M and Y at 1×1 and K larger); craft-jpeg reads those instead, and they +/// decode to the inks they were written with. The others of these files read as before. +#[test] +fn cmyk_frames_zune_jpeg_panics_on_decode() { + use jpeg_encoder::{ColorType, Encoder, SamplingFactor as S}; + let mut seen = 0; + for (s, progressive) in + [(S::F_2_2, true), (S::R_4_2_0, true), (S::F_4_1, false), (S::F_4_2, false), (S::F_2_4, false), (S::R_4_1_1, false), (S::R_4_1_0, false)] + { + for (w, h) in [(1u16, 1u16), (9, 1), (1, 9), (17, 13), (40, 40)] { + let inks: Vec = (0..usize::from(w) * usize::from(h)).flat_map(|_| [30u8, 90, 150, 210]).collect(); + let mut out = Vec::new(); + let mut e = Encoder::new(&mut out, 100); + e.set_sampling_factor(s); + e.set_progressive(progressive); + e.encode(&inks, w, h, ColorType::Cmyk).unwrap(); + let zune = std::panic::catch_unwind(|| { + let mut d = zune_jpeg::JpegDecoder::new(zune_core::bytestream::ZCursor::new(&out[..])); + d.decode().is_ok() + }); + if zune.is_ok() { + // Read or refused by zune-jpeg as before; no panic either way. + let _ = decode(&out); + continue; + } + seen += 1; + let img = decode(&out).unwrap_or_else(|e| panic!("{s:?} {w}x{h}: {e}")); + assert_eq!((img.dimensions(), img.layout()), ((u32::from(w), u32::from(h)), ChannelLayout::Cmyk)); + assert!(img.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 3), "{s:?} progressive {progressive} {w}x{h}"); + } + } + assert!(seen >= 10, "zune-jpeg panicked on {seen} of them"); +} From 58d6844dadee792e0d6ad845987b17c6ac8855ce Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:58:21 +0100 Subject: [PATCH 09/11] raw: carry the linearization table through DngImage views Main's writer (#1745) reads linearized/linearization_table from the image; the borrowed view needs the same fields so the merge compiles. --- crates/raw/src/dngwrite.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index b2eba18a3..2b8061949 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -173,6 +173,11 @@ pub struct DngImage<'a> { pub cinema: &'a crate::CinemaTags, /// Optional application-supplied UniqueCameraModel fallback when make/model are absent. pub unknown_camera_model: Option<&'a str>, + /// Whether `data` has already been mapped through `linearization_table` (see + /// [`RawImage::linearized`]). + pub linearized: bool, + /// A LinearizationTable still to be applied by the reader; written only when `linearized` is false. + pub linearization_table: Option<&'a [u16]>, } impl<'a> From<&'a RawImage> for DngImage<'a> { @@ -197,6 +202,8 @@ impl<'a> From<&'a RawImage> for DngImage<'a> { unique_model: raw.camera.unique_model.as_deref(), cinema: &raw.cinema, unknown_camera_model: None, + linearized: raw.linearized, + linearization_table: raw.linearization_table.as_deref(), } } } @@ -338,7 +345,7 @@ fn write_image(raw: &DngImage<'_>, opts: &DngWriteOptions, lossy: Option Date: Mon, 16 Nov 2026 02:17:34 +0100 Subject: [PATCH 10/11] tga, dpx: test the RGBA8 encoders' allocation-failure path Follow-up to #1631 (CodeRabbit): the fallible output reservation moves into a small helper whose OutOfMemory mapping is covered by a test. --- crates/dpx/src/rgba8_dpx10.rs | 15 +++++++++++++-- crates/tga/src/rgba8_raw.rs | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/crates/dpx/src/rgba8_dpx10.rs b/crates/dpx/src/rgba8_dpx10.rs index 6554a1eb8..eabab2ed0 100644 --- a/crates/dpx/src/rgba8_dpx10.rs +++ b/crates/dpx/src/rgba8_dpx10.rs @@ -9,6 +9,13 @@ pub enum Rgba8EncodeError { /// The output buffer could not be allocated. OutOfMemory, } +/// An empty buffer with room for `len` bytes, or [`Rgba8EncodeError::OutOfMemory`] when the +/// allocation fails (instead of aborting the process). +fn output_buffer(len: usize) -> std::result::Result, Rgba8EncodeError> { + let mut out = Vec::new(); + out.try_reserve_exact(len).map_err(|_| Rgba8EncodeError::OutOfMemory)?; + Ok(out) +} fn checked_len(rgba: &[u8], w: u32, h: u32) -> std::result::Result { let n = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(4)).ok_or(Rgba8EncodeError::FrameSize)?; if w == 0 || h == 0 || rgba.len() != n { @@ -86,8 +93,7 @@ pub fn encode_rgba8_dpx10(rgba: &[u8], w: u32, h: u32, creator: &str) -> std::re hd[1928] = 0; // The caller already holds the frame; reserve the output fallibly so a large export reports // an error instead of aborting the process. - let mut out = Vec::new(); - out.try_reserve_exact(size).map_err(|_| Rgba8EncodeError::OutOfMemory)?; + let mut out = output_buffer(size)?; out.extend_from_slice(&hd); let ten = |v: u8| (u32::from(v) << 2) | (u32::from(v) >> 6); for &[r, g, b, _] in rgba.as_chunks::<4>().0 { @@ -107,4 +113,9 @@ mod tests { assert_eq!(&bytes[2048..], &((1023u32 << 22) | (514u32 << 2)).to_be_bytes()); assert_eq!(encode_rgba8_dpx10(&[], 0, 1, ""), Err(Rgba8EncodeError::FrameSize)); } + #[test] + fn a_failed_output_allocation_is_an_error() { + assert_eq!(output_buffer(usize::MAX), Err(Rgba8EncodeError::OutOfMemory)); + assert!(output_buffer(16).is_ok_and(|b| b.is_empty() && b.capacity() >= 16)); + } } diff --git a/crates/tga/src/rgba8_raw.rs b/crates/tga/src/rgba8_raw.rs index 40cc256c7..a671462c9 100644 --- a/crates/tga/src/rgba8_raw.rs +++ b/crates/tga/src/rgba8_raw.rs @@ -9,6 +9,13 @@ pub enum Rgba8EncodeError { /// The output buffer could not be allocated. OutOfMemory, } +/// An empty buffer with room for `len` bytes, or [`Rgba8EncodeError::OutOfMemory`] when the +/// allocation fails (instead of aborting the process). +fn output_buffer(len: usize) -> std::result::Result, Rgba8EncodeError> { + let mut out = Vec::new(); + out.try_reserve_exact(len).map_err(|_| Rgba8EncodeError::OutOfMemory)?; + Ok(out) +} fn checked_len(rgba: &[u8], w: u32, h: u32) -> std::result::Result { let n = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(4)).ok_or(Rgba8EncodeError::FrameSize)?; if w == 0 || h == 0 || rgba.len() != n { @@ -29,8 +36,7 @@ pub fn encode_rgba8_raw(rgba: &[u8], w: u32, h: u32, alpha: bool) -> std::result // The caller already holds the frame; reserve the second, output-sized buffer fallibly so a // large export reports an error instead of aborting the process. let len = (w as usize * h as usize).checked_mul(bpp).and_then(|n| n.checked_add(18 + 26)).ok_or(Rgba8EncodeError::FrameSize)?; - let mut out = Vec::new(); - out.try_reserve_exact(len).map_err(|_| Rgba8EncodeError::OutOfMemory)?; + let mut out = output_buffer(len)?; out.extend_from_slice(&[0, 0, 2]); // no ID, no colour map, uncompressed true-colour out.extend_from_slice(&[0; 5]); // colour map specification (unused) out.extend_from_slice(&[0, 0, 0, 0]); // x / y origin @@ -63,4 +69,9 @@ mod tests { assert_eq!(&bytes[bytes.len() - 18..], b"TRUEVISION-XFILE.\0"); assert_eq!(encode_rgba8_raw(&[], 0, 1, false), Err(Rgba8EncodeError::FrameSize)); } + #[test] + fn a_failed_output_allocation_is_an_error() { + assert_eq!(output_buffer(usize::MAX), Err(Rgba8EncodeError::OutOfMemory)); + assert!(output_buffer(16).is_ok_and(|b| b.is_empty() && b.capacity() >= 16)); + } } From 831c080ac579236a62b7f9e4aac8f6efe39bc349 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:42:09 +0100 Subject: [PATCH 11/11] dpx: write the reservation helper and its test independently of tga's --- crates/dpx/src/rgba8_dpx10.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/dpx/src/rgba8_dpx10.rs b/crates/dpx/src/rgba8_dpx10.rs index eabab2ed0..8dec304ec 100644 --- a/crates/dpx/src/rgba8_dpx10.rs +++ b/crates/dpx/src/rgba8_dpx10.rs @@ -9,12 +9,13 @@ pub enum Rgba8EncodeError { /// The output buffer could not be allocated. OutOfMemory, } -/// An empty buffer with room for `len` bytes, or [`Rgba8EncodeError::OutOfMemory`] when the -/// allocation fails (instead of aborting the process). -fn output_buffer(len: usize) -> std::result::Result, Rgba8EncodeError> { - let mut out = Vec::new(); - out.try_reserve_exact(len).map_err(|_| Rgba8EncodeError::OutOfMemory)?; - Ok(out) +/// Reserves the DPX output up front; a failed reservation becomes [`Rgba8EncodeError::OutOfMemory`]. +fn output_buffer(size: usize) -> std::result::Result, Rgba8EncodeError> { + let mut buf = Vec::new(); + match buf.try_reserve_exact(size) { + Ok(()) => Ok(buf), + Err(_) => Err(Rgba8EncodeError::OutOfMemory), + } } fn checked_len(rgba: &[u8], w: u32, h: u32) -> std::result::Result { let n = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(4)).ok_or(Rgba8EncodeError::FrameSize)?; @@ -114,8 +115,8 @@ mod tests { assert_eq!(encode_rgba8_dpx10(&[], 0, 1, ""), Err(Rgba8EncodeError::FrameSize)); } #[test] - fn a_failed_output_allocation_is_an_error() { - assert_eq!(output_buffer(usize::MAX), Err(Rgba8EncodeError::OutOfMemory)); - assert!(output_buffer(16).is_ok_and(|b| b.is_empty() && b.capacity() >= 16)); + fn output_reservation_failure_maps_to_out_of_memory() { + assert!(matches!(output_buffer(usize::MAX), Err(Rgba8EncodeError::OutOfMemory))); + assert_eq!(output_buffer(DPX_HEADER).map(|b| b.capacity() >= DPX_HEADER), Ok(true)); } }