diff --git a/crates/metadata/src/cr3.rs b/crates/metadata/src/cr3.rs index a7538e88b..4ae189a80 100644 --- a/crates/metadata/src/cr3.rs +++ b/crates/metadata/src/cr3.rs @@ -351,28 +351,70 @@ fn trak(bytes: &[u8], t: &BoxRef, budget: &mut usize) -> Option { } } let stbl = stbl?; - let (mut kind, mut size, mut offset) = (None, None, None); + let (mut kind, mut size, mut offset, mut mapping, mut sample_count) = (None, None, None, None, 0); for b in boxes(bytes, stbl.body, stbl.end, budget) { let body = bytes.get(b.body..b.end)?; match &b.kind { // full box: version/flags, entry count, entries b"stsd" if be32(body, 4)? > 0 => kind = boxes(bytes, b.body + 8, b.end, budget).first().map(|e| sample_entry(bytes, e, budget)), + b"stsc" => { + // Only the first sample is exposed. It must map chunk 1 to + // the first sample description; don't trust offsets without + // that mapping, or a damaged/raw track could become a JPEG. + // A malformed mapping invalidates sample data, not the track's + // identity: dropping the largest raw track could select a reduced + // raw successfully and disguise corruption in the main image. + mapping = (|| { + let count = be32(body, 4)? as usize; + if be32(body, 0)? != 0 || count == 0 || count > body.len().saturating_sub(8) / 12 { + return None; + } + Some((be32(body, 8)?, be32(body, 12)?, be32(body, 16)?)) + })(); + } // version/flags, sample_size (0 = per-sample table), count, sizes b"stsz" => { - size = match be32(body, 4)? { - 0 if be32(body, 8)? > 0 => be32(body, 12), - 0 => None, - n if be32(body, 8)? > 0 => Some(n), + // A truncated size table invalidates data without hiding a known track. + sample_count = be32(body, 8).unwrap_or(0); + size = match be32(body, 4) { + // A first entry cannot justify an incomplete per-sample + // table, nor an impossible constant-size sample count. + Some(0) if sample_count > 0 && sample_count as usize <= body.len().saturating_sub(12) / 4 => be32(body, 12), + Some(n) + if n > 0 + && sample_count > 0 + && u64::from(n).checked_mul(u64::from(sample_count)).is_some_and(|total| total <= bytes.len() as u64) => + { + Some(n) + } _ => None, } } - b"co64" if be32(body, 4)? > 0 => offset = be64(body, 8).and_then(|v| usize::try_from(v).ok()), - b"stco" if be32(body, 4)? > 0 => offset = be32(body, 8).map(|v| v as usize), + // Missing or truncated offset-table counts deny sample access, but the known + // track must remain available to the largest-raw-track selector. + b"co64" => { + offset = match be32(body, 4) { + Some(count) if count > 0 && count as usize <= body.len().saturating_sub(8) / 8 => { + be64(body, 8).and_then(|v| usize::try_from(v).ok()) + } + _ => None, + }; + } + b"stco" => { + offset = match be32(body, 4) { + Some(count) if count > 0 && count as usize <= body.len().saturating_sub(8) / 4 => be32(body, 8).map(|v| v as usize), + _ => None, + }; + } _ => {} } } - let data = match (offset, size) { - (Some(o), Some(s)) if o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) => Some((o, s as usize)), + let data = match (offset, size, mapping) { + (Some(o), Some(s), Some((1, samples, 1))) + if samples > 0 && samples <= sample_count && o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) => + { + Some((o, s as usize)) + } _ => None, }; Some(Cr3Track { kind: kind?, data }) @@ -495,6 +537,13 @@ mod tests { bx(b"CRAW", &b) } fn trak(entry: Vec, offset: u64, size: u32) -> Vec { + let stsc = [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>(); + trak_with_mapping(entry, offset, size, &stsc) + } + fn trak_with_mapping(entry: Vec, offset: u64, size: u32, mapping: &[u8]) -> Vec { + trak_with_table(entry, offset, size, b"stsc", mapping) + } + fn trak_with_table(entry: Vec, offset: u64, size: u32, table: &[u8; 4], payload: &[u8]) -> Vec { let mut stsd = 1u32.to_be_bytes().to_vec(); stsd.extend(entry); let mut stsz = 0u32.to_be_bytes().to_vec(); @@ -502,7 +551,15 @@ mod tests { stsz.extend(size.to_be_bytes()); let mut co64 = 1u32.to_be_bytes().to_vec(); co64.extend(offset.to_be_bytes()); - let stbl = [full(b"stsd", &stsd), full(b"stsz", &stsz), full(b"co64", &co64)].concat(); + let stsc = [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>(); + let offsets = if table == b"stco" { full(b"stco", payload) } else { full(b"co64", if table == b"co64" { payload } else { &co64 }) }; + let stbl = [ + full(b"stsd", &stsd), + full(b"stsc", if table == b"stsc" { payload } else { &stsc }), + full(b"stsz", if table == b"stsz" { payload } else { &stsz }), + offsets, + ] + .concat(); bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl)))) } fn tiff_with(tag: u16, v: craft_tiff::Value) -> Vec { @@ -543,6 +600,97 @@ mod tests { assert_eq!(c.tracks[1].data, Some((4112, 32))); } + #[test] + fn first_sample_requires_a_complete_valid_sample_to_chunk_mapping() { + let file = |track: Vec| { + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + file.extend(bx(b"moov", &track)); + file.resize(4112, 0); + file + }; + let valid = trak(craw(16, 16, &bx(b"JPEG", &[0; 4])), 4096, 16); + assert_eq!(parse_cr3(&file(valid.clone())).unwrap().tracks[0].data, Some((4096, 16))); + let at = valid.windows(4).position(|bytes| bytes == b"stsc").unwrap(); + // Missing table, impossible first chunk, zero samples, a different + // description and a count that extends beyond this box all deny data. + for (field, value) in [(0, u32::from_be_bytes(*b"xxxx")), (8, 0), (8, 2), (12, 0), (12, 2), (16, 0), (16, 2), (4, u32::MAX)] { + let mut damaged = valid.clone(); + let offset = if field == 0 { at } else { at + 4 + field }; + damaged[offset..offset + 4].copy_from_slice(&value.to_be_bytes()); + let bytes = file(damaged); + let parsed = parse_cr3(&bytes).unwrap(); + assert_eq!(parsed.tracks.len(), 1, "invalid mapping must not erase an identifiable track"); + assert_eq!(parsed.tracks[0].kind, Cr3TrackKind::Jpeg); + assert!(parsed.tracks[0].data.is_none()); + } + for entry in [craw(16, 16, &[]), craw(16, 16, &bx(b"CMP1", &[0xff, 0x10, 0, 0x30]))] { + let bytes = file(trak(entry, 4096, 16)); + let parsed = parse_cr3(&bytes).unwrap(); + assert!(!matches!(parsed.tracks[0].kind, Cr3TrackKind::Jpeg)); + } + } + + #[test] + fn truncated_sample_mapping_preserves_larger_raw_track_identity_and_valid_reduced_track() { + let reduced = trak(craw(4, 4, &bx(b"CMP1", &compression_payload(0))), 4096, 16); + for table in [b"stsc", b"stsz", b"co64", b"stco"] { + for payload in [Vec::new(), vec![0; 4], 1u32.to_be_bytes().to_vec()] { + // full() supplies version/flags; an empty payload leaves only + // those four bytes, with the table's required count missing. + let main = trak_with_table(craw(8, 4, &bx(b"CMP1", &compression_payload(0))), 4112, 32, table, &payload); + for main_first in [false, true] { + let tracks = if main_first { [main.clone(), reduced.clone()].concat() } else { [reduced.clone(), main.clone()].concat() }; + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + file.extend(bx(b"moov", &tracks)); + file.resize(4144, 0); + let parsed = parse_cr3(&file).unwrap(); + assert_eq!(parsed.tracks.len(), 2, "truncated {table:?} cannot hide the larger track"); + let (main, reduced) = if main_first { (&parsed.tracks[0], &parsed.tracks[1]) } else { (&parsed.tracks[1], &parsed.tracks[0]) }; + assert!(matches!(reduced.kind, Cr3TrackKind::Raw { width: 4, height: 4, .. })); + assert_eq!(reduced.data, Some((4096, 16))); + assert!(matches!(main.kind, Cr3TrackKind::Raw { width: 8, height: 4, cmp1: Some(_), .. })); + assert!(main.data.is_none()); + assert!(main.compression(&file).is_some(), "only sample location is invalid"); + } + } + let mut offsets = 1u32.to_be_bytes().to_vec(); + offsets.extend(4112u64.to_be_bytes()); + let valid = match table { + b"stsc" => [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>(), + b"stsz" => [0u32, 1, 32].into_iter().flat_map(u32::to_be_bytes).collect(), + b"stco" => [1u32, 4112].into_iter().flat_map(u32::to_be_bytes).collect(), + _ => offsets, + }; + let main = trak_with_table(craw(8, 4, &bx(b"CMP1", &compression_payload(0))), 4112, 32, table, &valid); + for main_first in [false, true] { + let tracks = if main_first { [main.clone(), reduced.clone()].concat() } else { [reduced.clone(), main.clone()].concat() }; + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + file.extend(bx(b"moov", &tracks)); + file.resize(4144, 0); + let parsed = parse_cr3(&file).unwrap(); + assert_eq!(parsed.tracks.len(), 2); + let main = &parsed.tracks[if main_first { 0 } else { 1 }]; + assert_eq!(main.data, Some((4112, 32)), "complete valid {table:?} stays usable"); + } + } + } + + #[test] + fn first_sample_size_requires_a_complete_bounded_sample_table() { + let valid = trak(craw(16, 16, &bx(b"JPEG", &[0; 4])), 4096, 16); + let at = valid.windows(4).position(|bytes| bytes == b"stsz").unwrap(); + for (constant, count, accepted) in [(0u32, 1u32, true), (16, 1, true), (0, 2, false), (0, u32::MAX, false), (16, u32::MAX, false)] { + let mut track = valid.clone(); + track[at + 8..at + 12].copy_from_slice(&constant.to_be_bytes()); + track[at + 12..at + 16].copy_from_slice(&count.to_be_bytes()); + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + file.extend(bx(b"moov", &track)); + file.resize(4112, 0); + let parsed = parse_cr3(&file).unwrap(); + assert_eq!(parsed.tracks[0].data, accepted.then_some((4096, 16)), "size {constant}, count {count}"); + } + } + #[test] fn hevc_preview_is_distinct_from_an_equal_size_raw_track() { let preview = trak(craw(6000, 4000, &bx(b"HEVC", &[0; 4])), 4096, 16); @@ -714,7 +862,10 @@ mod tests { let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); file.extend(bx(b"moov", &bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl)))))); file.resize(512, 0); - assert!(parse_cr3(&file).unwrap().tracks.is_empty()); + let parsed = parse_cr3(&file).unwrap(); + assert_eq!(parsed.tracks.len(), 1, "a truncated table must not hide the identifiable raw track"); + assert!(matches!(parsed.tracks[0].kind, Cr3TrackKind::Raw { width: 16, height: 16, .. })); + assert!(parsed.tracks[0].data.is_none(), "the empty stsz must not borrow size bytes from the following co64 box"); } #[test] diff --git a/crates/pcraft-bundle/src/lib.rs b/crates/pcraft-bundle/src/lib.rs index acbe739b6..1368fa747 100644 --- a/crates/pcraft-bundle/src/lib.rs +++ b/crates/pcraft-bundle/src/lib.rs @@ -370,7 +370,8 @@ impl DirectoryBundleWriter { /// Saves an entry set incrementally. Existing objects are reused only after a hash check; /// unchanged file signatures may skip checks until they enter the rolling verification /// window. The manifest is atomically published last, then stale previews and objects are - /// collected while the bundle lock is held. + /// collected while the bundle lock is held. Collection is best-effort after publication: + /// inaccessible or unsafe leftovers cannot turn a committed save into a reported failure. pub fn save_directory(&mut self, root: &Path, bundle: ZipBundleWriter) -> Result { if !bundle.entries.contains_key(craft_pcraft_format::MANIFEST) { return Err(Error::InvalidBundle("manifest not set".into())); @@ -401,6 +402,10 @@ impl DirectoryBundleWriter { Err(std::fs::TryLockError::Error(_)) => break None, } }; + // A known unsafe manifest cannot be published. Refuse it before writing objects or + // previews, and still recheck immediately before publication below. + let manifest_path = root.join(craft_pcraft_format::MANIFEST); + refuse_symlink(&manifest_path)?; self.generation = self.generation.saturating_add(1); let generation = self.generation; let recheck = self.rolling_recheck(&bundle.entries); @@ -449,43 +454,33 @@ impl DirectoryBundleWriter { } let manifest = bundle.entries.get(craft_pcraft_format::MANIFEST).ok_or_else(|| Error::InvalidBundle("manifest not set".into()))?; - let manifest_path = root.join(craft_pcraft_format::MANIFEST); refuse_symlink(&manifest_path)?; craft_atomic::atomic_write(&manifest_path, manifest).map_err(io_error)?; stats.written = stats.written.saturating_add(1); - let mut removed_paths = Vec::new(); + // The durable manifest is the commit boundary. Cleanup owns only canonical regular + // object files and previews; leave unsafe, inaccessible or locked leftovers for later. for kind in [craft_pcraft_format::TILES_DIR, craft_pcraft_format::BLOBS_DIR] { let dir = root.join(kind); - if !dir.exists() { + if !cleanup_directory(&dir) { continue; } - refuse_symlink(&dir)?; - for entry in fs::read_dir(&dir).map_err(io_error)? { - let entry = entry.map_err(io_error)?; + let Ok(entries) = fs::read_dir(&dir) else { continue }; + for entry in entries { + let Ok(entry) = entry else { continue }; let name = entry.file_name(); let path = format!("{kind}/{}", name.to_string_lossy()); - if is_object_path(&path) && !bundle.entries.contains_key(&path) { - let metadata = fs::symlink_metadata(entry.path()).map_err(io_error)?; - if metadata.file_type().is_symlink() || !metadata.is_file() { - return Err(Error::InvalidBundle(format!("refusing to collect non-file object {path}"))); - } - fs::remove_file(entry.path()).map_err(io_error)?; - removed_paths.push(path); + if is_object_path(&path) && !bundle.entries.contains_key(&path) && remove_cleanup_file(&entry.path()) { stats.removed = stats.removed.saturating_add(1); } } } for path in [craft_pcraft_format::THUMB, craft_pcraft_format::COMPOSITE, craft_pcraft_format::FULL_COMPOSITE] { if !bundle.entries.contains_key(path) { - let target = root.join(path); - refuse_symlink(&target)?; - if fs::symlink_metadata(&target).is_ok() { - fs::remove_file(&target).map_err(io_error)?; - } + let _ = remove_cleanup_file(&root.join(path)); } } - self.verified.retain(|path, _| bundle.entries.contains_key(path) && !removed_paths.contains(path)); + self.verified.retain(|path, _| bundle.entries.contains_key(path)); Ok(stats) } @@ -510,7 +505,32 @@ impl DirectoryBundleWriter { } fn is_object_path(path: &str) -> bool { - path.starts_with("tiles/") || path.starts_with("blobs/") + valid_object_path(path) +} + +/// Do not traverse symlinked object/preview directories, including Windows junctions. +fn cleanup_directory(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir() && !cleanup_link(&metadata)) +} + +fn cleanup_link(metadata: &fs::Metadata) -> bool { + #[cfg(windows)] + { + use std::os::windows::fs::MetadataExt; + metadata.file_attributes() & 0x400 != 0 // FILE_ATTRIBUTE_REPARSE_POINT includes junctions. + } + #[cfg(not(windows))] + { + metadata.file_type().is_symlink() + } +} + +/// Cleanup never follows or removes a directory/link and never reports a post-commit failure. +fn remove_cleanup_file(path: &Path) -> bool { + if path.parent().is_none_or(|parent| !cleanup_directory(parent)) { + return false; + } + fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_file() && !cleanup_link(&metadata)) && fs::remove_file(path).is_ok() } fn file_signature(path: &Path) -> Result> { diff --git a/crates/pcraft-bundle/tests/directory_cleanup.rs b/crates/pcraft-bundle/tests/directory_cleanup.rs new file mode 100644 index 000000000..29015bd23 --- /dev/null +++ b/crates/pcraft-bundle/tests/directory_cleanup.rs @@ -0,0 +1,344 @@ +//! Publication succeeds independently of best-effort cleanup of the previous entry set. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use craft_pcraft_bundle::{DirectoryBundle, DirectoryBundleWriter, Limits, ObjectKind, ZipBundleWriter}; +use craft_pcraft_format::{COMPOSITE, FULL_COMPOSITE, MANIFEST, THUMB}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +const OLD: &[u8] = br#"{"revision":1}"#; +const NEW: &[u8] = br#"{"revision":2}"#; + +struct Temporary(PathBuf); + +impl Temporary { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + let path = std::env::temp_dir().join(format!("craft-bundle-cleanup-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed))); + fs::create_dir(&path).unwrap(); + Self(path) + } + + fn bundle(&self) -> PathBuf { + self.0.join("document.pcraft") + } +} + +impl Drop for Temporary { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn entries(manifest: &[u8]) -> ZipBundleWriter { + let mut bundle = ZipBundleWriter::default(); + bundle.set_manifest(manifest.to_vec()).unwrap(); + bundle +} + +fn assert_manifest(root: &Path, expected: &[u8]) { + assert_eq!(fs::read(root.join(MANIFEST)).unwrap(), expected); + assert_eq!(DirectoryBundle::open(root, Limits::default()).unwrap().manifest().unwrap(), expected); +} + +#[test] +fn non_file_stale_object_does_not_fail_the_committed_save() { + let temp = Temporary::new(); + let root = temp.bundle(); + let mut writer = DirectoryBundleWriter::default(); + writer.save_directory(&root, entries(OLD)).unwrap(); + let stuck = root.join("tiles").join(format!("{}.zst", "0".repeat(64))); + fs::create_dir_all(stuck.join("inner")).unwrap(); + fs::write(stuck.join("inner/keep"), b"not an object file").unwrap(); + + let stats = writer.save_directory(&root, entries(NEW)).expect("cleanup cannot fail after manifest publication"); + assert_manifest(&root, NEW); + assert_eq!(stats.removed, 0); + assert_eq!(fs::read(stuck.join("inner/keep")).unwrap(), b"not an object file"); +} + +#[test] +fn cleanup_removes_only_canonical_stale_regular_objects_and_reuses_current_objects() { + let temp = Temporary::new(); + let root = temp.bundle(); + let mut writer = DirectoryBundleWriter::default(); + let mut first = entries(OLD); + let retained = first.add_object(ObjectKind::Tile, b"still referenced").unwrap(); + let stale = first.add_object(ObjectKind::Blob, b"old blob").unwrap(); + for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] { + first.set_preview(preview, b"old preview".to_vec()).unwrap(); + } + writer.save_directory(&root, first).unwrap(); + let retained_bytes = fs::read(root.join(&retained)).unwrap(); + let unrelated = [ + "tiles/personal-notes.zst".to_string(), + "blobs/012345.zst".to_string(), + format!("blobs/{}.zst", "A".repeat(64)), + format!("tiles/{}.zst.backup", "b".repeat(64)), + ]; + for path in &unrelated { + fs::write(root.join(path), b"unrelated").unwrap(); + } + let nested = root.join("blobs/not-an-object-folder"); + fs::create_dir(&nested).unwrap(); + fs::write(nested.join(format!("{}.zst", "c".repeat(64))), b"unrelated nested file").unwrap(); + let mut second = entries(NEW); + assert_eq!(second.add_object(ObjectKind::Tile, b"still referenced").unwrap(), retained); + let stats = writer.save_directory(&root, second).unwrap(); + + assert_manifest(&root, NEW); + assert_eq!(stats.removed, 1); + assert_eq!(stats.reused, 1); + assert_eq!(stats.reused_objects.as_slice(), std::slice::from_ref(&retained)); + assert!(stats.written_objects.is_empty()); + assert!(!root.join(stale).exists()); + assert_eq!(fs::read(root.join(retained)).unwrap(), retained_bytes); + for path in unrelated { + assert_eq!(fs::read(root.join(path)).unwrap(), b"unrelated"); + } + assert!(nested.is_dir()); + for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] { + assert!(!root.join(preview).exists()); + } +} + +#[test] +fn stale_preview_directories_are_preserved_after_commit() { + let temp = Temporary::new(); + let root = temp.bundle(); + let mut writer = DirectoryBundleWriter::default(); + writer.save_directory(&root, entries(OLD)).unwrap(); + for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] { + fs::create_dir_all(root.join(preview)).unwrap(); + fs::write(root.join(preview).join("keep"), b"directory contents").unwrap(); + } + let stats = writer.save_directory(&root, entries(NEW)).unwrap(); + assert_manifest(&root, NEW); + assert_eq!(stats.removed, 0); + for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] { + assert_eq!(fs::read(root.join(preview).join("keep")).unwrap(), b"directory contents"); + } +} + +#[test] +fn required_non_file_objects_and_previews_fail_before_manifest_publication() { + let temp = Temporary::new(); + let root = temp.bundle(); + let mut writer = DirectoryBundleWriter::default(); + writer.save_directory(&root, entries(OLD)).unwrap(); + let mut changed = entries(NEW); + let object = changed.add_object(ObjectKind::Tile, b"required object").unwrap(); + fs::create_dir_all(root.join(&object)).unwrap(); + assert!(writer.save_directory(&root, changed).is_err()); + assert_manifest(&root, OLD); + + for preview in [THUMB, COMPOSITE] { + fs::create_dir_all(root.join(preview)).unwrap(); + let mut changed = entries(NEW); + changed.set_preview(preview, b"required preview".to_vec()).unwrap(); + assert!(writer.save_directory(&root, changed).is_err()); + assert_manifest(&root, OLD); + } +} + +#[cfg(windows)] +#[test] +fn locked_stale_object_and_preview_do_not_fail_commit_and_are_collected_on_retry() { + use std::fs::OpenOptions; + use std::os::windows::fs::OpenOptionsExt; + let temp = Temporary::new(); + let root = temp.bundle(); + let mut writer = DirectoryBundleWriter::default(); + let mut first = entries(OLD); + let stale = first.add_object(ObjectKind::Tile, b"locked stale tile").unwrap(); + first.set_preview(FULL_COMPOSITE, b"locked preview".to_vec()).unwrap(); + writer.save_directory(&root, first).unwrap(); + let tile_lock = OpenOptions::new().read(true).share_mode(0).open(root.join(&stale)).unwrap(); + let preview_lock = OpenOptions::new().read(true).share_mode(0).open(root.join(FULL_COMPOSITE)).unwrap(); + let stats = writer.save_directory(&root, entries(NEW)).unwrap(); + assert_manifest(&root, NEW); + assert_eq!(stats.removed, 0); + assert!(root.join(&stale).exists()); + assert!(root.join(FULL_COMPOSITE).exists()); + drop(tile_lock); + drop(preview_lock); + let retry = writer.save_directory(&root, entries(NEW)).unwrap(); + assert_manifest(&root, NEW); + assert_eq!(retry.removed, 1); + assert!(!root.join(stale).exists()); + assert!(!root.join(FULL_COMPOSITE).exists()); +} + +#[cfg(unix)] +#[test] +fn cleanup_preserves_symlinks_and_does_not_traverse_linked_directories() { + use std::os::unix::fs::symlink; + let temp = Temporary::new(); + let root = temp.bundle(); + let outside = temp.0.join("outside"); + fs::create_dir(&outside).unwrap(); + let object_name = format!("{}.zst", "d".repeat(64)); + fs::write(outside.join(&object_name), b"outside object").unwrap(); + fs::write(outside.join("preview.png"), b"outside preview").unwrap(); + let mut writer = DirectoryBundleWriter::default(); + writer.save_directory(&root, entries(OLD)).unwrap(); + symlink(&outside, root.join("blobs")).unwrap(); + symlink(&outside, root.join("composite")).unwrap(); + symlink(outside.join("preview.png"), root.join(THUMB)).unwrap(); + fs::create_dir(root.join("tiles")).unwrap(); + let linked_object = root.join("tiles").join(&object_name); + symlink(outside.join(&object_name), &linked_object).unwrap(); + let stats = writer.save_directory(&root, entries(NEW)).unwrap(); + assert_manifest(&root, NEW); + assert_eq!(stats.removed, 0); + assert_eq!(fs::read(outside.join(object_name)).unwrap(), b"outside object"); + assert_eq!(fs::read(outside.join("preview.png")).unwrap(), b"outside preview"); + for link in [root.join("blobs"), root.join("composite"), root.join(THUMB), linked_object] { + assert!(fs::symlink_metadata(link).unwrap().file_type().is_symlink()); + } +} + +#[cfg(unix)] +#[test] +fn required_symlink_objects_and_previews_still_fail_before_commit() { + use std::os::unix::fs::symlink; + let temp = Temporary::new(); + let root = temp.bundle(); + let outside = temp.0.join("outside-file"); + fs::write(&outside, b"unchanged outside bytes").unwrap(); + let mut writer = DirectoryBundleWriter::default(); + writer.save_directory(&root, entries(OLD)).unwrap(); + let mut changed = entries(NEW); + let object = changed.add_object(ObjectKind::Tile, b"required object").unwrap(); + fs::create_dir(root.join("tiles")).unwrap(); + symlink(&outside, root.join(object)).unwrap(); + assert!(writer.save_directory(&root, changed).is_err()); + assert_manifest(&root, OLD); + symlink(&outside, root.join(THUMB)).unwrap(); + let mut changed = entries(NEW); + changed.set_preview(THUMB, b"new preview".to_vec()).unwrap(); + assert!(writer.save_directory(&root, changed).is_err()); + assert_manifest(&root, OLD); + assert_eq!(fs::read(outside).unwrap(), b"unchanged outside bytes"); +} + +#[cfg(windows)] +#[test] +fn cleanup_does_not_traverse_windows_directory_junctions() { + use std::os::windows::process::CommandExt; + struct Junction(PathBuf); + impl Drop for Junction { + fn drop(&mut self) { + let _ = fs::remove_dir(&self.0); + } + } + fn junction(link: PathBuf, target: &Path) -> Junction { + // Fixture-owned absolute paths only; junctions require no symlink privilege. + assert!(link.is_absolute() && target.is_absolute()); + let output = std::process::Command::new("cmd") + .args(["/C", "mklink", "/J"]) + .arg(&link) + .arg(target) + .creation_flags(0x0800_0000) // CREATE_NO_WINDOW + .output() + .unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + Junction(link) + } + let temp = Temporary::new(); + let root = temp.bundle(); + let outside = temp.0.join("outside"); + fs::create_dir(&outside).unwrap(); + let object_name = format!("{}.zst", "e".repeat(64)); + fs::write(outside.join(&object_name), b"outside object").unwrap(); + fs::write(outside.join("preview.png"), b"outside preview").unwrap(); + fs::write(outside.join("full.png"), b"outside full preview").unwrap(); + let mut writer = DirectoryBundleWriter::default(); + writer.save_directory(&root, entries(OLD)).unwrap(); + let _objects = junction(root.join("blobs"), &outside); + let _previews = junction(root.join("composite"), &outside); + let stats = writer.save_directory(&root, entries(NEW)).unwrap(); + assert_manifest(&root, NEW); + assert_eq!(stats.removed, 0); + assert_eq!(fs::read(outside.join(object_name)).unwrap(), b"outside object"); + assert_eq!(fs::read(outside.join("preview.png")).unwrap(), b"outside preview"); + assert_eq!(fs::read(outside.join("full.png")).unwrap(), b"outside full preview"); + assert!(root.join("blobs").exists()); + assert!(root.join("composite").exists()); +} + +#[cfg(any(unix, windows))] +#[test] +fn manifest_symlink_is_refused_before_writing_any_objects_or_previews() { + let temp = Temporary::new(); + let root = temp.bundle(); + let mut writer = DirectoryBundleWriter::default(); + writer.save_directory(&root, entries(OLD)).unwrap(); + assert_manifest(&root, OLD); // Ordinary publication remains supported. + let external = temp.0.join("external-manifest.json"); + fs::write(&external, b"unchanged external manifest").unwrap(); + fs::remove_file(root.join(MANIFEST)).unwrap(); + #[cfg(unix)] + std::os::unix::fs::symlink(&external, root.join(MANIFEST)).unwrap(); + #[cfg(windows)] + if let Err(error) = std::os::windows::fs::symlink_file(&external, root.join(MANIFEST)) { + if error.raw_os_error() == Some(1314) || error.kind() == std::io::ErrorKind::Unsupported { + eprintln!("SKIP manifest symlink fixture: {error}"); + return; + } + panic!("create manifest symlink fixture: {error}"); + } + let mut changed = entries(NEW); + changed.add_object(ObjectKind::Tile, b"must not write tile").unwrap(); + changed.add_object(ObjectKind::Blob, b"must not write blob").unwrap(); + changed.set_preview(THUMB, b"must not write thumbnail".to_vec()).unwrap(); + changed.set_preview(FULL_COMPOSITE, b"must not write composite".to_vec()).unwrap(); + assert!(writer.save_directory(&root, changed).is_err()); + assert_eq!(fs::read(&external).unwrap(), b"unchanged external manifest"); + assert!(fs::symlink_metadata(root.join(MANIFEST)).unwrap().file_type().is_symlink()); + for unwritten in ["tiles", "blobs", THUMB, "composite"] { + assert!(!root.join(unwritten).exists(), "manifest refusal must precede {unwritten} publication"); + } +} + +#[cfg(unix)] +#[test] +fn chosen_root_symlink_cleanup_works_with_search_only_root_and_composite_permissions() { + use std::os::unix::fs::{PermissionsExt, symlink}; + struct Restore(PathBuf, fs::Permissions); + impl Restore { + fn restrict(path: &Path) -> Self { + let old = fs::metadata(path).unwrap().permissions(); + fs::set_permissions(path, fs::Permissions::from_mode(0o300)).unwrap(); + Self(path.to_path_buf(), old) + } + } + impl Drop for Restore { + fn drop(&mut self) { + let _ = fs::set_permissions(&self.0, self.1.clone()); + } + } + let temp = Temporary::new(); + let root = temp.bundle(); + let mut writer = DirectoryBundleWriter::default(); + let mut first = entries(OLD); + let stale = first.add_object(ObjectKind::Blob, b"stale blob").unwrap(); + first.set_preview(THUMB, b"stale thumbnail".to_vec()).unwrap(); + first.set_preview(COMPOSITE, b"stale preview".to_vec()).unwrap(); + writer.save_directory(&root, first).unwrap(); + let chosen = temp.0.join("chosen.pcraft"); + symlink(&root, &chosen).unwrap(); + let composite_mode = Restore::restrict(&root.join("composite")); + let root_mode = Restore::restrict(&root); + let result = writer.save_directory(&chosen, entries(NEW)); + drop(root_mode); + drop(composite_mode); + let stats = result.expect("chosen root resolves before cleanup; traversal does not require directory read permission"); + assert_manifest(&root, NEW); + assert_eq!(stats.removed, 1); + assert!(!root.join(stale).exists()); + assert!(!root.join(THUMB).exists()); + assert!(!root.join(COMPOSITE).exists()); + assert!(fs::symlink_metadata(chosen).unwrap().file_type().is_symlink()); +} diff --git a/crates/raw/src/arw.rs b/crates/raw/src/arw.rs index 639fc0032..1b25aacda 100644 --- a/crates/raw/src/arw.rs +++ b/crates/raw/src/arw.rs @@ -629,6 +629,19 @@ fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> { .max_by_key(|i| i.u64(t::IMAGE_WIDTH).unwrap_or(0).saturating_mul(i.u64(t::IMAGE_LENGTH).unwrap_or(0))) } +fn valid_arw2_curve(curve: &[u64]) -> bool { + curve.len() == 4 && curve.iter().all(|&v| v <= 16383) && curve.windows(2).all(|v| (v[0] >> 2) < (v[1] >> 2)) +} + +fn declared_arw2_curve(raw: &Ifd, compression: u16, one_byte_per_sample: bool) -> Result>> { + let curve = raw.u64s(TONE_CURVE); + if compression == 32767 && one_byte_per_sample && curve.as_ref().is_some_and(|curve| !valid_arw2_curve(curve)) { + // Reordering or clamping malformed breakpoints fabricates a different transfer curve. + return Err(RawError::Unsupported("Sony ARW2 with an invalid tone curve".into())); + } + Ok(curve) +} + pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result { let tiff = Tiff::parse(bytes)?; let ifd0 = tiff.ifds.first().ok_or_else(|| RawError::Corrupt("TIFF without IFDs".into()))?; @@ -645,6 +658,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result (RawData::U16(read_ycbcr_tiles(bytes, &info, raw, mode, limits)?), 14), @@ -720,13 +734,16 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result= (w * h * 2) as u64 { 14 } else { out_bits }; - let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) { + let (cfa, assumed_cfa) = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) { (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => { - Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 } + (Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }, false) } - _ => Cfa::bayer_static("RGGB"), + _ => (Cfa::bayer_static("RGGB"), true), }; - let default_black = if scale_bits >= 14 { 512.0 } else { 128.0 }; + let curve_black = (info.compression == 32767 && one_byte_per_sample) + .then(|| declared_curve.as_ref().and_then(|curve| code_curve(curve).get(256).copied()).map(f32::from)) + .flatten(); + let default_black = curve_black.unwrap_or(if scale_bits >= 14 { 512.0 } else { 128.0 }); // the decrypted SR2SubIFD, read once and only when a plain black level or white balance is missing let sr2_cell = std::cell::OnceCell::new(); let sr2 = || sr2_cell.get_or_init(|| crate::sr2::SubIfd::read(bytes, ifd0, tiff.order)).as_ref(); @@ -741,7 +758,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result (BlackLevel::uniform(default_black + YCBCR_OFFSET), crate::LevelSource::Fallback), _ => match (scale_bits >= 14).then(|| sr2().and_then(sr2_black)).flatten() { Some(b) => (BlackLevel::uniform(b), crate::LevelSource::Metadata), - None => (BlackLevel::uniform(default_black), crate::LevelSource::Fallback), + None => (BlackLevel::uniform(default_black), curve_black.map_or(crate::LevelSource::Fallback, |_| crate::LevelSource::Metadata)), }, }; // ARW6 samples are in 2 x 14-bit units, so the file's 14-bit levels double; without the tag the white is the @@ -842,6 +859,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Vec { + arw_with_strip_and_cfa(strip, None) + } + + fn arw_with_strip_and_cfa(strip: usize, cfa: Option<(Vec, Vec)>) -> Vec { use craft_tiff::{IfdBuilder, ImageData, TiffWriter, Value}; let mut raw = IfdBuilder::new(); raw.set(t::MAKE, Value::Ascii("SONY".into())); @@ -921,11 +945,37 @@ mod tests { raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])); raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])); raw.set(t::COMPRESSION, Value::Short(vec![32767])); + if let Some((dimensions, pattern)) = cfa { + raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(dimensions)); + raw.set(t::CFA_PATTERN_EP, Value::Byte(pattern)); + } raw.set(TONE_CURVE, Value::Short(vec![8000, 10400, 12900, 14100])); raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0u8; strip]] }); TiffWriter::default().write(&[raw]).unwrap() } + #[test] + fn cfa_fallback_notices_follow_missing_malformed_and_valid_source_tags() { + for (tags, expected, assumed) in [ + (None, "RGGB", true), + (Some((vec![1, 4], vec![0, 1, 1, 2])), "RGGB", true), + (Some((vec![2, 2], vec![0, 1, 3, 2])), "RGGB", true), + (Some((vec![2, 2], vec![0, 1, 2])), "RGGB", true), + (Some((vec![2, 2], vec![1, 0, 2, 1])), "GRBG", false), + ] { + let file = arw_with_strip_and_cfa(32 * 4, tags); + let header = decode(&file, Mode::Header, &Limits::default()).unwrap(); + let full = decode(&file, Mode::Full, &Limits::default()).unwrap(); + assert_eq!(header.info(), full.info()); + for image in [&header, &full] { + assert_eq!(image.cfa.as_ref().unwrap().name(), expected); + assert_eq!(image.notices.assumed_cfa, assumed); + } + assert!(header.data.is_empty()); + assert_eq!(full.data.len(), 32 * 4); + } + } + /// Strip byte counts that add up past `u64::MAX` (LONG8 `[u64::MAX, 1]`) are an error or a picture, not an overflow. #[test] fn strip_byte_counts_past_u64_max_do_not_overflow() { @@ -1110,6 +1160,25 @@ mod tests { let _ = code_curve(&[60000, 1, 0, 0]); } + #[test] + fn arw2_rejects_non_increasing_declared_curve_in_header_and_full_decode() { + let valid = arw_with_strip(32 * 4); + assert!(crate::decode(&valid).is_ok()); + let mut damaged = valid.clone(); + let tiff = Tiff::parse(&damaged).unwrap(); + let curve = raw_ifd(&tiff).unwrap().get(TONE_CURVE).unwrap(); + let at = curve.offset as usize; + for (dst, value) in damaged[at..at + 8].as_chunks_mut::<2>().0.iter_mut().zip([9000u16, 8000, 12000, 13000]) { + dst.copy_from_slice(&value.to_le_bytes()); + } + for mode in [Mode::Header, Mode::Full] { + assert!(matches!(decode(&damaged, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("curve"))); + let image = decode(&valid, mode, &Limits::default()).unwrap(); + assert_eq!(image.black.mean(), 512.0); + assert_eq!(image.level_sources.black, crate::LevelSource::Metadata, "the declared curve maps the black code"); + } + } + #[test] fn decipher_inverts_the_cube_substitution() { let mut seen = [false; 256]; diff --git a/crates/raw/src/cr2.rs b/crates/raw/src/cr2.rs index ec98ea48e..025384582 100644 --- a/crates/raw/src/cr2.rs +++ b/crates/raw/src/cr2.rs @@ -10,8 +10,11 @@ //! - Colour filter layout: IFD#3 tag `0xc5e0` (`CR2CFAPattern` in the ExifTool EXIF tag-name docs): 1 = RGGB, //! 2 = BGGR, 3 = GBRG, 4 = GRBG, anchored at the top-left of the full decoded sensor (masked borders included). //! It differs by model (e.g. 3 on the 50D/60D/7D/550D/5D Mark II, 1 on the 40D/5D Mark III/6D/5DS R), so it is -//! read per file (no model table); files without the tag fall back to RGGB (storytold/lightcraft#85). The parity of the +//! read per file first. Without a known tag, observed model IDs provide the fallback; scene-derived +//! measurements are not authoritative enough to establish sensor phase. The parity of the //! `SensorInfo` borders does *not* predict it (RGGB files come with both even and odd top borders). +//! Header-only probes do not decode pixels for this fallback: without the tag their reported CFA +//! is assumed and can refine on full decode, as indicated by `RawNotices::assumed_cfa`. //! - `SensorInfo` (maker note `0x00e0`): sensor width/height and the left/top/right/bottom borders of the image //! area. Light already reaches some columns left of that border (EOS 6D: masked columns 0–70, border 84; 10–36 //! such columns on every corpus body), so the masked columns that give the black level are measured from the @@ -38,6 +41,7 @@ const SRAW_TYPE: u16 = 0xc6c5; /// `SRAW_TYPE` value of the YCbCr frames of sRAW / mRAW (IFD2's preview image carries 3). const SRAW_YCC: u32 = 4; const CR2_CFA_PATTERN: u16 = 0xc5e0; +const CANON_MODEL_ID: u16 = 0x0010; const SENSOR_INFO: u16 = 0x00e0; const COLOR_BALANCE: u16 = 0x4001; /// ShotInfo, whose value 7 is the white-balance setting. @@ -61,6 +65,13 @@ fn cfa_from_tag(v: Option) -> Option { Some(Cfa::bayer_static(name)) } +/// Fallback for files without CR2CFAPattern. Image content cannot establish sensor phase safely. +fn fallback_cfa(_data: &[u16], _width: usize, _area: Rect, _black: f32, _white: f32, model: Option) -> (Cfa, bool) { + // Observed CC0 raw.pixls.us captures: 5D II, 7D, 550D, 600D and 1200D. + let green_first = [0x8000_0218, 0x8000_0250, 0x8000_0270, 0x8000_0286, 0x8000_0327].into_iter().any(|id| model == Some(id)); + (Cfa::bayer_static(if green_first { "GBRG" } else { "RGGB" }), true) +} + /// The 16-bit words of a maker-note array. Canon writes `ColorData` as SHORT in most models, but as UNDEFINED /// bytes (count = byte length, e.g. 5120) in the PowerShot / EOS M models with ColorData versions -3 and -4; /// those bytes are the same 16-bit words in the maker note's byte order, so they are paired up, not read one by @@ -198,6 +209,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result { let total = (fw as u64) @@ -270,12 +282,19 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result= 2 => black_from_columns_sourced(&data, width, cols, active.y..active.y + active.height, active), _ => (BlackLevel::uniform(0.0), crate::LevelSource::Fallback), }; + let model = mn.as_ref().and_then(|m| m.ifd.u64(CANON_MODEL_ID)); + let (cfa, assumed_cfa) = match tagged_cfa { + Some(cfa) => (cfa, false), + None => { + let (cfa, guessed) = fallback_cfa(&data, width, active, black.mean(), white, model); + (cfa, guessed || mode == Mode::Header) + } + }; let img = RawImage { format: RawFormat::Cr2, width, @@ -298,6 +317,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result = (0..w * h) + .map(|i| { + let (x, y) = (i % w, i / w); + let texture = ((x / 2 + y / 2) % 17) as u16; + if i % 97 == 0 { + return 13000; + } + 100 + if (y & 1) == red_row { + if x & 1 == 0 { 500 + texture } else { 1000 + texture } + } else if x & 1 == 0 { + 1000 + texture + } else { + 667 + texture + } + }) + .collect(); + for (left, top) in [(16, 4), (17, 4), (16, 5), (17, 5)] { + let area = Rect::new(left, top, w - left, h - top); + let (cfa, assumed) = fallback_cfa(&samples, w, area, 100.0, 4000.0, None); + assert_eq!(cfa.name(), "RGGB"); + assert!(assumed); + } + let mut bytes = synthetic_cr2_of(&samples, w, h, None, None); + let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0]; + let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap(); + // The explicit tag remains authoritative even when the observed sensor phase disagrees. + bytes[at + 8..at + 12].copy_from_slice(&4u32.to_le_bytes()); + let explicit = crate::decode(&bytes).unwrap(); + assert_eq!(explicit.cfa.as_ref().unwrap().name(), "GRBG"); + assert_eq!(explicit.info(), crate::probe_info(&bytes).unwrap()); + assert!(!explicit.notices.assumed_cfa); + bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes()); + let full = crate::decode(&bytes).unwrap(); + assert_eq!(full.cfa.as_ref().unwrap().name(), "RGGB"); + assert!(full.notices.assumed_cfa); + let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap(); + assert!(header.notices.assumed_cfa); + assert!(matches!(header.data, RawData::U16(ref v) if v.is_empty())); + assert_eq!(full.data, RawData::U16(samples)); + } + } + + #[test] + fn inconclusive_cfa_measurement_keeps_observed_model_fallback_and_safe_geometry() { + let data = vec![512; 64 * 32]; + let area = Rect::new(0, 0, 64, 32); + assert_eq!(fallback_cfa(&data, 64, area, 512.0, 16000.0, Some(0x8000_0250)).0.name(), "GBRG"); + assert!(fallback_cfa(&data, 64, area, 512.0, 16000.0, Some(0x8000_0250)).1); + assert_eq!(fallback_cfa(&data, 64, area, 512.0, 16000.0, None).0.name(), "RGGB"); + assert!(fallback_cfa(&data, 64, area, 512.0, 16000.0, None).1); + let gradient: Vec = (0..192 * 96) + .map(|i| { + let (x, y) = (i % 192, i / 192); + if y % 2 == 0 { + if x % 2 == 0 { 500 } else { 1000 + 8 * x as u16 } + } else if x % 2 == 0 { + 1000 + 8 * x as u16 + } else { + 500 + } + }) + .collect(); + let (cfa, assumed) = fallback_cfa(&gradient, 192, Rect::new(0, 0, 192, 96), 0.0, 4000.0, None); + assert_eq!(cfa.name(), "RGGB"); + assert!(assumed); + let mut bytes = synthetic_cr2_of(&gradient, 192, 96, None, None); + let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0]; + let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap(); + // Exercise the decoder's missing-tag fallback, not the fixture's default CFA tag. + bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes()); + let decoded = crate::decode(&bytes).unwrap(); + assert_eq!(decoded.cfa.as_ref().unwrap().name(), "RGGB"); + assert!(decoded.notices.assumed_cfa); + } + + #[test] + fn model_cfa_fallback_is_disclosed_in_header_and_full_decodes() { + // A single inline LONG maker-note entry needs no external offsets. + let mut note = Vec::new(); + note.extend_from_slice(&1u16.to_le_bytes()); + note.extend_from_slice(&CANON_MODEL_ID.to_le_bytes()); + note.extend_from_slice(&4u16.to_le_bytes()); + note.extend_from_slice(&1u32.to_le_bytes()); + note.extend_from_slice(&0x8000_0250u32.to_le_bytes()); + note.extend_from_slice(&0u32.to_le_bytes()); + let samples = vec![512u16; 64 * 32]; + let mut bytes = synthetic_cr2_of(&samples, 64, 32, None, Some(note)); + let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0]; + let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap(); + // Remove only the CFA tag; the model ID and encoded samples stay intact. + bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes()); + let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap(); + let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap(); + for image in [&header, &full] { + assert_eq!(image.cfa.as_ref().unwrap().name(), "GBRG"); + assert!(image.notices.assumed_cfa); + } + assert_eq!(header.info(), full.info()); + assert_eq!(full.data, RawData::U16(samples)); + assert!(header.data.is_empty()); + } + #[test] fn wb_probe() { let mut v = vec![0u64; 120]; diff --git a/crates/raw/src/cr2/sraw.rs b/crates/raw/src/cr2/sraw.rs index 7ad98906b..4b02881d2 100644 --- a/crates/raw/src/cr2/sraw.rs +++ b/crates/raw/src/cr2/sraw.rs @@ -393,6 +393,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode, limits: & camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(), cinema: Default::default(), lens_metadata: Default::default(), + notices: Default::default(), level_sources: Default::default(), }; img.validate_for(mode)?; diff --git a/crates/raw/src/cr3.rs b/crates/raw/src/cr3.rs index 956446c6e..14bec6132 100644 --- a/crates/raw/src/cr3.rs +++ b/crates/raw/src/cr3.rs @@ -274,6 +274,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Vec { + let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>(); + track_with_mapping(entry, offset, size, &mapping) + } + + fn track_with_mapping(entry: &[u8], offset: u64, size: u32, mapping: &[u8]) -> Vec { + track_with_table(entry, offset, size, b"stsc", mapping) + } + + fn track_with_table(entry: &[u8], offset: u64, size: u32, table: &[u8; 4], payload: &[u8]) -> Vec { let mut stsd = vec![0u8; 4]; stsd.extend(1u32.to_be_bytes()); stsd.extend(entry); @@ -413,7 +423,15 @@ mod tests { let mut co64 = vec![0u8; 4]; co64.extend(1u32.to_be_bytes()); co64.extend(offset.to_be_bytes()); - let tables = [box_bytes(b"stsd", &stsd), box_bytes(b"stsz", &stsz), box_bytes(b"co64", &co64)].concat(); + let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>(); + let offsets = if table == b"stco" { box_bytes(b"stco", payload) } else { box_bytes(b"co64", if table == b"co64" { payload } else { &co64 }) }; + let tables = [ + box_bytes(b"stsd", &stsd), + box_bytes(b"stsc", if table == b"stsc" { payload } else { &mapping }), + box_bytes(b"stsz", if table == b"stsz" { payload } else { &stsz }), + offsets, + ] + .concat(); box_bytes(b"trak", &box_bytes(b"mdia", &box_bytes(b"minf", &box_bytes(b"stbl", &tables)))) } @@ -636,6 +654,105 @@ mod tests { assert!(matches!(decode(&missing, Mode::Header, &Limits::default()), Err(RawError::Corrupt(_)))); } + /// Independently framed constant-value CRX: each 4x4 tile has four 2x2 + /// planes whose one-byte 0x1e entropy streams reconstruct sensor value8193. + fn flat_raw_track(tiles: u32) -> (Vec, Vec) { + let mut sample = Vec::new(); + let marker = |out: &mut Vec, code: u16, size: u32, flags: u32| { + out.extend(code.to_be_bytes()); + out.extend(8u16.to_be_bytes()); + out.extend(size.to_be_bytes()); + out.extend(flags.to_be_bytes()); + }; + for tile in 0..tiles { + marker(&mut sample, 0xff01, 4, tile << 16); + for plane in 0..4 { + marker(&mut sample, 0xff02, 1, (plane << 28) | 0x0800_0000); + marker(&mut sample, 0xff03, 1, 0x0020_0000); + } + } + let header_size = sample.len() as u32; + sample.extend(std::iter::repeat_n(0x1e, tiles as usize * 4)); + let mut coding = vec![0u8; 52]; + coding[..2].copy_from_slice(&0xff00u16.to_be_bytes()); + coding[2..4].copy_from_slice(&48u16.to_be_bytes()); + coding[4..6].copy_from_slice(&0x100u16.to_be_bytes()); + for (offset, value) in [(8, tiles * 4), (12, 4), (16, 4), (20, 4), (28, header_size)] { + coding[offset..offset + 4].copy_from_slice(&value.to_be_bytes()); + } + coding[24] = 14; + coding[25] = 0x40; + let mut entry = vec![0u8; 82]; + entry[24..26].copy_from_slice(&((tiles * 4) as u16).to_be_bytes()); + entry[26..28].copy_from_slice(&4u16.to_be_bytes()); + entry.extend(box_bytes(b"CMP1", &coding)); + (box_bytes(b"CRAW", &entry), sample) + } + + #[test] + fn truncated_main_sample_mapping_cannot_silently_select_a_valid_reduced_raw() { + let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>(); + let (reduced_entry, reduced_sample) = flat_raw_track(1); + let (main_entry, main_sample) = flat_raw_track(2); + let main_offset = 4096 + reduced_sample.len(); + let file = |main_table: Option<(&[u8; 4], &[u8])>, main_first: bool| { + let reduced = track_with_mapping(&reduced_entry, 4096, reduced_sample.len() as u32, &mapping); + let tracks = if let Some((table, payload)) = main_table { + let main = track_with_table(&main_entry, main_offset as u64, main_sample.len() as u32, table, payload); + if main_first { [main, reduced].concat() } else { [reduced, main].concat() } + } else { + reduced + }; + let mut bytes = box_bytes(b"ftyp", b"crx \0\0\0\x01crx isom"); + bytes.extend(box_bytes(b"moov", &tracks)); + assert!(bytes.len() < 4096); + bytes.resize(4096, 0); + bytes.extend_from_slice(&reduced_sample); + bytes.extend_from_slice(&main_sample); + bytes + }; + let mut co64 = vec![0u8; 4]; + co64.extend(1u32.to_be_bytes()); + co64.extend((main_offset as u64).to_be_bytes()); + let valid_tables = [ + (b"stsc", mapping.clone()), + (b"stsz", [0u32, main_sample.len() as u32, 1].into_iter().flat_map(u32::to_be_bytes).collect()), + (b"co64", co64), + (b"stco", [0u32, 1, main_offset as u32].into_iter().flat_map(u32::to_be_bytes).collect()), + ]; + for mode in [Mode::Header, Mode::Full] { + // The smaller source is genuinely decodable, so dropping the larger + // track would return success rather than merely a different failure. + let reduced = decode(&file(None, false), mode, &Limits::default()).unwrap(); + assert_eq!((reduced.width, reduced.height), (4, 4)); + if mode == Mode::Full { + assert_eq!(reduced.data, RawData::U16(vec![8193; 16])); + } + for main_first in [false, true] { + for (table, payload) in &valid_tables { + let valid = decode(&file(Some((table, payload)), main_first), mode, &Limits::default()).unwrap(); + assert_eq!((valid.width, valid.height), (8, 4)); + if mode == Mode::Full { + assert_eq!(valid.data, RawData::U16(vec![8193; 32])); + } + for truncated in [vec![], vec![0; 4], vec![0, 0, 0, 0, 0, 0, 0, 1]] { + let bytes = file(Some((table, &truncated)), main_first); + let container = parse_cr3(&bytes).unwrap(); + assert_eq!(container.tracks.len(), 2, "mapping damage cannot erase the main track"); + let selected = main_track(&container.tracks).unwrap(); + assert!(matches!(selected.kind, Cr3TrackKind::Raw { width: 8, height: 4, .. })); + assert!(selected.data.is_none()); + assert!(selected.compression(&bytes).is_some()); + assert!( + matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Corrupt(message)) if message.contains("sample")), + "{mode:?}/{table:?}: main mapping damage cannot decode the reduced source" + ); + } + } + } + } + } + #[test] fn color_data_uses_versioned_as_shot_offsets() { for (version, wb_at, black_at, white_at) in @@ -712,6 +829,7 @@ mod tests { camera: Camera::default(), cinema: Default::default(), lens_metadata: Default::default(), + notices: Default::default(), level_sources: Default::default(), } } diff --git a/crates/raw/src/crw.rs b/crates/raw/src/crw.rs index 690966751..b51c04261 100644 --- a/crates/raw/src/crw.rs +++ b/crates/raw/src/crw.rs @@ -65,6 +65,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, caller: &Limits) -> Result bool { if !raw.contains(t::AS_SHOT_NEUTRAL) && !tiff.ifds.first().is_some_and(|i| i.contains(t::AS_SHOT_NEUTRAL)) { return false; } - let Ok(info) = raw.image() else { return false }; + let Ok(info) = raw.image() else { + // Recognition must not turn a damaged, otherwise supported plain CFA + // image into an unsupported container. Let decode_as report its layout + // error; these tags alone never decompress or allocate pixel samples. + return raw.u16(t::COMPRESSION).unwrap_or(1) == t::compression::NONE + && raw.u16(t::SAMPLES_PER_PIXEL).unwrap_or(1) == 1 + && (1..=16).contains(&raw.u16(t::BITS_PER_SAMPLE).unwrap_or(1)) + && raw.u16(t::SAMPLE_FORMAT).unwrap_or(1) != 3; + }; if info.samples_per_pixel != 1 || !(1..=16).contains(&info.bits()) || info.sample_format == 3 { return false; } @@ -274,8 +282,13 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f let bits = info.bits() as u32; // DNG 1.4 stores lossy JPEG at 8 bits per sample: the tiles are 8-bit baseline JPEGs whatever the tag says, // and reading them as a deeper sample would put the picture at the bottom of its range - if info.compression == t::compression::LOSSY_JPEG && bits != 8 { - return Err(RawError::Unsupported(format!("lossy JPEG DNG with {bits}-bit samples (DNG allows only 8)"))); + if info.compression == t::compression::LOSSY_JPEG { + if bits != 8 { + return Err(RawError::Unsupported(format!("lossy JPEG DNG with {bits}-bit samples (DNG allows only 8)"))); + } + if !matches!(cpp, 1 | 3) { + return Err(RawError::Unsupported(format!("lossy JPEG DNG with {cpp} samples per pixel"))); + } } let mut data = read_image_in(mode, bytes, &info, tiff.order, Packing::Msb, limits)?; @@ -312,6 +325,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f // (a DOUBLE past `f32::MAX` casts to infinity, which would black the picture out: ignored like a zero one) let mut white: Vec = levels(t::WHITE_LEVEL).f64s(t::WHITE_LEVEL).unwrap_or_default().into_iter().map(|v| v as f32).filter(|v| v.is_finite() && *v > 0.0).collect(); + let white_source = if !white.is_empty() || table_max.is_some() { LevelSource::Metadata } else { LevelSource::Fallback }; if white.is_empty() { white = vec![default_white]; } @@ -333,7 +347,13 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f }; // (a FLOAT / DOUBLE tag can hold NaN or infinity: such a level reads as 0, in its place) let level = |v: f64| Some(v as f32).filter(|v| v.is_finite()).unwrap_or(0.0); - let mut values: Vec = levels(t::BLACK_LEVEL).f64s(t::BLACK_LEVEL).unwrap_or_default().into_iter().map(level).collect(); + let declared_black = levels(t::BLACK_LEVEL).f64s(t::BLACK_LEVEL).unwrap_or_default(); + let black_source = if !declared_black.is_empty() && declared_black.iter().all(|value| (*value as f32).is_finite()) { + LevelSource::Metadata + } else { + LevelSource::Fallback + }; + let mut values: Vec = declared_black.into_iter().map(level).collect(); // A TIFF that is not a DNG may state four black levels without `BlackLevelRepeatDim`: one per site of its 2 × 2 // CFA, row-major (PhotoCraft's reading of Hasselblad files). In a DNG the count has to match the repeat. let (br, bc) = if format != RawFormat::Dng && cpp == 1 && values.len() == 4 && !levels(t::BLACK_LEVEL).contains(t::BLACK_LEVEL_REPEAT_DIM) { @@ -364,7 +384,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f } _ => Rect::new(0, 0, full_w, full_h), }; - let crop = default_crop(raw, active_area); + let (crop, default_crop_adjusted) = default_crop(raw, active_area, full_w, full_h); let (active_area, crop) = if (full_w, full_h) == (w, h) { (active_area, crop) } else { @@ -378,6 +398,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f (area, scale(crop).clipped(area.width, area.height)) }; + let mut assumed_cfa = false; let cfa = if info.photometric == photometric::CFA { if raw.u16(t::CFA_LAYOUT).unwrap_or(1) != 1 { return Err(RawError::Unsupported("non-rectangular CFA layout".into())); @@ -391,7 +412,10 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f let assumed = [0u8, 1, 1, 2]; let pat = match raw.bytes(t::CFA_PATTERN_EP) { Some(p) => p, - None if format != RawFormat::Dng && (rows, cols) == (2, 2) => &assumed, + None if format != RawFormat::Dng && (rows, cols) == (2, 2) => { + assumed_cfa = true; + &assumed + } None => return Err(RawError::Corrupt("missing CFAPattern".into())), }; if pat.len() != rows * cols { @@ -422,6 +446,9 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f let opcodes = crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) }; + let color = color_data(ifd0, raw, tiff.order); + let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2)) + && color.profile.gain_table_map.is_none(); let img = RawImage { format, width: w, @@ -435,7 +462,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f active_area, crop, orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)), - color: color_data(ifd0, raw, tiff.order), + color, wb_multipliers: None, linearized, pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()), @@ -443,7 +470,8 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f camera: Camera::from_ifd0(ifd0), cinema: cinema_tags(ifd0, raw), lens_metadata: Default::default(), - level_sources: Default::default(), + level_sources: LevelSources { black: black_source, white: white_source }, + notices: RawNotices { default_crop_adjusted, gain_table_omitted, assumed_cfa }, }; img.validate_for(mode)?; Ok(img) @@ -472,33 +500,45 @@ fn cinema_tags(ifd0: &Ifd, raw: &Ifd) -> CinemaTags { } } -/// The default crop, relative to the active area. Each tag has its own default (DNG 1.7.1, `DefaultCropOrigin` -/// and `DefaultCropSize`): the origin (0, 0), the size the whole area, so a file with only a size crops from the -/// area's corner. A tag that is present must hold two finite, non-negative values, and the crop must start inside -/// the area and be at least one pixel each way; otherwise the crop is the whole active area. -fn default_crop(raw: &Ifd, active_area: Rect) -> Rect { +/// The default crop is relative to ActiveArea. DNG 1.7.1 defines independent +/// defaults: origin (0, 0), size ImageWidth/ImageLength. An unusable optional crop +/// preserves the entire active image. Valid partly overlapping rectangles retain the +/// existing clipped-crop policy, with an explicit notice rather than silent adjustment. +fn default_crop(raw: &Ifd, active_area: Rect, full_width: usize, full_height: usize) -> (Rect, bool) { let whole = Rect::new(0, 0, active_area.width, active_area.height); - let pair = |tag: u16, default: [f64; 2]| -> Option<[f64; 2]> { - if !raw.contains(tag) { - return Some(default); - } - match raw.f64s(tag).as_deref() { - Some([a, b]) if a.is_finite() && b.is_finite() && *a >= 0.0 && *b >= 0.0 => Some([*a, *b]), - _ => None, - } - }; - let (Some([x, y]), Some([cw, ch])) = - (pair(t::DEFAULT_CROP_ORIGIN, [0.0, 0.0]), pair(t::DEFAULT_CROP_SIZE, [active_area.width as f64, active_area.height as f64])) - else { - return whole; - }; - if cw < 1.0 || ch < 1.0 { - return whole; + if !raw.contains(t::DEFAULT_CROP_ORIGIN) && !raw.contains(t::DEFAULT_CROP_SIZE) { + return (whole, false); } - // (`as usize` saturates, and `clipped` holds the origin to the area) - let crop = - Rect::new(x.round() as usize, y.round() as usize, cw.round() as usize, ch.round() as usize).clipped(active_area.width, active_area.height); - if crop.width == 0 || crop.height == 0 { whole } else { crop } + let pair = |tag: u16, default: [f64; 2]| -> Option<[usize; 2]> { + let values = if raw.contains(tag) { + match raw.f64s(tag).as_deref() { + Some([a, b]) => [*a, *b], + _ => return None, + } + } else { + default + }; + // Float-to-integer casts saturate. Refuse nonfinite/unrepresentable numbers before + // casting so a hostile origin cannot become a seemingly valid clipped rectangle. + if values.iter().any(|value| !value.is_finite() || value.round() >= usize::MAX as f64) { + return None; + } + if values.iter().any(|value| *value < 0.0) { + return None; + } + Some(values.map(|value| value.round() as usize)) + }; + let (Some([x, y]), Some([width, height])) = + (pair(t::DEFAULT_CROP_ORIGIN, [0.0, 0.0]), pair(t::DEFAULT_CROP_SIZE, [full_width as f64, full_height as f64])) + else { + return (whole, true); + }; + let requested = Rect::new(x, y, width, height); + let crop = requested.clipped(active_area.width, active_area.height); + if crop.width == 0 || crop.height == 0 { + return (whole, true); + } + (crop, crop != requested) } /// Pixel height over width from `DefaultScale` (horizontal, vertical): only a clear 1.5 to 4 ratio counts (the rows stretch; a wider-than-tall pixel is left alone). @@ -509,6 +549,9 @@ fn default_scale_aspect(scale: Option<&[f64]>) -> f32 { } } +#[cfg(test)] +mod compat_review_tests; + #[cfg(test)] mod tests { #[test] @@ -621,3 +664,7 @@ mod tests { assert!(deinterleave(&mut hostile, 8, 6, 1, 1 << 20, 1).is_err()); } } + +#[cfg(all(test, feature = "hasselblad"))] +#[path = "dng/probe_review_tests.rs"] +mod probe_review_tests; diff --git a/crates/raw/src/dng/compat_review_tests.rs b/crates/raw/src/dng/compat_review_tests.rs new file mode 100644 index 000000000..48300cbf7 --- /dev/null +++ b/crates/raw/src/dng/compat_review_tests.rs @@ -0,0 +1,120 @@ +use super::*; +use craft_tiff::{Entry, IfdBuilder, ImageData, TiffWriter}; + +fn ifd(tags: Vec<(u16, Value)>) -> Ifd { + let mut entries: Vec<_> = tags.into_iter().map(|(tag, value)| Entry { tag, value, offset: 0 }).collect(); + entries.sort_by_key(|entry| entry.tag); + Ifd { entries, ..Default::default() } +} + +#[test] +fn optional_crop_adjustments_are_explicit_and_valid_clipping_is_preserved() { + let active = Rect::new(3, 2, 11, 8); + let whole = Rect::new(0, 0, 11, 8); + let clipped = ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![10, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]); + assert_eq!(default_crop(&clipped, active, 16, 12), (Rect::new(10, 0, 1, 4), true)); + assert_eq!(default_crop(&ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![1, 1]))]), active, 16, 12), (Rect::new(1, 1, 10, 7), true)); + for fields in [ + vec![(t::DEFAULT_CROP_SIZE, Value::Long(vec![4]))], + vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![0, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![0, 4]))], + ] { + assert_eq!(default_crop(&ifd(fields), active, 16, 12), (whole, true)); + } + assert_eq!(default_crop(&Ifd::default(), active, 16, 12), (whole, false)); + assert_eq!(default_crop(&ifd(vec![(t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]), active, 16, 12), (Rect::new(0, 0, 5, 4), false)); + assert_eq!( + default_crop(&ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![2, 1])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]), active, 16, 12), + (Rect::new(2, 1, 5, 4), false) + ); +} + +#[test] +fn hostile_crop_coordinates_fall_back_explicitly_before_saturating_casts() { + for bad in [1e300, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + for tag in [t::DEFAULT_CROP_ORIGIN, t::DEFAULT_CROP_SIZE] { + let raw = ifd(vec![(tag, Value::Double(vec![bad, 1.]))]); + assert_eq!(default_crop(&raw, Rect::new(0, 0, 16, 12), 16, 12), (Rect::new(0, 0, 16, 12), true)); + } + } +} + +fn linear(extra: &[(u16, Value)]) -> Vec { + let mut raw = IfdBuilder::new(); + for (tag, value) in [ + (t::IMAGE_WIDTH, Value::Long(vec![8])), + (t::IMAGE_LENGTH, Value::Long(vec![6])), + (t::BITS_PER_SAMPLE, Value::Short(vec![16])), + (t::SAMPLES_PER_PIXEL, Value::Short(vec![1])), + (t::PHOTOMETRIC, Value::Short(vec![photometric::LINEAR_RAW])), + (t::COMPRESSION, Value::Short(vec![1])), + ] { + raw.set(tag, value); + } + for (tag, value) in extra { + raw.set(*tag, value.clone()); + } + raw.set_image(ImageData::Strips { rows_per_strip: 6, strips: vec![(0u16..48).flat_map(u16::to_le_bytes).collect()] }); + TiffWriter::new(ByteOrder::Little, false).write(&[raw]).unwrap() +} + +#[test] +fn dng_and_generic_cfa_tiff_report_actual_level_sources_instead_of_unknown() { + for format in [RawFormat::Dng, RawFormat::CfaTiff] { + let missing = decode_as(&linear(&[]), Mode::Full, format, &Limits::default()).unwrap(); + assert_eq!(missing.level_sources, LevelSources { black: LevelSource::Fallback, white: LevelSource::Fallback }); + let declared = decode_as( + &linear(&[(t::BLACK_LEVEL, Value::Short(vec![17])), (t::WHITE_LEVEL, Value::Short(vec![1000]))]), + Mode::Full, + format, + &Limits::default(), + ) + .unwrap(); + assert_eq!(declared.level_sources, LevelSources { black: LevelSource::Metadata, white: LevelSource::Metadata }); + assert_eq!(declared.black.values, vec![17.]); + assert_eq!(declared.white, vec![1000.]); + let invalid = decode_as( + &linear(&[(t::BLACK_LEVEL, Value::Double(vec![f64::NAN])), (t::WHITE_LEVEL, Value::Double(vec![f64::INFINITY]))]), + Mode::Full, + format, + &Limits::default(), + ) + .unwrap(); + assert_eq!(invalid.level_sources, LevelSources { black: LevelSource::Fallback, white: LevelSource::Fallback }); + } + let table = decode_as(&linear(&[(t::LINEARIZATION_TABLE, Value::Short(vec![0, 1000]))]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap(); + assert_eq!(table.white, vec![1000.]); + assert_eq!(table.level_sources.white, LevelSource::Metadata); +} + +#[test] +fn decoded_notices_record_actual_crop_gain_table_and_cfa_decisions() { + let plain = decode_as(&linear(&[]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap(); + assert_eq!(plain.notices, RawNotices::default()); + let adjusted = decode_as( + &linear(&[(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![7, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]), + Mode::Full, + RawFormat::Dng, + &Limits::default(), + ) + .unwrap(); + assert_eq!(adjusted.crop, Rect::new(7, 0, 1, 4)); + assert!(adjusted.notices.default_crop_adjusted); + let invalid_gain = + decode_as(&linear(&[(t::PROFILE_GAIN_TABLE_MAP, Value::Undefined(vec![0; 16]))]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap(); + assert!(invalid_gain.notices.gain_table_omitted); + assert!(invalid_gain.color.profile.gain_table_map.is_none()); + let assumed = + decode_as(&linear(&[(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA]))]), Mode::Full, RawFormat::CfaTiff, &Limits::default()).unwrap(); + assert!(assumed.notices.assumed_cfa); + assert_eq!(assumed.cfa.as_ref().unwrap().pattern, vec![0, 1, 1, 2]); + let tagged = decode_as( + &linear(&[(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])), (t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2]))]), + Mode::Full, + RawFormat::Dng, + &Limits::default(), + ) + .unwrap(); + assert!(!tagged.notices.assumed_cfa); + let limits = Limits { max_samples: 47, ..Default::default() }; + assert!(matches!(decode_as(&linear(&[]), Mode::Full, RawFormat::Dng, &limits), Err(RawError::Limit(_)))); +} diff --git a/crates/raw/src/dng/probe_review_tests.rs b/crates/raw/src/dng/probe_review_tests.rs new file mode 100644 index 000000000..8e54e8db6 --- /dev/null +++ b/crates/raw/src/dng/probe_review_tests.rs @@ -0,0 +1,64 @@ +use super::*; +use craft_tiff::{IfdBuilder, ImageData, TiffWriter}; + +#[allow(clippy::unwrap_used)] // Test fixture construction must fail the test if its independent writer fails. +fn plain() -> Vec { + let mut raw = IfdBuilder::new(); + for (tag, value) in [ + (t::IMAGE_WIDTH, Value::Long(vec![2])), + (t::IMAGE_LENGTH, Value::Long(vec![2])), + (t::BITS_PER_SAMPLE, Value::Short(vec![16])), + (t::SAMPLES_PER_PIXEL, Value::Short(vec![1])), + (t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])), + (t::COMPRESSION, Value::Short(vec![1])), + (t::AS_SHOT_NEUTRAL, Value::Rational(vec![(1, 2), (1, 1), (2, 3)])), + ] { + raw.set(tag, value); + } + raw.set_image(ImageData::Strips { rows_per_strip: 2, strips: vec![[17u16, 31, 53, 79].into_iter().flat_map(u16::to_le_bytes).collect()] }); + TiffWriter::new(ByteOrder::Little, false).write(&[raw]).unwrap() +} + +#[allow(clippy::unwrap_used)] // Only known tags in this module's synthetic TIFF are inspected. +fn entry(bytes: &[u8], tag: u16) -> usize { + let at = u32::from_le_bytes(bytes[4..8].try_into().unwrap()) as usize; + let count = u16::from_le_bytes(bytes[at..at + 2].try_into().unwrap()) as usize; + (0..count).map(|i| at + 2 + i * 12).find(|&i| u16::from_le_bytes(bytes[i..i + 2].try_into().unwrap()) == tag).unwrap() +} + +#[test] +fn plain_cfa_probe_keeps_malformed_geometry_and_tables_in_the_supported_decoder() { + let valid = plain(); + assert_eq!(crate::probe(&valid), Some(RawFormat::CfaTiff)); + let decoded = crate::decode(&valid).unwrap(); + assert_eq!(decoded.data, RawData::U16(vec![17, 31, 53, 79])); + for tag in [t::IMAGE_WIDTH, t::IMAGE_LENGTH] { + let mut damaged = valid.clone(); + let at = entry(&damaged, tag); + damaged[at + 8..at + 12].copy_from_slice(&0u32.to_le_bytes()); + assert_eq!(crate::probe(&damaged), Some(RawFormat::CfaTiff)); + assert!(matches!(crate::probe_info(&damaged), Err(RawError::Tiff(_))), "zero geometry remains malformed before pixel decode"); + assert!(matches!(crate::decode(&damaged), Err(RawError::Tiff(_)))); + } + let mut missing = valid.clone(); + let at = entry(&missing, t::STRIP_OFFSETS); + missing[at..at + 2].copy_from_slice(&0xf123u16.to_le_bytes()); + assert_eq!(crate::probe(&missing), Some(RawFormat::CfaTiff)); + assert!(matches!(crate::probe_info(&missing), Err(RawError::Tiff(_)))); + assert!(matches!(crate::decode(&missing), Err(RawError::Tiff(_)))); +} + +#[test] +fn plain_cfa_probe_does_not_claim_unknown_tiff_codings_or_missing_color_contracts() { + for tag in [t::AS_SHOT_NEUTRAL, t::COMPRESSION] { + let mut unknown = plain(); + let at = entry(&unknown, tag); + if tag == t::COMPRESSION { + unknown[at + 8..at + 10].copy_from_slice(&65000u16.to_le_bytes()); + } else { + unknown[at..at + 2].copy_from_slice(&0xf123u16.to_le_bytes()); + } + assert_eq!(crate::probe(&unknown), Some(RawFormat::OtherTiff)); + assert!(matches!(crate::decode(&unknown), Err(RawError::Unsupported(_)))); + } +} diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index d7f6cdedd..8953f94ea 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -462,6 +462,7 @@ mod tests { cinema: Default::default(), lens_metadata: Default::default(), level_sources: Default::default(), + notices: crate::RawNotices::default(), }; let opts = |tile: u32| DngWriteOptions { compression: DngCompression::Deflate { tile, half: false }, ..DngWriteOptions::default() }; let hostile = write_dng(&raw, &opts(u32::MAX)).expect("the clamped tile writes"); diff --git a/crates/raw/src/erf.rs b/crates/raw/src/erf.rs index 40190ca20..fcdfa43cd 100644 --- a/crates/raw/src/erf.rs +++ b/crates/raw/src/erf.rs @@ -128,6 +128,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result, } +/// Bounded diagnostics for decisions actually made by a raw decoder. An absent optional tag +/// does not itself set a notice; hosts can translate these flags into their own warning text +/// without inferring a fallback from missing metadata or retaining source-controlled strings. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct RawNotices { + /// Explicit default-crop metadata needed correction or a fallback to the active area. + pub default_crop_adjusted: bool, + /// A source profile gain table could not be used and was omitted from the decoded image. + pub gain_table_omitted: bool, + /// The CFA layout was selected by a decoder fallback instead of usable source metadata. + pub assumed_cfa: bool, +} + /// A decoded raw image. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct RawImage { @@ -1041,6 +1055,9 @@ pub struct RawImage { /// measured levels are placeholders. #[serde(default)] pub level_sources: LevelSources, + /// Fixed-size decoder decisions, separate from ordinary missing optional metadata. + #[serde(default)] + pub notices: RawNotices, } impl RawImage { @@ -1173,3 +1190,21 @@ mod checked_arithmetic_tests { assert_eq!(Orientation::from_exif(9), Orientation::Normal); } } + +#[cfg(test)] +mod raw_notices_tests { + use super::*; + + #[test] + fn missing_notices_default_to_no_claims_and_explicit_flags_are_independent() { + type Error = serde::de::value::Error; + let empty = serde::de::value::MapDeserializer::<_, Error>::new(std::iter::empty::<(&str, bool)>()); + assert_eq!(RawNotices::deserialize(empty).unwrap(), RawNotices::default()); + let one = serde::de::value::MapDeserializer::<_, Error>::new([("assumed_cfa", true)].into_iter()); + let notices = RawNotices::deserialize(one).unwrap(); + let copied = notices; + assert_eq!(notices, copied); + assert_eq!(notices, RawNotices { assumed_cfa: true, default_crop_adjusted: false, gain_table_omitted: false }); + assert!(!std::mem::needs_drop::(), "decoder notices own no allocated strings"); + } +} diff --git a/crates/raw/src/mef.rs b/crates/raw/src/mef.rs index 5cd1fb705..36ba18e0f 100644 --- a/crates/raw/src/mef.rs +++ b/crates/raw/src/mef.rs @@ -128,6 +128,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Result BlackLevel { - repeat_rows: 2, - repeat_cols: 2, - values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(), - ..Default::default() + let (black, black_source) = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() { + Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => ( + BlackLevel { + repeat_rows: 2, + repeat_cols: 2, + values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(), + ..Default::default() + }, + crate::LevelSource::Metadata, + ), + _ => match nrw_black(mn.as_ref(), bits) { + Some(value) => (BlackLevel::uniform(value), crate::LevelSource::Metadata), + None => { + let (value, source) = be16_black(&info, bytes.len() as u64, mn.as_ref(), samples); + (BlackLevel::uniform(value), source) + } }, - _ => BlackLevel::uniform(nrw_black(mn.as_ref(), bits).unwrap_or_else(|| be16_black(&info, bytes.len() as u64, mn.as_ref(), samples))), }; let wb = mn .as_ref() .and_then(|m| m.ifd.f64s(WB_RB_LEVELS)) .filter(|v| v.len() >= 2 && v[0] > 0.1 && v[1] > 0.1 && v[0] < 10.0 && v[1] < 10.0) .map(|v| [v[0] as f32, 1.0, v[1] as f32]); - let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) { + let (cfa, assumed_cfa) = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) { (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => { - Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 } + (Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }, false) } - _ => Cfa::bayer_static("RGGB"), + _ => (Cfa::bayer_static("RGGB"), true), }; let model = ifd0.string(t::MODEL).unwrap_or_default(); let body = measured_body(&model, w, h, bits); let active_w = trailing_masked_columns(samples, w, bits); // the white level is measured from the whole image (or known for a measured body); the probe (no samples in its // result) takes the nominal one - let white = match (body.and_then(|b| b.white), mode) { - (Some(white), _) => white, - (None, Mode::Full) => white_from_data(samples, bits), - (None, Mode::Header) => ((1u32 << bits.clamp(1, 16)) - 1) as f32, + let (white, white_source) = match (body.and_then(|b| b.white), mode) { + (Some(white), _) => (white, crate::LevelSource::Fallback), + (None, Mode::Full) => white_from_data_sourced(samples, bits), + (None, Mode::Header) => (((1u32 << bits.clamp(1, 16)) - 1) as f32, crate::LevelSource::Fallback), }; let crop = match body { Some(&MeasuredBody { crop: (x, y, cw, ch), .. }) if x + cw <= active_w && y + ch <= h => Rect::new(x, y, cw, ch), @@ -329,7 +338,8 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Opt } let stride = strip / h; let tight = w.checked_mul(bits as usize)?.div_ceil(8); - let tight = if bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4) { tight } else { return None }; + // Unmarked data must still match a measured complete-group layout: do not guess a packing for compressed + // data. An explicit packed maker mode permits a final partial group, byte-aligned independently on each row. + if !stated && !(bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4)) { + return None; + } let padded = tight.checked_next_multiple_of(16)?; let lsb_ok = match bits { 12 => stride == tight || stride == padded, @@ -436,14 +450,14 @@ fn be16_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option, samples: &[u16]) -> f32 { +fn be16_black(info: &ImageInfo, file_len: u64, mn: Option<&makernote::MakerNote>, samples: &[u16]) -> (f32, crate::LevelSource) { let has_table = mn.is_some_and(|m| m.ifd.bytes(LINEARIZATION_TABLE).is_some()); let single = matches!(info.chunks(file_len).as_slice(), [c] if be16_layout(info.width as usize, info.height as usize, info.bits() as u32, c.len).is_some()); if info.compression != t::compression::NIKON || has_table || info.samples_per_pixel != 1 || !single { - return 0.0; + return (0.0, crate::LevelSource::Fallback); } let darkest = samples.iter().copied().min().unwrap_or(0); - BE16_BLACK.min(f32::from(darkest)) + (BE16_BLACK.min(f32::from(darkest)), crate::LevelSource::Inferred) } /// Unpack a [`PackedLayout`] strip into `w * h` samples. Short input leaves the missing samples zero. @@ -524,7 +538,7 @@ fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>, } /// A "compressed" strip of packed samples: the strip size must match a measured layout exactly. `stated` is set -/// when the maker note names the packed mode, which also allows 14-bit rows without padding. +/// when the maker note names the packed mode, which also allows partial groups and 14-bit rows without padding. fn packed(bytes: &[u8], info: &ImageInfo, rows: usize, stated: bool, limits: &Limits) -> Result { let (w, h) = (info.width as usize, info.height as usize); let bits = info.bits() as u32; @@ -663,6 +677,7 @@ fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::ima cinema: Default::default(), lens_metadata: Default::default(), level_sources: Default::default(), + notices: Default::default(), }; img.validate_for(mode)?; Ok(img) @@ -725,6 +740,21 @@ mod tests { TiffWriter::new(order, false).write(&[ifd0]).unwrap() } + #[test] + fn level_provenance_distinguishes_missing_black_from_recorded_black() { + let strip: Vec = [1000u16; 16].into_iter().flat_map(u16::to_be_bytes).collect(); + let fallback = nef(1, 12, vec![strip.clone()], 4, 4, 4); + let mut note = IfdBuilder::new(); + note.set(BLACK_LEVEL, Value::Short(vec![400; 4])); + let recorded = nef_with_note(1, 12, vec![strip], 4, 4, 4, Some(note)); + for mode in [Mode::Header, Mode::Full] { + let missing = decode(&fallback, mode, &Limits::default()).unwrap(); + assert_eq!((missing.black.mean(), missing.level_sources.black), (0.0, crate::LevelSource::Fallback)); + let stated = decode(&recorded, mode, &Limits::default()).unwrap(); + assert_eq!((stated.black.mean(), stated.level_sources.black), (100.0, crate::LevelSource::Metadata)); + } + } + /// A small uncompressed NEF with a real NikonNEFInfo TIFF block, without binary fixtures. fn nikon_distortion_file(model: &str, mode: u8) -> Vec { use crate::nef_lens::tests::{distortion_block, nef_info_blob, offset_block}; @@ -1527,4 +1557,95 @@ mod tests { } } } + + mod partial_row_tests { + use super::*; + + // Independent wire oracle: place each sample bit directly, leaving nonzero tail bits and padding. + fn image(data: &[u16], w: usize, bits: u32, padded: bool) -> Vec { + let tight = (w * bits as usize).div_ceil(8); + let stride = if padded { tight.next_multiple_of(16) } else { tight }; + let mut bytes = vec![0xA5; stride * (data.len() / w)]; + for (i, &sample) in data.iter().enumerate() { + for bit in 0..bits as usize { + let at = (i % w) * bits as usize + bit; + let byte = &mut bytes[(i / w) * stride + at / 8]; + *byte = (*byte & !(1 << (at % 8))) | (((sample >> bit) as u8 & 1) << (at % 8)); + } + } + bytes + } + + fn note(order: ByteOrder, modern: bool, mode: u16) -> IfdBuilder { + let mut n = IfdBuilder::new(); + if modern { + let mut record = b"01000400\0\0".to_vec(); + record.extend(if order == ByteOrder::Big { mode.to_be_bytes() } else { mode.to_le_bytes() }); + n.set(NEF_COMPRESSION_RECORD, Value::Undefined(record)); + } else { + n.set(NEF_COMPRESSION, Value::Short(vec![mode])); + } + n + } + + #[test] + fn declared_packed_partial_rows_preserve_every_sample() { + let h = 5; + for (bits, mode) in [(12u32, 6), (14, 10)] { + for w in [1, 3, 5, 22, 23] { + let data = samples(w * h, bits); + for padded in [false, true] { + let wire = image(&data, w, bits, padded); + if w == 22 && bits == 14 { + assert_eq!(wire.len() / h, if padded { 48 } else { 39 }); + } + for order in [ByteOrder::Big, ByteOrder::Little] { + for modern in [false, true] { + let file = nef_in( + order, + 34713, + bits as u16, + vec![wire.clone()], + w as u32, + h as u32, + h as u32, + Some(note(order, modern, mode)), + ); + let raw = crate::decode(&file).unwrap(); + assert_eq!((raw.width, raw.height, raw.cpp), (w, h, 1)); + assert!(!raw.notices.assumed_cfa, "valid CFA tag is retained"); + let RawData::U16(got) = raw.data else { panic!("integer samples") }; + assert_eq!(got, data, "{bits}-bit width {w}, padded {padded}, {order:?}, modern {modern}"); + } + } + } + } + } + } + + #[test] + fn partial_rows_keep_mode_depth_and_strip_guards() { + let (w, h) = (22, 5); + let data = samples(w * h, 14); + for padded in [false, true] { + let wire = image(&data, w, 14, padded); + let file = |bytes, mode: Option| { + nef_with_note(34713, 14, vec![bytes], w as u32, h as u32, h as u32, mode.map(|m| note(ByteOrder::Big, false, m))) + }; + for mode in [None, Some(0), Some(6), Some(13), Some(14)] { + assert!(matches!(crate::decode(&file(wire.clone(), mode)), Err(RawError::Unsupported(_))), "mode {mode:?}"); + } + let good = file(wire.clone(), Some(10)); + assert!(crate::decode(&good).is_ok()); + let short = truncated_tail_strip(good, wire.len() - 1); + assert!(matches!(crate::decode(&short), Err(RawError::Corrupt(_)))); + for len in [wire.len() - 1, wire.len() + 1] { + assert!(matches!(crate::decode(&file(vec![0; len], Some(10))), Err(RawError::Unsupported(_)))); + } + } + for (w, h, bits) in [(0, 5, 14), (22, 0, 14), (22, 5, 13), (usize::MAX, 5, 14)] { + assert!(packed_layout(w, h, bits, u64::MAX, true).is_none()); + } + } + } } diff --git a/crates/raw/src/orf.rs b/crates/raw/src/orf.rs index ebfa28eb2..566e56fed 100644 --- a/crates/raw/src/orf.rs +++ b/crates/raw/src/orf.rs @@ -207,6 +207,19 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result 0 && *n <= limits.samples()).ok_or(RawError::Limit("image too large"))?; + // A partial TIFF directory is damage, not evidence of an unsupported compressed layout. + // Do not infer packing from an absent strip table or from clipped sensor bytes. + if info.compression == 1 + && (info.offsets.is_empty() + || info.offsets.len() != info.byte_counts.len() + || info + .offsets + .iter() + .zip(&info.byte_counts) + .any(|(&at, &len)| len == 0 || at.checked_add(len).is_none_or(|end| end > bytes.len() as u64))) + { + return Err(RawError::Corrupt("ORF strip table is missing or outside file".into())); + } let chunks = info.chunks(bytes.len() as u64); let total = chunks.iter().fold(0u64, |a, c| a.saturating_add(c.len)); let stated = cfa_from_exif(&tiff); @@ -302,6 +315,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Rect::new(0, 0, w, h), }; + let assumed_cfa = stated.is_none(); let cfa = stated.unwrap_or_else(|| cfa_from_data(samples, w, active)); let (black, black_source) = match ip.as_ref().and_then(|i| i.f64s(BLACK)).as_deref() { Some(v @ [_, _, _, _]) => { @@ -348,6 +362,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result = strip.as_chunks::<2>().0.iter().map(|word| u16::from_le_bytes(*word)).collect(); + for (tag, assumed, pattern) in [ + (None, true, "RGGB"), + (Some(&[2, 0, 2, 0, 0, 1][..]), true, "RGGB"), + (Some(&[2, 0, 2, 0, 0, 1, 1, 3][..]), true, "RGGB"), + (Some(BGGR), false, "BGGR"), + (Some(GRBG), false, "GRBG"), + ] { + let bytes = orf_with(w as u32, h as u32, 16, strip.clone(), tag); + for mode in [Mode::Header, Mode::Full] { + let image = decode(&bytes, mode, &Limits::default()).unwrap(); + assert_eq!(image.cfa, Some(Cfa::bayer_static(pattern))); + assert_eq!(image.data, RawData::U16(expected.clone()), "word16 samples remain unchanged"); + assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() }); + } + } + } + #[test] fn packed_12_bit_probe_agrees_with_decode() { let (w, h) = (16usize, 4usize); diff --git a/crates/raw/src/pef.rs b/crates/raw/src/pef.rs index a33750db5..9dc3a89f1 100644 --- a/crates/raw/src/pef.rs +++ b/crates/raw/src/pef.rs @@ -194,6 +194,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result r, None => dark_trimmed(samples, w, hgt, black_point.as_deref().map(|v| v.iter().sum::() / 4.0), white), }; + let mut assumed_cfa = false; let cfa = match (ifd0.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), ifd0.bytes(t::CFA_PATTERN_EP)) { (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => { Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 } @@ -202,7 +203,10 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result cfa_from_exif(&tiff).unwrap_or_else(|| Cfa::bayer_static("BGGR")), + _ => cfa_from_exif(&tiff).unwrap_or_else(|| { + assumed_cfa = true; + Cfa::bayer_static("BGGR") + }), }; let black = match black_point.as_deref() { Some(v @ [_, _, _, _]) => { @@ -240,6 +244,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result = (0..16).map(|i| 100 + i * 7).collect(); + let valid_exif = &[2, 0, 2, 0, 1, 0, 2, 1][..]; + let invalid_exif = &[2, 0, 2, 0, 1, 0][..]; + for (primary, exif, assumed, pattern) in [ + (None, None, true, "BGGR"), + (Some([0, 1, 1, 3]), None, true, "BGGR"), + (None, Some(invalid_exif), true, "BGGR"), + (Some([0, 1, 1, 3]), Some(invalid_exif), true, "BGGR"), + (Some([0, 1, 1, 3]), Some(valid_exif), false, "GRBG"), + (None, Some(valid_exif), false, "GRBG"), + (Some([0, 1, 1, 2]), None, false, "RGGB"), + (Some([0, 1, 1, 2]), Some(valid_exif), false, "RGGB"), + ] { + let mut raw = IfdBuilder::new(); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![4])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![4])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16])); + raw.set(t::COMPRESSION, Value::Short(vec![1])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![32803])); + raw.set(t::MAKE, Value::Ascii("PENTAX Corporation".into())); + if let Some(pattern) = primary { + raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2])); + raw.set(t::CFA_PATTERN_EP, Value::Byte(pattern.to_vec())); + } + if let Some(pattern) = exif { + raw.set_child(t::EXIF_IFD, IfdBuilder::new().with(crate::EXIF_CFA_PATTERN, Value::Undefined(pattern.to_vec()))); + } + raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![expected.iter().flat_map(|value| value.to_be_bytes()).collect()] }); + let bytes = TiffWriter::new(ByteOrder::Big, false).write(&[raw]).unwrap(); + for mode in [Mode::Header, Mode::Full] { + let image = decode(&bytes, mode, &Limits::default()).unwrap(); + assert_eq!(image.cfa, Some(Cfa::bayer_static(pattern))); + assert_eq!(image.data, RawData::U16(expected.clone())); + assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() }); + } + } + } + /// A `w x h` big-endian 14-bit PEF of plain 16-bit words without area tags, whose maker note has only a /// BlackPoint of `black`, with `px(x, y)` as the samples. fn pef_with_black_point(w: usize, h: usize, black: u16, px: impl Fn(usize, usize) -> u16) -> Vec { diff --git a/crates/raw/src/preview.rs b/crates/raw/src/preview.rs index 48fa4e4b1..bd66f58c8 100644 --- a/crates/raw/src/preview.rs +++ b/crates/raw/src/preview.rs @@ -643,9 +643,15 @@ fn cr3_preview(bytes: &[u8]) -> Option<&[u8]> { /// A CR3's full-size JPEG track, when it holds a DCT JPEG. #[cfg(feature = "cr3")] fn cr3_full_jpeg(bytes: &[u8]) -> Option<&[u8]> { - let full = craft_metadata::cr3::parse_cr3(bytes) - .and_then(|c| c.tracks.iter().find(|t| t.kind == craft_metadata::cr3::Cr3TrackKind::Jpeg).and_then(|t| t.data)); - full.and_then(|(at, len)| bytes.get(at..at.checked_add(len)?)).filter(|j| is_dct_jpeg(j)) + let cr3 = craft_metadata::cr3::parse_cr3(bytes)?; + // An unusable preview track must not hide a later usable one. Share the marker budget + // across candidates so overlapping hostile track ranges cannot cause quadratic walks. + let mut work = CR3_MAX_WALK; + cr3.tracks.iter().filter(|t| t.kind == craft_metadata::cr3::Cr3TrackKind::Jpeg).find_map(|t| { + let (at, len) = t.data?; + let jpeg = bytes.get(at..at.checked_add(len)?)?; + first_frame_within(jpeg, &mut work).filter(|(m, _)| matches!(m, 0xc0..=0xc2)).map(|_| jpeg) + }) } /// Marker-walk budget of the CR3 fallback: candidates can overlap (each claiming the rest of the file) and diff --git a/crates/raw/src/preview/tests.rs b/crates/raw/src/preview/tests.rs index 19018d9b1..e04c8574f 100644 --- a/crates/raw/src/preview/tests.rs +++ b/crates/raw/src/preview/tests.rs @@ -582,7 +582,19 @@ fn cr3_prefers_the_full_size_jpeg_track() { let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat()); let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat()); let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat()); - let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat())))); + let trak = bx( + b"trak", + &bx( + b"mdia", + &bx( + b"minf", + &bx( + b"stbl", + &[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>()), stsz, co64].concat(), + ), + ), + ), + ); file.extend(bx(b"moov", &trak)); file.resize(at as usize, 0); file.extend_from_slice(&big); @@ -592,6 +604,48 @@ fn cr3_prefers_the_full_size_jpeg_track() { assert_eq!(embedded_preview(&file).unwrap(), small); } +#[cfg(feature = "cr3")] +#[test] +fn cr3_skips_bad_preview_tracks_but_never_uses_sensor_tracks_as_jpeg() { + let bx = |kind: &[u8; 4], body: &[u8]| -> Vec { [&((body.len() + 8) as u32).to_be_bytes()[..], kind, body].concat() }; + let full = |kind: &[u8; 4], body: &[u8]| bx(kind, &[&[0u8; 4][..], body].concat()); + let track = |jpeg: bool, at: u64, len: u32| { + let mut entry = vec![0u8; 82]; + if jpeg { + entry.extend(bx(b"JPEG", &[0; 4])); + } + let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &entry)].concat()); + let stsz = full(b"stsz", &[len.to_be_bytes(), 1u32.to_be_bytes()].concat()); + let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat()); + bx( + b"trak", + &bx( + b"mdia", + &bx( + b"minf", + &bx( + b"stbl", + &[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>()), stsz, co64].concat(), + ), + ), + ), + ) + }; + let good = fake_jpeg(100); + for first in [track(true, u64::MAX, 32), track(true, 4096, 32)] { + let mut bytes = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + bytes.extend(bx(b"moov", &[first, track(true, 4128, good.len() as u32)].concat())); + bytes.resize(4128, 0); + bytes.extend(&good); + assert_eq!(embedded_preview(&bytes).as_deref(), Some(good.as_slice())); + } + let mut bytes = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + bytes.extend(bx(b"moov", &track(false, 4096, good.len() as u32))); + bytes.resize(4096, 0); + bytes.extend(&good); + assert_eq!(embedded_preview(&bytes), None, "a sensor sample is not a rendered preview even if its bytes look like JPEG"); +} + #[cfg(feature = "cr3")] /// A CR3 whose `PRVW` box holds `prvw` and whose JPEG track holds `big`. fn cr3_with(prvw_jpeg: &[u8], big: &[u8]) -> Vec { @@ -608,7 +662,19 @@ fn cr3_with(prvw_jpeg: &[u8], big: &[u8]) -> Vec { let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat()); let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat()); let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat()); - let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat())))); + let trak = bx( + b"trak", + &bx( + b"mdia", + &bx( + b"minf", + &bx( + b"stbl", + &[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>()), stsz, co64].concat(), + ), + ), + ), + ); file.extend(bx(b"moov", &trak)); file.resize(at as usize, 0); file.extend_from_slice(big); @@ -656,7 +722,19 @@ fn the_smallest_preview_covering_the_size_is_chosen() { let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat()); let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat()); let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat()); - let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat())))); + let trak = bx( + b"trak", + &bx( + b"mdia", + &bx( + b"minf", + &bx( + b"stbl", + &[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>()), stsz, co64].concat(), + ), + ), + ), + ); file.extend(bx(b"moov", &trak)); file.resize(at as usize, 0); file.extend_from_slice(&big); @@ -769,7 +847,19 @@ fn a_cr3_with_two_prvw_boxes_offers_both() { let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat()); let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat()); let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat()); - let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat())))); + let trak = bx( + b"trak", + &bx( + b"mdia", + &bx( + b"minf", + &bx( + b"stbl", + &[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::>()), stsz, co64].concat(), + ), + ), + ), + ); file.extend(bx(b"moov", &trak)); // (the track in the media data, as a camera writes it, after a `free` box up to it) let pad = at as usize - 16 - file.len(); diff --git a/crates/raw/src/raf.rs b/crates/raw/src/raf.rs index f75fa160b..a3e407454 100644 --- a/crates/raw/src/raf.rs +++ b/crates/raw/src/raf.rs @@ -18,7 +18,10 @@ //! - CFA: the X-Trans layout record (36 bytes, 0 = R, 1 = G, 2 = B) is stored in reverse order and, reversed, //! is anchored at raw pixel (0, 0) — verified on X-Trans I and III samples by minimising the difference //! between horizontally / vertically adjacent "green" samples over all 72 orientations/phases, and by colour -//! renders. Bayer bodies (X-A series) use RGGB at (0, 0), verified by colour renders. +//! renders. Bayer FujiLayout (`0x0130`) gives the reversed 2×2 pattern in its low two bits +//! (0 = R, 1/3 = G, 2 = B); absent layout records retain the assumed RGGB fallback. +//! - RawExposureBias (`0x9650`) is a signed big-endian 16-bit fraction; its negative is the +//! baseline exposure used to restore the camera's dynamic-range exposure reduction. //! - Striped predictive compression is decoded by `rafc`; see `docs/raf-compression.md`. use crate::shared::white_from_data_sourced; @@ -87,7 +90,7 @@ fn pair(h: &Header, tag: u16) -> Option<(usize, usize)> { pub(crate) fn raw_ifd(raw: &[u8]) -> Result { let tiff = Tiff::parse(raw).map_err(|_| RawError::Unsupported("RAF without a raw IFD (older FinePix layout)".into()))?; let off = tiff.ifds.first().and_then(|i| i.u64(0xf000)).ok_or_else(|| RawError::Unsupported("RAF without a raw IFD".into()))?; - let (ifd, _) = craft_tiff::parse_ifd_at(raw, off, ByteOrder::Little, 0, false, &Default::default())?; + let (ifd, _) = craft_tiff::parse_ifd_at(raw, off, tiff.order, 0, false, &Default::default())?; Ok(ifd) } @@ -122,14 +125,42 @@ fn camera_name(b: &[u8]) -> String { String::from_utf8_lossy(name.get(..end).unwrap_or_default()).trim().to_string() } -/// The CFA: X-Trans from the (reversed) layout record, else RGGB. -fn cfa(h: &Header) -> Cfa { - match record(h, XTRANS_LAYOUT) { - Some(l) if l.len() == 36 && l.iter().all(|&c| c <= 2) => { - Cfa { width: 6, height: 6, pattern: l.iter().rev().copied().collect(), origin_x: 0, origin_y: 0 } +/// Layout records are reversed and anchored at sensor (0, 0). FujiLayout's low +/// two bits encode Bayer colour; both 1 and 3 denote green. +fn cfa(h: &Header) -> Result { + if let Some(l) = record(h, XTRANS_LAYOUT) { + if l.len() != 36 || l.iter().any(|&c| c > 2) { + return Err(RawError::Unsupported("Fujifilm RAF with an unknown X-Trans layout".into())); } - _ => Cfa::bayer_static("RGGB"), + return Ok(Cfa { width: 6, height: 6, pattern: l.iter().rev().copied().collect(), origin_x: 0, origin_y: 0 }); } + let Some(l) = record(h, 0x0130) else { return Ok(Cfa::bayer_static("RGGB")) }; + let cfa = Cfa { width: 2, height: 2, pattern: l.iter().rev().map(|&c| if c & 3 == 3 { 1 } else { c & 3 }).collect(), origin_x: 0, origin_y: 0 }; + // Colour counts alone also accept adjacent green sites; the two greens + // must occupy opposite corners of the 2x2 Bayer repeat. + if l.len() != 4 || !matches!(cfa.pattern.as_slice(), [0, 1, 1, 2] | [2, 1, 1, 0] | [1, 0, 2, 1] | [1, 2, 0, 1]) { + return Err(RawError::Unsupported("Fujifilm RAF with an unknown Bayer layout".into())); + } + Ok(cfa) +} + +/// Fuji RawExposureBias is a signed big-endian 16-bit fraction, undone like +/// DNG BaselineExposure. Invalid or implausible metadata does not alter exposure. +fn baseline_exposure(h: &Header) -> f64 { + record(h, 0x9650) + .and_then(|d| { + let num = i16::from_be_bytes(d.get(..2)?.try_into().ok()?); + let den = i16::from_be_bytes(d.get(2..4)?.try_into().ok()?); + (den != 0).then(|| -f64::from(num) / f64::from(den)) + }) + .filter(|v| v.is_finite() && v.abs() <= 8.0) + .unwrap_or(0.0) +} + +/// The codec fixes the green-site geometry; exchanging the two non-green +/// planes only exchanges their colour labels, so RGGB and BGGR are both valid. +fn compressed_layout_supported(cfa: &Cfa) -> bool { + *cfa == Cfa::xtrans() || *cfa == Cfa::bayer_static("RGGB") || *cfa == Cfa::bayer_static("BGGR") } /// Unpack one row of `bits`-bit samples in Fujifilm's packing. `words32`: the raw IFD's `0xf006` flag is set, the @@ -137,7 +168,7 @@ fn cfa(h: &Header) -> Cfa { /// LSB-first bit stream, or (other depths) the same 32-bit words. pub(crate) fn unpack_row(src: &[u8], bits: u32, words32: bool, out: &mut [u16]) { match bits { - 16 => out.iter_mut().zip(src.as_chunks::<2>().0).for_each(|(o, c)| *o = u16::from_le_bytes([c[0], c[1]])), + 16 if !words32 => out.iter_mut().zip(src.as_chunks::<2>().0).for_each(|(o, c)| *o = u16::from_le_bytes([c[0], c[1]])), 12 if !words32 => unpack_lsb(src, 12, out), _ => unpack_words32_msb(src, ByteOrder::Little, bits, out), } @@ -171,7 +202,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result false, @@ -180,8 +211,13 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result= (n as u64) * 2 { 16 } else { bits }; - let packed_row = w.checked_mul(store as usize).map(|n| n.div_ceil(8)).ok_or(RawError::Limit("RAF row too large"))?; + // An explicit packed-word flag wins over the size heuristic: padding in short + // packed rows can occupy as many bytes as unpacked 16-bit samples. + // Unflagged samples may use 16-bit words (e.g. 14-bit X-T20). + let store = if !words32 && len as u64 >= (n as u64) * 2 { 16 } else { bits }; + let row_bits = w.checked_mul(store as usize).ok_or(RawError::Limit("RAF row too large"))?; + // Word-packed rows store the high bits in the last little-endian word's final bytes. + // Reading only ceil(bits / 8) bytes would lose those samples, not just padding. + let packed_row = if words32 || (store != 16 && store != 12) { + row_bits.div_ceil(32).checked_mul(4).ok_or(RawError::Limit("RAF row too large"))? + } else { + row_bits.div_ceil(8) + }; let stride = match get(0xf005)?.unwrap_or(0) { 0 => packed_row, words => words.checked_mul(4).ok_or(RawError::Limit("RAF stride too large"))?.max(packed_row), @@ -206,7 +251,11 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Result = (0..w * h).map(|i| ((i * 977 + 101) % (1 << bits)) as u16).collect(); + // Independently place each sample bit into whole little-endian 32-bit row words. + let strip: Vec = px + .chunks(w as usize) + .flat_map(|row| { + let mut words = vec![0u32; (row.len() * bits as usize).div_ceil(32)]; + for (i, &sample) in row.iter().enumerate() { + for bit in 0..bits as usize { + let at = i * bits as usize + bit; + words[at / 32] |= u32::from((sample >> (bits as usize - 1 - bit)) & 1) << (31 - at % 32); + } + } + words.into_iter().flat_map(u32::to_le_bytes).collect::>() + }) + .collect(); + let bytes = raf_flagged(w, h, bits, strip, None, &[], flag); + assert_eq!(crate::decode(&bytes).unwrap().data, RawData::U16(px)); + assert!(crate::decode(&bytes[..bytes.len() - 1]).is_err()); + } + } + + #[test] + fn big_endian_words_and_bayer_layout_preserve_exact_sensor_values() { + use craft_tiff::{IfdBuilder, TiffWriter, Value}; + let (w, h) = (6u32, 4u32); + let px: Vec = (0..w * h).map(|i| (i * 977 + 123) as u16).collect(); + for order in [ByteOrder::Little, ByteOrder::Big] { + let build = |at| { + let mut sensor = IfdBuilder::new(); + for (tag, value) in [(0xf001, w), (0xf002, h), (0xf003, 16), (0xf007, at), (0xf008, w * h * 2)] { + sensor.set(tag, Value::Long(vec![value])); + } + let mut root = IfdBuilder::new(); + root.set_child(0xf000, sensor); + TiffWriter::new(order, false).write(&[root]).unwrap() + }; + let mut raw = build(build(0).len() as u32); + raw.extend(px.iter().flat_map(|v| if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() })); + let mut bytes = raf(w, h, 16, vec![0; (w * h * 2) as usize], None, &[]); + let raw_at = be32(&bytes, 100).unwrap(); + bytes.truncate(raw_at); + bytes[104..108].copy_from_slice(&(raw.len() as u32).to_be_bytes()); + bytes.extend(raw); + // A four-site Bayer record is reversed in storage, just like the X-Trans record. + let dir_at = be32(&bytes, 92).unwrap(); + let mut records = bytes[dir_at..raw_at].to_vec(); + let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1; + records[..4].copy_from_slice(&n.to_be_bytes()); + records.extend_from_slice(&[0x01, 0x30, 0, 4, 0x08, 0x09, 0x0b, 0x0a]); + bytes.splice(dir_at..raw_at, records.clone()); + bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes()); + bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes()); + let image = crate::decode(&bytes).unwrap(); + assert_eq!(image.data, RawData::U16(px.clone())); + assert_eq!(image.cfa.unwrap().name(), "BGGR"); + } + } + + #[test] + fn raw_exposure_bias_is_preserved_as_baseline_exposure_and_invalid_values_are_ignored() { + for (num, den, expected) in [(-72i16, 100i16, 0.72f64), (1, 0, 0.0), (32767, 1, 0.0)] { + let mut bytes = raf(6, 4, 16, pack(&[1000; 24], 16), None, &[]); + let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap()); + let mut records = bytes[dir_at..raw_at].to_vec(); + let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1; + records[..4].copy_from_slice(&n.to_be_bytes()); + records.extend_from_slice(&[0x96, 0x50, 0, 4]); + records.extend(num.to_be_bytes()); + records.extend(den.to_be_bytes()); + bytes.splice(dir_at..raw_at, records.clone()); + bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes()); + bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes()); + for mode in [Mode::Header, Mode::Full] { + assert!((decode(&bytes, mode, &Limits::default()).unwrap().color.baseline_exposure - expected).abs() < 1e-9); + } + } + } + + #[test] + fn invalid_declared_cfa_layouts_are_not_silently_replaced() { + for (tag, data) in [(0x0130, vec![8, 8, 10, 10]), (0x0130, vec![1, 2, 1, 0]), (0x0131, vec![3; 36])] { + let header = Header { jpeg: None, records: vec![(tag, data.as_slice())], raw: None }; + assert!(matches!(cfa(&header), Err(RawError::Unsupported(_)))); + } + } + + fn with_bayer_record(mut bytes: Vec, pattern: [u8; 4]) -> Vec { + let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap()); + let mut records = bytes[dir_at..raw_at].to_vec(); + let count = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1; + records[..4].copy_from_slice(&count.to_be_bytes()); + records.extend_from_slice(&[0x01, 0x30, 0, 4]); + records.extend_from_slice(&pattern); + bytes.splice(dir_at..raw_at, records.clone()); + bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes()); + bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes()); + bytes + } + + #[test] + fn non_bayer_geometry_is_rejected_in_header_and_full_modes() { + // Reversal gives RGBG: correct colour counts, but both greens in one column. + let bytes = with_bayer_record(raf(6, 4, 16, pack(&[1000; 24], 16), None, &[]), [1, 2, 1, 0]); + for mode in [Mode::Header, Mode::Full] { + assert!(matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("Bayer layout"))); + } + } + + #[test] + fn all_four_bayer_geometries_preserve_the_stored_samples() { + let pixels: Vec = (0..24).map(|i| 123 + i * 977).collect(); + for (stored, expected) in [([2, 1, 1, 0], "RGGB"), ([0, 1, 1, 2], "BGGR"), ([1, 2, 0, 1], "GRBG"), ([1, 0, 2, 1], "GBRG")] { + let bytes = with_bayer_record(raf(6, 4, 16, pack(&pixels, 16), None, &[]), stored); + for mode in [Mode::Header, Mode::Full] { + let image = decode(&bytes, mode, &Limits::default()).unwrap(); + assert_eq!((image.width, image.height), (6, 4)); + assert_eq!(image.cfa.as_ref().unwrap().name(), expected); + assert_eq!(image.data, RawData::U16(if mode == Mode::Full { pixels.clone() } else { Vec::new() })); + } + } + } + + #[test] + fn compressed_bayer_requires_the_layout_assumed_by_its_predictor() { + let strip = b"IS\x01\x00\x0e\0\x12\x06\0\x03\x0c\x03\0\x02\0\x03".to_vec(); + let mut bytes = raf(780, 18, 14, strip, None, &[]); + let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap()); + let mut records = bytes[dir_at..raw_at].to_vec(); + let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1; + records[..4].copy_from_slice(&n.to_be_bytes()); + // Reversed GRBG: a valid Bayer pattern, but not the RGGB order of this codec. + records.extend_from_slice(&[0x01, 0x30, 0, 4, 0x0b, 0x0a, 0x08, 0x09]); + bytes.splice(dir_at..raw_at, records.clone()); + bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes()); + bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes()); + for mode in [Mode::Header, Mode::Full] { + assert!(matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("layout"))); + } + } + + #[test] + fn compressed_layout_accepts_both_red_blue_orders_without_accepting_wrong_green_phase() { + for name in ["RGGB", "BGGR"] { + assert!(compressed_layout_supported(&Cfa::bayer_static(name)), "{name}"); + } + for name in ["GRBG", "GBRG"] { + assert!(!compressed_layout_supported(&Cfa::bayer_static(name)), "{name}"); + } + let mut other = Cfa::xtrans(); + assert!(compressed_layout_supported(&other)); + other.pattern.swap(0, 2); + assert!(!compressed_layout_supported(&other)); + } + + #[test] + fn unknown_compressed_signature_is_unsupported_but_identified_stream_damage_is_corrupt() { + let header = b"IS\x01\x00\x0e\0\x12\x06\0\x03\x0c\x03\0\x02\0\x03".to_vec(); + let known = raf(780, 18, 14, header, None, &[]); + let raw_at = be32(&known, 100).unwrap(); + let strip_at = raw_at + raw_ifd(&known[raw_at..]).unwrap().u64(0xf007).unwrap() as usize; + let mut unknown = known.clone(); + unknown[strip_at..strip_at + 2].copy_from_slice(b"XX"); + for mode in [Mode::Header, Mode::Full] { + assert!(matches!(decode(&unknown, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("signature"))); + assert!( + matches!(decode(&known, mode, &Limits::default()), Err(RawError::Corrupt(_))), + "identified but truncated IS stream must remain corrupt" + ); + } + } + /// Raw IFD flag `0xf006` selects the 32-bit-word packing at 12 bits; without it 12 bits are an LSB-first stream. #[test] fn flag_selects_word_packed_12_bit() { diff --git a/crates/raw/src/rw2.rs b/crates/raw/src/rw2.rs index 9984ed987..1dc1a885e 100644 --- a/crates/raw/src/rw2.rs +++ b/crates/raw/src/rw2.rs @@ -615,11 +615,18 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Rect::new(0, 0, active.width, active.height), }; - if !data.is_empty() { + // Packed formats 2/5/7 are lossless sample containers: zero is an ordinary + // code, not evidence of a mapped-out photosite. The documented zero repair + // applies to the predictive format 4 and older word containers only. + if !data.is_empty() && matches!(encoding, Encoding::Words | Encoding::Blocks { kind: Block::Format4, .. }) { fill_defects(&mut data, w, active); } - let cfa = cfa(ifd0.u16(CFA_PATTERN).unwrap_or(1)); + let cfa_code = ifd0.u16(CFA_PATTERN); + // RGGB is still usable when metadata is absent or unknown, but callers + // must be able to distinguish that fallback from a recorded sensor phase. + let assumed_cfa = !matches!(cfa_code, Some(1..=4)); + let cfa = cfa(cfa_code.unwrap_or(1)); let read = BLACK_RGB.map(|tag| ifd0.f64(tag).filter(|v| v.is_finite() && *v >= 0.0)); let tags = read.map(|v| v.unwrap_or(0.0) as f32); let offset = match encoding { @@ -671,6 +678,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result = (0..72).map(|i| 17 + i * 113).collect(); + let payload = pack(&pixels, 14); + for (code, expected, assumed) in [ + (None, "RGGB", true), + (Some(0), "RGGB", true), + (Some(5), "RGGB", true), + (Some(u16::MAX), "RGGB", true), + (Some(1), "RGGB", false), + (Some(2), "GRBG", false), + (Some(3), "GBRG", false), + (Some(4), "BGGR", false), + ] { + let mut tags = vec![short(BITS, 14), short(RAW_FORMAT, 7)]; + if let Some(code) = code { + tags.push(short(CFA_PATTERN, code)); + } + let bytes = file(18, 4, &tags, &payload, &|off| vec![(RAW_OFFSET, Value::Long(vec![off]))]); + for mode in [Mode::Header, Mode::Full] { + let image = decode(&bytes, mode, &Limits::default()).unwrap(); + assert_eq!((image.width, image.height), (18, 4)); + assert_eq!(image.cfa.as_ref().unwrap().name(), expected, "code {code:?}"); + assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() }, "code {code:?}, {mode:?}"); + assert_eq!(samples(&image), if mode == Mode::Full { pixels.as_slice() } else { &[] }); + } + } + } + + #[test] + fn lossless_packed_formats_preserve_zero_codes_inside_the_active_area() { + let (w, h) = (90usize, 12usize); + for (format, bits, rotated) in [(5u16, 12u32, true), (5, 14, true), (7, 14, false), (2, 12, false)] { + let mut px = vec![1000u16; w * h]; + px[5 * w + 20] = 0; + let payload = pack(&px, bits); + let bytes = rw2(w as u16, h as u16, bits as u16, format, 100, &if rotated { rotate(payload) } else { payload }); + assert_eq!(samples(&crate::decode(&bytes).unwrap()), px, "format {format}/{bits}"); + } + } + #[test] fn levels_crop_defects_and_cfa() { let (w, h) = (90usize, 60usize); let mut px: Vec = (0..w * h).map(|i| 300 + (i % 7) as u16).collect(); - // a saturation plateau sets the white level; zeros inside the active area are defects + // A saturation plateau sets the white level. In the older word container, + // isolated zeros inside the active area are documented mapped-out defects. px[10 * w..20 * w].fill(3900); px[30 * w + 40] = 0; px[1] = 0; // in the border: left alone - let mut tags = vec![short(CFA_PATTERN, 2), short(BITS, 12), short(RAW_FORMAT, 5)]; + let mut tags = vec![short(CFA_PATTERN, 2), short(BITS, 12), short(COMPRESSION, 34828)]; tags.extend([0x002f, 0x0030, 0x0031, 0x0032].iter().zip([12u16, 10, 52, 80]).map(|(&tag, v)| short(tag, v))); tags.extend(LINEARITY_LIMIT.map(|tag| short(tag, 2000))); // recorded limits are not used - let bytes = file(w as u16, h as u16, &tags, &rotate(pack(&px, 12)), &|off| vec![(RAW_OFFSET, Value::Long(vec![off]))]); + // Compression 34828 stores left-justified samples in little-endian u16 words. + let words: Vec = px.iter().flat_map(|&v| (v << 4).to_le_bytes()).collect(); + let bytes = file(w as u16, h as u16, &tags, &words, &|off| vec![(t::STRIP_OFFSETS, Value::Long(vec![off]))]); let r = crate::decode(&bytes).unwrap(); let d = samples(&r); assert_eq!(d[30 * w + 40], ((px[30 * w + 38] as u32 + px[30 * w + 42] as u32 + px[28 * w + 40] as u32 + px[32 * w + 40] as u32) / 4) as u16); diff --git a/crates/raw/src/srw.rs b/crates/raw/src/srw.rs index c78ef1f6f..c9f9a0ce6 100644 --- a/crates/raw/src/srw.rs +++ b/crates/raw/src/srw.rs @@ -192,6 +192,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result read_image(data, info, order, packing, limits), crate::Mode::Header => { check_image(data, info, limits)?; + check_uncompressed_chunks(data, info, &info.chunks(data.len() as u64), packing)?; // a lossless-JPEG layout the decoder rejects (subsampled components, e.g. Sony's // lossless M/S sizes) must fail here too, as the full decode will; likewise a JPEG XL // chunk whose headers don't match the IFD @@ -151,6 +152,7 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin let bits = info.bits() as u32; let float = info.sample_format == 3; let chunks = info.chunks(data.len() as u64); + check_uncompressed_chunks(data, info, &chunks, packing)?; let planar = info.planar == 2 && cpp > 1; let ccpp = if planar { 1 } else { cpp }; // The chunks are decoded and copied into the output in groups whose combined nominal size stays within @@ -222,6 +224,26 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin Ok(if float { RawData::F32(out_f32) } else { RawData::U16(out_u16) }) } +/// Framing precedes allocation even in header-only probes: missing source rows may not +/// be reported as a decodable image and later silently filled with zero samples. +fn check_uncompressed_chunks(data: &[u8], info: &ImageInfo, chunks: &[Chunk], packing: Packing) -> Result<()> { + if info.compression != comp::NONE { + return Ok(()); + } + let cpp = if info.planar == 2 { 1 } else { info.samples_per_pixel as usize }; + for chunk in chunks { + let src = chunk_bytes(data, chunk).ok_or_else(|| RawError::Corrupt("uncompressed chunk is outside the file".into()))?; + let (cw, _, vw, vh) = chunk_dims(info, chunk); + let row = packed_row_bytes(cw as u64 * cpp as u64, info.bits() as u32, packing, info.sample_format == 3); + let last = packed_row_bytes(vw as u64 * cpp as u64, info.bits() as u32, packing, info.sample_format == 3); + let need = row.saturating_mul(vh.saturating_sub(1) as u64).saturating_add(last); + if (src.len() as u64) < need { + return Err(RawError::Corrupt(format!("an uncompressed chunk has {} bytes, expected at least {need}", src.len()))); + } + } + Ok(()) +} + /// Bytes that `n` samples take in a row of an uncompressed or inflated chunk, laid out as /// [`unpack_chunk`] reads them: whole bytes or 16-bit words, or 32-bit words for `Words32Msb`. fn packed_row_bytes(n: u64, bits: u32, packing: Packing, float: bool) -> u64 { @@ -285,10 +307,12 @@ fn decode_chunk( } match info.compression { comp::NONE => { - // uncompressed: missing rows are filled with zeros, but a chunk holding a small fraction of what its - // samples need is no picture: refused before the nominal chunk is allocated (storytold/lightcraft#1512) - let need = (n as u64).saturating_mul(u64::from(bits)) / 8; - if (src.len() as u64).saturating_mul(64) < need { + // Require every visible sample. Nominal edge-tile padding may be absent, but + // missing image rows must not turn a truncated file into a zero-filled success. + let row_bytes = packed_row_bytes(cw as u64 * cpp as u64, bits, packing, float); + let last_row = packed_row_bytes(vw as u64 * cpp as u64, bits, packing, float); + let need = row_bytes.saturating_mul(vh.saturating_sub(1) as u64).saturating_add(last_row); + if (src.len() as u64) < need { return Err(RawError::Corrupt(format!("an uncompressed chunk of {n} samples has only {} bytes", src.len()))); } unpack_chunk(src, order, packing, cw, ch, cpp, bits, float, 1) @@ -314,13 +338,7 @@ fn decode_chunk( let mcu = |v: usize| v.div_ceil(32).saturating_mul(32); let tile_budget = mcu(cw).saturating_mul(mcu(ch)).saturating_mul(if cpp == 1 { 1 } else { 3 }).clamp(1, limits.max_lossy_tile_samples.max(1)); - let (px, jw, jh, jc) = lossy_jpeg(src, cpp, tile_budget)?; - // An edge tile's JPEG may be smaller than the nominal tile, but it has to cover the part of the tile - // inside the image: one that doesn't would leave a black region in a picture reported as decoded - // (PhotoCraft's and FilmCraft's readers refuse a JPEG smaller than its tile; this keeps the edge case). - if jw < vw || jh < vh { - return Err(RawError::Corrupt(format!("a lossy JPEG of {jw} × {jh} does not cover the {vw} × {vh} of its tile inside the image"))); - } + let (px, jw, jh, jc) = lossy_jpeg_with_tile(src, cpp, tile_budget, Some((cw, ch, vw, vh)))?; // the buffer holds the rows the decoded JPEG covers (the caller zero-fills the tile's // remainder and reads only rows it fills): a tile-many-larger decoded JPEG does not // get the nominal tile's full buffer (storytold/lightcraft#1517) @@ -408,9 +426,21 @@ const MAX_PADDING_FACTOR: usize = 32; #[cfg(feature = "dng")] const MAX_WRITER_TILE_SAMPLES: usize = 2048 * 2048; -#[cfg(feature = "dng")] +#[cfg(all(test, feature = "dng"))] /// Decode one baseline JPEG chunk to 8-bit samples: (samples, width, height, channels). pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<(Vec, usize, usize, usize)> { + lossy_jpeg_with_tile(src, cpp, max_samples, None) +} + +/// Validate the actual frame before decode allocates its output. Edge tiles may store +/// a smaller frame covering all visible samples or bounded whole-MCU padding. +#[cfg(feature = "dng")] +fn lossy_jpeg_with_tile( + src: &[u8], + cpp: usize, + max_samples: usize, + tile: Option<(usize, usize, usize, usize)>, +) -> Result<(Vec, usize, usize, usize)> { use zune_core::bytestream::ZCursor; use zune_core::colorspace::ColorSpace; use zune_core::options::DecoderOptions; @@ -422,7 +452,27 @@ pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<( d.decode_headers().map_err(|e| RawError::Corrupt(format!("lossy JPEG tile: {e}")))?; let info = d.info().ok_or_else(|| RawError::Corrupt("lossy JPEG tile without a header".into()))?; let (w, h) = (info.width as usize, info.height as usize); - let c = if cpp == 1 { 1 } else { 3 }; + if !matches!(cpp, 1 | 3) { + return Err(RawError::Unsupported(format!("lossy JPEG with {cpp} samples per pixel"))); + } + if usize::from(info.components) != cpp { + return Err(RawError::Corrupt(format!("lossy JPEG has {} components, expected {cpp}", info.components))); + } + if let Some((cw, ch, vw, vh)) = tile { + if w < vw || h < vh { + return Err(RawError::Corrupt(format!("a lossy JPEG of {w} x {h} does not cover the {vw} x {vh} of its tile inside the image"))); + } + // Writers can store a whole MCU around an edge tile. Keep the documented + // 32-pixel padding allowance without letting one tiny tile decode an arbitrary frame. + let mcu_extent = |value: usize| value.checked_add(31).map(|rounded| rounded / 32).and_then(|blocks| blocks.checked_mul(32)); + let (Some(max_w), Some(max_h)) = (mcu_extent(cw), mcu_extent(ch)) else { + return Err(RawError::Corrupt("lossy JPEG tile dimensions overflow MCU padding".into())); + }; + if w > max_w || h > max_h { + return Err(RawError::Corrupt(format!("a lossy JPEG of {w} x {h} exceeds its {max_w} x {max_h} MCU-padded tile dimensions"))); + } + } + let c = cpp; let out = w as u64 * h as u64 * c as u64; if w == 0 || h == 0 || out > max_samples as u64 { return Err(RawError::Corrupt(format!("a lossy JPEG of {w} × {h} × {c} would exceed the {max_samples} output samples allowed"))); @@ -519,6 +569,12 @@ fn unpack_chunk( } } else { undo_diff_u16(row, cpp * factor); + // Horizontal differences wrap in the declared sample precision, not in + // the u16 container used by the unpacker (TIFF 6.0 Predictor 2). + if bits < 16 { + let mask = ((1u32 << bits) - 1) as u16; + row.iter_mut().for_each(|sample| *sample &= mask); + } } } } @@ -982,3 +1038,6 @@ mod budget_tests { assert_eq!(decode_groups(&big.chunks(64), 1, 1 << 20), vec![0..1]); } } + +#[cfg(all(test, feature = "dng"))] +mod compat_review_tests; diff --git a/crates/raw/src/tiffraw/compat_review_tests.rs b/crates/raw/src/tiffraw/compat_review_tests.rs new file mode 100644 index 000000000..ae5da8f2f --- /dev/null +++ b/crates/raw/src/tiffraw/compat_review_tests.rs @@ -0,0 +1,115 @@ +use super::*; +use craft_tiff::image::Layout; + +fn image(width: u32, height: u32, bits: u16, cpp: u16, compression: u16, len: usize) -> ImageInfo { + ImageInfo { + width, + height, + bits_per_sample: vec![bits; cpp as usize], + samples_per_pixel: cpp, + compression, + photometric: 34892, + planar: 1, + predictor: 1, + sample_format: 1, + new_subfile_type: 0, + layout: Layout::Strips { rows_per_strip: height }, + offsets: vec![0], + byte_counts: vec![len as u64], + } +} + +#[test] +fn truncated_uncompressed_rows_are_rejected_in_full_and_header_modes() { + for (bits, packing, float, bytes) in [ + (8, Packing::Msb, false, 8), + (12, Packing::Msb, false, 12), + (12, Packing::Word16, false, 16), + (16, Packing::Msb, false, 16), + (32, Packing::Msb, true, 32), + ] { + let data = vec![0u8; bytes]; + let mut info = image(4, 2, bits, 1, comp::NONE, bytes); + info.sample_format = if float { 3 } else { 1 }; + assert!(read_image_in(crate::Mode::Full, &data, &info, ByteOrder::Little, packing, &Limits::default()).is_ok()); + info.byte_counts[0] -= 1; + for mode in [crate::Mode::Full, crate::Mode::Header] { + assert!( + matches!(read_image_in(mode, &data[..bytes - 1], &info, ByteOrder::Little, packing, &Limits::default()), Err(RawError::Corrupt(message)) if message.contains("uncompressed chunk")), + "{bits}/{float}/{mode:?}" + ); + } + } +} + +#[test] +fn edge_padding_may_be_absent_but_visible_uncompressed_samples_must_all_exist() { + let data = vec![31u8; 16 * 7 + 8]; + let mut info = image(8, 8, 8, 1, comp::NONE, data.len()); + info.layout = Layout::Tiles { tile_width: 16, tile_height: 16 }; + let RawData::U16(values) = read_image(&data, &info, ByteOrder::Little, Packing::Msb, &Limits::default()).unwrap() else { panic!("integer") }; + assert_eq!(values, vec![31; 64]); + info.byte_counts[0] -= 1; + assert!(read_image(&data[..data.len() - 1], &info, ByteOrder::Little, Packing::Msb, &Limits::default()).is_err()); +} + +#[test] +fn packed_predictor_two_wraps_at_declared_twelve_bits() { + // Samples [250, 10, 4090, 20]; differences modulo4096 are 0FA,F10,FF0,01A. + let packed = [0x0f, 0xaf, 0x10, 0xff, 0x00, 0x1a]; + let ChunkPx::U16(values) = unpack_chunk(&packed, ByteOrder::Little, Packing::Msb, 4, 1, 1, 12, false, 2).unwrap() else { panic!("integer") }; + assert_eq!(values, vec![250, 10, 4090, 20]); +} + +#[allow(clippy::unwrap_used)] // Test-only fixture encoder; failed construction must fail the test. +fn jpeg(cpp: usize, width: u16, height: u16) -> Vec { + let mut data = Vec::new(); + let color = if cpp == 1 { jpeg_encoder::ColorType::Luma } else { jpeg_encoder::ColorType::Rgb }; + jpeg_encoder::Encoder::new(&mut data, 100).encode(&vec![90; width as usize * height as usize * cpp], width, height, color).unwrap(); + data +} + +#[test] +fn baseline_jpeg_components_are_checked_before_forced_colorspace_conversion() { + for (actual, expected) in [(3, 1), (1, 3)] { + let data = jpeg(actual, 8, 8); + let error = lossy_jpeg(&data, expected, 1024).unwrap_err(); + assert!(matches!(error, RawError::Corrupt(message) if message.contains(&format!("{actual} components")))); + } + assert!(matches!(lossy_jpeg(&jpeg(3, 8, 8), 4, 1024), Err(RawError::Unsupported(message)) if message.contains("4 samples"))); + for cpp in [1, 3] { + assert_eq!(lossy_jpeg(&jpeg(cpp, 8, 8), cpp, 1024).unwrap().3, cpp); + } +} + +#[test] +fn jpeg_tile_geometry_and_limits_are_checked_at_the_header_with_valid_edge_controls() { + let data = jpeg(3, 8, 8); + assert!(lossy_jpeg_with_tile(&data, 3, 1024, Some((16, 16, 8, 8))).is_ok()); + assert!( + matches!(lossy_jpeg_with_tile(&data, 3, 1024, Some((16, 16, 9, 8))), Err(RawError::Corrupt(message)) if message.contains("does not cover")) + ); + assert!(matches!(lossy_jpeg_with_tile(&data, 3, 32, Some((16, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("would exceed"))); + // Mutate only SOF dimensions, retaining a valid header but no entropy for the claimed frame. + // The geometry error proves refusal precedes decode's short/corrupt entropy failure. + for (width, height) in [(33u16, 8u16), (8, 33), (256, 256)] { + let mut oversized = data.clone(); + let sof = oversized.windows(2).position(|bytes| bytes == [0xff, 0xc0]).unwrap(); + oversized[sof + 5..sof + 7].copy_from_slice(&height.to_be_bytes()); + oversized[sof + 7..sof + 9].copy_from_slice(&width.to_be_bytes()); + assert!( + matches!(lossy_jpeg_with_tile(&oversized, 3, 1 << 20, Some((16, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("exceeds its")) + ); + } + // A genuine full MCU remains valid even when only a 4x4 part of the nominal16 tile is visible. + let padded = jpeg(3, 32, 32); + assert!(lossy_jpeg_with_tile(&padded, 3, 32 * 32 * 3, Some((16, 16, 4, 4))).is_ok()); + assert!( + matches!(lossy_jpeg_with_tile(&data, 3, 1 << 20, Some((usize::MAX, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("overflow MCU padding")) + ); + let mut info = image(8, 8, 8, 3, comp::LOSSY_JPEG, data.len()); + info.layout = Layout::Tiles { tile_width: 16, tile_height: 16 }; + let RawData::U16(values) = read_image(&data, &info, ByteOrder::Little, Packing::Msb, &Limits::default()).unwrap() else { panic!("integer") }; + assert_eq!(values.len(), 8 * 8 * 3); + assert!(values.iter().all(|sample| (i32::from(*sample) - 90).abs() <= 3)); +} diff --git a/crates/raw/tests/it/dng.rs b/crates/raw/tests/it/dng.rs index 3b7f4bc0a..216937862 100644 --- a/crates/raw/tests/it/dng.rs +++ b/crates/raw/tests/it/dng.rs @@ -64,6 +64,7 @@ fn synthetic(w: usize, h: usize, cfa: Option, cpp: usize) -> RawImage { cinema: Default::default(), lens_metadata: Default::default(), level_sources: Default::default(), + notices: craft_raw::RawNotices::default(), } } @@ -948,18 +949,31 @@ fn malformed_semantic_masks_are_skipped() { let mut zero = mask_ifd(2, 2, "zero size", vec![0; 4]); zero.set(t::IMAGE_WIDTH, Value::Long(vec![0])); ifd0.add_sub_ifd(zero); + // A 4x4 8-bit mask needs 16 visible samples; three bytes must not be + // accepted as a synthetic mask with thirteen zero-filled samples. let mut short = mask_ifd(4, 4, "short strip", vec![200; 3]); short.set(t::MASK_SUB_AREA, Value::Long(vec![u32::MAX, u32::MAX, u32::MAX, u32::MAX])); ifd0.add_sub_ifd(short); + let mut valid = mask_ifd(4, 2, "valid crop", vec![10, 20, 30, 40, 50, 60, 70, 255]); + valid.set(t::MASK_SUB_AREA, Value::Long(vec![1, 2, 8, 4])); + ifd0.add_sub_ifd(valid); let mut outside = mask_ifd(2, 2, "sub area too small", vec![255; 4]); outside.set(t::MASK_SUB_AREA, Value::Long(vec![3, 0, 2, 4])); ifd0.add_sub_ifd(outside); }); - assert_eq!(decode(&bytes).unwrap().cpp, 3); + let raw = decode(&bytes).unwrap(); + let baseline = decode(&profile_dng(ByteOrder::Little, |_| {})).unwrap(); + assert_eq!((raw.width, raw.height, raw.cpp), (8, 6, 3)); + assert_eq!(raw.data, baseline.data, "optional damaged masks must not change base image samples"); let masks = craft_raw::semantic_masks(&bytes); let names: Vec<&str> = masks.iter().map(|m| m.name.as_str()).collect(); - assert_eq!(names, ["short strip", "sub area too small"]); - assert!(masks.iter().all(|m| m.sub_area.is_none()), "sub areas the mask doesn't fit are ignored"); + assert_eq!(names, ["valid crop", "sub area too small"], "the truncated mask is omitted, and later usable masks survive"); + assert_eq!(masks[0].data, [10u16, 20, 30, 40, 50, 60, 70, 255].map(|sample| sample * 257)); + assert_eq!(masks[0].sub_area, Some(craft_raw::semantic::MaskSubArea { top: 1, left: 2, full_width: 8, full_height: 4 })); + assert!((masks[0].value_at(2.5, 1.5) - 10. / 255.).abs() < 1e-4); + assert_eq!(masks[0].value_at(1.5, 1.5), 0.); + assert!(masks[1].sub_area.is_none(), "the complete mask survives while a sub area it doesn't fit is ignored"); + assert_eq!(masks[1].data, vec![65535; 4]); for m in &masks { assert_eq!(m.data.len(), m.width * m.height); } diff --git a/crates/raw/tests/it/robust.rs b/crates/raw/tests/it/robust.rs index b2adb26e0..5ad1d1b32 100644 --- a/crates/raw/tests/it/robust.rs +++ b/crates/raw/tests/it/robust.rs @@ -28,6 +28,7 @@ fn sample(comp: DngCompression) -> Vec { cinema: Default::default(), lens_metadata: Default::default(), level_sources: Default::default(), + notices: craft_raw::RawNotices::default(), }; craft_raw::write_dng(&raw, &DngWriteOptions { compression: comp, ..Default::default() }).unwrap() } diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index 9cb004d89..2f67edbda 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -302,6 +302,7 @@ struct Used { raster_limit: AtomicUsize, invalid_svg: AtomicUsize, svg_limit: AtomicUsize, + svg_resource_limit: AtomicUsize, } /// What resolving one drawing's embedded images used or refused. @@ -323,6 +324,10 @@ pub struct ImageUsage { pub invalid_svg_images: usize, /// Embedded SVG data refused by byte or nesting limits. pub svg_limit_images: usize, + /// Subset of `svg_limit_images` refused by unpacked byte or embedded-document-chain limits. + /// XML/tree-depth omissions and invalid or corrupt data do not set this counter. Consumers + /// can reject resource overruns while still allowing an optional, over-deep child to be omitted. + pub svg_resource_limit_images: usize, } /// A data-URL resolver and the accounting for the one drawing it serves. @@ -350,6 +355,7 @@ impl ImageResolver { raster_limit: AtomicUsize::new(0), invalid_svg: AtomicUsize::new(0), svg_limit: AtomicUsize::new(0), + svg_resource_limit: AtomicUsize::new(0), }), } } @@ -376,6 +382,7 @@ impl ImageResolver { raster_limit_images: self.used.raster_limit.load(Ordering::Relaxed), invalid_svg_images: self.used.invalid_svg.load(Ordering::Relaxed), svg_limit_images: self.used.svg_limit.load(Ordering::Relaxed), + svg_resource_limit_images: self.used.svg_resource_limit.load(Ordering::Relaxed), } } } @@ -438,6 +445,7 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc>, format: craft_ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'_>) -> Option { if used.depth.load(Ordering::Relaxed) >= limits.embedded_depth { used.svg_limit.fetch_add(1, Ordering::Relaxed); + used.svg_resource_limit.fetch_add(1, Ordering::Relaxed); return None; } let left = limits.embedded_bytes.saturating_sub(used.bytes.load(Ordering::Relaxed)); @@ -445,15 +453,37 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<' Ok(text) => text, // What a refused document unpacked before it was refused was work too: it uses up what // is left, so the next image can't repeat it. - Err(_) => { + Err(error) => { used.bytes.fetch_add(left, Ordering::Relaxed); used.svg_limit.fetch_add(1, Ordering::Relaxed); + if matches!(error, svgz::Error::TooLarge(_)) { + used.svg_resource_limit.fetch_add(1, Ordering::Relaxed); + } return None; } }; + // The raw/inflated bytes can fit while UTF-16/Latin-1 decoding expands + // beyond the aggregate allowance. A refusal consumes the remaining work. + if text.len() > left { + used.bytes.fetch_add(left, Ordering::Relaxed); + used.svg_limit.fetch_add(1, Ordering::Relaxed); + used.svg_resource_limit.fetch_add(1, Ordering::Relaxed); + return None; + } used.bytes.fetch_add(text.len(), Ordering::Relaxed); let xml = match checked(&text, limits) { Ok(xml) => xml, + // Decoding UTF-16 or Latin-1 can exceed the text cap after the + // raw/inflated bytes fitted. This is a resource refusal, not invalid XML. + Err(Error::Text(svgz::Error::TooLarge(_))) => { + used.svg_limit.fetch_add(1, Ordering::Relaxed); + used.svg_resource_limit.fetch_add(1, Ordering::Relaxed); + return None; + } + Err(Error::TooDeep(_)) => { + used.svg_limit.fetch_add(1, Ordering::Relaxed); + return None; + } Err(_) => { used.invalid_svg.fetch_add(1, Ordering::Relaxed); return None; diff --git a/crates/svg/src/tree/tests.rs b/crates/svg/src/tree/tests.rs index 59b42dacc..80681cad4 100644 --- a/crates/svg/src/tree/tests.rs +++ b/crates/svg/src/tree/tests.rs @@ -245,6 +245,195 @@ fn embedded_svgs_are_read_within_their_budgets() { assert_eq!(found(&svg(r#""#), shallow()), [false; 0]); } +mod resource_limit_usage { + use super::*; + + fn data_image(bytes: &[u8]) -> String { + let encoded: String = bytes.iter().map(|byte| format!("%{byte:02X}")).collect(); + format!(r#""#) + } + + fn packed(text: &str, gzip: bool) -> Vec { + if gzip { svgz::compress(text.as_bytes()) } else { text.as_bytes().to_vec() } + } + + /// The outer document has its own generous text allowance, so tiny resolver limits + /// test the public embedded admission contract rather than rejecting the outer XML. + fn resolved(text: &str, limits: Limits, outer_depth: usize) -> (Vec, ImageUsage) { + let resolver = ImageResolver::new(limits); + resolver.set_document_depth(outer_depth); + let options = usvg::Options { image_href_resolver: resolver.resolver(), ..Default::default() }; + let tree = parse(text.as_bytes(), &shallow(), &options).unwrap(); + let mut found = Vec::new(); + images(tree.root(), &mut found); + (found, resolver.usage()) + } + + fn expanding_encoded_children() -> Vec<(&'static str, Vec)> { + let latin = + format!("{}", svg(&format!("{}", "é".repeat(100)))); + let latin = latin.chars().map(|c| u8::try_from(u32::from(c)).unwrap()).collect(); + let wide = svg(&format!("{}", "ࠀ".repeat(400))); + let mut little = vec![0xff, 0xfe]; + little.extend(wide.encode_utf16().flat_map(u16::to_le_bytes)); + let mut big = vec![0xfe, 0xff]; + big.extend(wide.encode_utf16().flat_map(u16::to_be_bytes)); + vec![("Latin-1", latin), ("UTF-16LE", little), ("UTF-16BE", big)] + } + + #[test] + fn decoded_text_expansion_is_a_resource_refusal_for_plain_and_gzip_children() { + for (encoding, raw) in expanding_encoded_children() { + let decoded = svgz::decode(&raw).unwrap(); + let cap = raw.len(); + assert!(decoded.len() > cap, "{encoding} fixture must expand after inflation"); + for gzip in [false, true] { + let data = if gzip { svgz::compress(&raw) } else { raw.clone() }; + assert_eq!(svgz::inflate(&data, cap).unwrap().len(), cap, "raw bytes fit exactly"); + assert_eq!(svgz::text(&data, cap).unwrap().len(), decoded.len(), "decoding itself succeeds"); + let limits = Limits { bytes: cap, embedded_bytes: decoded.len(), ..shallow() }; + assert!(matches!(checked(&decoded, &limits), Err(Error::Text(svgz::Error::TooLarge(n))) if n == cap)); + let image = data_image(&data); + let (found, usage) = resolved(&svg(&image), limits, 1); + assert!(found.is_empty(), "{encoding}, gzip {gzip}"); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (1, 1, 0)); + assert_eq!(usage.embedded_svg_bytes, decoded.len(), "spent decoded work remains charged"); + let (found, usage) = resolved(&svg(&format!("{image}{image}")), limits, 1); + assert!(found.is_empty()); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (2, 2, 0)); + assert_eq!(usage.embedded_svg_bytes, decoded.len(), "a refused sibling cannot repeat exhausted work"); + } + } + } + + #[test] + fn decoded_text_expansion_respects_aggregate_bytes_even_when_each_child_fits() { + for (encoding, raw) in expanding_encoded_children() { + let decoded = svgz::decode(&raw).unwrap(); + let limits = Limits { bytes: decoded.len(), embedded_bytes: raw.len(), ..shallow() }; + assert!(decoded.len() > limits.embedded_bytes); + checked(&decoded, &limits).unwrap(); // The independent per-image cap allows this complete XML. + for gzip in [false, true] { + let data = if gzip { svgz::compress(&raw) } else { raw.clone() }; + assert_eq!(svgz::inflate(&data, limits.embedded_bytes).unwrap().len(), raw.len()); + let image = data_image(&data); + let (found, usage) = resolved(&svg(&image), limits, 1); + assert!(found.is_empty(), "{encoding}, gzip {gzip}"); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (1, 1, 0)); + assert_eq!(usage.embedded_svg_bytes, limits.embedded_bytes, "refusal consumes exactly the remaining allowance"); + let (found, usage) = resolved(&svg(&format!("{image}{image}")), limits, 1); + assert!(found.is_empty()); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (2, 2, 0)); + assert_eq!(usage.embedded_svg_bytes, limits.embedded_bytes, "a later child cannot repeat exhausted work"); + let (found, usage) = resolved(&svg(&image), Limits { embedded_bytes: decoded.len(), ..limits }, 1); + assert_eq!(found, [true]); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 0)); + assert_eq!(usage.embedded_svg_bytes, decoded.len()); + } + } + } + + #[test] + fn decoded_text_expansion_at_the_cap_is_valid_and_malformed_xml_stays_invalid() { + for (encoding, raw) in expanding_encoded_children() { + let decoded = svgz::decode(&raw).unwrap(); + for gzip in [false, true] { + let data = if gzip { svgz::compress(&raw) } else { raw.clone() }; + let limits = Limits { bytes: decoded.len(), embedded_bytes: decoded.len(), ..shallow() }; + let (found, usage) = resolved(&svg(&data_image(&data)), limits, 1); + assert_eq!(found, [true], "{encoding}, gzip {gzip}"); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 0)); + assert_eq!(usage.embedded_svg_bytes, decoded.len()); + // Preserve the declared/BOM encoding when making the XML malformed; + // re-encoding Latin-1 text as UTF-8 would test a second expansion instead. + let closing_bytes = if encoding == "Latin-1" { 6 } else { 12 }; + let malformed = &raw[..raw.len() - closing_bytes]; + let data = if gzip { svgz::compress(malformed) } else { malformed.to_vec() }; + let (found, usage) = resolved(&svg(&data_image(&data)), limits, 1); + assert!(found.is_empty()); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 1)); + } + } + } + + #[test] + fn embedded_chain_caps_are_explicit_resource_refusals_for_plain_and_gzip_children() { + for gzip in [false, true] { + let inner = svg(""); + let middle = svg(&data_image(&packed(&inner, gzip))); + let outer = svg(&data_image(&packed(&middle, gzip))); + let (found, usage) = resolved(&outer, Limits { embedded_depth: 1, ..shallow() }, 1); + assert_eq!(found, [true]); + assert_eq!(usage.embedded_svg_depth, 1); + assert_eq!(usage.svg_limit_images, 1, "strict consumers still see the existing limit flag"); + assert_eq!(usage.svg_resource_limit_images, 1, "a chain cap must not become a successful partial bounded preview"); + let (found, usage) = resolved(&outer, Limits { embedded_depth: 2, ..shallow() }, 1); + assert_eq!(found, [true, true]); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0)); + } + } + + #[test] + fn embedded_per_image_byte_caps_are_resource_refusals_for_plain_and_gzip_children() { + let inner = svg(""); + for gzip in [false, true] { + let outer = svg(&data_image(&packed(&inner, gzip))); + let (_, usage) = resolved(&outer, Limits { bytes: inner.len() - 1, embedded_bytes: inner.len() * 3, ..shallow() }, 1); + assert_eq!(usage.embedded_svg_count, 0); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (1, 1)); + let (found, usage) = resolved(&outer, Limits { bytes: inner.len(), ..shallow() }, 1); + assert_eq!(found, [true]); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0)); + } + } + + #[test] + fn aggregate_embedded_bytes_are_resource_refusals_even_when_each_child_fits() { + let inner = svg(""); + for gzip in [false, true] { + let image = data_image(&packed(&inner, gzip)); + let outer = svg(&format!("{image}{image}")); + let cap = inner.len() * 2 - 1; + let (found, usage) = resolved(&outer, Limits { embedded_bytes: cap, ..shallow() }, 1); + assert_eq!(found, [true]); + assert_eq!(usage.embedded_svg_bytes, cap, "a refused child still spends the remaining work allowance"); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (1, 1)); + let (found, usage) = resolved(&outer, Limits { embedded_bytes: inner.len() * 2, ..shallow() }, 1); + assert_eq!(found, [true, true]); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0)); + } + } + + #[test] + fn combined_xml_depth_omissions_remain_distinct_from_resource_refusals() { + let inner = svg(""); + for gzip in [false, true] { + let image = data_image(&packed(&inner, gzip)); + let outer = svg(&format!("{}{}{}", "".repeat(4), image, "".repeat(4))); + let (found, usage) = resolved(&outer, Limits { tree_depth: 6, ..shallow() }, 5); + assert!(found.is_empty()); + assert_eq!(usage.svg_limit_images, 1, "strict consumers must continue to reject this omission"); + assert_eq!(usage.svg_resource_limit_images, 0, "non-strict consumers may omit this child without disguising a byte/chain overrun"); + let (found, usage) = resolved(&svg(&image), Limits { tree_depth: 6, ..shallow() }, 1); + assert_eq!(found, [true]); + assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0)); + } + } + + #[test] + fn invalid_and_corrupt_children_do_not_claim_a_resource_limit() { + let mut corrupt = svgz::compress(svg("").as_bytes()); + let crc = corrupt.len() - 8; + corrupt[crc] ^= 1; + for data in [b"not XML".to_vec(), corrupt] { + let (found, usage) = resolved(&svg(&data_image(&data)), shallow(), 1); + assert!(found.is_empty()); + assert!(usage.invalid_svg_images > 0 || usage.svg_limit_images > 0, "strict consumers still observe unreadable content"); + assert_eq!(usage.svg_resource_limit_images, 0); + } + } +} + mod draw_checks { use super::*;