Merge pull request #1300 from Terra854/codex/format-adapter-safety

Preserve native bundle saves and validate SVG/RAW decoding
This commit is contained in:
FredApps authored and GitHub committed 2026-11-15 06:17:29 +01:00
commit ad8faa1c13
32 files changed
+2261 -145

No files matched your search

+162 -11
View File
@@ -351,28 +351,70 @@ fn trak(bytes: &[u8], t: &BoxRef, budget: &mut usize) -> Option<Cr3Track> {
}
}
let stbl = stbl?;
let (mut kind, mut size, mut offset) = (None, None, None);
let (mut kind, mut size, mut offset, mut mapping, mut sample_count) = (None, None, None, None, 0);
for b in boxes(bytes, stbl.body, stbl.end, budget) {
let body = bytes.get(b.body..b.end)?;
match &b.kind {
// full box: version/flags, entry count, entries
b"stsd" if be32(body, 4)? > 0 => kind = boxes(bytes, b.body + 8, b.end, budget).first().map(|e| sample_entry(bytes, e, budget)),
b"stsc" => {
// Only the first sample is exposed. It must map chunk 1 to
// the first sample description; don't trust offsets without
// that mapping, or a damaged/raw track could become a JPEG.
// A malformed mapping invalidates sample data, not the track's
// identity: dropping the largest raw track could select a reduced
// raw successfully and disguise corruption in the main image.
mapping = (|| {
let count = be32(body, 4)? as usize;
if be32(body, 0)? != 0 || count == 0 || count > body.len().saturating_sub(8) / 12 {
return None;
}
Some((be32(body, 8)?, be32(body, 12)?, be32(body, 16)?))
})();
}
// version/flags, sample_size (0 = per-sample table), count, sizes
b"stsz" => {
size = match be32(body, 4)? {
0 if be32(body, 8)? > 0 => be32(body, 12),
0 => None,
n if be32(body, 8)? > 0 => Some(n),
// A truncated size table invalidates data without hiding a known track.
sample_count = be32(body, 8).unwrap_or(0);
size = match be32(body, 4) {
// A first entry cannot justify an incomplete per-sample
// table, nor an impossible constant-size sample count.
Some(0) if sample_count > 0 && sample_count as usize <= body.len().saturating_sub(12) / 4 => be32(body, 12),
Some(n)
if n > 0
&& sample_count > 0
&& u64::from(n).checked_mul(u64::from(sample_count)).is_some_and(|total| total <= bytes.len() as u64) =>
{
Some(n)
}
_ => None,
}
}
b"co64" if be32(body, 4)? > 0 => offset = be64(body, 8).and_then(|v| usize::try_from(v).ok()),
b"stco" if be32(body, 4)? > 0 => offset = be32(body, 8).map(|v| v as usize),
// Missing or truncated offset-table counts deny sample access, but the known
// track must remain available to the largest-raw-track selector.
b"co64" => {
offset = match be32(body, 4) {
Some(count) if count > 0 && count as usize <= body.len().saturating_sub(8) / 8 => {
be64(body, 8).and_then(|v| usize::try_from(v).ok())
}
_ => None,
};
}
b"stco" => {
offset = match be32(body, 4) {
Some(count) if count > 0 && count as usize <= body.len().saturating_sub(8) / 4 => be32(body, 8).map(|v| v as usize),
_ => None,
};
}
_ => {}
}
}
let data = match (offset, size) {
(Some(o), Some(s)) if o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) => Some((o, s as usize)),
let data = match (offset, size, mapping) {
(Some(o), Some(s), Some((1, samples, 1)))
if samples > 0 && samples <= sample_count && o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) =>
{
Some((o, s as usize))
}
_ => None,
};
Some(Cr3Track { kind: kind?, data })
@@ -495,6 +537,13 @@ mod tests {
bx(b"CRAW", &b)
}
fn trak(entry: Vec<u8>, offset: u64, size: u32) -> Vec<u8> {
let stsc = [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
trak_with_mapping(entry, offset, size, &stsc)
}
fn trak_with_mapping(entry: Vec<u8>, offset: u64, size: u32, mapping: &[u8]) -> Vec<u8> {
trak_with_table(entry, offset, size, b"stsc", mapping)
}
fn trak_with_table(entry: Vec<u8>, offset: u64, size: u32, table: &[u8; 4], payload: &[u8]) -> Vec<u8> {
let mut stsd = 1u32.to_be_bytes().to_vec();
stsd.extend(entry);
let mut stsz = 0u32.to_be_bytes().to_vec();
@@ -502,7 +551,15 @@ mod tests {
stsz.extend(size.to_be_bytes());
let mut co64 = 1u32.to_be_bytes().to_vec();
co64.extend(offset.to_be_bytes());
let stbl = [full(b"stsd", &stsd), full(b"stsz", &stsz), full(b"co64", &co64)].concat();
let stsc = [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
let offsets = if table == b"stco" { full(b"stco", payload) } else { full(b"co64", if table == b"co64" { payload } else { &co64 }) };
let stbl = [
full(b"stsd", &stsd),
full(b"stsc", if table == b"stsc" { payload } else { &stsc }),
full(b"stsz", if table == b"stsz" { payload } else { &stsz }),
offsets,
]
.concat();
bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl))))
}
fn tiff_with(tag: u16, v: craft_tiff::Value) -> Vec<u8> {
@@ -543,6 +600,97 @@ mod tests {
assert_eq!(c.tracks[1].data, Some((4112, 32)));
}
#[test]
fn first_sample_requires_a_complete_valid_sample_to_chunk_mapping() {
let file = |track: Vec<u8>| {
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &track));
file.resize(4112, 0);
file
};
let valid = trak(craw(16, 16, &bx(b"JPEG", &[0; 4])), 4096, 16);
assert_eq!(parse_cr3(&file(valid.clone())).unwrap().tracks[0].data, Some((4096, 16)));
let at = valid.windows(4).position(|bytes| bytes == b"stsc").unwrap();
// Missing table, impossible first chunk, zero samples, a different
// description and a count that extends beyond this box all deny data.
for (field, value) in [(0, u32::from_be_bytes(*b"xxxx")), (8, 0), (8, 2), (12, 0), (12, 2), (16, 0), (16, 2), (4, u32::MAX)] {
let mut damaged = valid.clone();
let offset = if field == 0 { at } else { at + 4 + field };
damaged[offset..offset + 4].copy_from_slice(&value.to_be_bytes());
let bytes = file(damaged);
let parsed = parse_cr3(&bytes).unwrap();
assert_eq!(parsed.tracks.len(), 1, "invalid mapping must not erase an identifiable track");
assert_eq!(parsed.tracks[0].kind, Cr3TrackKind::Jpeg);
assert!(parsed.tracks[0].data.is_none());
}
for entry in [craw(16, 16, &[]), craw(16, 16, &bx(b"CMP1", &[0xff, 0x10, 0, 0x30]))] {
let bytes = file(trak(entry, 4096, 16));
let parsed = parse_cr3(&bytes).unwrap();
assert!(!matches!(parsed.tracks[0].kind, Cr3TrackKind::Jpeg));
}
}
#[test]
fn truncated_sample_mapping_preserves_larger_raw_track_identity_and_valid_reduced_track() {
let reduced = trak(craw(4, 4, &bx(b"CMP1", &compression_payload(0))), 4096, 16);
for table in [b"stsc", b"stsz", b"co64", b"stco"] {
for payload in [Vec::new(), vec![0; 4], 1u32.to_be_bytes().to_vec()] {
// full() supplies version/flags; an empty payload leaves only
// those four bytes, with the table's required count missing.
let main = trak_with_table(craw(8, 4, &bx(b"CMP1", &compression_payload(0))), 4112, 32, table, &payload);
for main_first in [false, true] {
let tracks = if main_first { [main.clone(), reduced.clone()].concat() } else { [reduced.clone(), main.clone()].concat() };
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &tracks));
file.resize(4144, 0);
let parsed = parse_cr3(&file).unwrap();
assert_eq!(parsed.tracks.len(), 2, "truncated {table:?} cannot hide the larger track");
let (main, reduced) = if main_first { (&parsed.tracks[0], &parsed.tracks[1]) } else { (&parsed.tracks[1], &parsed.tracks[0]) };
assert!(matches!(reduced.kind, Cr3TrackKind::Raw { width: 4, height: 4, .. }));
assert_eq!(reduced.data, Some((4096, 16)));
assert!(matches!(main.kind, Cr3TrackKind::Raw { width: 8, height: 4, cmp1: Some(_), .. }));
assert!(main.data.is_none());
assert!(main.compression(&file).is_some(), "only sample location is invalid");
}
}
let mut offsets = 1u32.to_be_bytes().to_vec();
offsets.extend(4112u64.to_be_bytes());
let valid = match table {
b"stsc" => [1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>(),
b"stsz" => [0u32, 1, 32].into_iter().flat_map(u32::to_be_bytes).collect(),
b"stco" => [1u32, 4112].into_iter().flat_map(u32::to_be_bytes).collect(),
_ => offsets,
};
let main = trak_with_table(craw(8, 4, &bx(b"CMP1", &compression_payload(0))), 4112, 32, table, &valid);
for main_first in [false, true] {
let tracks = if main_first { [main.clone(), reduced.clone()].concat() } else { [reduced.clone(), main.clone()].concat() };
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &tracks));
file.resize(4144, 0);
let parsed = parse_cr3(&file).unwrap();
assert_eq!(parsed.tracks.len(), 2);
let main = &parsed.tracks[if main_first { 0 } else { 1 }];
assert_eq!(main.data, Some((4112, 32)), "complete valid {table:?} stays usable");
}
}
}
#[test]
fn first_sample_size_requires_a_complete_bounded_sample_table() {
let valid = trak(craw(16, 16, &bx(b"JPEG", &[0; 4])), 4096, 16);
let at = valid.windows(4).position(|bytes| bytes == b"stsz").unwrap();
for (constant, count, accepted) in [(0u32, 1u32, true), (16, 1, true), (0, 2, false), (0, u32::MAX, false), (16, u32::MAX, false)] {
let mut track = valid.clone();
track[at + 8..at + 12].copy_from_slice(&constant.to_be_bytes());
track[at + 12..at + 16].copy_from_slice(&count.to_be_bytes());
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &track));
file.resize(4112, 0);
let parsed = parse_cr3(&file).unwrap();
assert_eq!(parsed.tracks[0].data, accepted.then_some((4096, 16)), "size {constant}, count {count}");
}
}
#[test]
fn hevc_preview_is_distinct_from_an_equal_size_raw_track() {
let preview = trak(craw(6000, 4000, &bx(b"HEVC", &[0; 4])), 4096, 16);
@@ -714,7 +862,10 @@ mod tests {
let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
file.extend(bx(b"moov", &bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl))))));
file.resize(512, 0);
assert!(parse_cr3(&file).unwrap().tracks.is_empty());
let parsed = parse_cr3(&file).unwrap();
assert_eq!(parsed.tracks.len(), 1, "a truncated table must not hide the identifiable raw track");
assert!(matches!(parsed.tracks[0].kind, Cr3TrackKind::Raw { width: 16, height: 16, .. }));
assert!(parsed.tracks[0].data.is_none(), "the empty stsz must not borrow size bytes from the following co64 box");
}
#[test]
+41 -21
View File
@@ -370,7 +370,8 @@ impl DirectoryBundleWriter {
/// Saves an entry set incrementally. Existing objects are reused only after a hash check;
/// unchanged file signatures may skip checks until they enter the rolling verification
/// window. The manifest is atomically published last, then stale previews and objects are
/// collected while the bundle lock is held.
/// collected while the bundle lock is held. Collection is best-effort after publication:
/// inaccessible or unsafe leftovers cannot turn a committed save into a reported failure.
pub fn save_directory(&mut self, root: &Path, bundle: ZipBundleWriter) -> Result<DirectoryStats> {
if !bundle.entries.contains_key(craft_pcraft_format::MANIFEST) {
return Err(Error::InvalidBundle("manifest not set".into()));
@@ -401,6 +402,10 @@ impl DirectoryBundleWriter {
Err(std::fs::TryLockError::Error(_)) => break None,
}
};
// A known unsafe manifest cannot be published. Refuse it before writing objects or
// previews, and still recheck immediately before publication below.
let manifest_path = root.join(craft_pcraft_format::MANIFEST);
refuse_symlink(&manifest_path)?;
self.generation = self.generation.saturating_add(1);
let generation = self.generation;
let recheck = self.rolling_recheck(&bundle.entries);
@@ -449,43 +454,33 @@ impl DirectoryBundleWriter {
}
let manifest = bundle.entries.get(craft_pcraft_format::MANIFEST).ok_or_else(|| Error::InvalidBundle("manifest not set".into()))?;
let manifest_path = root.join(craft_pcraft_format::MANIFEST);
refuse_symlink(&manifest_path)?;
craft_atomic::atomic_write(&manifest_path, manifest).map_err(io_error)?;
stats.written = stats.written.saturating_add(1);
let mut removed_paths = Vec::new();
// The durable manifest is the commit boundary. Cleanup owns only canonical regular
// object files and previews; leave unsafe, inaccessible or locked leftovers for later.
for kind in [craft_pcraft_format::TILES_DIR, craft_pcraft_format::BLOBS_DIR] {
let dir = root.join(kind);
if !dir.exists() {
if !cleanup_directory(&dir) {
continue;
}
refuse_symlink(&dir)?;
for entry in fs::read_dir(&dir).map_err(io_error)? {
let entry = entry.map_err(io_error)?;
let Ok(entries) = fs::read_dir(&dir) else { continue };
for entry in entries {
let Ok(entry) = entry else { continue };
let name = entry.file_name();
let path = format!("{kind}/{}", name.to_string_lossy());
if is_object_path(&path) && !bundle.entries.contains_key(&path) {
let metadata = fs::symlink_metadata(entry.path()).map_err(io_error)?;
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Err(Error::InvalidBundle(format!("refusing to collect non-file object {path}")));
}
fs::remove_file(entry.path()).map_err(io_error)?;
removed_paths.push(path);
if is_object_path(&path) && !bundle.entries.contains_key(&path) && remove_cleanup_file(&entry.path()) {
stats.removed = stats.removed.saturating_add(1);
}
}
}
for path in [craft_pcraft_format::THUMB, craft_pcraft_format::COMPOSITE, craft_pcraft_format::FULL_COMPOSITE] {
if !bundle.entries.contains_key(path) {
let target = root.join(path);
refuse_symlink(&target)?;
if fs::symlink_metadata(&target).is_ok() {
fs::remove_file(&target).map_err(io_error)?;
}
let _ = remove_cleanup_file(&root.join(path));
}
}
self.verified.retain(|path, _| bundle.entries.contains_key(path) && !removed_paths.contains(path));
self.verified.retain(|path, _| bundle.entries.contains_key(path));
Ok(stats)
}
@@ -510,7 +505,32 @@ impl DirectoryBundleWriter {
}
fn is_object_path(path: &str) -> bool {
path.starts_with("tiles/") || path.starts_with("blobs/")
valid_object_path(path)
}
/// Do not traverse symlinked object/preview directories, including Windows junctions.
fn cleanup_directory(path: &Path) -> bool {
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir() && !cleanup_link(&metadata))
}
fn cleanup_link(metadata: &fs::Metadata) -> bool {
#[cfg(windows)]
{
use std::os::windows::fs::MetadataExt;
metadata.file_attributes() & 0x400 != 0 // FILE_ATTRIBUTE_REPARSE_POINT includes junctions.
}
#[cfg(not(windows))]
{
metadata.file_type().is_symlink()
}
}
/// Cleanup never follows or removes a directory/link and never reports a post-commit failure.
fn remove_cleanup_file(path: &Path) -> bool {
if path.parent().is_none_or(|parent| !cleanup_directory(parent)) {
return false;
}
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_file() && !cleanup_link(&metadata)) && fs::remove_file(path).is_ok()
}
fn file_signature(path: &Path) -> Result<Option<FileSignature>> {
@@ -0,0 +1,344 @@
//! Publication succeeds independently of best-effort cleanup of the previous entry set.
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use craft_pcraft_bundle::{DirectoryBundle, DirectoryBundleWriter, Limits, ObjectKind, ZipBundleWriter};
use craft_pcraft_format::{COMPOSITE, FULL_COMPOSITE, MANIFEST, THUMB};
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
const OLD: &[u8] = br#"{"revision":1}"#;
const NEW: &[u8] = br#"{"revision":2}"#;
struct Temporary(PathBuf);
impl Temporary {
fn new() -> Self {
static NEXT: AtomicU64 = AtomicU64::new(0);
let path = std::env::temp_dir().join(format!("craft-bundle-cleanup-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed)));
fs::create_dir(&path).unwrap();
Self(path)
}
fn bundle(&self) -> PathBuf {
self.0.join("document.pcraft")
}
}
impl Drop for Temporary {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
fn entries(manifest: &[u8]) -> ZipBundleWriter {
let mut bundle = ZipBundleWriter::default();
bundle.set_manifest(manifest.to_vec()).unwrap();
bundle
}
fn assert_manifest(root: &Path, expected: &[u8]) {
assert_eq!(fs::read(root.join(MANIFEST)).unwrap(), expected);
assert_eq!(DirectoryBundle::open(root, Limits::default()).unwrap().manifest().unwrap(), expected);
}
#[test]
fn non_file_stale_object_does_not_fail_the_committed_save() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
let stuck = root.join("tiles").join(format!("{}.zst", "0".repeat(64)));
fs::create_dir_all(stuck.join("inner")).unwrap();
fs::write(stuck.join("inner/keep"), b"not an object file").unwrap();
let stats = writer.save_directory(&root, entries(NEW)).expect("cleanup cannot fail after manifest publication");
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
assert_eq!(fs::read(stuck.join("inner/keep")).unwrap(), b"not an object file");
}
#[test]
fn cleanup_removes_only_canonical_stale_regular_objects_and_reuses_current_objects() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
let mut first = entries(OLD);
let retained = first.add_object(ObjectKind::Tile, b"still referenced").unwrap();
let stale = first.add_object(ObjectKind::Blob, b"old blob").unwrap();
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
first.set_preview(preview, b"old preview".to_vec()).unwrap();
}
writer.save_directory(&root, first).unwrap();
let retained_bytes = fs::read(root.join(&retained)).unwrap();
let unrelated = [
"tiles/personal-notes.zst".to_string(),
"blobs/012345.zst".to_string(),
format!("blobs/{}.zst", "A".repeat(64)),
format!("tiles/{}.zst.backup", "b".repeat(64)),
];
for path in &unrelated {
fs::write(root.join(path), b"unrelated").unwrap();
}
let nested = root.join("blobs/not-an-object-folder");
fs::create_dir(&nested).unwrap();
fs::write(nested.join(format!("{}.zst", "c".repeat(64))), b"unrelated nested file").unwrap();
let mut second = entries(NEW);
assert_eq!(second.add_object(ObjectKind::Tile, b"still referenced").unwrap(), retained);
let stats = writer.save_directory(&root, second).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 1);
assert_eq!(stats.reused, 1);
assert_eq!(stats.reused_objects.as_slice(), std::slice::from_ref(&retained));
assert!(stats.written_objects.is_empty());
assert!(!root.join(stale).exists());
assert_eq!(fs::read(root.join(retained)).unwrap(), retained_bytes);
for path in unrelated {
assert_eq!(fs::read(root.join(path)).unwrap(), b"unrelated");
}
assert!(nested.is_dir());
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
assert!(!root.join(preview).exists());
}
}
#[test]
fn stale_preview_directories_are_preserved_after_commit() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
fs::create_dir_all(root.join(preview)).unwrap();
fs::write(root.join(preview).join("keep"), b"directory contents").unwrap();
}
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
for preview in [THUMB, COMPOSITE, FULL_COMPOSITE] {
assert_eq!(fs::read(root.join(preview).join("keep")).unwrap(), b"directory contents");
}
}
#[test]
fn required_non_file_objects_and_previews_fail_before_manifest_publication() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
let mut changed = entries(NEW);
let object = changed.add_object(ObjectKind::Tile, b"required object").unwrap();
fs::create_dir_all(root.join(&object)).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_manifest(&root, OLD);
for preview in [THUMB, COMPOSITE] {
fs::create_dir_all(root.join(preview)).unwrap();
let mut changed = entries(NEW);
changed.set_preview(preview, b"required preview".to_vec()).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_manifest(&root, OLD);
}
}
#[cfg(windows)]
#[test]
fn locked_stale_object_and_preview_do_not_fail_commit_and_are_collected_on_retry() {
use std::fs::OpenOptions;
use std::os::windows::fs::OpenOptionsExt;
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
let mut first = entries(OLD);
let stale = first.add_object(ObjectKind::Tile, b"locked stale tile").unwrap();
first.set_preview(FULL_COMPOSITE, b"locked preview".to_vec()).unwrap();
writer.save_directory(&root, first).unwrap();
let tile_lock = OpenOptions::new().read(true).share_mode(0).open(root.join(&stale)).unwrap();
let preview_lock = OpenOptions::new().read(true).share_mode(0).open(root.join(FULL_COMPOSITE)).unwrap();
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
assert!(root.join(&stale).exists());
assert!(root.join(FULL_COMPOSITE).exists());
drop(tile_lock);
drop(preview_lock);
let retry = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(retry.removed, 1);
assert!(!root.join(stale).exists());
assert!(!root.join(FULL_COMPOSITE).exists());
}
#[cfg(unix)]
#[test]
fn cleanup_preserves_symlinks_and_does_not_traverse_linked_directories() {
use std::os::unix::fs::symlink;
let temp = Temporary::new();
let root = temp.bundle();
let outside = temp.0.join("outside");
fs::create_dir(&outside).unwrap();
let object_name = format!("{}.zst", "d".repeat(64));
fs::write(outside.join(&object_name), b"outside object").unwrap();
fs::write(outside.join("preview.png"), b"outside preview").unwrap();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
symlink(&outside, root.join("blobs")).unwrap();
symlink(&outside, root.join("composite")).unwrap();
symlink(outside.join("preview.png"), root.join(THUMB)).unwrap();
fs::create_dir(root.join("tiles")).unwrap();
let linked_object = root.join("tiles").join(&object_name);
symlink(outside.join(&object_name), &linked_object).unwrap();
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
assert_eq!(fs::read(outside.join(object_name)).unwrap(), b"outside object");
assert_eq!(fs::read(outside.join("preview.png")).unwrap(), b"outside preview");
for link in [root.join("blobs"), root.join("composite"), root.join(THUMB), linked_object] {
assert!(fs::symlink_metadata(link).unwrap().file_type().is_symlink());
}
}
#[cfg(unix)]
#[test]
fn required_symlink_objects_and_previews_still_fail_before_commit() {
use std::os::unix::fs::symlink;
let temp = Temporary::new();
let root = temp.bundle();
let outside = temp.0.join("outside-file");
fs::write(&outside, b"unchanged outside bytes").unwrap();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
let mut changed = entries(NEW);
let object = changed.add_object(ObjectKind::Tile, b"required object").unwrap();
fs::create_dir(root.join("tiles")).unwrap();
symlink(&outside, root.join(object)).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_manifest(&root, OLD);
symlink(&outside, root.join(THUMB)).unwrap();
let mut changed = entries(NEW);
changed.set_preview(THUMB, b"new preview".to_vec()).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_manifest(&root, OLD);
assert_eq!(fs::read(outside).unwrap(), b"unchanged outside bytes");
}
#[cfg(windows)]
#[test]
fn cleanup_does_not_traverse_windows_directory_junctions() {
use std::os::windows::process::CommandExt;
struct Junction(PathBuf);
impl Drop for Junction {
fn drop(&mut self) {
let _ = fs::remove_dir(&self.0);
}
}
fn junction(link: PathBuf, target: &Path) -> Junction {
// Fixture-owned absolute paths only; junctions require no symlink privilege.
assert!(link.is_absolute() && target.is_absolute());
let output = std::process::Command::new("cmd")
.args(["/C", "mklink", "/J"])
.arg(&link)
.arg(target)
.creation_flags(0x0800_0000) // CREATE_NO_WINDOW
.output()
.unwrap();
assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr));
Junction(link)
}
let temp = Temporary::new();
let root = temp.bundle();
let outside = temp.0.join("outside");
fs::create_dir(&outside).unwrap();
let object_name = format!("{}.zst", "e".repeat(64));
fs::write(outside.join(&object_name), b"outside object").unwrap();
fs::write(outside.join("preview.png"), b"outside preview").unwrap();
fs::write(outside.join("full.png"), b"outside full preview").unwrap();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
let _objects = junction(root.join("blobs"), &outside);
let _previews = junction(root.join("composite"), &outside);
let stats = writer.save_directory(&root, entries(NEW)).unwrap();
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 0);
assert_eq!(fs::read(outside.join(object_name)).unwrap(), b"outside object");
assert_eq!(fs::read(outside.join("preview.png")).unwrap(), b"outside preview");
assert_eq!(fs::read(outside.join("full.png")).unwrap(), b"outside full preview");
assert!(root.join("blobs").exists());
assert!(root.join("composite").exists());
}
#[cfg(any(unix, windows))]
#[test]
fn manifest_symlink_is_refused_before_writing_any_objects_or_previews() {
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
writer.save_directory(&root, entries(OLD)).unwrap();
assert_manifest(&root, OLD); // Ordinary publication remains supported.
let external = temp.0.join("external-manifest.json");
fs::write(&external, b"unchanged external manifest").unwrap();
fs::remove_file(root.join(MANIFEST)).unwrap();
#[cfg(unix)]
std::os::unix::fs::symlink(&external, root.join(MANIFEST)).unwrap();
#[cfg(windows)]
if let Err(error) = std::os::windows::fs::symlink_file(&external, root.join(MANIFEST)) {
if error.raw_os_error() == Some(1314) || error.kind() == std::io::ErrorKind::Unsupported {
eprintln!("SKIP manifest symlink fixture: {error}");
return;
}
panic!("create manifest symlink fixture: {error}");
}
let mut changed = entries(NEW);
changed.add_object(ObjectKind::Tile, b"must not write tile").unwrap();
changed.add_object(ObjectKind::Blob, b"must not write blob").unwrap();
changed.set_preview(THUMB, b"must not write thumbnail".to_vec()).unwrap();
changed.set_preview(FULL_COMPOSITE, b"must not write composite".to_vec()).unwrap();
assert!(writer.save_directory(&root, changed).is_err());
assert_eq!(fs::read(&external).unwrap(), b"unchanged external manifest");
assert!(fs::symlink_metadata(root.join(MANIFEST)).unwrap().file_type().is_symlink());
for unwritten in ["tiles", "blobs", THUMB, "composite"] {
assert!(!root.join(unwritten).exists(), "manifest refusal must precede {unwritten} publication");
}
}
#[cfg(unix)]
#[test]
fn chosen_root_symlink_cleanup_works_with_search_only_root_and_composite_permissions() {
use std::os::unix::fs::{PermissionsExt, symlink};
struct Restore(PathBuf, fs::Permissions);
impl Restore {
fn restrict(path: &Path) -> Self {
let old = fs::metadata(path).unwrap().permissions();
fs::set_permissions(path, fs::Permissions::from_mode(0o300)).unwrap();
Self(path.to_path_buf(), old)
}
}
impl Drop for Restore {
fn drop(&mut self) {
let _ = fs::set_permissions(&self.0, self.1.clone());
}
}
let temp = Temporary::new();
let root = temp.bundle();
let mut writer = DirectoryBundleWriter::default();
let mut first = entries(OLD);
let stale = first.add_object(ObjectKind::Blob, b"stale blob").unwrap();
first.set_preview(THUMB, b"stale thumbnail".to_vec()).unwrap();
first.set_preview(COMPOSITE, b"stale preview".to_vec()).unwrap();
writer.save_directory(&root, first).unwrap();
let chosen = temp.0.join("chosen.pcraft");
symlink(&root, &chosen).unwrap();
let composite_mode = Restore::restrict(&root.join("composite"));
let root_mode = Restore::restrict(&root);
let result = writer.save_directory(&chosen, entries(NEW));
drop(root_mode);
drop(composite_mode);
let stats = result.expect("chosen root resolves before cleanup; traversal does not require directory read permission");
assert_manifest(&root, NEW);
assert_eq!(stats.removed, 1);
assert!(!root.join(stale).exists());
assert!(!root.join(THUMB).exists());
assert!(!root.join(COMPOSITE).exists());
assert!(fs::symlink_metadata(chosen).unwrap().file_type().is_symlink());
}
+74 -5
View File
@@ -629,6 +629,19 @@ fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> {
.max_by_key(|i| i.u64(t::IMAGE_WIDTH).unwrap_or(0).saturating_mul(i.u64(t::IMAGE_LENGTH).unwrap_or(0)))
}
fn valid_arw2_curve(curve: &[u64]) -> bool {
curve.len() == 4 && curve.iter().all(|&v| v <= 16383) && curve.windows(2).all(|v| (v[0] >> 2) < (v[1] >> 2))
}
fn declared_arw2_curve(raw: &Ifd, compression: u16, one_byte_per_sample: bool) -> Result<Option<Vec<u64>>> {
let curve = raw.u64s(TONE_CURVE);
if compression == 32767 && one_byte_per_sample && curve.as_ref().is_some_and(|curve| !valid_arw2_curve(curve)) {
// Reordering or clamping malformed breakpoints fabricates a different transfer curve.
return Err(RawError::Unsupported("Sony ARW2 with an invalid tone curve".into()));
}
Ok(curve)
}
pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawImage> {
let tiff = Tiff::parse(bytes)?;
let ifd0 = tiff.ifds.first().ok_or_else(|| RawError::Corrupt("TIFF without IFDs".into()))?;
@@ -645,6 +658,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// archive measure exactly width x height. A200/A230/A350 files carry a strip 4-23 % larger than that (and A290/A390 one
// smaller), which is a different packing, so they are refused rather than read with the wrong layout.
let one_byte_per_sample = chunks.len() == 1 && strip_len == samples;
let declared_curve = declared_arw2_curve(raw, info.compression, one_byte_per_sample)?;
let packed12 = info.compression == 32767 && is_packed12(&info, chunks.len(), strip_len);
let (data, out_bits) = match info.compression {
7 if linear_rgb => (RawData::U16(read_ycbcr_tiles(bytes, &info, raw, mode, limits)?), 14),
@@ -720,13 +734,16 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// agree too. Genuinely packed 12-bit strips retain their 12-bit scale.
let scale_bits = if info.compression == 1 && out_bits < 14 && strip_len >= (w * h * 2) as u64 { 14 } else { out_bits };
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
let (cfa, assumed_cfa) = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => {
Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }
(Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }, false)
}
_ => Cfa::bayer_static("RGGB"),
_ => (Cfa::bayer_static("RGGB"), true),
};
let default_black = if scale_bits >= 14 { 512.0 } else { 128.0 };
let curve_black = (info.compression == 32767 && one_byte_per_sample)
.then(|| declared_curve.as_ref().and_then(|curve| code_curve(curve).get(256).copied()).map(f32::from))
.flatten();
let default_black = curve_black.unwrap_or(if scale_bits >= 14 { 512.0 } else { 128.0 });
// the decrypted SR2SubIFD, read once and only when a plain black level or white balance is missing
let sr2_cell = std::cell::OnceCell::new();
let sr2 = || sr2_cell.get_or_init(|| crate::sr2::SubIfd::read(bytes, ifd0, tiff.order)).as_ref();
@@ -741,7 +758,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
_ if linear_rgb => (BlackLevel::uniform(default_black + YCBCR_OFFSET), crate::LevelSource::Fallback),
_ => match (scale_bits >= 14).then(|| sr2().and_then(sr2_black)).flatten() {
Some(b) => (BlackLevel::uniform(b), crate::LevelSource::Metadata),
None => (BlackLevel::uniform(default_black), crate::LevelSource::Fallback),
None => (BlackLevel::uniform(default_black), curve_black.map_or(crate::LevelSource::Fallback, |_| crate::LevelSource::Metadata)),
},
};
// ARW6 samples are in 2 x 14-bit units, so the file's 14-bit levels double; without the tag the white is the
@@ -842,6 +859,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: LensMetadata { sony: sony_lens },
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices { assumed_cfa: assumed_cfa && !linear_rgb, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -873,6 +891,8 @@ mod tests {
assert_eq!(header.info(), full.info());
assert_eq!((full.width, full.height, full.cpp), (4, 4, 3));
assert!(full.cfa.is_none());
assert!(!header.notices.assumed_cfa);
assert!(!full.notices.assumed_cfa);
assert_eq!(full.data.len(), 48);
assert_eq!(full.wb_multipliers, Some([1.0; 3]));
// the YCbCr data sit 512 above the recorded sensor black level; white moves with it
@@ -911,6 +931,10 @@ mod tests {
/// A 32767-compressed ARW whose single strip is `strip` bytes for a 32 x 4 image.
fn arw_with_strip(strip: usize) -> Vec<u8> {
arw_with_strip_and_cfa(strip, None)
}
fn arw_with_strip_and_cfa(strip: usize, cfa: Option<(Vec<u16>, Vec<u8>)>) -> Vec<u8> {
use craft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
let mut raw = IfdBuilder::new();
raw.set(t::MAKE, Value::Ascii("SONY".into()));
@@ -921,11 +945,37 @@ mod tests {
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA]));
raw.set(t::COMPRESSION, Value::Short(vec![32767]));
if let Some((dimensions, pattern)) = cfa {
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(dimensions));
raw.set(t::CFA_PATTERN_EP, Value::Byte(pattern));
}
raw.set(TONE_CURVE, Value::Short(vec![8000, 10400, 12900, 14100]));
raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0u8; strip]] });
TiffWriter::default().write(&[raw]).unwrap()
}
#[test]
fn cfa_fallback_notices_follow_missing_malformed_and_valid_source_tags() {
for (tags, expected, assumed) in [
(None, "RGGB", true),
(Some((vec![1, 4], vec![0, 1, 1, 2])), "RGGB", true),
(Some((vec![2, 2], vec![0, 1, 3, 2])), "RGGB", true),
(Some((vec![2, 2], vec![0, 1, 2])), "RGGB", true),
(Some((vec![2, 2], vec![1, 0, 2, 1])), "GRBG", false),
] {
let file = arw_with_strip_and_cfa(32 * 4, tags);
let header = decode(&file, Mode::Header, &Limits::default()).unwrap();
let full = decode(&file, Mode::Full, &Limits::default()).unwrap();
assert_eq!(header.info(), full.info());
for image in [&header, &full] {
assert_eq!(image.cfa.as_ref().unwrap().name(), expected);
assert_eq!(image.notices.assumed_cfa, assumed);
}
assert!(header.data.is_empty());
assert_eq!(full.data.len(), 32 * 4);
}
}
/// Strip byte counts that add up past `u64::MAX` (LONG8 `[u64::MAX, 1]`) are an error or a picture, not an overflow.
#[test]
fn strip_byte_counts_past_u64_max_do_not_overflow() {
@@ -1110,6 +1160,25 @@ mod tests {
let _ = code_curve(&[60000, 1, 0, 0]);
}
#[test]
fn arw2_rejects_non_increasing_declared_curve_in_header_and_full_decode() {
let valid = arw_with_strip(32 * 4);
assert!(crate::decode(&valid).is_ok());
let mut damaged = valid.clone();
let tiff = Tiff::parse(&damaged).unwrap();
let curve = raw_ifd(&tiff).unwrap().get(TONE_CURVE).unwrap();
let at = curve.offset as usize;
for (dst, value) in damaged[at..at + 8].as_chunks_mut::<2>().0.iter_mut().zip([9000u16, 8000, 12000, 13000]) {
dst.copy_from_slice(&value.to_le_bytes());
}
for mode in [Mode::Header, Mode::Full] {
assert!(matches!(decode(&damaged, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("curve")));
let image = decode(&valid, mode, &Limits::default()).unwrap();
assert_eq!(image.black.mean(), 512.0);
assert_eq!(image.level_sources.black, crate::LevelSource::Metadata, "the declared curve maps the black code");
}
}
#[test]
fn decipher_inverts_the_cube_substitution() {
let mut seen = [false; 256];
+128 -2
View File
@@ -10,8 +10,11 @@
//! - Colour filter layout: IFD#3 tag `0xc5e0` (`CR2CFAPattern` in the ExifTool EXIF tag-name docs): 1 = RGGB,
//! 2 = BGGR, 3 = GBRG, 4 = GRBG, anchored at the top-left of the full decoded sensor (masked borders included).
//! It differs by model (e.g. 3 on the 50D/60D/7D/550D/5D Mark II, 1 on the 40D/5D Mark III/6D/5DS R), so it is
//! read per file (no model table); files without the tag fall back to RGGB (storytold/lightcraft#85). The parity of the
//! read per file first. Without a known tag, observed model IDs provide the fallback; scene-derived
//! measurements are not authoritative enough to establish sensor phase. The parity of the
//! `SensorInfo` borders does *not* predict it (RGGB files come with both even and odd top borders).
//! Header-only probes do not decode pixels for this fallback: without the tag their reported CFA
//! is assumed and can refine on full decode, as indicated by `RawNotices::assumed_cfa`.
//! - `SensorInfo` (maker note `0x00e0`): sensor width/height and the left/top/right/bottom borders of the image
//! area. Light already reaches some columns left of that border (EOS 6D: masked columns 0–70, border 84; 10–36
//! such columns on every corpus body), so the masked columns that give the black level are measured from the
@@ -38,6 +41,7 @@ const SRAW_TYPE: u16 = 0xc6c5;
/// `SRAW_TYPE` value of the YCbCr frames of sRAW / mRAW (IFD2's preview image carries 3).
const SRAW_YCC: u32 = 4;
const CR2_CFA_PATTERN: u16 = 0xc5e0;
const CANON_MODEL_ID: u16 = 0x0010;
const SENSOR_INFO: u16 = 0x00e0;
const COLOR_BALANCE: u16 = 0x4001;
/// ShotInfo, whose value 7 is the white-balance setting.
@@ -61,6 +65,13 @@ fn cfa_from_tag(v: Option<u64>) -> Option<Cfa> {
Some(Cfa::bayer_static(name))
}
/// Fallback for files without CR2CFAPattern. Image content cannot establish sensor phase safely.
fn fallback_cfa(_data: &[u16], _width: usize, _area: Rect, _black: f32, _white: f32, model: Option<u64>) -> (Cfa, bool) {
// Observed CC0 raw.pixls.us captures: 5D II, 7D, 550D, 600D and 1200D.
let green_first = [0x8000_0218, 0x8000_0250, 0x8000_0270, 0x8000_0286, 0x8000_0327].into_iter().any(|id| model == Some(id));
(Cfa::bayer_static(if green_first { "GBRG" } else { "RGGB" }), true)
}
/// The 16-bit words of a maker-note array. Canon writes `ColorData` as SHORT in most models, but as UNDEFINED
/// bytes (count = byte length, e.g. 5120) in the PowerShot / EOS M models with ColorData versions -3 and -4;
/// those bytes are the same 16-bit words in the maker note's byte order, so they are paired up, not read one by
@@ -198,6 +209,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
let chunk = craft_tiff::image::Chunk { index: 0, x: 0, y: 0, width: 0, height: 0, plane: 0, offset: off, len };
let src = chunk_bytes(bytes, &chunk).ok_or_else(|| RawError::Corrupt("raw strip outside file".into()))?;
let (fw, fh, nc, prec) = ljpeg::frame_info(src)?;
let tagged_cfa = cfa_from_tag(raw.u64(CR2_CFA_PATTERN));
let frame = match mode {
Mode::Full => {
let total = (fw as u64)
@@ -270,12 +282,19 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
let color_data = mn.as_ref().and_then(|m| m.ifd.value(COLOR_BALANCE).map(|v| words(v, m.order)));
let kelvin_presets = color_data.as_deref().and_then(kelvin_presets_from_color_data);
let wb = color_data.as_deref().and_then(wb_from_color_balance).or_else(|| wb_from_wb_info(mn.as_ref()?));
let cfa = cfa_from_tag(raw.u64(CR2_CFA_PATTERN)).unwrap_or_else(|| Cfa::bayer_static("RGGB"));
let (white, white_source) = white_from_data_sourced(&active_rows(&data, width, active), prec as u32);
let (black, black_source) = match masked_columns(&data, width, active, white) {
cols if cols.len() >= 2 => black_from_columns_sourced(&data, width, cols, active.y..active.y + active.height, active),
_ => (BlackLevel::uniform(0.0), crate::LevelSource::Fallback),
};
let model = mn.as_ref().and_then(|m| m.ifd.u64(CANON_MODEL_ID));
let (cfa, assumed_cfa) = match tagged_cfa {
Some(cfa) => (cfa, false),
None => {
let (cfa, guessed) = fallback_cfa(&data, width, active, black.mean(), white, model);
(cfa, guessed || mode == Mode::Header)
}
};
let img = RawImage {
format: RawFormat::Cr2,
width,
@@ -298,6 +317,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -397,6 +417,112 @@ pub(crate) mod tests {
assert_eq!(name(None), None);
}
#[test]
fn absent_cfa_tag_uses_safe_phase_without_shifting_for_crop_parity() {
let (w, h) = (192usize, 96usize);
for red_row in [0usize, 1] {
let samples: Vec<u16> = (0..w * h)
.map(|i| {
let (x, y) = (i % w, i / w);
let texture = ((x / 2 + y / 2) % 17) as u16;
if i % 97 == 0 {
return 13000;
}
100 + if (y & 1) == red_row {
if x & 1 == 0 { 500 + texture } else { 1000 + texture }
} else if x & 1 == 0 {
1000 + texture
} else {
667 + texture
}
})
.collect();
for (left, top) in [(16, 4), (17, 4), (16, 5), (17, 5)] {
let area = Rect::new(left, top, w - left, h - top);
let (cfa, assumed) = fallback_cfa(&samples, w, area, 100.0, 4000.0, None);
assert_eq!(cfa.name(), "RGGB");
assert!(assumed);
}
let mut bytes = synthetic_cr2_of(&samples, w, h, None, None);
let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0];
let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap();
// The explicit tag remains authoritative even when the observed sensor phase disagrees.
bytes[at + 8..at + 12].copy_from_slice(&4u32.to_le_bytes());
let explicit = crate::decode(&bytes).unwrap();
assert_eq!(explicit.cfa.as_ref().unwrap().name(), "GRBG");
assert_eq!(explicit.info(), crate::probe_info(&bytes).unwrap());
assert!(!explicit.notices.assumed_cfa);
bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes());
let full = crate::decode(&bytes).unwrap();
assert_eq!(full.cfa.as_ref().unwrap().name(), "RGGB");
assert!(full.notices.assumed_cfa);
let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap();
assert!(header.notices.assumed_cfa);
assert!(matches!(header.data, RawData::U16(ref v) if v.is_empty()));
assert_eq!(full.data, RawData::U16(samples));
}
}
#[test]
fn inconclusive_cfa_measurement_keeps_observed_model_fallback_and_safe_geometry() {
let data = vec![512; 64 * 32];
let area = Rect::new(0, 0, 64, 32);
assert_eq!(fallback_cfa(&data, 64, area, 512.0, 16000.0, Some(0x8000_0250)).0.name(), "GBRG");
assert!(fallback_cfa(&data, 64, area, 512.0, 16000.0, Some(0x8000_0250)).1);
assert_eq!(fallback_cfa(&data, 64, area, 512.0, 16000.0, None).0.name(), "RGGB");
assert!(fallback_cfa(&data, 64, area, 512.0, 16000.0, None).1);
let gradient: Vec<u16> = (0..192 * 96)
.map(|i| {
let (x, y) = (i % 192, i / 192);
if y % 2 == 0 {
if x % 2 == 0 { 500 } else { 1000 + 8 * x as u16 }
} else if x % 2 == 0 {
1000 + 8 * x as u16
} else {
500
}
})
.collect();
let (cfa, assumed) = fallback_cfa(&gradient, 192, Rect::new(0, 0, 192, 96), 0.0, 4000.0, None);
assert_eq!(cfa.name(), "RGGB");
assert!(assumed);
let mut bytes = synthetic_cr2_of(&gradient, 192, 96, None, None);
let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0];
let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap();
// Exercise the decoder's missing-tag fallback, not the fixture's default CFA tag.
bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes());
let decoded = crate::decode(&bytes).unwrap();
assert_eq!(decoded.cfa.as_ref().unwrap().name(), "RGGB");
assert!(decoded.notices.assumed_cfa);
}
#[test]
fn model_cfa_fallback_is_disclosed_in_header_and_full_decodes() {
// A single inline LONG maker-note entry needs no external offsets.
let mut note = Vec::new();
note.extend_from_slice(&1u16.to_le_bytes());
note.extend_from_slice(&CANON_MODEL_ID.to_le_bytes());
note.extend_from_slice(&4u16.to_le_bytes());
note.extend_from_slice(&1u32.to_le_bytes());
note.extend_from_slice(&0x8000_0250u32.to_le_bytes());
note.extend_from_slice(&0u32.to_le_bytes());
let samples = vec![512u16; 64 * 32];
let mut bytes = synthetic_cr2_of(&samples, 64, 32, None, Some(note));
let marker = [0xe0, 0xc5, 4, 0, 1, 0, 0, 0];
let at = bytes.windows(marker.len()).position(|b| b == marker).unwrap();
// Remove only the CFA tag; the model ID and encoded samples stay intact.
bytes[at..at + 2].copy_from_slice(&0xc5dfu16.to_le_bytes());
let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap();
let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap();
for image in [&header, &full] {
assert_eq!(image.cfa.as_ref().unwrap().name(), "GBRG");
assert!(image.notices.assumed_cfa);
}
assert_eq!(header.info(), full.info());
assert_eq!(full.data, RawData::U16(samples));
assert!(header.data.is_empty());
}
#[test]
fn wb_probe() {
let mut v = vec![0u64; 120];
+1
View File
@@ -393,6 +393,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode, limits: &
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: Default::default(),
level_sources: Default::default(),
};
img.validate_for(mode)?;
+119 -1
View File
@@ -274,6 +274,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
camera: ifd0.map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: Default::default(),
level_sources: Default::default(),
};
// Canon's stated white level can sit below the sensor's true saturation (EOS RP: 14558
@@ -404,6 +405,15 @@ mod tests {
}
fn track(entry: &[u8], offset: u64, size: u32) -> Vec<u8> {
let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
track_with_mapping(entry, offset, size, &mapping)
}
fn track_with_mapping(entry: &[u8], offset: u64, size: u32, mapping: &[u8]) -> Vec<u8> {
track_with_table(entry, offset, size, b"stsc", mapping)
}
fn track_with_table(entry: &[u8], offset: u64, size: u32, table: &[u8; 4], payload: &[u8]) -> Vec<u8> {
let mut stsd = vec![0u8; 4];
stsd.extend(1u32.to_be_bytes());
stsd.extend(entry);
@@ -413,7 +423,15 @@ mod tests {
let mut co64 = vec![0u8; 4];
co64.extend(1u32.to_be_bytes());
co64.extend(offset.to_be_bytes());
let tables = [box_bytes(b"stsd", &stsd), box_bytes(b"stsz", &stsz), box_bytes(b"co64", &co64)].concat();
let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
let offsets = if table == b"stco" { box_bytes(b"stco", payload) } else { box_bytes(b"co64", if table == b"co64" { payload } else { &co64 }) };
let tables = [
box_bytes(b"stsd", &stsd),
box_bytes(b"stsc", if table == b"stsc" { payload } else { &mapping }),
box_bytes(b"stsz", if table == b"stsz" { payload } else { &stsz }),
offsets,
]
.concat();
box_bytes(b"trak", &box_bytes(b"mdia", &box_bytes(b"minf", &box_bytes(b"stbl", &tables))))
}
@@ -636,6 +654,105 @@ mod tests {
assert!(matches!(decode(&missing, Mode::Header, &Limits::default()), Err(RawError::Corrupt(_))));
}
/// Independently framed constant-value CRX: each 4x4 tile has four 2x2
/// planes whose one-byte 0x1e entropy streams reconstruct sensor value8193.
fn flat_raw_track(tiles: u32) -> (Vec<u8>, Vec<u8>) {
let mut sample = Vec::new();
let marker = |out: &mut Vec<u8>, code: u16, size: u32, flags: u32| {
out.extend(code.to_be_bytes());
out.extend(8u16.to_be_bytes());
out.extend(size.to_be_bytes());
out.extend(flags.to_be_bytes());
};
for tile in 0..tiles {
marker(&mut sample, 0xff01, 4, tile << 16);
for plane in 0..4 {
marker(&mut sample, 0xff02, 1, (plane << 28) | 0x0800_0000);
marker(&mut sample, 0xff03, 1, 0x0020_0000);
}
}
let header_size = sample.len() as u32;
sample.extend(std::iter::repeat_n(0x1e, tiles as usize * 4));
let mut coding = vec![0u8; 52];
coding[..2].copy_from_slice(&0xff00u16.to_be_bytes());
coding[2..4].copy_from_slice(&48u16.to_be_bytes());
coding[4..6].copy_from_slice(&0x100u16.to_be_bytes());
for (offset, value) in [(8, tiles * 4), (12, 4), (16, 4), (20, 4), (28, header_size)] {
coding[offset..offset + 4].copy_from_slice(&value.to_be_bytes());
}
coding[24] = 14;
coding[25] = 0x40;
let mut entry = vec![0u8; 82];
entry[24..26].copy_from_slice(&((tiles * 4) as u16).to_be_bytes());
entry[26..28].copy_from_slice(&4u16.to_be_bytes());
entry.extend(box_bytes(b"CMP1", &coding));
(box_bytes(b"CRAW", &entry), sample)
}
#[test]
fn truncated_main_sample_mapping_cannot_silently_select_a_valid_reduced_raw() {
let mapping = [0u32, 1, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>();
let (reduced_entry, reduced_sample) = flat_raw_track(1);
let (main_entry, main_sample) = flat_raw_track(2);
let main_offset = 4096 + reduced_sample.len();
let file = |main_table: Option<(&[u8; 4], &[u8])>, main_first: bool| {
let reduced = track_with_mapping(&reduced_entry, 4096, reduced_sample.len() as u32, &mapping);
let tracks = if let Some((table, payload)) = main_table {
let main = track_with_table(&main_entry, main_offset as u64, main_sample.len() as u32, table, payload);
if main_first { [main, reduced].concat() } else { [reduced, main].concat() }
} else {
reduced
};
let mut bytes = box_bytes(b"ftyp", b"crx \0\0\0\x01crx isom");
bytes.extend(box_bytes(b"moov", &tracks));
assert!(bytes.len() < 4096);
bytes.resize(4096, 0);
bytes.extend_from_slice(&reduced_sample);
bytes.extend_from_slice(&main_sample);
bytes
};
let mut co64 = vec![0u8; 4];
co64.extend(1u32.to_be_bytes());
co64.extend((main_offset as u64).to_be_bytes());
let valid_tables = [
(b"stsc", mapping.clone()),
(b"stsz", [0u32, main_sample.len() as u32, 1].into_iter().flat_map(u32::to_be_bytes).collect()),
(b"co64", co64),
(b"stco", [0u32, 1, main_offset as u32].into_iter().flat_map(u32::to_be_bytes).collect()),
];
for mode in [Mode::Header, Mode::Full] {
// The smaller source is genuinely decodable, so dropping the larger
// track would return success rather than merely a different failure.
let reduced = decode(&file(None, false), mode, &Limits::default()).unwrap();
assert_eq!((reduced.width, reduced.height), (4, 4));
if mode == Mode::Full {
assert_eq!(reduced.data, RawData::U16(vec![8193; 16]));
}
for main_first in [false, true] {
for (table, payload) in &valid_tables {
let valid = decode(&file(Some((table, payload)), main_first), mode, &Limits::default()).unwrap();
assert_eq!((valid.width, valid.height), (8, 4));
if mode == Mode::Full {
assert_eq!(valid.data, RawData::U16(vec![8193; 32]));
}
for truncated in [vec![], vec![0; 4], vec![0, 0, 0, 0, 0, 0, 0, 1]] {
let bytes = file(Some((table, &truncated)), main_first);
let container = parse_cr3(&bytes).unwrap();
assert_eq!(container.tracks.len(), 2, "mapping damage cannot erase the main track");
let selected = main_track(&container.tracks).unwrap();
assert!(matches!(selected.kind, Cr3TrackKind::Raw { width: 8, height: 4, .. }));
assert!(selected.data.is_none());
assert!(selected.compression(&bytes).is_some());
assert!(
matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Corrupt(message)) if message.contains("sample")),
"{mode:?}/{table:?}: main mapping damage cannot decode the reduced source"
);
}
}
}
}
}
#[test]
fn color_data_uses_versioned_as_shot_offsets() {
for (version, wb_at, black_at, white_at) in
@@ -712,6 +829,7 @@ mod tests {
camera: Camera::default(),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: Default::default(),
level_sources: Default::default(),
}
}
+1
View File
@@ -65,6 +65,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, caller: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices::default(),
};
img.validate_for(mode)?;
Ok(img)
+82 -35
View File
@@ -5,8 +5,8 @@ use crate::gaintable::GainTableMap;
use crate::profile::{HsvTable, ProfileLook, ToneCurve};
use crate::tiffraw::{Packing, read_image_in};
use crate::{
BlackLevel, Camera, Cfa, CinemaTags, ColorData, Limits, Matrix3, Mode, Orientation, RawData, RawError, RawFormat, RawImage, Rect, Result,
Timecode, opcodes,
BlackLevel, Camera, Cfa, CinemaTags, ColorData, LevelSource, LevelSources, Limits, Matrix3, Mode, Orientation, RawData, RawError, RawFormat,
RawImage, RawNotices, Rect, Result, Timecode, opcodes,
};
use craft_tiff::tags::{self as t, photometric};
use craft_tiff::{ByteOrder, Ifd, Tiff, Value};
@@ -178,7 +178,15 @@ pub(crate) fn is_plain_cfa_tiff(tiff: &Tiff, bytes: &[u8]) -> bool {
if !raw.contains(t::AS_SHOT_NEUTRAL) && !tiff.ifds.first().is_some_and(|i| i.contains(t::AS_SHOT_NEUTRAL)) {
return false;
}
let Ok(info) = raw.image() else { return false };
let Ok(info) = raw.image() else {
// Recognition must not turn a damaged, otherwise supported plain CFA
// image into an unsupported container. Let decode_as report its layout
// error; these tags alone never decompress or allocate pixel samples.
return raw.u16(t::COMPRESSION).unwrap_or(1) == t::compression::NONE
&& raw.u16(t::SAMPLES_PER_PIXEL).unwrap_or(1) == 1
&& (1..=16).contains(&raw.u16(t::BITS_PER_SAMPLE).unwrap_or(1))
&& raw.u16(t::SAMPLE_FORMAT).unwrap_or(1) != 3;
};
if info.samples_per_pixel != 1 || !(1..=16).contains(&info.bits()) || info.sample_format == 3 {
return false;
}
@@ -274,8 +282,13 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
let bits = info.bits() as u32;
// DNG 1.4 stores lossy JPEG at 8 bits per sample: the tiles are 8-bit baseline JPEGs whatever the tag says,
// and reading them as a deeper sample would put the picture at the bottom of its range
if info.compression == t::compression::LOSSY_JPEG && bits != 8 {
return Err(RawError::Unsupported(format!("lossy JPEG DNG with {bits}-bit samples (DNG allows only 8)")));
if info.compression == t::compression::LOSSY_JPEG {
if bits != 8 {
return Err(RawError::Unsupported(format!("lossy JPEG DNG with {bits}-bit samples (DNG allows only 8)")));
}
if !matches!(cpp, 1 | 3) {
return Err(RawError::Unsupported(format!("lossy JPEG DNG with {cpp} samples per pixel")));
}
}
let mut data = read_image_in(mode, bytes, &info, tiff.order, Packing::Msb, limits)?;
@@ -312,6 +325,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
// (a DOUBLE past `f32::MAX` casts to infinity, which would black the picture out: ignored like a zero one)
let mut white: Vec<f32> =
levels(t::WHITE_LEVEL).f64s(t::WHITE_LEVEL).unwrap_or_default().into_iter().map(|v| v as f32).filter(|v| v.is_finite() && *v > 0.0).collect();
let white_source = if !white.is_empty() || table_max.is_some() { LevelSource::Metadata } else { LevelSource::Fallback };
if white.is_empty() {
white = vec![default_white];
}
@@ -333,7 +347,13 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
};
// (a FLOAT / DOUBLE tag can hold NaN or infinity: such a level reads as 0, in its place)
let level = |v: f64| Some(v as f32).filter(|v| v.is_finite()).unwrap_or(0.0);
let mut values: Vec<f32> = levels(t::BLACK_LEVEL).f64s(t::BLACK_LEVEL).unwrap_or_default().into_iter().map(level).collect();
let declared_black = levels(t::BLACK_LEVEL).f64s(t::BLACK_LEVEL).unwrap_or_default();
let black_source = if !declared_black.is_empty() && declared_black.iter().all(|value| (*value as f32).is_finite()) {
LevelSource::Metadata
} else {
LevelSource::Fallback
};
let mut values: Vec<f32> = declared_black.into_iter().map(level).collect();
// A TIFF that is not a DNG may state four black levels without `BlackLevelRepeatDim`: one per site of its 2 × 2
// CFA, row-major (PhotoCraft's reading of Hasselblad files). In a DNG the count has to match the repeat.
let (br, bc) = if format != RawFormat::Dng && cpp == 1 && values.len() == 4 && !levels(t::BLACK_LEVEL).contains(t::BLACK_LEVEL_REPEAT_DIM) {
@@ -364,7 +384,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
}
_ => Rect::new(0, 0, full_w, full_h),
};
let crop = default_crop(raw, active_area);
let (crop, default_crop_adjusted) = default_crop(raw, active_area, full_w, full_h);
let (active_area, crop) = if (full_w, full_h) == (w, h) {
(active_area, crop)
} else {
@@ -378,6 +398,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
(area, scale(crop).clipped(area.width, area.height))
};
let mut assumed_cfa = false;
let cfa = if info.photometric == photometric::CFA {
if raw.u16(t::CFA_LAYOUT).unwrap_or(1) != 1 {
return Err(RawError::Unsupported("non-rectangular CFA layout".into()));
@@ -391,7 +412,10 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
let assumed = [0u8, 1, 1, 2];
let pat = match raw.bytes(t::CFA_PATTERN_EP) {
Some(p) => p,
None if format != RawFormat::Dng && (rows, cols) == (2, 2) => &assumed,
None if format != RawFormat::Dng && (rows, cols) == (2, 2) => {
assumed_cfa = true;
&assumed
}
None => return Err(RawError::Corrupt("missing CFAPattern".into())),
};
if pat.len() != rows * cols {
@@ -422,6 +446,9 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
let opcodes =
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
let color = color_data(ifd0, raw, tiff.order);
let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2))
&& color.profile.gain_table_map.is_none();
let img = RawImage {
format,
width: w,
@@ -435,7 +462,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
active_area,
crop,
orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)),
color: color_data(ifd0, raw, tiff.order),
color,
wb_multipliers: None,
linearized,
pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()),
@@ -443,7 +470,8 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
camera: Camera::from_ifd0(ifd0),
cinema: cinema_tags(ifd0, raw),
lens_metadata: Default::default(),
level_sources: Default::default(),
level_sources: LevelSources { black: black_source, white: white_source },
notices: RawNotices { default_crop_adjusted, gain_table_omitted, assumed_cfa },
};
img.validate_for(mode)?;
Ok(img)
@@ -472,33 +500,45 @@ fn cinema_tags(ifd0: &Ifd, raw: &Ifd) -> CinemaTags {
}
}
/// The default crop, relative to the active area. Each tag has its own default (DNG 1.7.1, `DefaultCropOrigin`
/// and `DefaultCropSize`): the origin (0, 0), the size the whole area, so a file with only a size crops from the
/// area's corner. A tag that is present must hold two finite, non-negative values, and the crop must start inside
/// the area and be at least one pixel each way; otherwise the crop is the whole active area.
fn default_crop(raw: &Ifd, active_area: Rect) -> Rect {
/// The default crop is relative to ActiveArea. DNG 1.7.1 defines independent
/// defaults: origin (0, 0), size ImageWidth/ImageLength. An unusable optional crop
/// preserves the entire active image. Valid partly overlapping rectangles retain the
/// existing clipped-crop policy, with an explicit notice rather than silent adjustment.
fn default_crop(raw: &Ifd, active_area: Rect, full_width: usize, full_height: usize) -> (Rect, bool) {
let whole = Rect::new(0, 0, active_area.width, active_area.height);
let pair = |tag: u16, default: [f64; 2]| -> Option<[f64; 2]> {
if !raw.contains(tag) {
return Some(default);
}
match raw.f64s(tag).as_deref() {
Some([a, b]) if a.is_finite() && b.is_finite() && *a >= 0.0 && *b >= 0.0 => Some([*a, *b]),
_ => None,
}
};
let (Some([x, y]), Some([cw, ch])) =
(pair(t::DEFAULT_CROP_ORIGIN, [0.0, 0.0]), pair(t::DEFAULT_CROP_SIZE, [active_area.width as f64, active_area.height as f64]))
else {
return whole;
};
if cw < 1.0 || ch < 1.0 {
return whole;
if !raw.contains(t::DEFAULT_CROP_ORIGIN) && !raw.contains(t::DEFAULT_CROP_SIZE) {
return (whole, false);
}
// (`as usize` saturates, and `clipped` holds the origin to the area)
let crop =
Rect::new(x.round() as usize, y.round() as usize, cw.round() as usize, ch.round() as usize).clipped(active_area.width, active_area.height);
if crop.width == 0 || crop.height == 0 { whole } else { crop }
let pair = |tag: u16, default: [f64; 2]| -> Option<[usize; 2]> {
let values = if raw.contains(tag) {
match raw.f64s(tag).as_deref() {
Some([a, b]) => [*a, *b],
_ => return None,
}
} else {
default
};
// Float-to-integer casts saturate. Refuse nonfinite/unrepresentable numbers before
// casting so a hostile origin cannot become a seemingly valid clipped rectangle.
if values.iter().any(|value| !value.is_finite() || value.round() >= usize::MAX as f64) {
return None;
}
if values.iter().any(|value| *value < 0.0) {
return None;
}
Some(values.map(|value| value.round() as usize))
};
let (Some([x, y]), Some([width, height])) =
(pair(t::DEFAULT_CROP_ORIGIN, [0.0, 0.0]), pair(t::DEFAULT_CROP_SIZE, [full_width as f64, full_height as f64]))
else {
return (whole, true);
};
let requested = Rect::new(x, y, width, height);
let crop = requested.clipped(active_area.width, active_area.height);
if crop.width == 0 || crop.height == 0 {
return (whole, true);
}
(crop, crop != requested)
}
/// Pixel height over width from `DefaultScale` (horizontal, vertical): only a clear 1.5 to 4 ratio counts (the rows stretch; a wider-than-tall pixel is left alone).
@@ -509,6 +549,9 @@ fn default_scale_aspect(scale: Option<&[f64]>) -> f32 {
}
}
#[cfg(test)]
mod compat_review_tests;
#[cfg(test)]
mod tests {
#[test]
@@ -621,3 +664,7 @@ mod tests {
assert!(deinterleave(&mut hostile, 8, 6, 1, 1 << 20, 1).is_err());
}
}
#[cfg(all(test, feature = "hasselblad"))]
#[path = "dng/probe_review_tests.rs"]
mod probe_review_tests;
+120
View File
@@ -0,0 +1,120 @@
use super::*;
use craft_tiff::{Entry, IfdBuilder, ImageData, TiffWriter};
fn ifd(tags: Vec<(u16, Value)>) -> Ifd {
let mut entries: Vec<_> = tags.into_iter().map(|(tag, value)| Entry { tag, value, offset: 0 }).collect();
entries.sort_by_key(|entry| entry.tag);
Ifd { entries, ..Default::default() }
}
#[test]
fn optional_crop_adjustments_are_explicit_and_valid_clipping_is_preserved() {
let active = Rect::new(3, 2, 11, 8);
let whole = Rect::new(0, 0, 11, 8);
let clipped = ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![10, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]);
assert_eq!(default_crop(&clipped, active, 16, 12), (Rect::new(10, 0, 1, 4), true));
assert_eq!(default_crop(&ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![1, 1]))]), active, 16, 12), (Rect::new(1, 1, 10, 7), true));
for fields in [
vec![(t::DEFAULT_CROP_SIZE, Value::Long(vec![4]))],
vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![0, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![0, 4]))],
] {
assert_eq!(default_crop(&ifd(fields), active, 16, 12), (whole, true));
}
assert_eq!(default_crop(&Ifd::default(), active, 16, 12), (whole, false));
assert_eq!(default_crop(&ifd(vec![(t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]), active, 16, 12), (Rect::new(0, 0, 5, 4), false));
assert_eq!(
default_crop(&ifd(vec![(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![2, 1])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]), active, 16, 12),
(Rect::new(2, 1, 5, 4), false)
);
}
#[test]
fn hostile_crop_coordinates_fall_back_explicitly_before_saturating_casts() {
for bad in [1e300, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
for tag in [t::DEFAULT_CROP_ORIGIN, t::DEFAULT_CROP_SIZE] {
let raw = ifd(vec![(tag, Value::Double(vec![bad, 1.]))]);
assert_eq!(default_crop(&raw, Rect::new(0, 0, 16, 12), 16, 12), (Rect::new(0, 0, 16, 12), true));
}
}
}
fn linear(extra: &[(u16, Value)]) -> Vec<u8> {
let mut raw = IfdBuilder::new();
for (tag, value) in [
(t::IMAGE_WIDTH, Value::Long(vec![8])),
(t::IMAGE_LENGTH, Value::Long(vec![6])),
(t::BITS_PER_SAMPLE, Value::Short(vec![16])),
(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])),
(t::PHOTOMETRIC, Value::Short(vec![photometric::LINEAR_RAW])),
(t::COMPRESSION, Value::Short(vec![1])),
] {
raw.set(tag, value);
}
for (tag, value) in extra {
raw.set(*tag, value.clone());
}
raw.set_image(ImageData::Strips { rows_per_strip: 6, strips: vec![(0u16..48).flat_map(u16::to_le_bytes).collect()] });
TiffWriter::new(ByteOrder::Little, false).write(&[raw]).unwrap()
}
#[test]
fn dng_and_generic_cfa_tiff_report_actual_level_sources_instead_of_unknown() {
for format in [RawFormat::Dng, RawFormat::CfaTiff] {
let missing = decode_as(&linear(&[]), Mode::Full, format, &Limits::default()).unwrap();
assert_eq!(missing.level_sources, LevelSources { black: LevelSource::Fallback, white: LevelSource::Fallback });
let declared = decode_as(
&linear(&[(t::BLACK_LEVEL, Value::Short(vec![17])), (t::WHITE_LEVEL, Value::Short(vec![1000]))]),
Mode::Full,
format,
&Limits::default(),
)
.unwrap();
assert_eq!(declared.level_sources, LevelSources { black: LevelSource::Metadata, white: LevelSource::Metadata });
assert_eq!(declared.black.values, vec![17.]);
assert_eq!(declared.white, vec![1000.]);
let invalid = decode_as(
&linear(&[(t::BLACK_LEVEL, Value::Double(vec![f64::NAN])), (t::WHITE_LEVEL, Value::Double(vec![f64::INFINITY]))]),
Mode::Full,
format,
&Limits::default(),
)
.unwrap();
assert_eq!(invalid.level_sources, LevelSources { black: LevelSource::Fallback, white: LevelSource::Fallback });
}
let table = decode_as(&linear(&[(t::LINEARIZATION_TABLE, Value::Short(vec![0, 1000]))]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap();
assert_eq!(table.white, vec![1000.]);
assert_eq!(table.level_sources.white, LevelSource::Metadata);
}
#[test]
fn decoded_notices_record_actual_crop_gain_table_and_cfa_decisions() {
let plain = decode_as(&linear(&[]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap();
assert_eq!(plain.notices, RawNotices::default());
let adjusted = decode_as(
&linear(&[(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![7, 0])), (t::DEFAULT_CROP_SIZE, Value::Long(vec![5, 4]))]),
Mode::Full,
RawFormat::Dng,
&Limits::default(),
)
.unwrap();
assert_eq!(adjusted.crop, Rect::new(7, 0, 1, 4));
assert!(adjusted.notices.default_crop_adjusted);
let invalid_gain =
decode_as(&linear(&[(t::PROFILE_GAIN_TABLE_MAP, Value::Undefined(vec![0; 16]))]), Mode::Full, RawFormat::Dng, &Limits::default()).unwrap();
assert!(invalid_gain.notices.gain_table_omitted);
assert!(invalid_gain.color.profile.gain_table_map.is_none());
let assumed =
decode_as(&linear(&[(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA]))]), Mode::Full, RawFormat::CfaTiff, &Limits::default()).unwrap();
assert!(assumed.notices.assumed_cfa);
assert_eq!(assumed.cfa.as_ref().unwrap().pattern, vec![0, 1, 1, 2]);
let tagged = decode_as(
&linear(&[(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])), (t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2]))]),
Mode::Full,
RawFormat::Dng,
&Limits::default(),
)
.unwrap();
assert!(!tagged.notices.assumed_cfa);
let limits = Limits { max_samples: 47, ..Default::default() };
assert!(matches!(decode_as(&linear(&[]), Mode::Full, RawFormat::Dng, &limits), Err(RawError::Limit(_))));
}
+64
View File
@@ -0,0 +1,64 @@
use super::*;
use craft_tiff::{IfdBuilder, ImageData, TiffWriter};
#[allow(clippy::unwrap_used)] // Test fixture construction must fail the test if its independent writer fails.
fn plain() -> Vec<u8> {
let mut raw = IfdBuilder::new();
for (tag, value) in [
(t::IMAGE_WIDTH, Value::Long(vec![2])),
(t::IMAGE_LENGTH, Value::Long(vec![2])),
(t::BITS_PER_SAMPLE, Value::Short(vec![16])),
(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])),
(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])),
(t::COMPRESSION, Value::Short(vec![1])),
(t::AS_SHOT_NEUTRAL, Value::Rational(vec![(1, 2), (1, 1), (2, 3)])),
] {
raw.set(tag, value);
}
raw.set_image(ImageData::Strips { rows_per_strip: 2, strips: vec![[17u16, 31, 53, 79].into_iter().flat_map(u16::to_le_bytes).collect()] });
TiffWriter::new(ByteOrder::Little, false).write(&[raw]).unwrap()
}
#[allow(clippy::unwrap_used)] // Only known tags in this module's synthetic TIFF are inspected.
fn entry(bytes: &[u8], tag: u16) -> usize {
let at = u32::from_le_bytes(bytes[4..8].try_into().unwrap()) as usize;
let count = u16::from_le_bytes(bytes[at..at + 2].try_into().unwrap()) as usize;
(0..count).map(|i| at + 2 + i * 12).find(|&i| u16::from_le_bytes(bytes[i..i + 2].try_into().unwrap()) == tag).unwrap()
}
#[test]
fn plain_cfa_probe_keeps_malformed_geometry_and_tables_in_the_supported_decoder() {
let valid = plain();
assert_eq!(crate::probe(&valid), Some(RawFormat::CfaTiff));
let decoded = crate::decode(&valid).unwrap();
assert_eq!(decoded.data, RawData::U16(vec![17, 31, 53, 79]));
for tag in [t::IMAGE_WIDTH, t::IMAGE_LENGTH] {
let mut damaged = valid.clone();
let at = entry(&damaged, tag);
damaged[at + 8..at + 12].copy_from_slice(&0u32.to_le_bytes());
assert_eq!(crate::probe(&damaged), Some(RawFormat::CfaTiff));
assert!(matches!(crate::probe_info(&damaged), Err(RawError::Tiff(_))), "zero geometry remains malformed before pixel decode");
assert!(matches!(crate::decode(&damaged), Err(RawError::Tiff(_))));
}
let mut missing = valid.clone();
let at = entry(&missing, t::STRIP_OFFSETS);
missing[at..at + 2].copy_from_slice(&0xf123u16.to_le_bytes());
assert_eq!(crate::probe(&missing), Some(RawFormat::CfaTiff));
assert!(matches!(crate::probe_info(&missing), Err(RawError::Tiff(_))));
assert!(matches!(crate::decode(&missing), Err(RawError::Tiff(_))));
}
#[test]
fn plain_cfa_probe_does_not_claim_unknown_tiff_codings_or_missing_color_contracts() {
for tag in [t::AS_SHOT_NEUTRAL, t::COMPRESSION] {
let mut unknown = plain();
let at = entry(&unknown, tag);
if tag == t::COMPRESSION {
unknown[at + 8..at + 10].copy_from_slice(&65000u16.to_le_bytes());
} else {
unknown[at..at + 2].copy_from_slice(&0xf123u16.to_le_bytes());
}
assert_eq!(crate::probe(&unknown), Some(RawFormat::OtherTiff));
assert!(matches!(crate::decode(&unknown), Err(RawError::Unsupported(_))));
}
}
+1
View File
@@ -462,6 +462,7 @@ mod tests {
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices::default(),
};
let opts = |tile: u32| DngWriteOptions { compression: DngCompression::Deflate { tile, half: false }, ..DngWriteOptions::default() };
let hostile = write_dng(&raw, &opts(u32::MAX)).expect("the clamped tile writes");
+1
View File
@@ -128,6 +128,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices::default(),
pixel_aspect: 1.0,
};
img.validate_for(mode)?;
+1
View File
@@ -121,6 +121,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices::default(),
};
img.validate_for(mode)?;
Ok(img)
+35
View File
@@ -997,6 +997,20 @@ pub struct ColorData {
pub kelvin_presets: Option<kelvin::KelvinPresets>,
}
/// Bounded diagnostics for decisions actually made by a raw decoder. An absent optional tag
/// does not itself set a notice; hosts can translate these flags into their own warning text
/// without inferring a fallback from missing metadata or retaining source-controlled strings.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct RawNotices {
/// Explicit default-crop metadata needed correction or a fallback to the active area.
pub default_crop_adjusted: bool,
/// A source profile gain table could not be used and was omitted from the decoded image.
pub gain_table_omitted: bool,
/// The CFA layout was selected by a decoder fallback instead of usable source metadata.
pub assumed_cfa: bool,
}
/// A decoded raw image.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct RawImage {
@@ -1041,6 +1055,9 @@ pub struct RawImage {
/// measured levels are placeholders.
#[serde(default)]
pub level_sources: LevelSources,
/// Fixed-size decoder decisions, separate from ordinary missing optional metadata.
#[serde(default)]
pub notices: RawNotices,
}
impl RawImage {
@@ -1173,3 +1190,21 @@ mod checked_arithmetic_tests {
assert_eq!(Orientation::from_exif(9), Orientation::Normal);
}
}
#[cfg(test)]
mod raw_notices_tests {
use super::*;
#[test]
fn missing_notices_default_to_no_claims_and_explicit_flags_are_independent() {
type Error = serde::de::value::Error;
let empty = serde::de::value::MapDeserializer::<_, Error>::new(std::iter::empty::<(&str, bool)>());
assert_eq!(RawNotices::deserialize(empty).unwrap(), RawNotices::default());
let one = serde::de::value::MapDeserializer::<_, Error>::new([("assumed_cfa", true)].into_iter());
let notices = RawNotices::deserialize(one).unwrap();
let copied = notices;
assert_eq!(notices, copied);
assert_eq!(notices, RawNotices { assumed_cfa: true, default_crop_adjusted: false, gain_table_omitted: false });
assert!(!std::mem::needs_drop::<RawNotices>(), "decoder notices own no allocated strings");
}
}
+1
View File
@@ -128,6 +128,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: crate::LevelSource::Fallback, white: white_source },
notices: crate::RawNotices::default(),
pixel_aspect: 1.0,
};
img.validate_for(mode)?;
+1
View File
@@ -170,6 +170,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: crate::LevelSource::Fallback, white: white_source },
notices: crate::RawNotices::default(),
pixel_aspect: 1.0,
};
img.validate_for(mode)?;
+144 -23
View File
@@ -13,7 +13,7 @@
//! refused.
use crate::nefc;
use crate::shared::white_from_data;
use crate::shared::white_from_data_sourced;
use crate::tiffraw::{Packing, check_image, read_image};
use crate::{BlackLevel, Camera, Cfa, ColorData, Limits, Mode, OpcodeLists, Orientation, RawData, RawError, RawFormat, RawImage, Rect, Result};
use craft_tiff::image::ImageInfo;
@@ -266,35 +266,44 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// D7500 and Z 50 store 600 / 1008 / 400 / 400 / 1008 while their darkest samples are 150 / ~252 / 99 / 99 / ~250
// (our measurements on CC0 raw.pixls.us samples; the same bodies' 14-bit files match the tag as is).
let black_scale = if bits == 12 { 0.25 } else { 1.0 };
let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() {
Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => BlackLevel {
repeat_rows: 2,
repeat_cols: 2,
values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(),
..Default::default()
let (black, black_source) = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() {
Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => (
BlackLevel {
repeat_rows: 2,
repeat_cols: 2,
values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(),
..Default::default()
},
crate::LevelSource::Metadata,
),
_ => match nrw_black(mn.as_ref(), bits) {
Some(value) => (BlackLevel::uniform(value), crate::LevelSource::Metadata),
None => {
let (value, source) = be16_black(&info, bytes.len() as u64, mn.as_ref(), samples);
(BlackLevel::uniform(value), source)
}
},
_ => BlackLevel::uniform(nrw_black(mn.as_ref(), bits).unwrap_or_else(|| be16_black(&info, bytes.len() as u64, mn.as_ref(), samples))),
};
let wb = mn
.as_ref()
.and_then(|m| m.ifd.f64s(WB_RB_LEVELS))
.filter(|v| v.len() >= 2 && v[0] > 0.1 && v[1] > 0.1 && v[0] < 10.0 && v[1] < 10.0)
.map(|v| [v[0] as f32, 1.0, v[1] as f32]);
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
let (cfa, assumed_cfa) = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => {
Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }
(Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }, false)
}
_ => Cfa::bayer_static("RGGB"),
_ => (Cfa::bayer_static("RGGB"), true),
};
let model = ifd0.string(t::MODEL).unwrap_or_default();
let body = measured_body(&model, w, h, bits);
let active_w = trailing_masked_columns(samples, w, bits);
// the white level is measured from the whole image (or known for a measured body); the probe (no samples in its
// result) takes the nominal one
let white = match (body.and_then(|b| b.white), mode) {
(Some(white), _) => white,
(None, Mode::Full) => white_from_data(samples, bits),
(None, Mode::Header) => ((1u32 << bits.clamp(1, 16)) - 1) as f32,
let (white, white_source) = match (body.and_then(|b| b.white), mode) {
(Some(white), _) => (white, crate::LevelSource::Fallback),
(None, Mode::Full) => white_from_data_sourced(samples, bits),
(None, Mode::Header) => (((1u32 << bits.clamp(1, 16)) - 1) as f32, crate::LevelSource::Fallback),
};
let crop = match body {
Some(&MeasuredBody { crop: (x, y, cw, ch), .. }) if x + cw <= active_w && y + ch <= h => Rect::new(x, y, cw, ch),
@@ -329,7 +338,8 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -378,8 +388,8 @@ fn uncompressed(
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum PackedLayout {
/// LSB-first bit stream of `bits` per sample (12: 3 bytes per 2 samples, 14: 7 bytes per 4), rows byte aligned.
/// 12-bit rows are `3w/2` bytes (older bodies) or padded to 16 bytes (Z 6, D6); 14-bit rows are `7w/4` bytes
/// padded to 16 bytes.
/// Rows use `ceil(bits * width / 8)` bytes, optionally padded to 16 bytes. Without an explicit packed maker
/// mode, recognition requires complete sample groups and 14-bit rows padded to 16 bytes.
Lsb { stride: usize },
/// D100: 16-byte chunks of fifteen data bytes holding ten MSB-first 12-bit samples, plus one zero pad byte. The
/// rows carry more samples than the IFD width (3040 against 3034); the extra columns are not image.
@@ -399,7 +409,11 @@ fn packed_layout(w: usize, h: usize, bits: u32, strip: u64, stated: bool) -> Opt
}
let stride = strip / h;
let tight = w.checked_mul(bits as usize)?.div_ceil(8);
let tight = if bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4) { tight } else { return None };
// Unmarked data must still match a measured complete-group layout: do not guess a packing for compressed
// data. An explicit packed maker mode permits a final partial group, byte-aligned independently on each row.
if !stated && !(bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4)) {
return None;
}
let padded = tight.checked_next_multiple_of(16)?;
let lsb_ok = match bits {
12 => stride == tight || stride == padded,
@@ -436,14 +450,14 @@ fn be16_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option<PackedLayout
/// 223), and the other 12-bit Coolpix NRWs state 200 in their NRW block; so 200, never above the darkest sample.
const BE16_BLACK: f32 = 200.0;
fn be16_black(info: &ImageInfo, file_len: u64, mn: Option<&makernote::MakerNote>, samples: &[u16]) -> f32 {
fn be16_black(info: &ImageInfo, file_len: u64, mn: Option<&makernote::MakerNote>, samples: &[u16]) -> (f32, crate::LevelSource) {
let has_table = mn.is_some_and(|m| m.ifd.bytes(LINEARIZATION_TABLE).is_some());
let single = matches!(info.chunks(file_len).as_slice(), [c] if be16_layout(info.width as usize, info.height as usize, info.bits() as u32, c.len).is_some());
if info.compression != t::compression::NIKON || has_table || info.samples_per_pixel != 1 || !single {
return 0.0;
return (0.0, crate::LevelSource::Fallback);
}
let darkest = samples.iter().copied().min().unwrap_or(0);
BE16_BLACK.min(f32::from(darkest))
(BE16_BLACK.min(f32::from(darkest)), crate::LevelSource::Inferred)
}
/// Unpack a [`PackedLayout`] strip into `w * h` samples. Short input leaves the missing samples zero.
@@ -524,7 +538,7 @@ fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>,
}
/// A "compressed" strip of packed samples: the strip size must match a measured layout exactly. `stated` is set
/// when the maker note names the packed mode, which also allows 14-bit rows without padding.
/// when the maker note names the packed mode, which also allows partial groups and 14-bit rows without padding.
fn packed(bytes: &[u8], info: &ImageInfo, rows: usize, stated: bool, limits: &Limits) -> Result<RawData> {
let (w, h) = (info.width as usize, info.height as usize);
let bits = info.bits() as u32;
@@ -663,6 +677,7 @@ fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::ima
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: Default::default(),
};
img.validate_for(mode)?;
Ok(img)
@@ -725,6 +740,21 @@ mod tests {
TiffWriter::new(order, false).write(&[ifd0]).unwrap()
}
#[test]
fn level_provenance_distinguishes_missing_black_from_recorded_black() {
let strip: Vec<u8> = [1000u16; 16].into_iter().flat_map(u16::to_be_bytes).collect();
let fallback = nef(1, 12, vec![strip.clone()], 4, 4, 4);
let mut note = IfdBuilder::new();
note.set(BLACK_LEVEL, Value::Short(vec![400; 4]));
let recorded = nef_with_note(1, 12, vec![strip], 4, 4, 4, Some(note));
for mode in [Mode::Header, Mode::Full] {
let missing = decode(&fallback, mode, &Limits::default()).unwrap();
assert_eq!((missing.black.mean(), missing.level_sources.black), (0.0, crate::LevelSource::Fallback));
let stated = decode(&recorded, mode, &Limits::default()).unwrap();
assert_eq!((stated.black.mean(), stated.level_sources.black), (100.0, crate::LevelSource::Metadata));
}
}
/// A small uncompressed NEF with a real NikonNEFInfo TIFF block, without binary fixtures.
fn nikon_distortion_file(model: &str, mode: u8) -> Vec<u8> {
use crate::nef_lens::tests::{distortion_block, nef_info_blob, offset_block};
@@ -1527,4 +1557,95 @@ mod tests {
}
}
}
mod partial_row_tests {
use super::*;
// Independent wire oracle: place each sample bit directly, leaving nonzero tail bits and padding.
fn image(data: &[u16], w: usize, bits: u32, padded: bool) -> Vec<u8> {
let tight = (w * bits as usize).div_ceil(8);
let stride = if padded { tight.next_multiple_of(16) } else { tight };
let mut bytes = vec![0xA5; stride * (data.len() / w)];
for (i, &sample) in data.iter().enumerate() {
for bit in 0..bits as usize {
let at = (i % w) * bits as usize + bit;
let byte = &mut bytes[(i / w) * stride + at / 8];
*byte = (*byte & !(1 << (at % 8))) | (((sample >> bit) as u8 & 1) << (at % 8));
}
}
bytes
}
fn note(order: ByteOrder, modern: bool, mode: u16) -> IfdBuilder {
let mut n = IfdBuilder::new();
if modern {
let mut record = b"01000400\0\0".to_vec();
record.extend(if order == ByteOrder::Big { mode.to_be_bytes() } else { mode.to_le_bytes() });
n.set(NEF_COMPRESSION_RECORD, Value::Undefined(record));
} else {
n.set(NEF_COMPRESSION, Value::Short(vec![mode]));
}
n
}
#[test]
fn declared_packed_partial_rows_preserve_every_sample() {
let h = 5;
for (bits, mode) in [(12u32, 6), (14, 10)] {
for w in [1, 3, 5, 22, 23] {
let data = samples(w * h, bits);
for padded in [false, true] {
let wire = image(&data, w, bits, padded);
if w == 22 && bits == 14 {
assert_eq!(wire.len() / h, if padded { 48 } else { 39 });
}
for order in [ByteOrder::Big, ByteOrder::Little] {
for modern in [false, true] {
let file = nef_in(
order,
34713,
bits as u16,
vec![wire.clone()],
w as u32,
h as u32,
h as u32,
Some(note(order, modern, mode)),
);
let raw = crate::decode(&file).unwrap();
assert_eq!((raw.width, raw.height, raw.cpp), (w, h, 1));
assert!(!raw.notices.assumed_cfa, "valid CFA tag is retained");
let RawData::U16(got) = raw.data else { panic!("integer samples") };
assert_eq!(got, data, "{bits}-bit width {w}, padded {padded}, {order:?}, modern {modern}");
}
}
}
}
}
}
#[test]
fn partial_rows_keep_mode_depth_and_strip_guards() {
let (w, h) = (22, 5);
let data = samples(w * h, 14);
for padded in [false, true] {
let wire = image(&data, w, 14, padded);
let file = |bytes, mode: Option<u16>| {
nef_with_note(34713, 14, vec![bytes], w as u32, h as u32, h as u32, mode.map(|m| note(ByteOrder::Big, false, m)))
};
for mode in [None, Some(0), Some(6), Some(13), Some(14)] {
assert!(matches!(crate::decode(&file(wire.clone(), mode)), Err(RawError::Unsupported(_))), "mode {mode:?}");
}
let good = file(wire.clone(), Some(10));
assert!(crate::decode(&good).is_ok());
let short = truncated_tail_strip(good, wire.len() - 1);
assert!(matches!(crate::decode(&short), Err(RawError::Corrupt(_))));
for len in [wire.len() - 1, wire.len() + 1] {
assert!(matches!(crate::decode(&file(vec![0; len], Some(10))), Err(RawError::Unsupported(_))));
}
}
for (w, h, bits) in [(0, 5, 14), (22, 0, 14), (22, 5, 13), (usize::MAX, 5, 14)] {
assert!(packed_layout(w, h, bits, u64::MAX, true).is_none());
}
}
}
}
+65
View File
@@ -207,6 +207,19 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
let info = ifd0.image()?;
let (w, h) = (info.width as usize, info.height as usize);
let n = w.checked_mul(h).filter(|n| *n > 0 && *n <= limits.samples()).ok_or(RawError::Limit("image too large"))?;
// A partial TIFF directory is damage, not evidence of an unsupported compressed layout.
// Do not infer packing from an absent strip table or from clipped sensor bytes.
if info.compression == 1
&& (info.offsets.is_empty()
|| info.offsets.len() != info.byte_counts.len()
|| info
.offsets
.iter()
.zip(&info.byte_counts)
.any(|(&at, &len)| len == 0 || at.checked_add(len).is_none_or(|end| end > bytes.len() as u64)))
{
return Err(RawError::Corrupt("ORF strip table is missing or outside file".into()));
}
let chunks = info.chunks(bytes.len() as u64);
let total = chunks.iter().fold(0u64, |a, c| a.saturating_add(c.len));
let stated = cfa_from_exif(&tiff);
@@ -302,6 +315,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
}
_ => Rect::new(0, 0, w, h),
};
let assumed_cfa = stated.is_none();
let cfa = stated.unwrap_or_else(|| cfa_from_data(samples, w, active));
let (black, black_source) = match ip.as_ref().and_then(|i| i.f64s(BLACK)).as_deref() {
Some(v @ [_, _, _, _]) => {
@@ -348,6 +362,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
level_sources: crate::LevelSources { black: black_source, white: white_source },
};
img.validate_for(mode)?;
@@ -416,6 +431,34 @@ mod tests {
}
}
#[test]
fn truncated_uncompressed_directory_and_strip_are_corrupt_not_compressed() {
let mut bytes = orf_with(4, 4, 16, vec![0; 32], Some(&[2, 0, 2, 0, 0, 1, 1, 2]));
// TiffWriter puts image chunks before its directory/metadata, so EOF is not a
// required sensor byte. Relocate the declared strip to EOF without changing it.
let directory = u32::from_le_bytes(bytes[4..8].try_into().unwrap()) as usize;
let count = u16::from_le_bytes(bytes[directory..directory + 2].try_into().unwrap()) as usize;
let entry = (0..count)
.map(|i| directory + 2 + i * 12)
.find(|&at| u16::from_le_bytes(bytes[at..at + 2].try_into().unwrap()) == t::STRIP_OFFSETS)
.unwrap();
let old_offset = u32::from_le_bytes(bytes[entry + 8..entry + 12].try_into().unwrap()) as usize;
let pixels = bytes[old_offset..old_offset + 32].to_vec();
let offset = bytes.len();
bytes.extend(pixels);
bytes[entry + 8..entry + 12].copy_from_slice(&(offset as u32).to_le_bytes());
assert!(crate::decode(&bytes).is_ok());
for cut in 0..bytes.len() {
for result in [crate::decode(&bytes[..cut]).map(|_| ()), crate::probe_info(&bytes[..cut]).map(|_| ())] {
assert!(!matches!(result, Err(RawError::Unsupported(ref why)) if why.contains("compressed")), "cut {cut}: {result:?}");
}
}
let result = crate::decode(&bytes[..bytes.len() - 1]);
assert!(matches!(result, Err(RawError::Corrupt(_))), "required final strip byte missing: {result:?}");
let result = crate::probe_info(&bytes[..bytes.len() - 1]);
assert!(matches!(result, Err(RawError::Corrupt(_))), "required final strip byte missing during probe: {result:?}");
}
/// Strip byte counts that add up past `u64::MAX` (LONG8 `[u64::MAX, 1]`) are an error or a picture, not an overflow.
#[test]
fn strip_byte_counts_past_u64_max_do_not_overflow() {
@@ -627,6 +670,28 @@ mod tests {
}
}
#[test]
fn cfa_notice_distinguishes_missing_and_unusable_exif_from_stated_patterns() {
let (w, h) = (160usize, 144usize);
let strip = mosaic(w, h, |cx, cy| [600, 1000, 1000, 300].map(|value| value + shade(cx, cy)));
let expected: Vec<u16> = strip.as_chunks::<2>().0.iter().map(|word| u16::from_le_bytes(*word)).collect();
for (tag, assumed, pattern) in [
(None, true, "RGGB"),
(Some(&[2, 0, 2, 0, 0, 1][..]), true, "RGGB"),
(Some(&[2, 0, 2, 0, 0, 1, 1, 3][..]), true, "RGGB"),
(Some(BGGR), false, "BGGR"),
(Some(GRBG), false, "GRBG"),
] {
let bytes = orf_with(w as u32, h as u32, 16, strip.clone(), tag);
for mode in [Mode::Header, Mode::Full] {
let image = decode(&bytes, mode, &Limits::default()).unwrap();
assert_eq!(image.cfa, Some(Cfa::bayer_static(pattern)));
assert_eq!(image.data, RawData::U16(expected.clone()), "word16 samples remain unchanged");
assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() });
}
}
}
#[test]
fn packed_12_bit_probe_agrees_with_decode() {
let (w, h) = (16usize, 4usize);
+47 -1
View File
@@ -194,6 +194,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
Some(r) => r,
None => dark_trimmed(samples, w, hgt, black_point.as_deref().map(|v| v.iter().sum::<f64>() / 4.0), white),
};
let mut assumed_cfa = false;
let cfa = match (ifd0.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), ifd0.bytes(t::CFA_PATTERN_EP)) {
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => {
Cfa { width: 2, height: 2, pattern: p.to_vec(), origin_x: 0, origin_y: 0 }
@@ -202,7 +203,10 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// of the image area: the *ist D (origin 19, 11) and the 645D (80, 59) only fit unshifted. The 12-bit 10 MP
// bodies and everything from the K-3 on say RGGB; the K-7 / K-5 / 645D generation says BGGR. Files without the
// tag keep BGGR.
_ => cfa_from_exif(&tiff).unwrap_or_else(|| Cfa::bayer_static("BGGR")),
_ => cfa_from_exif(&tiff).unwrap_or_else(|| {
assumed_cfa = true;
Cfa::bayer_static("BGGR")
}),
};
let black = match black_point.as_deref() {
Some(v @ [_, _, _, _]) => {
@@ -240,6 +244,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -487,6 +492,47 @@ mod tests {
assert_eq!(img.cfa, Some(Cfa::bayer_static("BGGR")));
}
#[test]
fn fixed_cfa_fallback_is_noticed_but_either_usable_metadata_form_is_not_assumed() {
use craft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
let expected: Vec<u16> = (0..16).map(|i| 100 + i * 7).collect();
let valid_exif = &[2, 0, 2, 0, 1, 0, 2, 1][..];
let invalid_exif = &[2, 0, 2, 0, 1, 0][..];
for (primary, exif, assumed, pattern) in [
(None, None, true, "BGGR"),
(Some([0, 1, 1, 3]), None, true, "BGGR"),
(None, Some(invalid_exif), true, "BGGR"),
(Some([0, 1, 1, 3]), Some(invalid_exif), true, "BGGR"),
(Some([0, 1, 1, 3]), Some(valid_exif), false, "GRBG"),
(None, Some(valid_exif), false, "GRBG"),
(Some([0, 1, 1, 2]), None, false, "RGGB"),
(Some([0, 1, 1, 2]), Some(valid_exif), false, "RGGB"),
] {
let mut raw = IfdBuilder::new();
raw.set(t::IMAGE_WIDTH, Value::Long(vec![4]));
raw.set(t::IMAGE_LENGTH, Value::Long(vec![4]));
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16]));
raw.set(t::COMPRESSION, Value::Short(vec![1]));
raw.set(t::PHOTOMETRIC, Value::Short(vec![32803]));
raw.set(t::MAKE, Value::Ascii("PENTAX Corporation".into()));
if let Some(pattern) = primary {
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2]));
raw.set(t::CFA_PATTERN_EP, Value::Byte(pattern.to_vec()));
}
if let Some(pattern) = exif {
raw.set_child(t::EXIF_IFD, IfdBuilder::new().with(crate::EXIF_CFA_PATTERN, Value::Undefined(pattern.to_vec())));
}
raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![expected.iter().flat_map(|value| value.to_be_bytes()).collect()] });
let bytes = TiffWriter::new(ByteOrder::Big, false).write(&[raw]).unwrap();
for mode in [Mode::Header, Mode::Full] {
let image = decode(&bytes, mode, &Limits::default()).unwrap();
assert_eq!(image.cfa, Some(Cfa::bayer_static(pattern)));
assert_eq!(image.data, RawData::U16(expected.clone()));
assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() });
}
}
}
/// A `w x h` big-endian 14-bit PEF of plain 16-bit words without area tags, whose maker note has only a
/// BlackPoint of `black`, with `px(x, y)` as the samples.
fn pef_with_black_point(w: usize, h: usize, black: u16, px: impl Fn(usize, usize) -> u16) -> Vec<u8> {
+9 -3
View File
@@ -643,9 +643,15 @@ fn cr3_preview(bytes: &[u8]) -> Option<&[u8]> {
/// A CR3's full-size JPEG track, when it holds a DCT JPEG.
#[cfg(feature = "cr3")]
fn cr3_full_jpeg(bytes: &[u8]) -> Option<&[u8]> {
let full = craft_metadata::cr3::parse_cr3(bytes)
.and_then(|c| c.tracks.iter().find(|t| t.kind == craft_metadata::cr3::Cr3TrackKind::Jpeg).and_then(|t| t.data));
full.and_then(|(at, len)| bytes.get(at..at.checked_add(len)?)).filter(|j| is_dct_jpeg(j))
let cr3 = craft_metadata::cr3::parse_cr3(bytes)?;
// An unusable preview track must not hide a later usable one. Share the marker budget
// across candidates so overlapping hostile track ranges cannot cause quadratic walks.
let mut work = CR3_MAX_WALK;
cr3.tracks.iter().filter(|t| t.kind == craft_metadata::cr3::Cr3TrackKind::Jpeg).find_map(|t| {
let (at, len) = t.data?;
let jpeg = bytes.get(at..at.checked_add(len)?)?;
first_frame_within(jpeg, &mut work).filter(|(m, _)| matches!(m, 0xc0..=0xc2)).map(|_| jpeg)
})
}
/// Marker-walk budget of the CR3 fallback: candidates can overlap (each claiming the rest of the file) and
+94 -4
View File
@@ -582,7 +582,19 @@ fn cr3_prefers_the_full_size_jpeg_track() {
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
let trak = bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
);
file.extend(bx(b"moov", &trak));
file.resize(at as usize, 0);
file.extend_from_slice(&big);
@@ -592,6 +604,48 @@ fn cr3_prefers_the_full_size_jpeg_track() {
assert_eq!(embedded_preview(&file).unwrap(), small);
}
#[cfg(feature = "cr3")]
#[test]
fn cr3_skips_bad_preview_tracks_but_never_uses_sensor_tracks_as_jpeg() {
let bx = |kind: &[u8; 4], body: &[u8]| -> Vec<u8> { [&((body.len() + 8) as u32).to_be_bytes()[..], kind, body].concat() };
let full = |kind: &[u8; 4], body: &[u8]| bx(kind, &[&[0u8; 4][..], body].concat());
let track = |jpeg: bool, at: u64, len: u32| {
let mut entry = vec![0u8; 82];
if jpeg {
entry.extend(bx(b"JPEG", &[0; 4]));
}
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &entry)].concat());
let stsz = full(b"stsz", &[len.to_be_bytes(), 1u32.to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
)
};
let good = fake_jpeg(100);
for first in [track(true, u64::MAX, 32), track(true, 4096, 32)] {
let mut bytes = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
bytes.extend(bx(b"moov", &[first, track(true, 4128, good.len() as u32)].concat()));
bytes.resize(4128, 0);
bytes.extend(&good);
assert_eq!(embedded_preview(&bytes).as_deref(), Some(good.as_slice()));
}
let mut bytes = bx(b"ftyp", b"crx \0\0\0\x01crx isom");
bytes.extend(bx(b"moov", &track(false, 4096, good.len() as u32)));
bytes.resize(4096, 0);
bytes.extend(&good);
assert_eq!(embedded_preview(&bytes), None, "a sensor sample is not a rendered preview even if its bytes look like JPEG");
}
#[cfg(feature = "cr3")]
/// A CR3 whose `PRVW` box holds `prvw` and whose JPEG track holds `big`.
fn cr3_with(prvw_jpeg: &[u8], big: &[u8]) -> Vec<u8> {
@@ -608,7 +662,19 @@ fn cr3_with(prvw_jpeg: &[u8], big: &[u8]) -> Vec<u8> {
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
let trak = bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
);
file.extend(bx(b"moov", &trak));
file.resize(at as usize, 0);
file.extend_from_slice(big);
@@ -656,7 +722,19 @@ fn the_smallest_preview_covering_the_size_is_chosen() {
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
let trak = bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
);
file.extend(bx(b"moov", &trak));
file.resize(at as usize, 0);
file.extend_from_slice(&big);
@@ -769,7 +847,19 @@ fn a_cr3_with_two_prvw_boxes_offers_both() {
let stsd = full(b"stsd", &[&1u32.to_be_bytes()[..], &bx(b"CRAW", &craw)].concat());
let stsz = full(b"stsz", &[0u32.to_be_bytes(), 1u32.to_be_bytes(), (big.len() as u32).to_be_bytes()].concat());
let co64 = full(b"co64", &[&1u32.to_be_bytes()[..], &at.to_be_bytes()].concat());
let trak = bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &[stsd, stsz, co64].concat()))));
let trak = bx(
b"trak",
&bx(
b"mdia",
&bx(
b"minf",
&bx(
b"stbl",
&[stsd, full(b"stsc", &[1u32, 1, 1, 1].into_iter().flat_map(u32::to_be_bytes).collect::<Vec<_>>()), stsz, co64].concat(),
),
),
),
);
file.extend(bx(b"moov", &trak));
// (the track in the media data, as a camera writes it, after a `free` box up to it)
let pad = at as usize - 16 - file.len();
+242 -17
View File
@@ -18,7 +18,10 @@
//! - CFA: the X-Trans layout record (36 bytes, 0 = R, 1 = G, 2 = B) is stored in reverse order and, reversed,
//! is anchored at raw pixel (0, 0) — verified on X-Trans I and III samples by minimising the difference
//! between horizontally / vertically adjacent "green" samples over all 72 orientations/phases, and by colour
//! renders. Bayer bodies (X-A series) use RGGB at (0, 0), verified by colour renders.
//! renders. Bayer FujiLayout (`0x0130`) gives the reversed 2×2 pattern in its low two bits
//! (0 = R, 1/3 = G, 2 = B); absent layout records retain the assumed RGGB fallback.
//! - RawExposureBias (`0x9650`) is a signed big-endian 16-bit fraction; its negative is the
//! baseline exposure used to restore the camera's dynamic-range exposure reduction.
//! - Striped predictive compression is decoded by `rafc`; see `docs/raf-compression.md`.
use crate::shared::white_from_data_sourced;
@@ -87,7 +90,7 @@ fn pair(h: &Header, tag: u16) -> Option<(usize, usize)> {
pub(crate) fn raw_ifd(raw: &[u8]) -> Result<Ifd> {
let tiff = Tiff::parse(raw).map_err(|_| RawError::Unsupported("RAF without a raw IFD (older FinePix layout)".into()))?;
let off = tiff.ifds.first().and_then(|i| i.u64(0xf000)).ok_or_else(|| RawError::Unsupported("RAF without a raw IFD".into()))?;
let (ifd, _) = craft_tiff::parse_ifd_at(raw, off, ByteOrder::Little, 0, false, &Default::default())?;
let (ifd, _) = craft_tiff::parse_ifd_at(raw, off, tiff.order, 0, false, &Default::default())?;
Ok(ifd)
}
@@ -122,14 +125,42 @@ fn camera_name(b: &[u8]) -> String {
String::from_utf8_lossy(name.get(..end).unwrap_or_default()).trim().to_string()
}
/// The CFA: X-Trans from the (reversed) layout record, else RGGB.
fn cfa(h: &Header) -> Cfa {
match record(h, XTRANS_LAYOUT) {
Some(l) if l.len() == 36 && l.iter().all(|&c| c <= 2) => {
Cfa { width: 6, height: 6, pattern: l.iter().rev().copied().collect(), origin_x: 0, origin_y: 0 }
/// Layout records are reversed and anchored at sensor (0, 0). FujiLayout's low
/// two bits encode Bayer colour; both 1 and 3 denote green.
fn cfa(h: &Header) -> Result<Cfa> {
if let Some(l) = record(h, XTRANS_LAYOUT) {
if l.len() != 36 || l.iter().any(|&c| c > 2) {
return Err(RawError::Unsupported("Fujifilm RAF with an unknown X-Trans layout".into()));
}
_ => Cfa::bayer_static("RGGB"),
return Ok(Cfa { width: 6, height: 6, pattern: l.iter().rev().copied().collect(), origin_x: 0, origin_y: 0 });
}
let Some(l) = record(h, 0x0130) else { return Ok(Cfa::bayer_static("RGGB")) };
let cfa = Cfa { width: 2, height: 2, pattern: l.iter().rev().map(|&c| if c & 3 == 3 { 1 } else { c & 3 }).collect(), origin_x: 0, origin_y: 0 };
// Colour counts alone also accept adjacent green sites; the two greens
// must occupy opposite corners of the 2x2 Bayer repeat.
if l.len() != 4 || !matches!(cfa.pattern.as_slice(), [0, 1, 1, 2] | [2, 1, 1, 0] | [1, 0, 2, 1] | [1, 2, 0, 1]) {
return Err(RawError::Unsupported("Fujifilm RAF with an unknown Bayer layout".into()));
}
Ok(cfa)
}
/// Fuji RawExposureBias is a signed big-endian 16-bit fraction, undone like
/// DNG BaselineExposure. Invalid or implausible metadata does not alter exposure.
fn baseline_exposure(h: &Header) -> f64 {
record(h, 0x9650)
.and_then(|d| {
let num = i16::from_be_bytes(d.get(..2)?.try_into().ok()?);
let den = i16::from_be_bytes(d.get(2..4)?.try_into().ok()?);
(den != 0).then(|| -f64::from(num) / f64::from(den))
})
.filter(|v| v.is_finite() && v.abs() <= 8.0)
.unwrap_or(0.0)
}
/// The codec fixes the green-site geometry; exchanging the two non-green
/// planes only exchanges their colour labels, so RGGB and BGGR are both valid.
fn compressed_layout_supported(cfa: &Cfa) -> bool {
*cfa == Cfa::xtrans() || *cfa == Cfa::bayer_static("RGGB") || *cfa == Cfa::bayer_static("BGGR")
}
/// Unpack one row of `bits`-bit samples in Fujifilm's packing. `words32`: the raw IFD's `0xf006` flag is set, the
@@ -137,7 +168,7 @@ fn cfa(h: &Header) -> Cfa {
/// LSB-first bit stream, or (other depths) the same 32-bit words.
pub(crate) fn unpack_row(src: &[u8], bits: u32, words32: bool, out: &mut [u16]) {
match bits {
16 => out.iter_mut().zip(src.as_chunks::<2>().0).for_each(|(o, c)| *o = u16::from_le_bytes([c[0], c[1]])),
16 if !words32 => out.iter_mut().zip(src.as_chunks::<2>().0).for_each(|(o, c)| *o = u16::from_le_bytes([c[0], c[1]])),
12 if !words32 => unpack_lsb(src, 12, out),
_ => unpack_words32_msb(src, ByteOrder::Little, bits, out),
}
@@ -171,7 +202,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
let src = raw
.get(off..off.checked_add(len).ok_or_else(|| RawError::Corrupt("RAF strip size overflow".into()))?)
.ok_or_else(|| RawError::Corrupt("RAF strip outside file".into()))?;
let cfa = cfa(&h);
let cfa = cfa(&h)?;
let compression = be32(bytes, 108).unwrap_or(0);
let compressed = match compression {
0 => false,
@@ -180,8 +211,13 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
};
let mut data = Vec::new();
if compressed {
if cfa.width == 6 && cfa != Cfa::xtrans() {
return Err(RawError::Unsupported("Fujifilm compressed RAF with an unknown X-Trans layout".into()));
if !compressed_layout_supported(&cfa) {
return Err(RawError::Unsupported("Fujifilm compressed RAF with an unsupported CFA layout".into()));
}
if !src.starts_with(b"IS") {
// A recognized RAF compression mode can wrap an unknown packing. Do not
// feed it to the IS codec; damage inside an identified IS stream stays Corrupt.
return Err(RawError::Unsupported("Fujifilm compressed RAF without a supported IS signature".into()));
}
let flag = if compression == 2 { 1 } else { 0 };
if src.get(2) != Some(&flag) {
@@ -190,9 +226,18 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
data = crate::rafc::decode(src, w, hgt, bits, cfa.width == 6, mode, limits)?;
} else {
let words32 = ifd.u64(PACKED_WORDS).is_some_and(|v| v != 0);
// Samples may use 16-bit words (e.g. 14-bit X-T20); otherwise they are packed at `bits`.
let store = if len as u64 >= (n as u64) * 2 { 16 } else { bits };
let packed_row = w.checked_mul(store as usize).map(|n| n.div_ceil(8)).ok_or(RawError::Limit("RAF row too large"))?;
// An explicit packed-word flag wins over the size heuristic: padding in short
// packed rows can occupy as many bytes as unpacked 16-bit samples.
// Unflagged samples may use 16-bit words (e.g. 14-bit X-T20).
let store = if !words32 && len as u64 >= (n as u64) * 2 { 16 } else { bits };
let row_bits = w.checked_mul(store as usize).ok_or(RawError::Limit("RAF row too large"))?;
// Word-packed rows store the high bits in the last little-endian word's final bytes.
// Reading only ceil(bits / 8) bytes would lose those samples, not just padding.
let packed_row = if words32 || (store != 16 && store != 12) {
row_bits.div_ceil(32).checked_mul(4).ok_or(RawError::Limit("RAF row too large"))?
} else {
row_bits.div_ceil(8)
};
let stride = match get(0xf005)?.unwrap_or(0) {
0 => packed_row,
words => words.checked_mul(4).ok_or(RawError::Limit("RAF stride too large"))?.max(packed_row),
@@ -206,7 +251,11 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
data.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
// `required <= src.len()` proves these row offsets fit, including the final row.
let s = src.get(y * stride..y * stride + packed_row).unwrap_or(&[]);
unpack_row(s, store, words32, row);
if !words32 && store == 16 && raw.starts_with(b"MM") {
crate::unpack::read_u16s(s, ByteOrder::Big, row);
} else {
unpack_row(s, store, words32, row);
}
});
}
}
@@ -246,7 +295,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
active_area: active,
crop: Rect::new(0, 0, active.width, active.height),
orientation,
color: ColorData { kelvin_presets, ..Default::default() },
color: ColorData { kelvin_presets, baseline_exposure: baseline_exposure(&h), ..Default::default() },
wb_multipliers: wb,
linearized: false,
pixel_aspect: 1.0,
@@ -256,6 +305,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
.unwrap_or_else(|| Camera { model: Some(camera_name(bytes)).filter(|n| !n.is_empty()), ..Default::default() }),
cinema: Default::default(),
lens_metadata: Default::default(),
notices: crate::RawNotices { assumed_cfa: record(&h, XTRANS_LAYOUT).is_none() && record(&h, 0x0130).is_none(), ..Default::default() },
level_sources: crate::LevelSources { black: black_source, white: white_source },
};
img.validate_for(mode)?;
@@ -480,6 +530,181 @@ pub(super) mod tests {
assert!(matches!(crate::decode(&bytes), Err(RawError::Unsupported(_))));
}
#[test]
fn word_packed_rows_keep_the_final_partial_word_and_reject_truncation() {
for (w, bits, flag) in [(30u32, 14u32, 0u32), (22, 12, 1), (6, 12, 1), (5, 16, 1)] {
let h = 6u32;
let px: Vec<u16> = (0..w * h).map(|i| ((i * 977 + 101) % (1 << bits)) as u16).collect();
// Independently place each sample bit into whole little-endian 32-bit row words.
let strip: Vec<u8> = px
.chunks(w as usize)
.flat_map(|row| {
let mut words = vec![0u32; (row.len() * bits as usize).div_ceil(32)];
for (i, &sample) in row.iter().enumerate() {
for bit in 0..bits as usize {
let at = i * bits as usize + bit;
words[at / 32] |= u32::from((sample >> (bits as usize - 1 - bit)) & 1) << (31 - at % 32);
}
}
words.into_iter().flat_map(u32::to_le_bytes).collect::<Vec<_>>()
})
.collect();
let bytes = raf_flagged(w, h, bits, strip, None, &[], flag);
assert_eq!(crate::decode(&bytes).unwrap().data, RawData::U16(px));
assert!(crate::decode(&bytes[..bytes.len() - 1]).is_err());
}
}
#[test]
fn big_endian_words_and_bayer_layout_preserve_exact_sensor_values() {
use craft_tiff::{IfdBuilder, TiffWriter, Value};
let (w, h) = (6u32, 4u32);
let px: Vec<u16> = (0..w * h).map(|i| (i * 977 + 123) as u16).collect();
for order in [ByteOrder::Little, ByteOrder::Big] {
let build = |at| {
let mut sensor = IfdBuilder::new();
for (tag, value) in [(0xf001, w), (0xf002, h), (0xf003, 16), (0xf007, at), (0xf008, w * h * 2)] {
sensor.set(tag, Value::Long(vec![value]));
}
let mut root = IfdBuilder::new();
root.set_child(0xf000, sensor);
TiffWriter::new(order, false).write(&[root]).unwrap()
};
let mut raw = build(build(0).len() as u32);
raw.extend(px.iter().flat_map(|v| if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() }));
let mut bytes = raf(w, h, 16, vec![0; (w * h * 2) as usize], None, &[]);
let raw_at = be32(&bytes, 100).unwrap();
bytes.truncate(raw_at);
bytes[104..108].copy_from_slice(&(raw.len() as u32).to_be_bytes());
bytes.extend(raw);
// A four-site Bayer record is reversed in storage, just like the X-Trans record.
let dir_at = be32(&bytes, 92).unwrap();
let mut records = bytes[dir_at..raw_at].to_vec();
let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
records[..4].copy_from_slice(&n.to_be_bytes());
records.extend_from_slice(&[0x01, 0x30, 0, 4, 0x08, 0x09, 0x0b, 0x0a]);
bytes.splice(dir_at..raw_at, records.clone());
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
let image = crate::decode(&bytes).unwrap();
assert_eq!(image.data, RawData::U16(px.clone()));
assert_eq!(image.cfa.unwrap().name(), "BGGR");
}
}
#[test]
fn raw_exposure_bias_is_preserved_as_baseline_exposure_and_invalid_values_are_ignored() {
for (num, den, expected) in [(-72i16, 100i16, 0.72f64), (1, 0, 0.0), (32767, 1, 0.0)] {
let mut bytes = raf(6, 4, 16, pack(&[1000; 24], 16), None, &[]);
let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap());
let mut records = bytes[dir_at..raw_at].to_vec();
let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
records[..4].copy_from_slice(&n.to_be_bytes());
records.extend_from_slice(&[0x96, 0x50, 0, 4]);
records.extend(num.to_be_bytes());
records.extend(den.to_be_bytes());
bytes.splice(dir_at..raw_at, records.clone());
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
for mode in [Mode::Header, Mode::Full] {
assert!((decode(&bytes, mode, &Limits::default()).unwrap().color.baseline_exposure - expected).abs() < 1e-9);
}
}
}
#[test]
fn invalid_declared_cfa_layouts_are_not_silently_replaced() {
for (tag, data) in [(0x0130, vec![8, 8, 10, 10]), (0x0130, vec![1, 2, 1, 0]), (0x0131, vec![3; 36])] {
let header = Header { jpeg: None, records: vec![(tag, data.as_slice())], raw: None };
assert!(matches!(cfa(&header), Err(RawError::Unsupported(_))));
}
}
fn with_bayer_record(mut bytes: Vec<u8>, pattern: [u8; 4]) -> Vec<u8> {
let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap());
let mut records = bytes[dir_at..raw_at].to_vec();
let count = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
records[..4].copy_from_slice(&count.to_be_bytes());
records.extend_from_slice(&[0x01, 0x30, 0, 4]);
records.extend_from_slice(&pattern);
bytes.splice(dir_at..raw_at, records.clone());
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
bytes
}
#[test]
fn non_bayer_geometry_is_rejected_in_header_and_full_modes() {
// Reversal gives RGBG: correct colour counts, but both greens in one column.
let bytes = with_bayer_record(raf(6, 4, 16, pack(&[1000; 24], 16), None, &[]), [1, 2, 1, 0]);
for mode in [Mode::Header, Mode::Full] {
assert!(matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("Bayer layout")));
}
}
#[test]
fn all_four_bayer_geometries_preserve_the_stored_samples() {
let pixels: Vec<u16> = (0..24).map(|i| 123 + i * 977).collect();
for (stored, expected) in [([2, 1, 1, 0], "RGGB"), ([0, 1, 1, 2], "BGGR"), ([1, 2, 0, 1], "GRBG"), ([1, 0, 2, 1], "GBRG")] {
let bytes = with_bayer_record(raf(6, 4, 16, pack(&pixels, 16), None, &[]), stored);
for mode in [Mode::Header, Mode::Full] {
let image = decode(&bytes, mode, &Limits::default()).unwrap();
assert_eq!((image.width, image.height), (6, 4));
assert_eq!(image.cfa.as_ref().unwrap().name(), expected);
assert_eq!(image.data, RawData::U16(if mode == Mode::Full { pixels.clone() } else { Vec::new() }));
}
}
}
#[test]
fn compressed_bayer_requires_the_layout_assumed_by_its_predictor() {
let strip = b"IS\x01\x00\x0e\0\x12\x06\0\x03\x0c\x03\0\x02\0\x03".to_vec();
let mut bytes = raf(780, 18, 14, strip, None, &[]);
let (dir_at, raw_at) = (be32(&bytes, 92).unwrap(), be32(&bytes, 100).unwrap());
let mut records = bytes[dir_at..raw_at].to_vec();
let n = u32::from_be_bytes(records[..4].try_into().unwrap()) + 1;
records[..4].copy_from_slice(&n.to_be_bytes());
// Reversed GRBG: a valid Bayer pattern, but not the RGGB order of this codec.
records.extend_from_slice(&[0x01, 0x30, 0, 4, 0x0b, 0x0a, 0x08, 0x09]);
bytes.splice(dir_at..raw_at, records.clone());
bytes[96..100].copy_from_slice(&(records.len() as u32).to_be_bytes());
bytes[100..104].copy_from_slice(&((raw_at + 8) as u32).to_be_bytes());
for mode in [Mode::Header, Mode::Full] {
assert!(matches!(decode(&bytes, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("layout")));
}
}
#[test]
fn compressed_layout_accepts_both_red_blue_orders_without_accepting_wrong_green_phase() {
for name in ["RGGB", "BGGR"] {
assert!(compressed_layout_supported(&Cfa::bayer_static(name)), "{name}");
}
for name in ["GRBG", "GBRG"] {
assert!(!compressed_layout_supported(&Cfa::bayer_static(name)), "{name}");
}
let mut other = Cfa::xtrans();
assert!(compressed_layout_supported(&other));
other.pattern.swap(0, 2);
assert!(!compressed_layout_supported(&other));
}
#[test]
fn unknown_compressed_signature_is_unsupported_but_identified_stream_damage_is_corrupt() {
let header = b"IS\x01\x00\x0e\0\x12\x06\0\x03\x0c\x03\0\x02\0\x03".to_vec();
let known = raf(780, 18, 14, header, None, &[]);
let raw_at = be32(&known, 100).unwrap();
let strip_at = raw_at + raw_ifd(&known[raw_at..]).unwrap().u64(0xf007).unwrap() as usize;
let mut unknown = known.clone();
unknown[strip_at..strip_at + 2].copy_from_slice(b"XX");
for mode in [Mode::Header, Mode::Full] {
assert!(matches!(decode(&unknown, mode, &Limits::default()), Err(RawError::Unsupported(why)) if why.contains("signature")));
assert!(
matches!(decode(&known, mode, &Limits::default()), Err(RawError::Corrupt(_))),
"identified but truncated IS stream must remain corrupt"
);
}
}
/// Raw IFD flag `0xf006` selects the 32-bit-word packing at 12 bits; without it 12 bits are an LSB-first stream.
#[test]
fn flag_selects_word_packed_12_bit() {
+57 -5
View File
@@ -615,11 +615,18 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
}
_ => Rect::new(0, 0, active.width, active.height),
};
if !data.is_empty() {
// Packed formats 2/5/7 are lossless sample containers: zero is an ordinary
// code, not evidence of a mapped-out photosite. The documented zero repair
// applies to the predictive format 4 and older word containers only.
if !data.is_empty() && matches!(encoding, Encoding::Words | Encoding::Blocks { kind: Block::Format4, .. }) {
fill_defects(&mut data, w, active);
}
let cfa = cfa(ifd0.u16(CFA_PATTERN).unwrap_or(1));
let cfa_code = ifd0.u16(CFA_PATTERN);
// RGGB is still usable when metadata is absent or unknown, but callers
// must be able to distinguish that fallback from a recorded sensor phase.
let assumed_cfa = !matches!(cfa_code, Some(1..=4));
let cfa = cfa(cfa_code.unwrap_or(1));
let read = BLACK_RGB.map(|tag| ifd0.f64(tag).filter(|v| v.is_finite() && *v >= 0.0));
let tags = read.map(|v| v.unwrap_or(0.0) as f32);
let offset = match encoding {
@@ -671,6 +678,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices { assumed_cfa, ..Default::default() },
};
img.validate_for(mode)?;
Ok(img)
@@ -1069,18 +1077,62 @@ mod tests {
}
}
#[test]
fn cfa_fallback_provenance_preserves_pixels_in_header_and_full_modes() {
let pixels: Vec<u16> = (0..72).map(|i| 17 + i * 113).collect();
let payload = pack(&pixels, 14);
for (code, expected, assumed) in [
(None, "RGGB", true),
(Some(0), "RGGB", true),
(Some(5), "RGGB", true),
(Some(u16::MAX), "RGGB", true),
(Some(1), "RGGB", false),
(Some(2), "GRBG", false),
(Some(3), "GBRG", false),
(Some(4), "BGGR", false),
] {
let mut tags = vec![short(BITS, 14), short(RAW_FORMAT, 7)];
if let Some(code) = code {
tags.push(short(CFA_PATTERN, code));
}
let bytes = file(18, 4, &tags, &payload, &|off| vec![(RAW_OFFSET, Value::Long(vec![off]))]);
for mode in [Mode::Header, Mode::Full] {
let image = decode(&bytes, mode, &Limits::default()).unwrap();
assert_eq!((image.width, image.height), (18, 4));
assert_eq!(image.cfa.as_ref().unwrap().name(), expected, "code {code:?}");
assert_eq!(image.notices, crate::RawNotices { assumed_cfa: assumed, ..Default::default() }, "code {code:?}, {mode:?}");
assert_eq!(samples(&image), if mode == Mode::Full { pixels.as_slice() } else { &[] });
}
}
}
#[test]
fn lossless_packed_formats_preserve_zero_codes_inside_the_active_area() {
let (w, h) = (90usize, 12usize);
for (format, bits, rotated) in [(5u16, 12u32, true), (5, 14, true), (7, 14, false), (2, 12, false)] {
let mut px = vec![1000u16; w * h];
px[5 * w + 20] = 0;
let payload = pack(&px, bits);
let bytes = rw2(w as u16, h as u16, bits as u16, format, 100, &if rotated { rotate(payload) } else { payload });
assert_eq!(samples(&crate::decode(&bytes).unwrap()), px, "format {format}/{bits}");
}
}
#[test]
fn levels_crop_defects_and_cfa() {
let (w, h) = (90usize, 60usize);
let mut px: Vec<u16> = (0..w * h).map(|i| 300 + (i % 7) as u16).collect();
// a saturation plateau sets the white level; zeros inside the active area are defects
// A saturation plateau sets the white level. In the older word container,
// isolated zeros inside the active area are documented mapped-out defects.
px[10 * w..20 * w].fill(3900);
px[30 * w + 40] = 0;
px[1] = 0; // in the border: left alone
let mut tags = vec![short(CFA_PATTERN, 2), short(BITS, 12), short(RAW_FORMAT, 5)];
let mut tags = vec![short(CFA_PATTERN, 2), short(BITS, 12), short(COMPRESSION, 34828)];
tags.extend([0x002f, 0x0030, 0x0031, 0x0032].iter().zip([12u16, 10, 52, 80]).map(|(&tag, v)| short(tag, v)));
tags.extend(LINEARITY_LIMIT.map(|tag| short(tag, 2000))); // recorded limits are not used
let bytes = file(w as u16, h as u16, &tags, &rotate(pack(&px, 12)), &|off| vec![(RAW_OFFSET, Value::Long(vec![off]))]);
// Compression 34828 stores left-justified samples in little-endian u16 words.
let words: Vec<u8> = px.iter().flat_map(|&v| (v << 4).to_le_bytes()).collect();
let bytes = file(w as u16, h as u16, &tags, &words, &|off| vec![(t::STRIP_OFFSETS, Value::Long(vec![off]))]);
let r = crate::decode(&bytes).unwrap();
let d = samples(&r);
assert_eq!(d[30 * w + 40], ((px[30 * w + 38] as u32 + px[30 * w + 42] as u32 + px[28 * w + 40] as u32 + px[32 * w + 40] as u32) / 4) as u16);
+1
View File
@@ -192,6 +192,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: crate::LevelSources { black: black_source, white: white_source },
notices: crate::RawNotices::default(),
};
img.validate_for(mode)?;
Ok(img)
+1
View File
@@ -49,6 +49,7 @@ fn an_active_area_that_wraps_past_the_image_is_corrupt() {
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: crate::RawNotices::default(),
pixel_aspect: 1.0,
};
assert!(raw.validate().is_err(), "`x + width` wrapped: the area is outside the image");
+72 -13
View File
@@ -115,6 +115,7 @@ pub(crate) fn read_image_in(
crate::Mode::Full => read_image(data, info, order, packing, limits),
crate::Mode::Header => {
check_image(data, info, limits)?;
check_uncompressed_chunks(data, info, &info.chunks(data.len() as u64), packing)?;
// a lossless-JPEG layout the decoder rejects (subsampled components, e.g. Sony's
// lossless M/S sizes) must fail here too, as the full decode will; likewise a JPEG XL
// chunk whose headers don't match the IFD
@@ -151,6 +152,7 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin
let bits = info.bits() as u32;
let float = info.sample_format == 3;
let chunks = info.chunks(data.len() as u64);
check_uncompressed_chunks(data, info, &chunks, packing)?;
let planar = info.planar == 2 && cpp > 1;
let ccpp = if planar { 1 } else { cpp };
// The chunks are decoded and copied into the output in groups whose combined nominal size stays within
@@ -222,6 +224,26 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin
Ok(if float { RawData::F32(out_f32) } else { RawData::U16(out_u16) })
}
/// Framing precedes allocation even in header-only probes: missing source rows may not
/// be reported as a decodable image and later silently filled with zero samples.
fn check_uncompressed_chunks(data: &[u8], info: &ImageInfo, chunks: &[Chunk], packing: Packing) -> Result<()> {
if info.compression != comp::NONE {
return Ok(());
}
let cpp = if info.planar == 2 { 1 } else { info.samples_per_pixel as usize };
for chunk in chunks {
let src = chunk_bytes(data, chunk).ok_or_else(|| RawError::Corrupt("uncompressed chunk is outside the file".into()))?;
let (cw, _, vw, vh) = chunk_dims(info, chunk);
let row = packed_row_bytes(cw as u64 * cpp as u64, info.bits() as u32, packing, info.sample_format == 3);
let last = packed_row_bytes(vw as u64 * cpp as u64, info.bits() as u32, packing, info.sample_format == 3);
let need = row.saturating_mul(vh.saturating_sub(1) as u64).saturating_add(last);
if (src.len() as u64) < need {
return Err(RawError::Corrupt(format!("an uncompressed chunk has {} bytes, expected at least {need}", src.len())));
}
}
Ok(())
}
/// Bytes that `n` samples take in a row of an uncompressed or inflated chunk, laid out as
/// [`unpack_chunk`] reads them: whole bytes or 16-bit words, or 32-bit words for `Words32Msb`.
fn packed_row_bytes(n: u64, bits: u32, packing: Packing, float: bool) -> u64 {
@@ -285,10 +307,12 @@ fn decode_chunk(
}
match info.compression {
comp::NONE => {
// uncompressed: missing rows are filled with zeros, but a chunk holding a small fraction of what its
// samples need is no picture: refused before the nominal chunk is allocated (storytold/lightcraft#1512)
let need = (n as u64).saturating_mul(u64::from(bits)) / 8;
if (src.len() as u64).saturating_mul(64) < need {
// Require every visible sample. Nominal edge-tile padding may be absent, but
// missing image rows must not turn a truncated file into a zero-filled success.
let row_bytes = packed_row_bytes(cw as u64 * cpp as u64, bits, packing, float);
let last_row = packed_row_bytes(vw as u64 * cpp as u64, bits, packing, float);
let need = row_bytes.saturating_mul(vh.saturating_sub(1) as u64).saturating_add(last_row);
if (src.len() as u64) < need {
return Err(RawError::Corrupt(format!("an uncompressed chunk of {n} samples has only {} bytes", src.len())));
}
unpack_chunk(src, order, packing, cw, ch, cpp, bits, float, 1)
@@ -314,13 +338,7 @@ fn decode_chunk(
let mcu = |v: usize| v.div_ceil(32).saturating_mul(32);
let tile_budget =
mcu(cw).saturating_mul(mcu(ch)).saturating_mul(if cpp == 1 { 1 } else { 3 }).clamp(1, limits.max_lossy_tile_samples.max(1));
let (px, jw, jh, jc) = lossy_jpeg(src, cpp, tile_budget)?;
// An edge tile's JPEG may be smaller than the nominal tile, but it has to cover the part of the tile
// inside the image: one that doesn't would leave a black region in a picture reported as decoded
// (PhotoCraft's and FilmCraft's readers refuse a JPEG smaller than its tile; this keeps the edge case).
if jw < vw || jh < vh {
return Err(RawError::Corrupt(format!("a lossy JPEG of {jw} × {jh} does not cover the {vw} × {vh} of its tile inside the image")));
}
let (px, jw, jh, jc) = lossy_jpeg_with_tile(src, cpp, tile_budget, Some((cw, ch, vw, vh)))?;
// the buffer holds the rows the decoded JPEG covers (the caller zero-fills the tile's
// remainder and reads only rows it fills): a tile-many-larger decoded JPEG does not
// get the nominal tile's full buffer (storytold/lightcraft#1517)
@@ -408,9 +426,21 @@ const MAX_PADDING_FACTOR: usize = 32;
#[cfg(feature = "dng")]
const MAX_WRITER_TILE_SAMPLES: usize = 2048 * 2048;
#[cfg(feature = "dng")]
#[cfg(all(test, feature = "dng"))]
/// Decode one baseline JPEG chunk to 8-bit samples: (samples, width, height, channels).
pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<(Vec<u8>, usize, usize, usize)> {
lossy_jpeg_with_tile(src, cpp, max_samples, None)
}
/// Validate the actual frame before decode allocates its output. Edge tiles may store
/// a smaller frame covering all visible samples or bounded whole-MCU padding.
#[cfg(feature = "dng")]
fn lossy_jpeg_with_tile(
src: &[u8],
cpp: usize,
max_samples: usize,
tile: Option<(usize, usize, usize, usize)>,
) -> Result<(Vec<u8>, usize, usize, usize)> {
use zune_core::bytestream::ZCursor;
use zune_core::colorspace::ColorSpace;
use zune_core::options::DecoderOptions;
@@ -422,7 +452,27 @@ pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<(
d.decode_headers().map_err(|e| RawError::Corrupt(format!("lossy JPEG tile: {e}")))?;
let info = d.info().ok_or_else(|| RawError::Corrupt("lossy JPEG tile without a header".into()))?;
let (w, h) = (info.width as usize, info.height as usize);
let c = if cpp == 1 { 1 } else { 3 };
if !matches!(cpp, 1 | 3) {
return Err(RawError::Unsupported(format!("lossy JPEG with {cpp} samples per pixel")));
}
if usize::from(info.components) != cpp {
return Err(RawError::Corrupt(format!("lossy JPEG has {} components, expected {cpp}", info.components)));
}
if let Some((cw, ch, vw, vh)) = tile {
if w < vw || h < vh {
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} x {h} does not cover the {vw} x {vh} of its tile inside the image")));
}
// Writers can store a whole MCU around an edge tile. Keep the documented
// 32-pixel padding allowance without letting one tiny tile decode an arbitrary frame.
let mcu_extent = |value: usize| value.checked_add(31).map(|rounded| rounded / 32).and_then(|blocks| blocks.checked_mul(32));
let (Some(max_w), Some(max_h)) = (mcu_extent(cw), mcu_extent(ch)) else {
return Err(RawError::Corrupt("lossy JPEG tile dimensions overflow MCU padding".into()));
};
if w > max_w || h > max_h {
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} x {h} exceeds its {max_w} x {max_h} MCU-padded tile dimensions")));
}
}
let c = cpp;
let out = w as u64 * h as u64 * c as u64;
if w == 0 || h == 0 || out > max_samples as u64 {
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} × {h} × {c} would exceed the {max_samples} output samples allowed")));
@@ -519,6 +569,12 @@ fn unpack_chunk(
}
} else {
undo_diff_u16(row, cpp * factor);
// Horizontal differences wrap in the declared sample precision, not in
// the u16 container used by the unpacker (TIFF 6.0 Predictor 2).
if bits < 16 {
let mask = ((1u32 << bits) - 1) as u16;
row.iter_mut().for_each(|sample| *sample &= mask);
}
}
}
}
@@ -982,3 +1038,6 @@ mod budget_tests {
assert_eq!(decode_groups(&big.chunks(64), 1, 1 << 20), vec![0..1]);
}
}
#[cfg(all(test, feature = "dng"))]
mod compat_review_tests;
@@ -0,0 +1,115 @@
use super::*;
use craft_tiff::image::Layout;
fn image(width: u32, height: u32, bits: u16, cpp: u16, compression: u16, len: usize) -> ImageInfo {
ImageInfo {
width,
height,
bits_per_sample: vec![bits; cpp as usize],
samples_per_pixel: cpp,
compression,
photometric: 34892,
planar: 1,
predictor: 1,
sample_format: 1,
new_subfile_type: 0,
layout: Layout::Strips { rows_per_strip: height },
offsets: vec![0],
byte_counts: vec![len as u64],
}
}
#[test]
fn truncated_uncompressed_rows_are_rejected_in_full_and_header_modes() {
for (bits, packing, float, bytes) in [
(8, Packing::Msb, false, 8),
(12, Packing::Msb, false, 12),
(12, Packing::Word16, false, 16),
(16, Packing::Msb, false, 16),
(32, Packing::Msb, true, 32),
] {
let data = vec![0u8; bytes];
let mut info = image(4, 2, bits, 1, comp::NONE, bytes);
info.sample_format = if float { 3 } else { 1 };
assert!(read_image_in(crate::Mode::Full, &data, &info, ByteOrder::Little, packing, &Limits::default()).is_ok());
info.byte_counts[0] -= 1;
for mode in [crate::Mode::Full, crate::Mode::Header] {
assert!(
matches!(read_image_in(mode, &data[..bytes - 1], &info, ByteOrder::Little, packing, &Limits::default()), Err(RawError::Corrupt(message)) if message.contains("uncompressed chunk")),
"{bits}/{float}/{mode:?}"
);
}
}
}
#[test]
fn edge_padding_may_be_absent_but_visible_uncompressed_samples_must_all_exist() {
let data = vec![31u8; 16 * 7 + 8];
let mut info = image(8, 8, 8, 1, comp::NONE, data.len());
info.layout = Layout::Tiles { tile_width: 16, tile_height: 16 };
let RawData::U16(values) = read_image(&data, &info, ByteOrder::Little, Packing::Msb, &Limits::default()).unwrap() else { panic!("integer") };
assert_eq!(values, vec![31; 64]);
info.byte_counts[0] -= 1;
assert!(read_image(&data[..data.len() - 1], &info, ByteOrder::Little, Packing::Msb, &Limits::default()).is_err());
}
#[test]
fn packed_predictor_two_wraps_at_declared_twelve_bits() {
// Samples [250, 10, 4090, 20]; differences modulo4096 are 0FA,F10,FF0,01A.
let packed = [0x0f, 0xaf, 0x10, 0xff, 0x00, 0x1a];
let ChunkPx::U16(values) = unpack_chunk(&packed, ByteOrder::Little, Packing::Msb, 4, 1, 1, 12, false, 2).unwrap() else { panic!("integer") };
assert_eq!(values, vec![250, 10, 4090, 20]);
}
#[allow(clippy::unwrap_used)] // Test-only fixture encoder; failed construction must fail the test.
fn jpeg(cpp: usize, width: u16, height: u16) -> Vec<u8> {
let mut data = Vec::new();
let color = if cpp == 1 { jpeg_encoder::ColorType::Luma } else { jpeg_encoder::ColorType::Rgb };
jpeg_encoder::Encoder::new(&mut data, 100).encode(&vec![90; width as usize * height as usize * cpp], width, height, color).unwrap();
data
}
#[test]
fn baseline_jpeg_components_are_checked_before_forced_colorspace_conversion() {
for (actual, expected) in [(3, 1), (1, 3)] {
let data = jpeg(actual, 8, 8);
let error = lossy_jpeg(&data, expected, 1024).unwrap_err();
assert!(matches!(error, RawError::Corrupt(message) if message.contains(&format!("{actual} components"))));
}
assert!(matches!(lossy_jpeg(&jpeg(3, 8, 8), 4, 1024), Err(RawError::Unsupported(message)) if message.contains("4 samples")));
for cpp in [1, 3] {
assert_eq!(lossy_jpeg(&jpeg(cpp, 8, 8), cpp, 1024).unwrap().3, cpp);
}
}
#[test]
fn jpeg_tile_geometry_and_limits_are_checked_at_the_header_with_valid_edge_controls() {
let data = jpeg(3, 8, 8);
assert!(lossy_jpeg_with_tile(&data, 3, 1024, Some((16, 16, 8, 8))).is_ok());
assert!(
matches!(lossy_jpeg_with_tile(&data, 3, 1024, Some((16, 16, 9, 8))), Err(RawError::Corrupt(message)) if message.contains("does not cover"))
);
assert!(matches!(lossy_jpeg_with_tile(&data, 3, 32, Some((16, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("would exceed")));
// Mutate only SOF dimensions, retaining a valid header but no entropy for the claimed frame.
// The geometry error proves refusal precedes decode's short/corrupt entropy failure.
for (width, height) in [(33u16, 8u16), (8, 33), (256, 256)] {
let mut oversized = data.clone();
let sof = oversized.windows(2).position(|bytes| bytes == [0xff, 0xc0]).unwrap();
oversized[sof + 5..sof + 7].copy_from_slice(&height.to_be_bytes());
oversized[sof + 7..sof + 9].copy_from_slice(&width.to_be_bytes());
assert!(
matches!(lossy_jpeg_with_tile(&oversized, 3, 1 << 20, Some((16, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("exceeds its"))
);
}
// A genuine full MCU remains valid even when only a 4x4 part of the nominal16 tile is visible.
let padded = jpeg(3, 32, 32);
assert!(lossy_jpeg_with_tile(&padded, 3, 32 * 32 * 3, Some((16, 16, 4, 4))).is_ok());
assert!(
matches!(lossy_jpeg_with_tile(&data, 3, 1 << 20, Some((usize::MAX, 16, 8, 8))), Err(RawError::Corrupt(message)) if message.contains("overflow MCU padding"))
);
let mut info = image(8, 8, 8, 3, comp::LOSSY_JPEG, data.len());
info.layout = Layout::Tiles { tile_width: 16, tile_height: 16 };
let RawData::U16(values) = read_image(&data, &info, ByteOrder::Little, Packing::Msb, &Limits::default()).unwrap() else { panic!("integer") };
assert_eq!(values.len(), 8 * 8 * 3);
assert!(values.iter().all(|sample| (i32::from(*sample) - 90).abs() <= 3));
}
+17 -3
View File
@@ -64,6 +64,7 @@ fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: craft_raw::RawNotices::default(),
}
}
@@ -948,18 +949,31 @@ fn malformed_semantic_masks_are_skipped() {
let mut zero = mask_ifd(2, 2, "zero size", vec![0; 4]);
zero.set(t::IMAGE_WIDTH, Value::Long(vec![0]));
ifd0.add_sub_ifd(zero);
// A 4x4 8-bit mask needs 16 visible samples; three bytes must not be
// accepted as a synthetic mask with thirteen zero-filled samples.
let mut short = mask_ifd(4, 4, "short strip", vec![200; 3]);
short.set(t::MASK_SUB_AREA, Value::Long(vec![u32::MAX, u32::MAX, u32::MAX, u32::MAX]));
ifd0.add_sub_ifd(short);
let mut valid = mask_ifd(4, 2, "valid crop", vec![10, 20, 30, 40, 50, 60, 70, 255]);
valid.set(t::MASK_SUB_AREA, Value::Long(vec![1, 2, 8, 4]));
ifd0.add_sub_ifd(valid);
let mut outside = mask_ifd(2, 2, "sub area too small", vec![255; 4]);
outside.set(t::MASK_SUB_AREA, Value::Long(vec![3, 0, 2, 4]));
ifd0.add_sub_ifd(outside);
});
assert_eq!(decode(&bytes).unwrap().cpp, 3);
let raw = decode(&bytes).unwrap();
let baseline = decode(&profile_dng(ByteOrder::Little, |_| {})).unwrap();
assert_eq!((raw.width, raw.height, raw.cpp), (8, 6, 3));
assert_eq!(raw.data, baseline.data, "optional damaged masks must not change base image samples");
let masks = craft_raw::semantic_masks(&bytes);
let names: Vec<&str> = masks.iter().map(|m| m.name.as_str()).collect();
assert_eq!(names, ["short strip", "sub area too small"]);
assert!(masks.iter().all(|m| m.sub_area.is_none()), "sub areas the mask doesn't fit are ignored");
assert_eq!(names, ["valid crop", "sub area too small"], "the truncated mask is omitted, and later usable masks survive");
assert_eq!(masks[0].data, [10u16, 20, 30, 40, 50, 60, 70, 255].map(|sample| sample * 257));
assert_eq!(masks[0].sub_area, Some(craft_raw::semantic::MaskSubArea { top: 1, left: 2, full_width: 8, full_height: 4 }));
assert!((masks[0].value_at(2.5, 1.5) - 10. / 255.).abs() < 1e-4);
assert_eq!(masks[0].value_at(1.5, 1.5), 0.);
assert!(masks[1].sub_area.is_none(), "the complete mask survives while a sub area it doesn't fit is ignored");
assert_eq!(masks[1].data, vec![65535; 4]);
for m in &masks {
assert_eq!(m.data.len(), m.width * m.height);
}
+1
View File
@@ -28,6 +28,7 @@ fn sample(comp: DngCompression) -> Vec<u8> {
cinema: Default::default(),
lens_metadata: Default::default(),
level_sources: Default::default(),
notices: craft_raw::RawNotices::default(),
};
craft_raw::write_dng(&raw, &DngWriteOptions { compression: comp, ..Default::default() }).unwrap()
}
+31 -1
View File
@@ -302,6 +302,7 @@ struct Used {
raster_limit: AtomicUsize,
invalid_svg: AtomicUsize,
svg_limit: AtomicUsize,
svg_resource_limit: AtomicUsize,
}
/// What resolving one drawing's embedded images used or refused.
@@ -323,6 +324,10 @@ pub struct ImageUsage {
pub invalid_svg_images: usize,
/// Embedded SVG data refused by byte or nesting limits.
pub svg_limit_images: usize,
/// Subset of `svg_limit_images` refused by unpacked byte or embedded-document-chain limits.
/// XML/tree-depth omissions and invalid or corrupt data do not set this counter. Consumers
/// can reject resource overruns while still allowing an optional, over-deep child to be omitted.
pub svg_resource_limit_images: usize,
}
/// A data-URL resolver and the accounting for the one drawing it serves.
@@ -350,6 +355,7 @@ impl ImageResolver {
raster_limit: AtomicUsize::new(0),
invalid_svg: AtomicUsize::new(0),
svg_limit: AtomicUsize::new(0),
svg_resource_limit: AtomicUsize::new(0),
}),
}
}
@@ -376,6 +382,7 @@ impl ImageResolver {
raster_limit_images: self.used.raster_limit.load(Ordering::Relaxed),
invalid_svg_images: self.used.invalid_svg.load(Ordering::Relaxed),
svg_limit_images: self.used.svg_limit.load(Ordering::Relaxed),
svg_resource_limit_images: self.used.svg_resource_limit.load(Ordering::Relaxed),
}
}
}
@@ -438,6 +445,7 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc<Vec<u8>>, format: craft_
fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'_>) -> Option<usvg::ImageKind> {
if used.depth.load(Ordering::Relaxed) >= limits.embedded_depth {
used.svg_limit.fetch_add(1, Ordering::Relaxed);
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
let left = limits.embedded_bytes.saturating_sub(used.bytes.load(Ordering::Relaxed));
@@ -445,15 +453,37 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'
Ok(text) => text,
// What a refused document unpacked before it was refused was work too: it uses up what
// is left, so the next image can't repeat it.
Err(_) => {
Err(error) => {
used.bytes.fetch_add(left, Ordering::Relaxed);
used.svg_limit.fetch_add(1, Ordering::Relaxed);
if matches!(error, svgz::Error::TooLarge(_)) {
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
}
return None;
}
};
// The raw/inflated bytes can fit while UTF-16/Latin-1 decoding expands
// beyond the aggregate allowance. A refusal consumes the remaining work.
if text.len() > left {
used.bytes.fetch_add(left, Ordering::Relaxed);
used.svg_limit.fetch_add(1, Ordering::Relaxed);
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
used.bytes.fetch_add(text.len(), Ordering::Relaxed);
let xml = match checked(&text, limits) {
Ok(xml) => xml,
// Decoding UTF-16 or Latin-1 can exceed the text cap after the
// raw/inflated bytes fitted. This is a resource refusal, not invalid XML.
Err(Error::Text(svgz::Error::TooLarge(_))) => {
used.svg_limit.fetch_add(1, Ordering::Relaxed);
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
Err(Error::TooDeep(_)) => {
used.svg_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
Err(_) => {
used.invalid_svg.fetch_add(1, Ordering::Relaxed);
return None;
+189
View File
@@ -245,6 +245,195 @@ fn embedded_svgs_are_read_within_their_budgets() {
assert_eq!(found(&svg(r#"<image width="5" height="5" xlink:href="/etc/passwd.svg"/>"#), shallow()), [false; 0]);
}
mod resource_limit_usage {
use super::*;
fn data_image(bytes: &[u8]) -> String {
let encoded: String = bytes.iter().map(|byte| format!("%{byte:02X}")).collect();
format!(r#"<image width="5" height="5" xlink:href="data:image/svg+xml,{encoded}"/>"#)
}
fn packed(text: &str, gzip: bool) -> Vec<u8> {
if gzip { svgz::compress(text.as_bytes()) } else { text.as_bytes().to_vec() }
}
/// The outer document has its own generous text allowance, so tiny resolver limits
/// test the public embedded admission contract rather than rejecting the outer XML.
fn resolved(text: &str, limits: Limits, outer_depth: usize) -> (Vec<bool>, ImageUsage) {
let resolver = ImageResolver::new(limits);
resolver.set_document_depth(outer_depth);
let options = usvg::Options { image_href_resolver: resolver.resolver(), ..Default::default() };
let tree = parse(text.as_bytes(), &shallow(), &options).unwrap();
let mut found = Vec::new();
images(tree.root(), &mut found);
(found, resolver.usage())
}
fn expanding_encoded_children() -> Vec<(&'static str, Vec<u8>)> {
let latin =
format!("<?xml version='1.0' encoding='ISO-8859-1'?>{}", svg(&format!("<title>{}</title><rect width='1' height='1'/>", "é".repeat(100))));
let latin = latin.chars().map(|c| u8::try_from(u32::from(c)).unwrap()).collect();
let wide = svg(&format!("<title>{}</title><rect width='1' height='1'/>", "ࠀ".repeat(400)));
let mut little = vec![0xff, 0xfe];
little.extend(wide.encode_utf16().flat_map(u16::to_le_bytes));
let mut big = vec![0xfe, 0xff];
big.extend(wide.encode_utf16().flat_map(u16::to_be_bytes));
vec![("Latin-1", latin), ("UTF-16LE", little), ("UTF-16BE", big)]
}
#[test]
fn decoded_text_expansion_is_a_resource_refusal_for_plain_and_gzip_children() {
for (encoding, raw) in expanding_encoded_children() {
let decoded = svgz::decode(&raw).unwrap();
let cap = raw.len();
assert!(decoded.len() > cap, "{encoding} fixture must expand after inflation");
for gzip in [false, true] {
let data = if gzip { svgz::compress(&raw) } else { raw.clone() };
assert_eq!(svgz::inflate(&data, cap).unwrap().len(), cap, "raw bytes fit exactly");
assert_eq!(svgz::text(&data, cap).unwrap().len(), decoded.len(), "decoding itself succeeds");
let limits = Limits { bytes: cap, embedded_bytes: decoded.len(), ..shallow() };
assert!(matches!(checked(&decoded, &limits), Err(Error::Text(svgz::Error::TooLarge(n))) if n == cap));
let image = data_image(&data);
let (found, usage) = resolved(&svg(&image), limits, 1);
assert!(found.is_empty(), "{encoding}, gzip {gzip}");
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (1, 1, 0));
assert_eq!(usage.embedded_svg_bytes, decoded.len(), "spent decoded work remains charged");
let (found, usage) = resolved(&svg(&format!("{image}{image}")), limits, 1);
assert!(found.is_empty());
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (2, 2, 0));
assert_eq!(usage.embedded_svg_bytes, decoded.len(), "a refused sibling cannot repeat exhausted work");
}
}
}
#[test]
fn decoded_text_expansion_respects_aggregate_bytes_even_when_each_child_fits() {
for (encoding, raw) in expanding_encoded_children() {
let decoded = svgz::decode(&raw).unwrap();
let limits = Limits { bytes: decoded.len(), embedded_bytes: raw.len(), ..shallow() };
assert!(decoded.len() > limits.embedded_bytes);
checked(&decoded, &limits).unwrap(); // The independent per-image cap allows this complete XML.
for gzip in [false, true] {
let data = if gzip { svgz::compress(&raw) } else { raw.clone() };
assert_eq!(svgz::inflate(&data, limits.embedded_bytes).unwrap().len(), raw.len());
let image = data_image(&data);
let (found, usage) = resolved(&svg(&image), limits, 1);
assert!(found.is_empty(), "{encoding}, gzip {gzip}");
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (1, 1, 0));
assert_eq!(usage.embedded_svg_bytes, limits.embedded_bytes, "refusal consumes exactly the remaining allowance");
let (found, usage) = resolved(&svg(&format!("{image}{image}")), limits, 1);
assert!(found.is_empty());
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (2, 2, 0));
assert_eq!(usage.embedded_svg_bytes, limits.embedded_bytes, "a later child cannot repeat exhausted work");
let (found, usage) = resolved(&svg(&image), Limits { embedded_bytes: decoded.len(), ..limits }, 1);
assert_eq!(found, [true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 0));
assert_eq!(usage.embedded_svg_bytes, decoded.len());
}
}
}
#[test]
fn decoded_text_expansion_at_the_cap_is_valid_and_malformed_xml_stays_invalid() {
for (encoding, raw) in expanding_encoded_children() {
let decoded = svgz::decode(&raw).unwrap();
for gzip in [false, true] {
let data = if gzip { svgz::compress(&raw) } else { raw.clone() };
let limits = Limits { bytes: decoded.len(), embedded_bytes: decoded.len(), ..shallow() };
let (found, usage) = resolved(&svg(&data_image(&data)), limits, 1);
assert_eq!(found, [true], "{encoding}, gzip {gzip}");
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 0));
assert_eq!(usage.embedded_svg_bytes, decoded.len());
// Preserve the declared/BOM encoding when making the XML malformed;
// re-encoding Latin-1 text as UTF-8 would test a second expansion instead.
let closing_bytes = if encoding == "Latin-1" { 6 } else { 12 };
let malformed = &raw[..raw.len() - closing_bytes];
let data = if gzip { svgz::compress(malformed) } else { malformed.to_vec() };
let (found, usage) = resolved(&svg(&data_image(&data)), limits, 1);
assert!(found.is_empty());
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0, 1));
}
}
}
#[test]
fn embedded_chain_caps_are_explicit_resource_refusals_for_plain_and_gzip_children() {
for gzip in [false, true] {
let inner = svg("<rect width='1' height='1'/>");
let middle = svg(&data_image(&packed(&inner, gzip)));
let outer = svg(&data_image(&packed(&middle, gzip)));
let (found, usage) = resolved(&outer, Limits { embedded_depth: 1, ..shallow() }, 1);
assert_eq!(found, [true]);
assert_eq!(usage.embedded_svg_depth, 1);
assert_eq!(usage.svg_limit_images, 1, "strict consumers still see the existing limit flag");
assert_eq!(usage.svg_resource_limit_images, 1, "a chain cap must not become a successful partial bounded preview");
let (found, usage) = resolved(&outer, Limits { embedded_depth: 2, ..shallow() }, 1);
assert_eq!(found, [true, true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
}
}
#[test]
fn embedded_per_image_byte_caps_are_resource_refusals_for_plain_and_gzip_children() {
let inner = svg("<rect width='1' height='1'/>");
for gzip in [false, true] {
let outer = svg(&data_image(&packed(&inner, gzip)));
let (_, usage) = resolved(&outer, Limits { bytes: inner.len() - 1, embedded_bytes: inner.len() * 3, ..shallow() }, 1);
assert_eq!(usage.embedded_svg_count, 0);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (1, 1));
let (found, usage) = resolved(&outer, Limits { bytes: inner.len(), ..shallow() }, 1);
assert_eq!(found, [true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
}
}
#[test]
fn aggregate_embedded_bytes_are_resource_refusals_even_when_each_child_fits() {
let inner = svg("<rect width='1' height='1'/>");
for gzip in [false, true] {
let image = data_image(&packed(&inner, gzip));
let outer = svg(&format!("{image}{image}"));
let cap = inner.len() * 2 - 1;
let (found, usage) = resolved(&outer, Limits { embedded_bytes: cap, ..shallow() }, 1);
assert_eq!(found, [true]);
assert_eq!(usage.embedded_svg_bytes, cap, "a refused child still spends the remaining work allowance");
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (1, 1));
let (found, usage) = resolved(&outer, Limits { embedded_bytes: inner.len() * 2, ..shallow() }, 1);
assert_eq!(found, [true, true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
}
}
#[test]
fn combined_xml_depth_omissions_remain_distinct_from_resource_refusals() {
let inner = svg("<rect width='1' height='1'/>");
for gzip in [false, true] {
let image = data_image(&packed(&inner, gzip));
let outer = svg(&format!("{}{}{}", "<g>".repeat(4), image, "</g>".repeat(4)));
let (found, usage) = resolved(&outer, Limits { tree_depth: 6, ..shallow() }, 5);
assert!(found.is_empty());
assert_eq!(usage.svg_limit_images, 1, "strict consumers must continue to reject this omission");
assert_eq!(usage.svg_resource_limit_images, 0, "non-strict consumers may omit this child without disguising a byte/chain overrun");
let (found, usage) = resolved(&svg(&image), Limits { tree_depth: 6, ..shallow() }, 1);
assert_eq!(found, [true]);
assert_eq!((usage.svg_limit_images, usage.svg_resource_limit_images), (0, 0));
}
}
#[test]
fn invalid_and_corrupt_children_do_not_claim_a_resource_limit() {
let mut corrupt = svgz::compress(svg("<rect width='1' height='1'/>").as_bytes());
let crc = corrupt.len() - 8;
corrupt[crc] ^= 1;
for data in [b"not XML".to_vec(), corrupt] {
let (found, usage) = resolved(&svg(&data_image(&data)), shallow(), 1);
assert!(found.is_empty());
assert!(usage.invalid_svg_images > 0 || usage.svg_limit_images > 0, "strict consumers still observe unreadable content");
assert_eq!(usage.svg_resource_limit_images, 0);
}
}
}
mod draw_checks {
use super::*;