Merge remote-tracking branch 'origin/main' into w1692

This commit is contained in:
FredApps committed 2026-11-16 02:13:24 +01:00
commit eb5eff868a
67 files changed
+3538 -167

No files matched your search

+1 -1
View File
@@ -23,5 +23,5 @@ jobs:
# The codec and format code behind features that are off by default. Without these the
# coverage build never compiles it, and new code there reads as untested.
cargo-features: >-
craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write
craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write,craft-image-codecs/sgi
secrets: inherit
Generated
+13
View File
@@ -1868,6 +1868,15 @@ dependencies = [
"image",
]
[[package]]
name = "craft-gridcraft-format"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-gzip"
version = "0.1.0"
@@ -1903,6 +1912,10 @@ dependencies = [
"proptest",
]
[[package]]
name = "craft-http-url"
version = "0.1.0"
[[package]]
name = "craft-i18n"
version = "0.1.0"
+2
View File
@@ -11,6 +11,7 @@ rust-version = "1.95"
repository = "https://github.com/storytold/craft-libs"
[workspace.dependencies]
craft-http-url = { path = "crates/http-url" }
craft-usd = { path = "crates/usd" }
craft-smime = { path = "crates/smime" }
craft-ber = { path = "crates/ber" }
@@ -40,6 +41,7 @@ bytemuck = "1"
kurbo = "0.13"
thiserror = "2"
craft-gzip = { path = "crates/gzip" }
craft-gridcraft-format = { path = "crates/gridcraft-format" }
craft-aaf = { path = "crates/aaf" }
craft-atomic = { path = "crates/atomic" }
craft-captions = { path = "crates/captions" }
+3
View File
@@ -176,6 +176,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
every frame's position, delay, disposal and transparent colour, the loop count and the first
frame's colour table, without decompressing pixels. Tells an animation from a still and whether
a file can hold the picture it declares before a decoder runs.
- [`craft-gridcraft-format`](crates/gridcraft-format/README.md): GridCraft's native JSON workbook
(`.json`, `.scjson`): the extensions, the reader, the pretty writer, and newer entries kept
unread and written back into a list. The workbook model and what follows a read stay in GridCraft.
- [`craft-hdr`](crates/hdr/src/lib.rs): Radiance HDR (RGBE) picture reader, from FilmCraft: flat,
old and new run-length scan lines and all eight axis orders, to linear float RGB. Scan lines are
validated before the picture is allocated.
+147
View File
@@ -0,0 +1,147 @@
//! Compatibility WAVE and AIFF wrappers for embedded PCM essence in interchange files.
//! PCM inside the model is interleaved little-endian signed integers (8-bit unsigned for WAVE).
pub use crate::{pcm_aiff_header as aiff_header, pcm_wav_file as wav_file, pcm_wav_header as wav_header};
/// PCM, channels, sample rate and bits of a WAVE file (integer PCM only).
pub fn parse_wav(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
if b.len() < 12 || &b[..4] != b"RIFF" || &b[8..12] != b"WAVE" {
return None;
}
let mut at = 12;
let mut fmt = None;
while at + 8 <= b.len() {
let id = &b[at..at + 4];
let len = u32::from_le_bytes(b[at + 4..at + 8].try_into().ok()?) as usize;
let body = &b[at + 8..(at + 8).saturating_add(len).min(b.len())];
if id == b"fmt " && body.len() >= 16 {
let tag = u16::from_le_bytes([body[0], body[1]]);
let ch = u16::from_le_bytes([body[2], body[3]]);
let sr = u32::from_le_bytes(body[4..8].try_into().ok()?);
let bits = u16::from_le_bytes([body[14], body[15]]);
if tag != 1 && tag != 0xFFFE {
return None;
}
if tag == 0xFFFE {
// The first two bytes of the sub-format GUID carry the real format code.
let sub = body.get(24..26)?;
if u16::from_le_bytes([sub[0], sub[1]]) != 1 {
return None;
}
}
fmt = Some((ch, sr, bits));
} else if id == b"data" {
let (ch, sr, bits) = fmt?;
return Some((body.to_vec(), ch, sr, bits));
}
at = at.checked_add(8usize.checked_add(len)?.checked_add(len & 1)?)?;
}
None
}
/// A complete AIFF file (big-endian PCM) from little-endian PCM.
pub fn aiff_file(pcm: &[u8], channels: u16, sample_rate: u32, bits: u16) -> Vec<u8> {
let bps = bits.div_ceil(8) as usize;
let frames = pcm.len() / (bps * channels.max(1) as usize).max(1);
let mut v = aiff_header(frames as u32, channels, sample_rate, bits);
for s in pcm.chunks_exact(bps.max(1)) {
if bps == 1 {
v.push(s[0] ^ 0x80); // WAVE 8-bit is unsigned, AIFF signed
} else {
v.extend(s.iter().rev());
}
}
if (v.len() - 8) & 1 == 1 {
v.push(0);
}
v
}
/// PCM (converted to little-endian), channels, sample rate and bits of an AIFF / AIFF-C (`NONE`
/// / `twos` / `sowt`) file.
pub fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
if b.len() < 12 || &b[..4] != b"FORM" || !(&b[8..12] == b"AIFF" || &b[8..12] == b"AIFC") {
return None;
}
let mut at = 12;
let mut comm = None;
while at + 8 <= b.len() {
let id = &b[at..at + 4];
let len = u32::from_be_bytes(b[at + 4..at + 8].try_into().ok()?) as usize;
let body = &b[at + 8..(at + 8).saturating_add(len).min(b.len())];
if id == b"COMM" && body.len() >= 18 {
let ch = u16::from_be_bytes([body[0], body[1]]);
let bits = u16::from_be_bytes([body[6], body[7]]);
let sr = from_extended(body[8..18].try_into().ok()?).round() as u32;
let little = body.len() >= 22 && &body[18..22] == b"sowt";
comm = Some((ch, sr, bits, little));
} else if id == b"SSND" && body.len() >= 8 {
let (ch, sr, bits, little) = comm?;
let off = u32::from_be_bytes(body[..4].try_into().ok()?) as usize;
let data = body.get(8usize.checked_add(off)?..)?;
let bps = bits.div_ceil(8) as usize;
let mut pcm = Vec::with_capacity(data.len());
for s in data.chunks_exact(bps.max(1)) {
if bps == 1 {
pcm.push(s[0] ^ 0x80);
} else if little {
pcm.extend_from_slice(s);
} else {
pcm.extend(s.iter().rev());
}
}
return Some((pcm, ch, sr, bits));
}
at = at.checked_add(8usize.checked_add(len)?.checked_add(len & 1)?)?;
}
None
}
fn from_extended(b: [u8; 10]) -> f64 {
let e = (u16::from_be_bytes([b[0], b[1]]) & 0x7FFF) as i32;
let m = u64::from_be_bytes(b[2..10].try_into().unwrap_or([0; 8]));
if e == 0 && m == 0 {
return 0.0;
}
m as f64 / (1u64 << 63) as f64 * 2f64.powi(e - 16383)
}
#[cfg(test)]
mod tests {
/// A mutated EDL whose clip ran for days produced an OMF descriptor for more than 4 GiB of
/// PCM: the RIFF size field overflowed ("attempt to add with overflow").
#[test]
fn huge_wav_header_saturates() {
let h = super::wav_header(u32::MAX, 2, u32::MAX, 24);
assert_eq!(&h[4..8], &u32::MAX.to_le_bytes());
}
use super::*;
#[test]
fn accepts_truncated_essence_summaries() {
let mut wav = wav_file(&[1, 2, 3, 4], 1, 48_000, 16);
wav.truncate(46);
assert_eq!(parse_wav(&wav), Some((vec![1, 2], 1, 48_000, 16)));
let mut aiff = aiff_file(&[1, 2, 3, 4], 1, 48_000, 16);
aiff.truncate(56);
assert_eq!(parse_aiff(&aiff), Some((vec![1, 2], 1, 48_000, 16)));
}
#[test]
fn wav_and_aiff_round_trip() {
let pcm: Vec<u8> = (0..48u8).collect(); // 8 stereo 24-bit frames
let w = wav_file(&pcm, 2, 48_000, 24);
assert_eq!(parse_wav(&w), Some((pcm.clone(), 2, 48_000, 24)));
let a = aiff_file(&pcm, 2, 44_100, 24);
assert_eq!(parse_aiff(&a), Some((pcm.clone(), 2, 44_100, 24)));
let p16: Vec<u8> = (0..32u8).collect();
assert_eq!(parse_aiff(&aiff_file(&p16, 1, 96_000, 16)), Some((p16, 1, 96_000, 16)));
assert!(parse_wav(b"RIFF").is_none());
assert!(parse_aiff(&a[..30]).is_none());
for cut in 0..w.len() {
let _ = parse_wav(&w[..cut]);
let _ = parse_aiff(&a[..cut.min(a.len())]);
}
}
}
+3
View File
@@ -76,6 +76,9 @@ pub mod wav_markers;
))]
mod write;
/// PCM WAVE and AIFF wrappers for interchange essence.
#[cfg(feature = "pcm-encode")]
pub mod essence_pcm;
/// Interleaved PCM packet decoding.
pub mod pcm_interleaved;
/// Range-backed PCM WAVE access and a 16-bit PCM writer.
+1
View File
@@ -31,6 +31,7 @@ pub mod math;
pub mod named;
pub mod pipeline;
pub mod profile;
pub mod sony_lut;
pub mod spi1d;
pub mod synth;
pub mod transform;
+171 -42
View File
@@ -150,62 +150,142 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> {
}
}
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
let mut title = String::new();
let (mut size3, mut size1) = (0usize, 0usize);
let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]);
let mut rows: Vec<f32> = Vec::new();
for line in text.lines() {
// Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an
// unquoted title's comment must not turn that comment into part of the title.
if let Some(quoted) = line
.trim_start()
.strip_prefix("TITLE")
.filter(|rest| rest.starts_with(char::is_whitespace))
.and_then(|rest| rest.trim_start().strip_prefix('"'))
/// A bounded, three-dimensional Cube profile with RGB rows in file order.
/// Unlike [`parse_cube`], this preserves the optional title and does not resample 1D curves.
#[derive(Clone, Debug, PartialEq)]
pub struct Cube3d {
pub title: Option<String>,
pub size: usize,
pub data: Vec<[f32; 3]>,
pub domain_min: [f32; 3],
pub domain_max: [f32; 3],
}
/// Read the simple 3D profile dialect used by camera-profile consumers.
///
/// `max_size` must be in 2..=256; the row budget is checked before each allocation.
/// Whole-line comments and literal titles are preserved. Unknown alphabetic directives
/// are ignored; 1D tables are refused. Finite domains are returned unchanged so the
/// consumer retains its existing policy for equal or reversed endpoints.
/// Numeric fields must consist of exactly three finite numbers (never silently skip tokens).
pub fn parse_cube_3d(text: &str, max_size: usize) -> Result<Cube3d, LutError> {
let p = read_cube(text, Some(max_size))?;
if p.size3 == 0 {
return err("not a 3D .cube LUT (no LUT_3D_SIZE)");
}
let expected = p.size3.checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
if p.rows.len() != expected {
return err(format!("expected {expected} entries, found {}", p.rows.len()));
}
Ok(Cube3d { title: p.title, size: p.size3, data: p.rows, domain_min: p.min, domain_max: p.max })
}
struct CubeRows {
title: Option<String>,
size3: usize,
size1: usize,
min: [f32; 3],
max: [f32; 3],
rows: Vec<[f32; 3]>,
}
fn cube_triplet(text: &str) -> Result<[f32; 3], LutError> {
let mut it = text.split_whitespace();
let mut row = [0.0; 3];
for value in &mut row {
*value = it.next().and_then(|s| s.parse::<f32>().ok()).filter(|v| v.is_finite()).ok_or_else(|| LutError(format!("bad line `{text}`")))?;
}
if it.next().is_some() {
return err(format!("bad line `{text}`"));
}
Ok(row)
}
fn read_cube(text: &str, profile_max: Option<usize>) -> Result<CubeRows, LutError> {
if profile_max.is_some_and(|n| !(2..=256).contains(&n)) {
return err("3D profile size limit must be in 2..=256");
}
let mut p = CubeRows { title: None, size3: 0, size1: 0, min: [0.0; 3], max: [1.0; 3], rows: Vec::new() };
let row_limit = profile_max.unwrap_or(256).checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
for raw in text.lines() {
let profile = profile_max.is_some();
let line = if profile { raw.trim() } else { raw.split('#').next().unwrap_or("").trim() };
// The standard dialect protects '#' inside quoted titles; profile titles are literal.
if !profile
&& let Some(quoted) =
raw.trim_start().strip_prefix("TITLE").filter(|r| r.starts_with(char::is_whitespace)).and_then(|r| r.trim_start().strip_prefix('"'))
{
let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
title = value.to_string();
let Some((title, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
p.title = Some(title.to_string());
continue;
}
let line = line.split('#').next().unwrap_or("").trim();
if line.is_empty() {
if line.is_empty() || line.starts_with('#') {
continue;
}
let mut it = line.split_whitespace();
let key = it.next().unwrap_or("");
let nums = |it: std::str::SplitWhitespace<'_>| -> Result<[f32; 3], LutError> {
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
if v.len() == 3 { Ok([v[0], v[1], v[2]]) } else { err(format!("bad line `{line}`")) }
let (key, rest) = if profile {
["TITLE", "LUT_3D_SIZE", "LUT_1D_SIZE", "DOMAIN_MIN", "DOMAIN_MAX"]
.into_iter()
.find_map(|key| line.strip_prefix(key).map(|rest| (key, rest.trim())))
.unwrap_or_else(|| line.split_once(char::is_whitespace).unwrap_or((line, "")))
} else {
line.split_once(char::is_whitespace).map_or((line, ""), |(key, rest)| (key, rest.trim()))
};
match key {
"TITLE" => title = line[5..].trim().trim_matches('"').to_string(),
"LUT_3D_SIZE" => size3 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_3D_SIZE".into()))?,
"LUT_1D_SIZE" => size1 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?,
"DOMAIN_MIN" => dmin = nums(it)?,
"DOMAIN_MAX" => dmax = nums(it)?,
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" => {
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
if v.len() == 2 {
dmin = [v[0]; 3];
dmax = [v[1]; 3];
"TITLE" => p.title = Some(rest.trim_matches('"').to_string()),
"LUT_3D_SIZE" => {
let value = if profile { rest } else { rest.split_whitespace().next().unwrap_or("") };
p.size3 = value.parse().map_err(|_| LutError(format!("bad LUT_3D_SIZE `{value}`")))?;
let limit = profile_max.unwrap_or(MAX_SIZE);
if !(2..=limit).contains(&p.size3) {
return err(format!("LUT_3D_SIZE {} out of range", p.size3));
}
}
k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => {
let mut v = vec![k.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?];
for value in it {
v.push(value.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?);
"LUT_1D_SIZE" => {
if profile {
return err("1D LUTs are not supported (only 3D .cube files)");
}
if v.len() != 3 {
return err(format!("bad data line `{line}`"));
p.size1 = rest.split_whitespace().next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?;
if p.size1 > row_limit {
return err("1D LUT exceeds row budget");
}
}
"DOMAIN_MIN" => p.min = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MIN".into()))?,
"DOMAIN_MAX" => p.max = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MAX".into()))?,
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" if !profile => {
let mut it = rest.split_whitespace();
let min = it.next().and_then(|s| s.parse::<f32>().ok());
let max = it.next().and_then(|s| s.parse::<f32>().ok());
if let (Some(min), Some(max), None) = (min, max, it.next()) {
p.min = [min; 3];
p.max = [max; 3];
}
}
_ => {
let numeric = key.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.');
let nonfinite = key.parse::<f32>().is_ok();
if numeric || nonfinite || (profile && !key.starts_with(|c: char| c.is_ascii_alphabetic())) {
let row = cube_triplet(line)?;
if p.rows.len() >= row_limit {
return err("LUT exceeds row budget");
}
// Explicit fallible growth prevents capacity growth past the configured budget.
if p.rows.len() == p.rows.capacity() {
let grow = (p.rows.capacity().max(1)).min(row_limit - p.rows.len());
p.rows.try_reserve_exact(grow).map_err(|_| LutError("LUT allocation failed".into()))?;
}
p.rows.push(row);
}
rows.extend(v);
}
// Unknown keywords (e.g. Resolve's LUT_IN_VIDEO_RANGE) are ignored, as the spec allows.
_ => {}
}
}
Ok(p)
}
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
let p = read_cube(text, None)?;
let (title, size3, size1, dmin, dmax) = (p.title.unwrap_or_default(), p.size3, p.size1, p.min, p.max);
let rows: Vec<f32> = p.rows.into_iter().flatten().collect();
let domain_ok = (0..3).all(|k| dmin[k] < dmax[k] && dmin[k].is_finite() && dmax[k].is_finite());
if (size3 > 0 || size1 >= 2) && !domain_ok {
return err(format!("DOMAIN_MIN {:?} must lie below DOMAIN_MAX {:?}", dmin, dmax));
@@ -1386,3 +1466,52 @@ LUT_3D_SIZE 2
assert_eq!(back.data, [0.0, 0.25, 1.0].repeat(8));
}
}
#[cfg(test)]
mod profile_cube_tests {
use super::*;
fn cube(prefix: &str) -> String {
format!("{prefix}\nLUT_3D_SIZE 2\n{}", "0.25 0.5 0.75\n".repeat(8))
}
#[test]
fn profile_preserves_optional_literal_title_and_domains() {
let no_title = parse_cube_3d(&cube(""), 256).unwrap();
assert_eq!(no_title.title, None);
let value = parse_cube_3d(&cube("TITLE \"look # literal\"\nDOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1"), 256).unwrap();
assert_eq!(value.title.as_deref(), Some("look # literal"));
assert_eq!(value.domain_min, [1.0, 0.0, 2.0]);
assert_eq!(value.domain_max, [0.0, 0.0, 1.0]);
assert_eq!(value.data, vec![[0.25, 0.5, 0.75]; 8]);
assert_eq!(parse_cube_3d(&cube("TITLE \"\""), 256).unwrap().title.as_deref(), Some(""));
assert!(parse_cube(&cube("DOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1")).is_err());
}
#[test]
fn profile_limits_are_checked_without_large_allocations() {
assert!(parse_cube_3d("LUT_3D_SIZE 256", 256).unwrap_err().0.contains("16777216"));
assert!(parse_cube_3d("LUT_3D_SIZE 257", 256).unwrap_err().0.contains("out of range"));
assert!(parse_cube_3d("LUT_3D_SIZE 256", 129).unwrap_err().0.contains("out of range"));
assert!(parse_cube_3d(&cube(""), usize::MAX).is_err());
assert!(parse_cube_3d(&cube(""), 1).is_err());
assert!(parse_cube_3d(&format!("{}0 0 0\n", cube("")), 2).unwrap_err().0.contains("budget"));
assert!(parse_cube_3d("LUT_1D_SIZE 2\n0 0 0\n1 1 1", 256).is_err());
}
#[test]
fn every_numeric_token_must_be_valid_and_finite() {
for row in ["0 ignored 0 0", "0 0 0 ignored", "NaN 0 0", "inf 0 0", "0 0 -inf", "1e39 0 0", "0 0", "0 0 0 0"] {
assert!(parse_cube_3d(&format!("LUT_3D_SIZE 2\n{row}\n{}", "0 0 0\n".repeat(7)), 256).is_err(), "{row}");
assert!(parse_cube_3d(&cube(&format!("DOMAIN_MIN {row}")), 256).is_err(), "{row}");
}
}
#[test]
fn standard_reader_keeps_quoted_comments_and_1d_expansion() {
assert_eq!(parse_cube(&cube("TITLE \"look # preserved\" # comment")).unwrap().title, "look # preserved");
let value = parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1").unwrap();
assert_eq!(value.size, 33);
assert!(parse_cube_3d(&cube("DOMAIN_MIN 0 0 0 # not a whole-line comment"), 256).is_err());
}
}
+82
View File
@@ -0,0 +1,82 @@
//! Sony Imageworks SPI3D and SPIMTX text readers, extracted from EffectCraft.
//!
//! These preserve the source reader's format tolerance. SPI3D sample values are not
//! clamped or sanitized; rendering policy belongs to the caller. Duplicate coordinates
//! retain their last value, and the total row count must match the lattice volume.
/// Maximum lattice side accepted by the source reader.
pub const MAX_SIDE: usize = 129;
/// Row-major homogeneous transform with a separate normalized code-value offset.
#[derive(Clone, Debug, PartialEq)]
pub struct Matrix {
pub matrix: [f64; 16],
pub offset: [f64; 4],
}
/// A cubic RGB table, red coordinate varying fastest.
#[derive(Clone, Debug, PartialEq)]
pub struct Lattice {
pub size: usize,
pub values: Vec<[f32; 3]>,
}
/// Read exactly twelve finite SPIMTX numbers, dividing the offset column by 65535.
/// Parsing uses a fixed array, not storage proportional to an oversized input.
pub fn parse_spimtx(text: &str) -> Option<Matrix> {
let mut words = text.split_whitespace();
let mut v = [0.0f64; 12];
for value in &mut v {
*value = words.next()?.parse().ok()?;
if !value.is_finite() {
return None;
}
}
if words.next().is_some() {
return None;
}
let mut matrix = [0.0; 16];
let mut offset = [0.0; 4];
for (r, row) in v.as_chunks::<4>().0.iter().enumerate() {
matrix[r * 4..r * 4 + 3].copy_from_slice(&row[..3]);
offset[r] = row[3] / 65535.0;
}
matrix[15] = 1.0;
Some(Matrix { matrix, offset })
}
/// Read a SPI3D table with a bounded 2..=129 cubic lattice and exactly six fields per row.
/// A `SPILUT` prefix is accepted and the second header line is ignored, as in EffectCraft.
pub fn parse_spi3d(text: &str) -> Option<Lattice> {
let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty());
if !lines.next()?.starts_with("SPILUT") {
return None;
}
lines.next()?;
let mut sizes = lines.next()?.split_whitespace();
let n: usize = sizes.next()?.parse().ok()?;
let g: usize = sizes.next()?.parse().ok()?;
let b: usize = sizes.next()?.parse().ok()?;
if sizes.next().is_some() || g != n || b != n || !(2..=MAX_SIDE).contains(&n) {
return None;
}
let count = n * n * n;
let mut values = vec![[0.0f32; 3]; count];
let mut seen = 0usize;
for line in lines {
if seen == count {
return None;
}
let mut fields = line.split_whitespace();
let r: usize = fields.next()?.parse().ok()?;
let g: usize = fields.next()?.parse().ok()?;
let b: usize = fields.next()?.parse().ok()?;
let color = [fields.next()?.parse().ok()?, fields.next()?.parse().ok()?, fields.next()?.parse().ok()?];
if fields.next().is_some() || r >= n || g >= n || b >= n {
return None;
}
*values.get_mut(r + g * n + b * n * n)? = color;
seen += 1;
}
(seen == count).then_some(Lattice { size: n, values })
}
+46
View File
@@ -0,0 +1,46 @@
use craft_cms::sony_lut::{MAX_SIDE, parse_spi3d, parse_spimtx};
#[test]
fn spimtx_is_finite_exact_and_scales_offsets() {
let m = parse_spimtx("2 0 0 6553.5\n0 3 0 -65535\n0 0 4 0").unwrap();
assert_eq!(m.matrix, [2.0, 0.0, 0.0, 0.0, 0.0, 3.0, 0.0, 0.0, 0.0, 0.0, 4.0, 0.0, 0.0, 0.0, 0.0, 1.0]);
assert_eq!(m.offset, [0.1, -1.0, 0.0, 0.0]);
for bad in ["", "1 0 0 0", "NaN 0 0 0 0 1 0 0 0 0 1 0", "inf 0 0 0 0 1 0 0 0 0 1 0", "1 0 0 0 0 1 0 0 0 0 1 0 9"] {
assert!(parse_spimtx(bad).is_none());
}
}
fn table() -> String {
let mut text = String::from("SPILUT 1.0\n3 3\n2 2 2\n");
// Reverse row order to prove coordinates, rather than row order, determine storage.
for i in (0..8).rev() {
let (r, g, b) = (i & 1, (i >> 1) & 1, i >> 2);
text.push_str(&format!("{r} {g} {b} {r} {g} {b}\n"));
}
text
}
#[test]
fn spi3d_red_fast_coordinates_and_limits() {
let text = table();
let lut = parse_spi3d(&text).unwrap();
assert_eq!(lut.size, 2);
for (i, row) in lut.values.iter().enumerate() {
assert_eq!(*row, [(i & 1) as f32, ((i >> 1) & 1) as f32, (i >> 2) as f32]);
}
assert!(parse_spi3d(&format!("{text}0 0 0 0 0 0\n")).is_none());
assert!(parse_spi3d("SPILUT 1.0\n3 3\n2 2 2\n0 0 0 0 0 0\n").is_none());
assert!(parse_spi3d(&format!("SPILUT 1.0\n3 3\n{0} {0} {0}\n", MAX_SIDE + 1)).is_none());
assert!(parse_spi3d(&text.replace("2 2 2", "2 2 3")).is_none());
assert!(parse_spi3d(&text.replace("0 0 0 0 0 0", "2 0 0 0 0 0")).is_none());
assert!(parse_spi3d(&text.replace("0 0 0 0 0 0", "0 0 0 0 0 0 extra")).is_none());
}
#[test]
fn inherited_tolerance_is_explicit() {
let text = "SPILUT alternative\nignored\n2 2 2\n".to_owned() + &"0 0 0 -1 2 NaN\n".repeat(8);
let lut = parse_spi3d(&text).unwrap();
assert_eq!(&lut.values[0][..2], &[-1.0, 2.0]);
assert!(lut.values[0][2].is_nan());
assert_eq!(lut.values[1], [0.0; 3]);
}
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "craft-gridcraft-format"
description = "GridCraft's native JSON workbook file: its extensions, reader and pretty writer"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
+17
View File
@@ -0,0 +1,17 @@
# craft-gridcraft-format
GridCraft's native workbook file is one pretty-printed UTF-8 JSON object, with a two-space indent
and no newline at the end. It is saved as `.json`, or as `.scjson` by older builds.
The workbook is a type parameter. GridCraft owns the model, and after a read it rebuilds the
model's indexes, drops cells outside the grid and reads the macros. This crate owns the bytes:
- `EXTENSIONS` and `is_extension` name the format.
- `read(bytes)` reads a workbook. A failure reads as "not a GridCraft JSON workbook: …".
- `write(workbook)` writes one with `serde_json`'s pretty printer.
- `write_with_kept(workbook, field, kept)` appends entries that a newer version wrote, and this
build kept without reading, to the top-level list `field`, as the file held them. With nothing
kept, the output is exactly `write`'s. Otherwise the workbook goes through a `serde_json::Value`
first, so its object keys come out in map order, as GridCraft has always written them.
There is no `unsafe` and no panics, and it builds for `wasm32-unknown-unknown`.
+59
View File
@@ -0,0 +1,59 @@
//! GridCraft's native workbook file: the workbook as one pretty-printed UTF-8 JSON object,
//! saved as `.json` (or `.scjson`, its older name).
//!
//! GridCraft owns the workbook model and what happens after a read: rebuilding its indexes,
//! dropping cells outside the grid, and reading its macros. This crate owns the bytes: which
//! extensions name the format, how the JSON is read and written, and how entries kept from a
//! newer version are written back into a list as the file held them.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::Value;
/// The extensions of the format, lower case. `scjson` is the name older builds saved.
pub const EXTENSIONS: [&str; 2] = ["json", "scjson"];
/// Whether `extension` (without the dot, any case) names the format.
pub fn is_extension(extension: &str) -> bool {
EXTENSIONS.iter().any(|e| e.eq_ignore_ascii_case(extension))
}
#[derive(Debug, thiserror::Error)]
pub enum Error {
/// The bytes are not a workbook: not JSON, or JSON of another shape.
#[error("not a GridCraft JSON workbook: {0}")]
Read(serde_json::Error),
/// The workbook could not be written as JSON.
#[error("{0}")]
Write(serde_json::Error),
}
/// Read a workbook. Unknown fields are left to the model's own serde rules.
pub fn read<T: DeserializeOwned>(bytes: &[u8]) -> Result<T, Error> {
serde_json::from_slice(bytes).map_err(Error::Read)
}
/// Write a workbook as pretty-printed JSON (two-space indent, no newline at the end).
pub fn write<T: Serialize>(workbook: &T) -> Result<Vec<u8>, Error> {
serde_json::to_vec_pretty(workbook).map_err(Error::Write)
}
/// [`write`], with `kept` appended to the top-level list `field` (made if absent): entries a
/// newer version wrote that this build kept without reading. With nothing kept this is exactly
/// [`write`]. Otherwise the workbook goes through a [`Value`] first, so its object keys are
/// written in `serde_json`'s map order.
pub fn write_with_kept<T: Serialize>(workbook: &T, field: &str, kept: &[Value]) -> Result<Vec<u8>, Error> {
if kept.is_empty() {
return write(workbook);
}
let mut v = serde_json::to_value(workbook).map_err(Error::Write)?;
if let Some(list) = v.as_object_mut().map(|o| o.entry(field).or_insert_with(|| Value::Array(vec![]))).and_then(Value::as_array_mut) {
list.extend(kept.iter().cloned());
}
serde_json::to_vec_pretty(&v).map_err(Error::Write)
}
#[cfg(test)]
mod tests;
+87
View File
@@ -0,0 +1,87 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use serde::Deserialize;
use serde_json::json;
use super::*;
/// A stand-in for an app's workbook: fields in declaration order, one of them a list.
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Book {
sheets: Vec<String>,
active: u32,
#[serde(default)]
macros: Vec<Value>,
}
fn book() -> Book {
Book { sheets: vec!["Sheet1".into(), "Été ✓".into()], active: 1, macros: vec![json!({"v": 1, "name": "Fill"})] }
}
#[test]
fn extensions_are_json_and_scjson_in_any_case() {
for yes in ["json", "scjson", "JSON", "ScJson"] {
assert!(is_extension(yes), "{yes}");
}
for no in ["", "xlsx", "jsonl", ".json", "scjs"] {
assert!(!is_extension(no), "{no}");
}
}
#[test]
fn a_workbook_is_written_pretty_in_field_order() {
let bytes = write(&book()).unwrap();
let text = String::from_utf8(bytes).unwrap();
assert_eq!(
text,
"{\n \"sheets\": [\n \"Sheet1\",\n \"Été ✓\"\n ],\n \"active\": 1,\n \"macros\": [\n {\n \"name\": \"Fill\",\n \"v\": 1\n }\n ]\n}"
);
assert_eq!(read::<Book>(text.as_bytes()).unwrap(), book());
}
#[test]
fn kept_entries_are_appended_as_the_file_held_them() {
let kept = [json!({"v": 9, "name": "Future", "extra": [1, 2]})];
let bytes = write_with_kept(&book(), "macros", &kept).unwrap();
let back: Book = read(&bytes).unwrap();
assert_eq!(back.macros, [json!({"v": 1, "name": "Fill"}), kept[0].clone()]);
// Through a JSON value, keys are in map order: `active` before `macros` before `sheets`.
let text = String::from_utf8(bytes).unwrap();
assert!(text.find("\"active\"") < text.find("\"macros\"") && text.find("\"macros\"") < text.find("\"sheets\""), "{text}");
// With nothing kept, the plain writer's bytes.
assert_eq!(write_with_kept(&book(), "macros", &[]).unwrap(), write(&book()).unwrap());
}
#[test]
fn kept_entries_make_the_list_when_it_is_absent() {
#[derive(Serialize)]
struct Bare {
sheets: Vec<String>,
}
let bytes = write_with_kept(&Bare { sheets: vec![] }, "macros", &[json!(1)]).unwrap();
assert_eq!(serde_json::from_slice::<Value>(&bytes).unwrap(), json!({"sheets": [], "macros": [1]}));
// A field that is not a list is left as it is.
let bytes = write_with_kept(&json!({"macros": "text"}), "macros", &[json!(1)]).unwrap();
assert_eq!(serde_json::from_slice::<Value>(&bytes).unwrap(), json!({"macros": "text"}));
}
#[test]
fn what_is_not_a_workbook_says_so() {
for bad in [&b""[..], b"{", b"[]", b"{\"sheets\": 7, \"active\": 0}", b"\xff\xfe"] {
let err = read::<Book>(bad).unwrap_err();
assert!(matches!(err, Error::Read(_)));
assert!(err.to_string().starts_with("not a GridCraft JSON workbook: "), "{err}");
}
assert_eq!(
read::<Book>(b"{\"sheets\": 7}").unwrap_err().to_string(),
"not a GridCraft JSON workbook: invalid type: integer `7`, expected a sequence at line 1 column 12"
);
}
#[test]
fn a_value_json_cannot_hold_is_a_write_error() {
use std::collections::BTreeMap;
let bad: BTreeMap<Vec<u8>, u8> = [(vec![1], 1)].into();
assert!(matches!(write(&bad), Err(Error::Write(_))));
assert!(matches!(write_with_kept(&bad, "macros", &[json!(1)]), Err(Error::Write(_))));
}
+11
View File
@@ -0,0 +1,11 @@
[package]
name = "craft-http-url"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license = "Apache-2.0"
repository.workspace = true
description = "Bounded HTTP URL parsing and redirect resolution without network access"
[lints]
workspace = true
+25
View File
@@ -0,0 +1,25 @@
# craft-http-url
HTTP/HTTPS URL parsing, dot-segment removal, relative redirects, HTTPS downgrade rejection,
and query-redacted display. No dependencies or network operations. Parsing rejects inputs
larger than 16 KiB and redirects are bounded before concatenation.
This is not an SSRF filter, DNS resolver or HTTP client. Local/private hosts are allowed.
Applications keep destination permission, authentication, TLS, request/response limits and
redirect-count policy. Public fields preserve the original API; manually constructed values
must not be treated as validated URLs.
## Attribution
Extracted from LightCraft `crates/fetch/src/http.rs` at
`2dcb258ba40738230a07484a95eb27dc84ed66fe` (Apache-2.0).
Original parser: Daniel Voshart, `7fa7506db46d69d609e85e39b338f33786ed0293`.
HTTPS downgrade protection: MarekSpodBiedry, `4a91a1c218bf3d0ac49cc73ebb60621a21ae55e2`.
URL normalization, scheme, fragment and authority fixes: NetterSilver, including
`edc369565eef92161b56b8f1e39ef53629c05489`,
`78f0e6e650444886069f4d4ae4e6593f221b98d4`,
`cee93496587d31c514aaeced3e6b8027cfdb41a0`,
`dee5f7910ad8c24cf28bab528f8d007550af2b23`,
`c3a5d9b7f1f837dc14c01942e3d0293ecbf36345` and
`13849307a9343504339d68b086f1a32cdcbbb457`.
Source-author commits are retained separately in this extraction.
+277
View File
@@ -0,0 +1,277 @@
//! Bounded HTTP/HTTPS URL parsing and redirect resolution, extracted from LightCraft.
//!
//! This is syntax handling, not a network permission or SSRF policy. It permits local/private
//! hosts and performs no DNS, socket or TLS operations. Callers own destination allowlists,
//! credential forwarding, redirect counts, connection deadlines and response limits.
//! Only parsed values have validated fields; constructing public fields bypasses parsing.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
/// A URL or redirect cannot be represented by this bounded HTTP URL syntax.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UrlError(pub String);
impl std::fmt::Display for UrlError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for UrlError {}
/// A parsed `http://` or `https://` URL.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Url {
pub tls: bool,
pub host: String,
pub port: u16,
/// Path and query, starting with `/`.
pub path: String,
}
/// Remove `.` and `..` segments from the path part of `path` (RFC 3986 section 5.2.4); the query is kept as is.
fn remove_dot_segments(path: &str) -> String {
let (p, query) = match path.find('?') {
Some(i) => (path.get(..i).unwrap_or_default(), path.get(i..).unwrap_or_default()),
None => (path, ""),
};
let mut out: Vec<&str> = Vec::new();
let mut segments = p.split('/').skip(1).peekable();
let mut trailing_slash = false;
while let Some(seg) = segments.next() {
let last = segments.peek().is_none();
match seg {
"." => trailing_slash = last,
".." => {
out.pop();
trailing_slash = last;
}
_ => {
out.push(seg);
trailing_slash = false;
}
}
}
let mut result = format!("/{}", out.join("/"));
if trailing_slash && !result.ends_with('/') {
result.push('/');
}
result.push_str(query);
result
}
/// `s` without the leading `prefix`, matched ASCII case-insensitively (RFC 3986 §3.1).
fn strip_scheme<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
let head = s.get(..prefix.len())?;
head.eq_ignore_ascii_case(prefix).then(|| s.get(prefix.len()..).unwrap_or_default())
}
impl Url {
pub fn parse(s: &str) -> Result<Url, UrlError> {
let bad = |why: &str| UrlError(why.to_string());
if s.len() > 16 * 1024 {
return Err(bad("URL exceeds 16 KiB"));
}
let s = s.trim();
if s.chars().any(|c| c.is_control() || c == ' ') {
return Err(bad("contains spaces or control characters"));
}
let (tls, rest) = if let Some(r) = strip_scheme(s, "https://") {
(true, r)
} else if let Some(r) = strip_scheme(s, "http://") {
(false, r)
} else {
return Err(bad("only http:// and https:// are supported"));
};
let rest = rest.split('#').next().unwrap_or_default();
let (authority, path) = match rest.find(['/', '?']) {
Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()),
None => (rest, ""),
};
let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") };
let path = remove_dot_segments(&path);
if authority.contains('@') {
return Err(bad("user names and passwords in URLs are not supported"));
}
let (host, port) = if let Some(v6) = authority.strip_prefix('[') {
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
let host = v6.get(..end).unwrap_or_default();
let after = v6.get(end + 1..).unwrap_or_default();
if !after.is_empty() && !after.starts_with(':') {
return Err(bad("unexpected text after IPv6 address"));
}
(host.to_string(), after.strip_prefix(':'))
} else {
match authority.rsplit_once(':') {
Some((h, p)) => (h.to_string(), Some(p)),
None => (authority.to_string(), None),
}
};
let port = match port {
Some(p) if p.bytes().all(|b| b.is_ascii_digit()) => p.parse::<u16>().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?,
Some(_) => return Err(bad("bad port")),
None if tls => 443,
None => 80,
};
if host.is_empty() || !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | ':' | '_')) {
return Err(bad("bad host name"));
}
Ok(Url { tls, host: host.to_ascii_lowercase(), port, path })
}
/// `Location` of a redirect, resolved against this URL.
pub fn join(&self, location: &str) -> Result<Url, UrlError> {
// Reject before concatenating a hostile Location with the base URL.
if location.len() > 16 * 1024 || self.host.len().saturating_add(self.path.len()) > 16 * 1024 {
return Err(UrlError("URL exceeds 16 KiB".into()));
}
let l = location.trim();
if strip_scheme(l, "http://").is_some() || strip_scheme(l, "https://").is_some() {
let url = Url::parse(l)?;
if self.tls && !url.tls {
return Err(UrlError("redirect from https to http refused".into()));
}
return Ok(url);
}
// another scheme (`ftp:`, `file:`, `data:`…)
if let Some((scheme, _)) = l.split_once(':')
&& !scheme.is_empty()
&& scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
{
return Err(UrlError(format!("redirect to an unsupported location `{scheme}:`")));
}
let scheme = if self.tls { "https" } else { "http" };
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
let base = format!("{scheme}://{host}:{}", self.port);
if let Some(rest) = l.strip_prefix("//") {
return Url::parse(&format!("{scheme}://{rest}"));
}
if l.starts_with('/') {
return Url::parse(&format!("{base}{l}"));
}
// empty or fragment-only: the same document
if l.is_empty() || l.starts_with('#') {
return Ok(self.clone());
}
// relative to the current directory
let dir = self.path.split('?').next().unwrap_or("/");
// a query-only reference keeps the whole path, file name included (RFC 3986 §5.2.2)
if l.starts_with('?') {
return Url::parse(&format!("{base}{dir}{l}"));
}
let dir = dir.rsplit_once('/').map_or("", |(d, _)| d);
Url::parse(&format!("{base}{dir}/{l}"))
}
/// Authority formatted for an HTTP Host field.
pub fn host_header(&self) -> String {
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
let default = if self.tls { 443 } else { 80 };
if self.port == default { host } else { format!("{host}:{}", self.port) }
}
/// The URL without its query (for messages: a query may hold a signed token).
pub fn display(&self) -> String {
let scheme = if self.tls { "https" } else { "http" };
format!("{scheme}://{}{}", self.host_header(), self.path.split('?').next().unwrap_or("/"))
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::{Url, UrlError};
#[test]
fn fragment_only_or_empty_location_keeps_file_name_and_query() {
let base = Url::parse("https://a.example/x/y/file?q").unwrap();
for l in ["#top", "", " "] {
let u = base.join(l).unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/x/y/file?q"), "{l:?}");
}
}
#[test]
fn rejects_oversized_and_control_character_urls() {
let base = Url::parse("https://example.com/path").unwrap();
assert!(base.join(&format!("#{}", "x".repeat(16 * 1024))).is_err());
assert!(base.join(&"x".repeat(16 * 1024 + 1)).is_err());
assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err());
assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err());
}
#[test]
fn join_query_only_keeps_file_name() {
let u = Url::parse("https://a.example/x/y/file?q=1").unwrap();
assert_eq!(u.join("?page=2").unwrap().path, "/x/y/file?page=2");
}
#[test]
fn join_removes_dot_segments() {
let base = Url::parse("https://a.example/x/y/file").unwrap();
for (loc, want) in
[("../z", "/x/z"), ("./z", "/x/y/z"), ("../../../z", "/z"), ("/a/./b/../c", "/a/c"), ("../z?q=1", "/x/z?q=1"), ("..", "/x/")]
{
assert_eq!(base.join(loc).unwrap().path, want, "{loc}");
}
}
#[test]
fn url_scheme_is_matched_case_insensitively() {
let u = Url::parse("HTTPS://A.example/z").unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/z"));
assert!(Url::parse("Http://a.example/").is_ok());
let j = Url::parse("https://a.example/start").unwrap().join("HTTPS://cdn.example/file").unwrap();
assert_eq!((j.tls, j.host.as_str(), j.path.as_str()), (true, "cdn.example", "/file"));
}
#[test]
fn parses_urls() {
let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "cdn.example.com", 443, "/models/sam3/model.safetensors?sig=1"));
assert_eq!(u.display(), "https://cdn.example.com/models/sam3/model.safetensors");
let u = Url::parse("http://127.0.0.1:8080").unwrap();
assert_eq!((u.tls, u.port, u.path.as_str()), (false, 8080, "/"));
let u = Url::parse("http://[::1]:9/a").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 9));
for bad in ["ftp://x/y", "https://", "https://a b/", "https://u:p@h/", "http://h:0/", "http://h:99999/", "https://h/\r\nX: y", "file:///etc"]
{
assert!(Url::parse(bad).is_err(), "{bad}");
}
}
#[test]
fn rejects_signed_ports() {
for bad in ["http://example.com:+80/x", "http://example.com:-80/x", "http://[::1]:+80/x"] {
assert!(Url::parse(bad).is_err(), "{bad}");
}
assert!(Url::parse("http://a.test/").unwrap().join("http://example.com:+80/x").is_err());
assert_eq!(Url::parse("http://example.com:80/x").unwrap().port, 80);
}
#[test]
fn rejects_text_after_a_bracketed_ipv6_host() {
for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] {
assert!(matches!(Url::parse(bad), Err(UrlError(_))), "{bad}");
}
let u = Url::parse("http://[::1]:8080/x").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 8080));
let u = Url::parse("http://[::1]/x").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 80));
}
#[test]
fn resolves_redirects() {
let u = Url::parse("https://a.example/x/y/file?q").unwrap();
assert_eq!(u.join("https://b.example/z").unwrap().host, "b.example");
assert_eq!(u.join("/root").unwrap().path, "/root");
assert_eq!(u.join("other").unwrap().path, "/x/y/other");
assert_eq!(u.join("//c.example/p").unwrap().host, "c.example");
assert!(u.join("ftp://nope").is_err());
// no downgrade from https to http, however the redirect is written
assert!(u.join("http://a.example/z").is_err());
let plain = Url::parse("http://127.0.0.1:8080/a").unwrap();
assert!(!plain.join("http://127.0.0.1:9/b").unwrap().tls);
assert!(plain.join("https://b.example/z").unwrap().tls);
}
}
+1
View File
@@ -13,6 +13,7 @@ workspace = true
[features]
corpus = []
default = []
sgi = []
avif = ["image/avif", "dep:ravif", "dep:rav1e", "dep:avif-serialize"]
heif = ["dep:craft-heif"]
# AVIF decode: craft-video's clean-room pure-Rust AV1 decoder. Native targets only (on wasm32 a
+1
View File
@@ -57,6 +57,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "<
| Cineon | yes | yes | U16 (10-bit code values read and written exactly through U16) | Gray, RGB | no | no | no | no | no | no | no | built in |
| DICOM | yes | **no** | U8, U16 | Gray, RGB | no | no | no | no | no | no | n/a | built in |
| Amiga IFF (ILBM/PBM) | yes | yes (24-bit ILBM) | U8 | RGB, RGBA | yes (1-bit mask plane on write) | no | no | no | no | no | no | built in |
| SGI (`.sgi`, `.rgb`, `.bw`) | with feature `sgi` | with feature `sgi` (RLE) | U8, U16 | Gray, Gray+A, RGB, RGBA | yes | no | no | no | no | no | no | `sgi` (off by default) |
| JPEG-LS (`.jls`) | yes (any NEAR; one scan per component) | yes (lossless) | U8, U16 | Gray, RGB | no | no | no | no | no | no | no | built in |
| AVIF | with feature `avif-decode` (native only) | only with feature `avif` | U8, U16 (10/12-bit decodes to U16; PQ/HLG to linear F32) | RGB, RGBA | yes (auxiliary alpha) | read (`colr` prof/rICC) | read | read | no | no | yes | `craft-video` AV1 decoder; `image` + `ravif` to write |
| HEIF/HEIC | with feature `heif` | **no** | U8, U16 (10/12-bit decodes to U16) | RGB, RGBA | yes (auxiliary alpha) | yes (`colr` prof) | yes | yes | no | no | n/a | `heic-rs` |
+2
View File
@@ -20,6 +20,8 @@ pub(crate) mod jxl;
pub(crate) mod pcx;
pub(crate) mod png;
pub(crate) mod pnm;
#[cfg(feature = "sgi")]
pub(crate) mod sgi;
pub(crate) mod tga;
pub(crate) mod tiff;
pub(crate) mod tiff_exif;
@@ -842,6 +842,10 @@ fn read_metadata(c: &Container) -> (Option<Vec<u8>>, Option<String>) {
/// Decodes the primary image. `keep_orientation` leaves the container's `clap`, `irot` and
/// `imir` unapplied (the stored pixels), as for HEIF.
pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<Image> {
decode_with_depth(bytes, limits, keep_orientation).map(|(image, _)| image)
}
pub(crate) fn decode_with_depth(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<(Image, u8)> {
let c = parse(bytes)?;
let id = c.primary;
let (w, h) = item_size(&c, id)?;
@@ -890,9 +894,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> R
// PQ or HLG: linear light in 32-bit float, as for HEIF.
if cicp.is_hdr() {
limits.check(w, h, layout, SampleType::F32)?;
return crate::cicp::linearize(img, cicp);
return crate::cicp::linearize(img, cicp).map(|image| (image, bits));
}
Ok(img)
Ok((img, bits))
}
#[cfg(test)]
@@ -1301,6 +1305,9 @@ mod tests {
let src = Image::from_u8(w as u32, h as u32, ChannelLayout::Rgb, card(w, h, 3)).unwrap();
let file = encode_avif_with_speed(&src, 95, 10).unwrap();
let img = crate::decode(&file).unwrap();
let (with_depth, depth) = crate::decode_avif_with_depth(&file, &crate::DecodeOptions::default()).unwrap();
assert_eq!(depth, 10);
assert_eq!(with_depth, img);
assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (48, 32, ChannelLayout::Rgb, SampleType::U16));
let diff = mean_abs_diff(&rgb8(&img), &card(w, h, 3));
assert!(diff < 2.5, "mean difference {diff}");
+395
View File
@@ -0,0 +1,395 @@
//! SGI image files (`.sgi`, `.rgb`, `.rgba`, `.bw`): 8- and 16-bit gray, gray + alpha, RGB and
//! RGBA, verbatim or run-length encoded.
//!
//! Read and write follow the published SGI image file format description: a 512-byte big-endian
//! header, then one plane per channel with the rows stored bottom to top. Palette-mapped files
//! (colormap ID other than 0) are rejected. The writer uses the run-length encoding.
use crate::error::CodecError;
use crate::fidelity::Plan;
use crate::format::Format;
use crate::image::{ChannelLayout, Image, SampleType};
use crate::options::Limits;
const F: Format = Format::Sgi;
const HEADER: usize = 512;
const MAGIC: u16 = 474;
fn bad(reason: impl std::fmt::Display) -> CodecError {
CodecError::malformed(F, reason)
}
fn be16(b: &[u8], i: usize) -> u16 {
u16::from_be_bytes([b[i], b[i + 1]])
}
fn be32(b: &[u8], i: usize) -> u32 {
u32::from_be_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]])
}
/// The fields of the header that decoding needs, already checked for plausibility.
struct Header {
rle: bool,
wide: bool,
width: u32,
height: u32,
planes: usize,
}
/// Parse the 512-byte header. `None` when the bytes are not a plausible SGI header.
fn header(b: &[u8]) -> Option<Header> {
if b.len() < HEADER || be16(b, 0) != MAGIC {
return None;
}
let (storage, bpc, dimension) = (b[2], b[3], be16(b, 4));
let (width, height, planes) = (u32::from(be16(b, 6)), u32::from(be16(b, 8)), usize::from(be16(b, 10)));
if storage > 1 || !(1..=2).contains(&bpc) || !(1..=3).contains(&dimension) {
return None;
}
// A one-dimensional image has one row, a two-dimensional one a single plane.
let (height, planes) = match dimension {
1 => (1, 1),
2 => (height, 1),
_ => (height, planes),
};
if width == 0 || height == 0 || !(1..=4).contains(&planes) {
return None;
}
Some(Header { rle: storage == 1, wide: bpc == 2, width, height, planes })
}
/// Whether `b` starts with a plausible SGI header (the magic number plus consistent fields).
pub(crate) fn looks_like_sgi(b: &[u8]) -> bool {
header(b).is_some()
}
fn layout_of(planes: usize) -> ChannelLayout {
match planes {
1 => ChannelLayout::Gray,
2 => ChannelLayout::GrayA,
3 => ChannelLayout::Rgb,
_ => ChannelLayout::Rgba,
}
}
/// Expand one run-length encoded row into exactly `out.len()` samples.
fn read_rle_row(src: &[u8], wide: bool, out: &mut [u16]) -> Result<(), CodecError> {
let step = if wide { 2 } else { 1 };
let sample = |pos: usize| -> Result<u16, CodecError> {
let s = src.get(pos..pos + step).ok_or_else(|| bad("truncated RLE row"))?;
Ok(if wide { u16::from_be_bytes([s[0], s[1]]) } else { u16::from(s[0]) })
};
let (mut pos, mut n) = (0usize, 0usize);
while n < out.len() {
let code = sample(pos)?;
pos += step;
let count = usize::from(code & 0x7f);
if count == 0 {
return Err(bad("RLE row ends before the image width"));
}
if count > out.len() - n {
return Err(bad("RLE packet runs past the end of the row"));
}
if code & 0x80 != 0 {
for slot in &mut out[n..n + count] {
*slot = sample(pos)?;
pos += step;
}
} else {
let value = sample(pos)?;
pos += step;
out[n..n + count].fill(value);
}
n += count;
}
Ok(())
}
pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result<Image, CodecError> {
let h = header(b).ok_or_else(|| bad("invalid SGI header"))?;
if be32(b, 104) != 0 {
return Err(CodecError::unsupported(F, "palette-mapped SGI files are not supported"));
}
let layout = layout_of(h.planes);
let sample = if h.wide { SampleType::U16 } else { SampleType::U8 };
limits.check(h.width, h.height, layout, sample)?;
let (w, ht, planes) = (h.width as usize, h.height as usize, h.planes);
let rows = ht.checked_mul(planes).ok_or_else(|| bad("image size overflow"))?;
let bps = if h.wide { 2 } else { 1 };
let total = w.checked_mul(rows).ok_or_else(|| bad("image size overflow"))?;
// Reject inputs that cannot hold the declared pixels before allocating anything for them.
let table = if h.rle { rows.checked_mul(8).ok_or_else(|| bad("image size overflow"))? } else { 0 };
let min_len = if h.rle {
// Both offset tables must fit; rows may share compressed data, so the data itself is
// checked row by row against the offsets below.
HEADER + table
} else {
HEADER + total.checked_mul(bps).ok_or_else(|| bad("image size overflow"))?
};
if b.len() < min_len {
return Err(bad("the file is too short for the declared image"));
}
let mut out = Vec::new();
out.try_reserve_exact(total).map_err(|_| bad("pixel buffer is too large"))?;
out.resize(total, 0u16);
let mut row = vec![0u16; w];
for z in 0..planes {
for y in 0..ht {
let index = y + z * ht;
if h.rle {
let start = be32(b, HEADER + index * 4) as usize;
let len = be32(b, HEADER + rows * 4 + index * 4) as usize;
let end = start
.checked_add(len)
.filter(|&e| e <= b.len() && start >= HEADER + table)
.ok_or_else(|| bad("RLE row lies outside the file"))?;
read_rle_row(&b[start..end], h.wide, &mut row)?;
} else {
let start = HEADER + index * w * bps;
let s = &b[start..start + w * bps];
for (x, v) in row.iter_mut().enumerate() {
*v = if h.wide { u16::from_be_bytes([s[2 * x], s[2 * x + 1]]) } else { u16::from(s[x]) };
}
}
// Rows are stored bottom to top; planes are separate.
let top_y = ht - 1 - y;
for (x, &v) in row.iter().enumerate() {
out[(top_y * w + x) * planes + z] = v;
}
}
}
if h.wide {
Image::from_u16(h.width, h.height, layout, &out)
} else {
Image::from_u8(h.width, h.height, layout, out.iter().map(|&v| v as u8).collect())
}
}
/// Append one run-length encoded row (one packet list and a terminator).
fn write_rle_row(row: &[u16], wide: bool, out: &mut Vec<u8>) {
let put = |out: &mut Vec<u8>, v: u16| {
if wide {
out.extend_from_slice(&v.to_be_bytes());
} else {
out.push(v as u8);
}
};
let mut i = 0;
while i < row.len() {
let mut run = 1;
while i + run < row.len() && run < 127 && row[i + run] == row[i] {
run += 1;
}
if run >= 2 {
put(out, run as u16);
put(out, row[i]);
i += run;
} else {
// A literal packet extends until the next run of two or more.
let mut end = i + 1;
while end < row.len() && end - i < 127 && !(end + 1 < row.len() && row[end] == row[end + 1]) {
end += 1;
}
put(out, 0x80 | (end - i) as u16);
for &v in &row[i..end] {
put(out, v);
}
i = end;
}
}
put(out, 0);
}
/// Writes a run-length encoded SGI file: 8 or 16 bits per channel, gray, gray + alpha, RGB or RGBA.
pub(crate) fn encode(src: &Image, plan: Plan) -> Result<Vec<u8>, CodecError> {
let enc = |msg: String| CodecError::encode(F, msg);
let img = src.converted(plan.layout, plan.sample);
let wide = match img.sample_type() {
SampleType::U8 => false,
SampleType::U16 => true,
s => return Err(enc(format!("SGI stores 8 or 16 bits per channel, not {s:?}"))),
};
let planes = img.layout().channels();
if !matches!(img.layout(), ChannelLayout::Gray | ChannelLayout::GrayA | ChannelLayout::Rgb | ChannelLayout::Rgba) {
return Err(enc(format!("no SGI layout for {:?}", img.layout())));
}
let (w, ht) = img.dimensions();
let (Ok(w16), Ok(h16)) = (u16::try_from(w), u16::try_from(ht)) else {
return Err(enc(format!("dimensions exceed 65535x65535: {w}x{ht}")));
};
let (w, ht) = (w as usize, ht as usize);
let bps = if wide { 2 } else { 1 };
let data = img.data();
if data.len() != w * ht * planes * bps {
return Err(enc("the image has the wrong number of samples".into()));
}
let sample_at = |i: usize| -> u16 { if wide { u16::from_ne_bytes([data[2 * i], data[2 * i + 1]]) } else { u16::from(data[i]) } };
let rows = ht * planes;
let table = HEADER;
let mut starts = Vec::with_capacity(rows);
let mut lens = Vec::with_capacity(rows);
let mut body = Vec::new();
let mut row = vec![0u16; w];
let base = u32::try_from(HEADER + rows * 8).map_err(|_| enc("image too large".into()))?;
for z in 0..planes {
for y in 0..ht {
let src_y = ht - 1 - y; // stored bottom to top
for (x, v) in row.iter_mut().enumerate() {
*v = sample_at((src_y * w + x) * planes + z);
}
let before = body.len();
write_rle_row(&row, wide, &mut body);
let start = u32::try_from(base as usize + before).map_err(|_| enc("image too large".into()))?;
starts.push(start);
lens.push((body.len() - before) as u32);
}
}
let mut out = Vec::with_capacity(table + rows * 8 + body.len());
out.extend_from_slice(&MAGIC.to_be_bytes());
out.push(1); // RLE
out.push(bps as u8);
out.extend_from_slice(&(if planes > 1 { 3u16 } else { 2u16 }).to_be_bytes());
out.extend_from_slice(&w16.to_be_bytes());
out.extend_from_slice(&h16.to_be_bytes());
out.extend_from_slice(&(planes as u16).to_be_bytes());
out.extend_from_slice(&0u32.to_be_bytes()); // pixmin
out.extend_from_slice(&(if wide { 65535u32 } else { 255 }).to_be_bytes()); // pixmax
out.resize(HEADER, 0); // dummy, image name, colormap 0, padding
for s in &starts {
out.extend_from_slice(&s.to_be_bytes());
}
for l in &lens {
out.extend_from_slice(&l.to_be_bytes());
}
out.extend_from_slice(&body);
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
fn gradient(w: u32, h: u32, layout: ChannelLayout) -> Image {
let ch = layout.channels();
let data = (0..w as usize * h as usize * ch).map(|i| ((i * 7 + i / 5) % 251) as u8).collect();
Image::from_u8(w, h, layout, data).unwrap()
}
#[test]
fn rle_round_trips_every_layout_in_8_bits() {
for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] {
let img = gradient(37, 11, layout);
let plan = Plan { layout, sample: SampleType::U8 };
let bytes = encode(&img, plan).unwrap();
assert!(looks_like_sgi(&bytes));
let back = decode(&bytes, &Limits::default()).unwrap();
assert_eq!(back.layout(), layout);
assert_eq!(back.data(), img.data());
}
}
#[test]
fn rle_round_trips_16_bits_with_long_runs() {
let (w, h) = (300u32, 4u32);
let samples: Vec<u16> = (0..w * h * 3).map(|i| if (i / 3) % 200 < 150 { 4242 } else { (i * 31) as u16 }).collect();
let img = Image::from_u16(w, h, ChannelLayout::Rgb, &samples).unwrap();
let bytes = encode(&img, Plan { layout: ChannelLayout::Rgb, sample: SampleType::U16 }).unwrap();
let back = decode(&bytes, &Limits::default()).unwrap();
assert_eq!(back.sample_type(), SampleType::U16);
assert_eq!(back.data(), img.data());
}
/// A hand-built verbatim file: two rows, one plane, stored bottom to top.
#[test]
fn verbatim_files_read_bottom_to_top() {
let mut f = vec![0u8; HEADER];
f[0..2].copy_from_slice(&MAGIC.to_be_bytes());
f[2] = 0; // verbatim
f[3] = 1;
f[4..6].copy_from_slice(&2u16.to_be_bytes());
f[6..8].copy_from_slice(&3u16.to_be_bytes()); // x
f[8..10].copy_from_slice(&2u16.to_be_bytes()); // y
f[10..12].copy_from_slice(&1u16.to_be_bytes()); // z
f.extend_from_slice(&[10, 11, 12, 20, 21, 22]); // bottom row first
let img = decode(&f, &Limits::default()).unwrap();
assert_eq!(img.layout(), ChannelLayout::Gray);
assert_eq!(img.data(), &[20, 21, 22, 10, 11, 12]);
}
#[test]
fn hostile_input_is_an_error_not_a_panic() {
let img = gradient(16, 8, ChannelLayout::Rgb);
let good = encode(&img, Plan { layout: ChannelLayout::Rgb, sample: SampleType::U8 }).unwrap();
// Every truncation fails cleanly.
for len in 0..good.len() {
let _ = decode(&good[..len], &Limits::default());
}
// A row table pointing outside the file, and a packet that overruns its row.
let mut bad_table = good.clone();
bad_table[HEADER..HEADER + 4].copy_from_slice(&u32::MAX.to_be_bytes());
assert!(decode(&bad_table, &Limits::default()).is_err());
let mut overrun = good.clone();
let start = be32(&overrun, HEADER) as usize;
overrun[start] = 127; // a run longer than the 16-pixel row
assert!(decode(&overrun, &Limits::default()).is_err());
// A huge declared size is stopped by the limits or the length check, before allocating.
let mut huge = good;
huge[6..8].copy_from_slice(&u16::MAX.to_be_bytes());
huge[8..10].copy_from_slice(&u16::MAX.to_be_bytes());
assert!(decode(&huge, &Limits::default()).is_err());
}
/// A 1-plane, 8-bit RLE file whose `rows` all start where `starts[i]` says, each `len` bytes long.
fn rle_file(w: u16, starts: &[u32], len: u32, payload: &[u8]) -> Vec<u8> {
let mut f = vec![0u8; HEADER];
f[0..2].copy_from_slice(&MAGIC.to_be_bytes());
f[2] = 1;
f[3] = 1;
f[4..6].copy_from_slice(&2u16.to_be_bytes());
f[6..8].copy_from_slice(&w.to_be_bytes());
f[8..10].copy_from_slice(&(starts.len() as u16).to_be_bytes());
f[10..12].copy_from_slice(&1u16.to_be_bytes());
for s in starts {
f.extend_from_slice(&s.to_be_bytes());
}
for _ in starts {
f.extend_from_slice(&len.to_be_bytes());
}
f.extend_from_slice(payload);
f
}
/// The format lets table entries share one compressed row: eight rows, one payload.
#[test]
fn rows_may_share_compressed_data() {
let table = HEADER + 8 * 8;
let f = rle_file(1, &[table as u32; 8], 3, &[0x81, 255, 0]);
assert_eq!(f.len(), table + 3);
let img = decode(&f, &Limits::default()).unwrap();
assert_eq!((img.dimensions(), img.data()), ((1, 8), &[255u8; 8][..]));
}
/// Compressed rows start after both tables: an offset into a table is refused.
#[test]
fn rows_may_not_start_inside_the_offset_tables() {
// 3x1: the tables end at 520; 515 reads the table's own bytes as a packet list.
let f = rle_file(3, &[515], 3, &[0, 0, 0, 0, 0]);
assert!(decode(&f, &Limits::default()).is_err());
}
/// FFmpeg's decoder, run as an oracle, also reads a row that ends with its last packet and no
/// zero terminator; so does this one.
#[test]
fn a_row_without_its_terminator_still_reads() {
let f = rle_file(1, &[520], 2, &[0x81, 42]);
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[42]);
}
#[test]
fn palette_mapped_files_are_unsupported() {
let img = gradient(4, 4, ChannelLayout::Gray);
let mut f = encode(&img, Plan { layout: ChannelLayout::Gray, sample: SampleType::U8 }).unwrap();
f[104..108].copy_from_slice(&1u32.to_be_bytes());
assert!(matches!(decode(&f, &Limits::default()), Err(CodecError::Unsupported { .. })));
}
}
+15 -2
View File
@@ -45,6 +45,8 @@ pub enum Format {
Iff,
/// JPEG 2000 (JP2 container or raw J2K codestream): read-only; see [`ASYMMETRIC_EXCEPTIONS`].
Jp2,
/// SGI image files (`.sgi`, `.rgb`, `.bw`): 8- and 16-bit gray, gray + alpha, RGB and RGBA.
Sgi,
/// JPEG-LS (`.jls`): lossless and near-lossless gray and RGB.
JpegLs,
}
@@ -118,7 +120,7 @@ const TIFF_LAYOUTS: &[ChannelLayout] = &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L::
impl Format {
/// Every format known to the crate (enabled or not).
pub const ALL: [Format; 25] = [
pub const ALL: [Format; 26] = [
Format::Png,
Format::Jpeg,
Format::Jp2,
@@ -143,6 +145,7 @@ impl Format {
Format::Cineon,
Format::Dicom,
Format::Iff,
Format::Sgi,
Format::JpegLs,
];
@@ -182,6 +185,7 @@ impl Format {
Format::Dicom => "DICOM",
Format::Iff => "Amiga IFF",
Format::Jp2 => "JPEG 2000",
Format::Sgi => "SGI",
Format::JpegLs => "JPEG-LS",
}
}
@@ -214,6 +218,7 @@ impl Format {
Format::Dicom => &["dcm", "dicom", "dc3"],
Format::Iff => &["iff", "lbm", "ilbm"],
Format::Jp2 => &["jp2", "j2k", "j2c", "jpf", "jpx"],
Format::Sgi => &["sgi", "rgb", "rgba", "bw"],
Format::JpegLs => &["jls"],
}
}
@@ -244,6 +249,7 @@ impl Format {
Format::Dicom => "application/dicom",
Format::Iff => "image/x-iff",
Format::Jp2 => "image/jp2",
Format::Sgi => "image/x-sgi",
Format::JpegLs => "image/jls",
}
}
@@ -251,7 +257,7 @@ impl Format {
/// Maximum encodable dimensions, if the format has a hard limit.
pub fn max_dimensions(self) -> Option<(u32, u32)> {
match self {
Format::Jpeg | Format::Gif | Format::Iff | Format::JpegLs => Some((65535, 65535)),
Format::Jpeg | Format::Gif | Format::Iff | Format::Sgi | Format::JpegLs => Some((65535, 65535)),
Format::WebP => Some((16384, 16384)),
Format::Ico => Some((256, 256)),
_ => None,
@@ -321,6 +327,9 @@ pub fn caps(format: Format) -> FormatCaps {
Format::Cineon => FormatCaps { depths: &[S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
Format::Dicom => FormatCaps { read: true, write: false, depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
Format::Iff => FormatCaps { depths: &[S::U8], layouts: &[L::Rgb, L::Rgba], alpha: true, ..base },
Format::Sgi => {
FormatCaps { read: cfg!(feature = "sgi"), write: cfg!(feature = "sgi"), depths: &[S::U8, S::U16], layouts: RGB_GRAY, alpha: true, ..base }
}
Format::JpegLs => FormatCaps { depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
Format::Jp2 => FormatCaps { read: true, write: false, layouts: &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L::Cmyk, L::CmykA], icc: true, ..base },
}
@@ -403,6 +412,10 @@ pub fn detect(bytes: &[u8]) -> Option<Format> {
if b.starts_with(b"/* XPM */") || b.starts_with(b"! XPM2\n") || b.starts_with(b"! XPM2\r\n") {
return Some(Format::Xpm);
}
#[cfg(feature = "sgi")]
if crate::codecs::sgi::looks_like_sgi(b) {
return Some(Format::Sgi);
}
if crate::codecs::pcx::looks_like_pcx(b) {
return Some(Format::Pcx);
}
+30
View File
@@ -67,6 +67,8 @@ pub use crate::resolution::{exif_resolution, export_exif, export_xmp, photoshop_
pub use crate::source_settings::{WebpEncoding, jpeg_quality, webp_encoding};
pub use half::f16;
#[cfg(feature = "sgi")]
use crate::codecs::sgi;
use crate::codecs::{dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm};
/// Detect the format and decode with default [`Limits`].
@@ -106,6 +108,16 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
Format::Cineon => dpx::decode_cineon(bytes, l),
Format::Dicom => dicom::decode(bytes, l),
Format::Iff => iff::decode(bytes, l),
Format::Sgi => {
#[cfg(feature = "sgi")]
{
sgi::decode(bytes, l)
}
#[cfg(not(feature = "sgi"))]
{
Err(CodecError::unsupported(format, "SGI support requires the sgi feature"))
}
}
Format::JpegLs => jpegls::decode(bytes, l),
Format::OpenExr => exr::decode(bytes, l),
Format::Heif => heif::decode(bytes, l, opts.keep_orientation),
@@ -120,6 +132,14 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
finish_format(format, bytes, img, opts)
}
/// Decode AVIF pixels and report their original AV1 bit depth (8, 10 or 12).
/// The returned image may use a wider sample type, particularly for linear HDR.
#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))]
pub fn decode_avif_with_depth(bytes: &[u8], opts: &DecodeOptions) -> Result<(Image, u8), CodecError> {
let (image, depth) = codecs::avif_decode::decode_with_depth(bytes, &opts.limits, opts.keep_orientation)?;
Ok((finish_format(Format::Avif, bytes, image, opts)?, depth))
}
/// What [`decode_as_with`] does with a PNG decoded in steps by [`PngDecodeRows`] (`img` is its
/// output): the resolution from the metadata, the upright orientation and the final limit check,
/// so the result equals [`decode_as_with`]'s on the same file.
@@ -473,6 +493,16 @@ pub fn encode(image: &Image, format: Format, opts: &EncodeOptions) -> Result<Vec
Format::Cineon => dpx::encode_cineon(image, plan),
Format::Dicom => Err(CodecError::unsupported(format, "DICOM is import-only; save as TIFF or PNG instead")),
Format::Iff => iff::encode(image, plan),
Format::Sgi => {
#[cfg(feature = "sgi")]
{
sgi::encode(image, plan)
}
#[cfg(not(feature = "sgi"))]
{
Err(CodecError::unsupported(format, "SGI support requires the sgi feature"))
}
}
Format::JpegLs => jpegls::encode(image, plan),
Format::OpenExr => exr::encode(image, plan, opts),
Format::Heif => Err(CodecError::unsupported(format, "encoding is not available in this build")),
@@ -62,6 +62,8 @@ fn default_build_formats_all_symmetric() {
Format::Dpx,
Format::Cineon,
Format::Iff,
#[cfg(feature = "sgi")]
Format::Sgi,
Format::JpegLs,
];
for f in expected {
@@ -224,6 +226,23 @@ fn detect_magic_iff() {
assert_eq!(detect(b"FORM\0\0\0\x14AIFF"), None);
assert!(decode_as(Format::Iff, b"FORM\0\0\0\x14ILBM").is_err());
}
#[cfg(feature = "sgi")]
#[test]
fn detect_magic_sgi() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
let img = Image::from_u8(5, 3, ChannelLayout::Rgba, (0..60).collect()).unwrap();
let bytes = encode(&img, Format::Sgi, &EncodeOptions::default()).unwrap();
assert_eq!(detect(&bytes), Some(Format::Sgi));
assert_eq!(from_extension("scan.RGB"), Some(Format::Sgi));
assert_eq!(from_extension("scan.bw"), Some(Format::Sgi));
assert_eq!(decode(&bytes).unwrap().data(), img.data());
assert!(!ASYMMETRIC_EXCEPTIONS.iter().any(|(f, _)| *f == Format::Sgi));
assert_eq!(Format::Sgi.max_dimensions(), Some((65535, 65535)));
// The magic number alone, without a full header, is not SGI.
assert_eq!(detect(&[0x01, 0xda, 0, 1, 0, 2]), None);
assert!(decode_as(Format::Sgi, &[0x01, 0xda]).is_err());
}
#[test]
fn detect_magic_jpeg_ls() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
@@ -328,3 +347,16 @@ fn extensions_unique_across_formats() {
all.dedup();
assert_eq!(all.len(), n);
}
#[cfg(not(feature = "sgi"))]
#[test]
fn sgi_requires_explicit_feature() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
assert!(!caps(Format::Sgi).read && !caps(Format::Sgi).write);
assert!(decode_as(Format::Sgi, &[]).is_err());
let image = Image::from_u8(1, 1, ChannelLayout::Gray, vec![0]).unwrap();
assert!(encode(&image, Format::Sgi, &EncodeOptions::default()).is_err());
let mut header = vec![0; 512];
header[..12].copy_from_slice(&[1, 218, 0, 1, 0, 2, 0, 1, 0, 1, 0, 1]);
assert_eq!(detect(&header), None);
}
+2 -2
View File
@@ -899,7 +899,7 @@ struct Layer {
impl Layer {
/// The layer and its flat array of `width` numbers per entry.
fn of(geom: &Record, element: &str, data: &str, index: &[&str], width: usize) -> Option<(Layer, Vec<f64>)> {
let e = geom.child(element)?;
let e = geom.children_named(element).min_by_key(|e| e.props.first().and_then(Prop::as_i64) != Some(0))?;
let mapping = match e.child("MappingInformationType").and_then(|m| m.str_at(0)).unwrap_or("ByPolygonVertex") {
"ByPolygonVertex" => Mapping::PolygonVertex,
"ByVertex" | "ByVertice" | "ByControlPoint" => Mapping::Vertex,
@@ -908,7 +908,7 @@ impl Layer {
_ => return None,
};
let reference = e.child("ReferenceInformationType").and_then(|m| m.str_at(0)).unwrap_or("Direct");
let index = (reference != "Direct").then(|| index.iter().find_map(|n| e.child(n)).map(Record::ints)).flatten();
let index = if matches!(reference, "IndexToDirect" | "Index") { Some(index.iter().find_map(|n| e.child(n))?.ints()) } else { None };
let data = e.child(data)?.floats();
Some((Layer { mapping, index, len: data.len() / width }, data))
}
+10
View File
@@ -628,6 +628,16 @@ fn fbx6_model_holds_its_mesh() {
}
}
#[test]
fn layer_zero_takes_precedence_and_an_indexed_layer_needs_indices() {
let layer_one = "LayerElementUV: 1 { MappingInformationType: \"ByPolygonVertex\" ReferenceInformationType: \"Direct\" UV: 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25 }\n";
let text = ASCII.replace("LayerElementUV: 0 {", &format!("{layer_one} LayerElementUV: 0 {{"));
let scene = parse(text.as_bytes()).unwrap();
assert_eq!(scene.meshes[0].uv_sets[0][0], [0.0, 1.0]);
let missing = ASCII.replace("UVIndex: *7", "UnusedIndex: *7");
assert!(parse(missing.as_bytes()).unwrap().meshes[0].uv_sets.is_empty());
}
#[test]
fn optional_inactive_pre_rotation_keeps_legacy_placements() {
let text = "FBXHeaderExtension: { FBXVersion: 7400 } Objects: { Model: 1, \"Model::A\", \"Null\" { Properties70: { P: \"PreRotation\", \"\", \"\", \"\", 0,0,90 } } }";
+4
View File
@@ -3,6 +3,10 @@
//! Callers map their page and color models to these neutral records. This module owns PDF object
//! layout, image streams, ICC/Separation resources, page labels, marks, xref and trailer syntax.
/// Multi-image 8/16-bit print pages with per-image ICC resources and explicit byte budgets.
#[path = "raster_print.rs"]
pub mod print;
use std::collections::BTreeSet;
use std::fmt::Write as _;
use std::io::Write as _;
+658
View File
@@ -0,0 +1,658 @@
//! A small PDF writer for print output: pages of a given size in points, each with raster images
//! placed on it, tagged with their ICC profile (`ICCBased`) so the numbers mean the same colour in
//! every reader and RIP, Flate-compressed (lossless) at 8 or 16 bits per component, or carried as
//! JPEG (`DCTDecode`) where small files matter more than exact samples. Optionally an output intent:
//! the printing condition (the printer / paper profile) the file was made for, as PDF/X readers
//! expect it in the catalog. Profiles shared by several images are written once.
//!
//! The file is PDF 1.6 (16-bit images need 1.5): classic cross-reference table, no object streams,
//! deterministic (no dates or IDs), so the same pages always give the same bytes.
//!
//! Encoded JPEG and Flate payloads are copied, not decoded or verified against declared
//! dimensions. Callers own their encoding validity and ICC/profile-channel compatibility.
//! An output intent describes a printing condition; this API does not certify PDF/X.
//! Byte limits bound accounted input and emitted output, not peak process memory or
//! caller-owned image construction. This is not the streaming slideshow writer.
//!
//! Sources: ISO 32000-1:2008 (§7.3 objects, §7.5 file structure, §8.6.5.5 ICCBased colour spaces,
//! §8.9 images, §14.11.5 output intents). Own design.
use std::collections::HashMap;
use std::io::Write as _;
use std::sync::Arc;
/// Pages per document: a print job beyond this is a mistake, not a book.
pub const MAX_PAGES: usize = 1000;
/// Images per page.
pub const MAX_IMAGES_PER_PAGE: usize = 256;
/// Largest page side in points (200 inches): more than any printer's roll.
pub const MAX_PAGE_POINTS: f64 = 14_400.0;
/// Largest image, in pixels per side.
pub const MAX_IMAGE_SIDE: u32 = 65_535;
/// Serialization budgets; input records already belong to the caller. These are not a
/// process-wide peak-memory guarantee. Repeated references count toward the input budget.
#[derive(Clone, Copy, Debug)]
pub struct Limits {
pub input_bytes: usize,
pub output_bytes: usize,
}
impl Default for Limits {
fn default() -> Self {
Self { input_bytes: 512 * 1024 * 1024, output_bytes: 512 * 1024 * 1024 }
}
}
/// How an image's samples are stored.
#[derive(Clone, Debug, PartialEq)]
pub enum ImageData {
/// Interleaved samples, `bits` (8 or 16) per component, 16-bit values big-endian: compressed
/// losslessly in the file.
Raw { bits: u8, samples: Vec<u8> },
/// A baseline JPEG stream, copied into the file as it is.
Jpeg(Vec<u8>),
/// Samples already compressed with zlib (see [`PdfImage::deflated`]), `bits` per component.
Flate { bits: u8, data: Vec<u8> },
}
/// A raster image and the colour space its samples are in.
#[derive(Clone, Debug, PartialEq)]
pub struct PdfImage {
pub width: u32,
pub height: u32,
/// Components per pixel: 1 (grey), 3 (RGB) or 4 (CMYK).
pub channels: u8,
pub data: ImageData,
/// The ICC profile the samples are in; without one the image is device colour.
pub icc: Option<Arc<Vec<u8>>>,
}
impl PdfImage {
/// An image from interleaved 16-bit samples.
pub fn from_u16(width: u32, height: u32, channels: u8, samples: &[u16], icc: Option<Arc<Vec<u8>>>) -> PdfImage {
let bytes = samples.iter().flat_map(|v| v.to_be_bytes()).collect();
PdfImage { width, height, channels, data: ImageData::Raw { bits: 16, samples: bytes }, icc }
}
/// The same image with its samples compressed now (checked first), so a long print doesn't
/// hold every page's uncompressed samples until the file is written.
pub fn deflated(self) -> Result<PdfImage, String> {
self.check()?;
if matches!(&self.data, ImageData::Raw { samples, .. } if samples.len() > Limits::default().input_bytes) {
return Err("PDF input byte limit exceeded".into());
}
let data = match self.data {
ImageData::Raw { bits, samples } => ImageData::Flate { bits, data: deflate(&samples, Limits::default().output_bytes)? },
other => other,
};
Ok(PdfImage { data, ..self })
}
fn check(&self) -> Result<(), String> {
if self.width == 0 || self.height == 0 || self.width > MAX_IMAGE_SIDE || self.height > MAX_IMAGE_SIDE {
return Err(format!("an image of {} × {} pixels can't be written", self.width, self.height));
}
if !matches!(self.channels, 1 | 3 | 4) {
return Err(format!("images have 1, 3 or 4 components, not {}", self.channels));
}
match &self.data {
ImageData::Raw { bits, samples } => {
if !matches!(bits, 8 | 16) {
return Err(format!("images have 8 or 16 bits per component, not {bits}"));
}
let want = u64::from(self.width) * u64::from(self.height) * u64::from(self.channels) * u64::from(bits / 8);
if samples.len() as u64 != want {
return Err(format!("the image needs {want} bytes of samples, not {}", samples.len()));
}
}
ImageData::Jpeg(data) => {
if !data.starts_with(&[0xFF, 0xD8]) {
return Err("the image isn't a JPEG stream".into());
}
}
ImageData::Flate { bits, data } => {
if !matches!(bits, 8 | 16) || data.is_empty() {
return Err("a compressed image has 8 or 16 bits per component and some data".into());
}
}
}
Ok(())
}
}
/// Where an image goes on its page, in points from the page's bottom-left corner.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Placement {
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
}
/// One page: its size in points and the images on it, drawn in order (later ones on top).
#[derive(Clone, Debug, PartialEq)]
pub struct PdfPage {
pub width: f64,
pub height: f64,
pub images: Vec<(PdfImage, Placement)>,
}
/// The printing condition the document is made for (PDF/X-style output intent).
#[derive(Clone, Debug, PartialEq)]
pub struct OutputIntent {
/// The printer / paper profile.
pub icc: Arc<Vec<u8>>,
/// Its components: 1, 3 or 4.
pub channels: u8,
/// A name for the condition (the profile's description).
pub condition: String,
}
/// Document-level settings.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct PdfInfo {
pub title: Option<String>,
pub output_intent: Option<OutputIntent>,
/// Application name, not a PDF/X conformance claim.
pub producer: String,
}
fn finite(v: f64) -> bool {
v.is_finite()
}
/// A number as PDF writes it: up to four decimals, no exponent, no trailing zeros.
fn num(v: f64) -> String {
let s = format!("{:.4}", if finite(v) { v } else { 0.0 });
let s = s.trim_end_matches('0').trim_end_matches('.');
if s == "-0" || s.is_empty() { "0".into() } else { s.into() }
}
/// A text string: plain ASCII as a literal string, anything else as UTF-16BE with a byte order mark.
fn text(s: &str) -> String {
if s.bytes().all(|b| (0x20..0x7f).contains(&b)) {
let mut out = String::with_capacity(s.len() + 2);
out.push('(');
for c in s.chars() {
if matches!(c, '(' | ')' | '\\') {
out.push('\\');
}
out.push(c);
}
out.push(')');
out
} else {
super::pdf_unicode_string(s)
}
}
struct Writer {
out: Vec<u8>,
offsets: Vec<usize>,
limit: usize,
}
impl Writer {
fn put(&mut self, bytes: &[u8]) -> Result<(), String> {
let end = self.out.len().checked_add(bytes.len()).ok_or("PDF output size overflow")?;
if end > self.limit {
return Err("PDF output byte limit exceeded".into());
}
self.out.try_reserve(bytes.len()).map_err(|_| "PDF output allocation failed")?;
self.out.extend_from_slice(bytes);
Ok(())
}
fn reserve(&mut self) -> Result<usize, String> {
self.offsets.try_reserve(1).map_err(|_| "PDF object allocation failed")?;
self.offsets.push(0);
Ok(self.offsets.len())
}
fn object(&mut self, id: usize, dict: &str, stream: Option<&[u8]>) -> Result<(), String> {
let pos = self.out.len();
*self.offsets.get_mut(id.checked_sub(1).ok_or("invalid PDF object")?).ok_or("invalid PDF object")? = pos;
self.put(format!("{id} 0 obj\n{dict}").as_bytes())?;
if let Some(data) = stream {
self.put(b"\nstream\n")?;
self.put(data)?;
self.put(b"\nendstream")?;
}
self.put(b"\nendobj\n")
}
}
fn deflate(data: &[u8], max: usize) -> Result<Vec<u8>, String> {
struct Sink {
bytes: Vec<u8>,
max: usize,
}
impl std::io::Write for Sink {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
if bytes.len() > self.max.saturating_sub(self.bytes.len()) {
return Err(std::io::Error::other("PDF compressed byte limit exceeded"));
}
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
self.bytes.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let mut encoder = flate2::write::ZlibEncoder::new(Sink { bytes: Vec::new(), max }, flate2::Compression::new(6));
encoder.write_all(data).map_err(|e| e.to_string())?;
Ok(encoder.finish().map_err(|e| e.to_string())?.bytes)
}
/// Write a document. Errors name the first page or image that can't be written.
pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result<Vec<u8>, String> {
write_with_limits(pages, info, Limits::default())
}
/// Write with explicit total input and serialized output budgets. ICC bytes are opaque:
/// callers validate their profile semantics; resources are keyed by bytes AND channels.
pub fn write_with_limits(pages: &[PdfPage], info: &PdfInfo, limits: Limits) -> Result<Vec<u8>, String> {
let mut input = 0usize;
let mut account = |n: usize| -> Result<(), String> {
input = input.checked_add(n).ok_or("PDF input size overflow")?;
if input > limits.input_bytes {
return Err("PDF input byte limit exceeded".into());
}
Ok(())
};
account(info.producer.len())?;
account(info.title.as_ref().map_or(0, String::len))?;
if let Some(intent) = &info.output_intent {
account(intent.icc.len())?;
account(intent.condition.len())?;
}
if pages.is_empty() {
return Err("there are no pages to write".into());
}
if pages.len() > MAX_PAGES {
return Err(format!("a document holds at most {MAX_PAGES} pages"));
}
for (n, page) in pages.iter().enumerate() {
let n = n + 1;
if !(finite(page.width) && finite(page.height))
|| page.width <= 0.0
|| page.height <= 0.0
|| page.width > MAX_PAGE_POINTS
|| page.height > MAX_PAGE_POINTS
{
return Err(format!("page {n} has no usable size"));
}
if page.images.len() > MAX_IMAGES_PER_PAGE {
return Err(format!("page {n} has more than {MAX_IMAGES_PER_PAGE} images"));
}
for (image, at) in &page.images {
account(match &image.data {
ImageData::Raw { samples, .. } => samples.len(),
ImageData::Jpeg(data) | ImageData::Flate { data, .. } => data.len(),
})?;
if let Some(icc) = &image.icc {
account(icc.len())?;
}
image.check().map_err(|e| format!("page {n}: {e}"))?;
if ![at.x, at.y, at.width, at.height].into_iter().all(finite) || at.width <= 0.0 || at.height <= 0.0 {
return Err(format!("page {n}: an image has no usable placement"));
}
}
}
if let Some(intent) = &info.output_intent
&& !matches!(intent.channels, 1 | 3 | 4)
{
return Err("the output intent's profile has 1, 3 or 4 components".into());
}
let mut w = Writer { out: Vec::new(), offsets: Vec::new(), limit: limits.output_bytes };
w.put(b"%PDF-1.6\n%\xE2\xE3\xCF\xD3\n")?;
let catalog = w.reserve()?;
let pages_id = w.reserve()?;
let info_id = w.reserve()?;
// ICC profiles, each written once (keyed by content) with its component count
let mut profiles: HashMap<(Arc<Vec<u8>>, u8), usize> = HashMap::new();
let mut profile = |w: &mut Writer, icc: &Arc<Vec<u8>>, n: u8| -> Result<usize, String> {
if let Some(&id) = profiles.get(&(Arc::clone(icc), n)) {
return Ok(id);
}
let id = w.reserve()?;
let data = deflate(icc, limits.output_bytes)?;
let alternate = match n {
1 => "/DeviceGray",
4 => "/DeviceCMYK",
_ => "/DeviceRGB",
};
w.object(id, &format!("<< /N {n} /Alternate {alternate} /Filter /FlateDecode /Length {} >>", data.len()), Some(&data))?;
profiles.try_reserve(1).map_err(|_| "PDF profile allocation failed")?;
profiles.insert((Arc::clone(icc), n), id);
Ok(id)
};
let intent_id = info.output_intent.as_ref().map(|intent| profile(&mut w, &intent.icc, intent.channels)).transpose()?;
let mut kids = Vec::with_capacity(pages.len());
for page in pages {
let page_id = w.reserve()?;
let mut xobjects = String::new();
let mut content = String::new();
for (k, (image, at)) in page.images.iter().enumerate() {
let space = match &image.icc {
Some(icc) => format!("[/ICCBased {} 0 R]", profile(&mut w, icc, image.channels)?),
None => match image.channels {
1 => "/DeviceGray".into(),
4 => "/DeviceCMYK".into(),
_ => "/DeviceRGB".into(),
},
};
let id = w.reserve()?;
let (bits, filter, data): (u8, &str, std::borrow::Cow<[u8]>) = match &image.data {
ImageData::Raw { bits, samples } => (*bits, "/FlateDecode", deflate(samples, limits.output_bytes)?.into()),
ImageData::Flate { bits, data } => (*bits, "/FlateDecode", data.as_slice().into()),
ImageData::Jpeg(jpeg) => (8, "/DCTDecode", jpeg.as_slice().into()),
};
w.object(
id,
&format!(
"<< /Type /XObject /Subtype /Image /Width {} /Height {} /ColorSpace {space} /BitsPerComponent {bits} /Filter {filter} /Length {} >>",
image.width,
image.height,
data.len()
),
Some(&data),
)?;
xobjects.push_str(&format!("/Im{k} {id} 0 R "));
content.push_str(&format!("q {} 0 0 {} {} {} cm /Im{k} Do Q\n", num(at.width), num(at.height), num(at.x), num(at.y)));
}
let content_id = w.reserve()?;
let stream = deflate(content.as_bytes(), limits.output_bytes)?;
w.object(content_id, &format!("<< /Filter /FlateDecode /Length {} >>", stream.len()), Some(&stream))?;
w.object(
page_id,
&format!(
"<< /Type /Page /Parent {pages_id} 0 R /MediaBox [0 0 {} {}] /Resources << /XObject << {xobjects}>> >> /Contents {content_id} 0 R >>",
num(page.width),
num(page.height)
),
None,
)?;
kids.push(format!("{page_id} 0 R"));
}
w.object(pages_id, &format!("<< /Type /Pages /Count {} /Kids [{}] >>", kids.len(), kids.join(" ")), None)?;
let intents = match (&info.output_intent, intent_id) {
(Some(intent), Some(id)) => format!(
" /OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX /OutputConditionIdentifier (Custom) /Info {} /DestOutputProfile {id} 0 R >>]",
text(&intent.condition)
),
_ => String::new(),
};
w.object(catalog, &format!("<< /Type /Catalog /Pages {pages_id} 0 R{intents} >>"), None)?;
let title = info.title.as_deref().map(|t| format!(" /Title {}", text(t))).unwrap_or_default();
w.object(info_id, &format!("<< /Producer {}{title} >>", text(&info.producer)), None)?;
let xref = w.out.len();
let mut table = format!("xref\n0 {}\n0000000000 65535 f \n", w.offsets.len() + 1);
for offset in &w.offsets {
table.push_str(&format!("{offset:010} 00000 n \n"));
}
table.push_str(&format!("trailer\n<< /Size {} /Root {catalog} 0 R /Info {info_id} 0 R >>\nstartxref\n{xref}\n%%EOF\n", w.offsets.len() + 1));
w.put(table.as_bytes())?;
Ok(w.out)
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use super::*;
/// The body of object `id` (dictionary and raw stream bytes), found through the xref table.
fn object(pdf: &[u8], id: usize) -> (String, Vec<u8>) {
// byte offsets: the streams are binary, so search the bytes, not a lossy string
let start = pdf.windows(10).rposition(|w| w == b"startxref\n").unwrap() + 10;
let tail = std::str::from_utf8(&pdf[start..]).unwrap();
let xref: usize = tail.lines().next().unwrap().trim().parse().unwrap();
let table = std::str::from_utf8(&pdf[xref..]).unwrap();
let line = table.lines().nth(2 + id).unwrap();
let offset: usize = line[..10].parse().unwrap();
let head = format!("{id} 0 obj\n");
assert_eq!(&pdf[offset..offset + head.len()], head.as_bytes(), "xref entry for {id}");
let rest = &pdf[offset + head.len()..];
let end = rest.windows(7).position(|w| w == b"\nendobj").unwrap();
let body = &rest[..end];
match body.windows(8).position(|w| w == b"\nstream\n") {
Some(p) => {
let data = &body[p + 8..body.len() - "\nendstream".len()];
(String::from_utf8_lossy(&body[..p]).into_owned(), data.to_vec())
}
None => (String::from_utf8_lossy(body).into_owned(), vec![]),
}
}
fn inflate(data: &[u8]) -> Vec<u8> {
{
use std::io::Read;
let mut out = Vec::new();
flate2::read::ZlibDecoder::new(data).read_to_end(&mut out).unwrap();
out
}
}
/// Object ids of every image XObject, in file order.
fn images(pdf: &[u8]) -> Vec<usize> {
let s = String::from_utf8_lossy(pdf);
s.match_indices(" 0 obj\n<< /Type /XObject /Subtype /Image").map(|(i, _)| s[..i].rsplit('\n').next().unwrap().parse().unwrap()).collect()
}
fn reference(dict: &str, key: &str) -> usize {
let at = dict.find(key).unwrap() + key.len();
dict[at..].trim_start().split(' ').next().unwrap().parse().unwrap()
}
fn srgb() -> Arc<Vec<u8>> {
Arc::new(b"opaque RGB profile fixture".to_vec())
}
fn page(images: Vec<(PdfImage, Placement)>) -> PdfPage {
PdfPage { width: 612.0, height: 792.0, images }
}
fn full(w: f64, h: f64) -> Placement {
Placement { x: 0.0, y: 0.0, width: w, height: h }
}
#[test]
fn a_16_bit_icc_image_round_trips() {
let samples: Vec<u16> = (0..2 * 3 * 3).map(|i| (i * 3851) as u16).collect();
let img = PdfImage::from_u16(3, 2, 3, &samples, Some(srgb()));
let pdf = write(&[page(vec![(img, full(612.0, 792.0))])], &PdfInfo::default()).unwrap();
assert!(pdf.starts_with(b"%PDF-1.6\n"));
assert!(pdf.ends_with(b"%%EOF\n"));
let ids = images(&pdf);
assert_eq!(ids.len(), 1);
let (dict, data) = object(&pdf, ids[0]);
assert!(dict.contains("/Width 3 /Height 2") && dict.contains("/BitsPerComponent 16") && dict.contains("/FlateDecode"), "{dict}");
let back: Vec<u16> = inflate(&data).chunks(2).map(|b| u16::from_be_bytes([b[0], b[1]])).collect();
assert_eq!(back, samples);
let (profile, icc) = object(&pdf, reference(&dict, "/ICCBased"));
assert!(profile.contains("/N 3") && profile.contains("/Alternate /DeviceRGB"), "{profile}");
assert_eq!(inflate(&icc), *srgb());
}
#[test]
fn pages_sizes_placements_and_a_shared_profile() {
let px = |v: u16| PdfImage::from_u16(1, 1, 3, &[v, v, v], Some(srgb()));
let at = Placement { x: 36.0, y: 72.5, width: 288.0, height: 216.0 };
let pages = [page(vec![(px(1), at), (px(2), full(612.0, 792.0))]), PdfPage { width: 841.89, height: 595.28, images: vec![(px(3), at)] }];
let pdf = write(&pages, &PdfInfo::default()).unwrap();
let s = String::from_utf8_lossy(&pdf);
assert!(s.contains("/Type /Pages /Count 2"), "two pages");
assert!(s.contains("/MediaBox [0 0 612 792]") && s.contains("/MediaBox [0 0 841.89 595.28]"));
// the sRGB profile is written once for all three images
assert_eq!(s.matches("/N 3 /Alternate").count(), 1);
let first: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
assert!(first.windows(2).all(|w| w[0] == w[1]), "{first:?}");
// placements reach the content streams
let contents: Vec<String> = s
.match_indices("/Contents ")
.map(|(i, _)| s[i + 10..].split(' ').next().unwrap().parse::<usize>().unwrap())
.map(|id| String::from_utf8(inflate(&object(&pdf, id).1)).unwrap())
.collect();
assert_eq!(contents[0], "q 288 0 0 216 36 72.5 cm /Im0 Do Q\nq 612 0 0 792 0 0 cm /Im1 Do Q\n");
assert_eq!(write(&pages, &PdfInfo::default()).unwrap(), pdf, "deterministic");
}
#[test]
fn output_intent_and_title() {
let cmyk_like = srgb();
let info = PdfInfo {
title: Some("Garden – prints (2)".into()),
producer: "Fixture".into(),
output_intent: Some(OutputIntent { icc: cmyk_like.clone(), channels: 3, condition: "Glossy (Photo) \\ test".into() }),
};
let img = PdfImage::from_u16(1, 1, 3, &[0, 0, 0], Some(cmyk_like));
let pdf = write(&[page(vec![(img, full(10.0, 10.0))])], &info).unwrap();
let (catalog, _) = object(&pdf, 1);
assert!(catalog.contains("/OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX"), "{catalog}");
assert!(catalog.contains("/Info (Glossy \\(Photo\\) \\\\ test)"), "{catalog}");
// the intent and the image share one profile object
let dest = reference(&catalog, "/DestOutputProfile");
assert_eq!(reference(&object(&pdf, images(&pdf)[0]).0, "/ICCBased"), dest);
let (info_dict, _) = object(&pdf, 3);
assert!(info_dict.contains("/Title <FEFF"), "non-ASCII titles are UTF-16: {info_dict}");
assert!(info_dict.contains("2013"), "the en dash: {info_dict}");
}
#[test]
fn cmyk_grey_jpeg_and_device_images() {
let jpeg = vec![0xFF, 0xD8, 0xFF, 0xD9];
let imgs = vec![
(
PdfImage { width: 1, height: 1, channels: 4, data: ImageData::Raw { bits: 8, samples: vec![0, 10, 20, 30] }, icc: None },
full(1.0, 1.0),
),
(PdfImage { width: 2, height: 1, channels: 1, data: ImageData::Raw { bits: 8, samples: vec![0, 255] }, icc: None }, full(1.0, 1.0)),
(PdfImage { width: 4, height: 4, channels: 3, data: ImageData::Jpeg(jpeg.clone()), icc: None }, full(1.0, 1.0)),
];
let pdf = write(&[page(imgs)], &PdfInfo::default()).unwrap();
let ids = images(&pdf);
let dicts: Vec<_> = ids.iter().map(|&id| object(&pdf, id)).collect();
assert!(dicts[0].0.contains("/ColorSpace /DeviceCMYK") && dicts[0].0.contains("/BitsPerComponent 8"));
assert!(dicts[1].0.contains("/ColorSpace /DeviceGray"));
assert!(dicts[2].0.contains("/DCTDecode") && dicts[2].0.contains("/DeviceRGB"));
assert_eq!(dicts[2].1, jpeg, "JPEG streams are copied as they are");
}
#[test]
fn compressing_early_writes_the_same_file() {
let samples: Vec<u16> = (0..4 * 3 * 3).map(|i| (i * 977) as u16).collect();
let raw = PdfImage::from_u16(4, 3, 3, &samples, Some(srgb()));
let early = raw.clone().deflated().unwrap();
assert!(matches!(early.data, ImageData::Flate { bits: 16, .. }));
let at = full(100.0, 75.0);
let a = write(&[page(vec![(raw, at)])], &PdfInfo::default()).unwrap();
let b = write(&[page(vec![(early, at)])], &PdfInfo::default()).unwrap();
assert_eq!(a, b);
let mut short = PdfImage::from_u16(4, 3, 3, &samples, None);
short.width = 5;
assert!(short.deflated().is_err(), "checked before compressing");
}
#[test]
fn hostile_documents_are_errors() {
let ok = || PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
let at = full(10.0, 10.0);
let bad = |pages: Vec<PdfPage>| write(&pages, &PdfInfo::default()).is_err();
assert!(bad(vec![]));
assert!(bad(vec![PdfPage { width: f64::NAN, height: 10.0, images: vec![] }]));
assert!(bad(vec![PdfPage { width: -1.0, height: 10.0, images: vec![] }]));
assert!(bad(vec![PdfPage { width: 1e9, height: 10.0, images: vec![] }]));
assert!(bad(vec![page(vec![(ok(), Placement { x: f64::INFINITY, ..at })])]));
assert!(bad(vec![page(vec![(ok(), Placement { width: 0.0, ..at })])]));
let mut short = ok();
short.data = ImageData::Raw { bits: 16, samples: vec![0; 5] };
assert!(bad(vec![page(vec![(short, at)])]));
let mut empty = ok();
empty.width = 0;
assert!(bad(vec![page(vec![(empty, at)])]));
let mut odd = ok();
odd.channels = 2;
assert!(bad(vec![page(vec![(odd, at)])]));
let mut not_jpeg = ok();
not_jpeg.data = ImageData::Jpeg(vec![1, 2, 3]);
assert!(bad(vec![page(vec![(not_jpeg, at)])]));
let too_many = (0..MAX_PAGES + 1).map(|_| page(vec![])).collect();
assert!(bad(too_many));
let info = PdfInfo {
producer: String::new(),
title: None,
output_intent: Some(OutputIntent { icc: srgb(), channels: 2, condition: String::new() }),
};
assert!(write(&[page(vec![])], &info).is_err());
}
#[test]
fn numbers_and_strings() {
assert_eq!(num(612.0), "612");
assert_eq!(num(72.5), "72.5");
assert_eq!(num(-0.00001), "0");
assert_eq!(num(f64::NAN), "0");
assert_eq!(num(1.0 / 3.0), "0.3333");
assert_eq!(text("a(b)c\\"), "(a\\(b\\)c\\\\)");
assert_eq!(text("é"), "<FEFF00E9>");
}
#[test]
fn distinct_profiles_and_component_counts_are_not_conflated() {
let common = Arc::new(b"opaque profile A".to_vec());
let different = Arc::new(b"opaque profile B".to_vec());
let image = |channels, profile| PdfImage {
width: 1,
height: 1,
channels,
data: ImageData::Raw { bits: 8, samples: vec![7; channels as usize] },
icc: Some(profile),
};
let pages = [page(vec![
(image(3, common.clone()), full(1.0, 1.0)),
(image(4, common.clone()), full(1.0, 1.0)),
(image(3, different.clone()), full(1.0, 1.0)),
(image(3, common.clone()), full(1.0, 1.0)),
])];
let pdf = write(&pages, &PdfInfo::default()).unwrap();
let refs: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
assert_ne!(refs[0], refs[1]);
assert_ne!(refs[0], refs[2]);
assert_eq!(refs[0], refs[3]);
for (i, channels, bytes) in [(0, 3, common.as_slice()), (1, 4, common.as_slice()), (2, 3, different.as_slice())] {
let (dict, data) = object(&pdf, refs[i]);
assert!(dict.contains(&format!("/N {channels}")));
assert_eq!(inflate(&data), bytes);
}
}
#[test]
fn gray_rgb_cmyk_raw_depths_preserve_samples() {
for channels in [1, 3, 4] {
for bits in [8, 16] {
let bytes: Vec<u8> = (0..channels * (bits / 8)).collect();
let image = PdfImage { width: 1, height: 1, channels, data: ImageData::Raw { bits, samples: bytes.clone() }, icc: None };
let pdf = write(&[page(vec![(image, full(1.0, 1.0))])], &PdfInfo::default()).unwrap();
let (dict, data) = object(&pdf, images(&pdf)[0]);
assert!(dict.contains(&format!("/BitsPerComponent {bits}")));
assert_eq!(inflate(&data), bytes);
}
}
}
#[test]
fn input_and_output_budgets_fail_without_partial_success() {
let image = PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
let pages = [page(vec![(image, full(1.0, 1.0))])];
let info = PdfInfo::default();
let expected = write(&pages, &info).unwrap();
assert_eq!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() }).unwrap(), expected);
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 5, output_bytes: expected.len() }).is_err());
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() - 1 }).is_err());
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: 0 }).is_err());
}
}
+1 -1
View File
@@ -33,7 +33,7 @@ The format layer ends where arithmetic on the picture begins.
|---|---|
| Recognising containers (`probe`), walking TIFF/IFD structure on `craft-tiff` | Black subtraction, white scaling, cropping to the active area |
| Decompressing the samples: unpacking, lossless JPEG, lossy JPEG, Deflate, JPEG XL | Demosaicing, binned previews, hot-pixel and highlight handling |
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values; left unapplied when `OpcodeList1` is present, so the develop pipeline can run that list on the stored codes first) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
| Black and white levels, CFA layout, active area, default crop, orientation, pixel aspect | The DNG colour model's arithmetic (illuminant interpolation, camera to working space) |
| Colour tags as data: matrices, calibration illuminants, as-shot neutral / white | White balance, Kelvin scales, spectral or fitted camera colour |
| Opcode lists parsed and written (`opcodes`), profile tables and gain table maps as data | Applying opcodes (gain maps, warps, bad pixels), profile look tables, tone curves |
+1
View File
@@ -852,6 +852,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// makes a neutral surface magenta. WB edits remain relative to this as-shot RGB.
wb_multipliers: if linear_rgb { Some([1.0; 3]) } else { wb },
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
// only validated components become opcodes (see `arw_lens`, and `arw_lens_rm6` for the ILCE-7RM6)
opcodes: OpcodeLists { list3: warp.into_iter().chain(rm6).collect(), ..Default::default() },
+1
View File
@@ -311,6 +311,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData { kelvin_presets, ..Default::default() },
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -388,6 +388,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode, limits: &
// the camera's white balance is already applied, as in Sony's linear YCbCr
wb_multipliers: Some([1.0; 3]),
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+2
View File
@@ -269,6 +269,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: levels.wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: ifd0.map(Camera::from_ifd0).unwrap_or_default(),
@@ -824,6 +825,7 @@ mod tests {
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Camera::default(),
+1
View File
@@ -59,6 +59,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, caller: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: h.wb_multipliers,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera,
+372
View File
@@ -0,0 +1,372 @@
//! DNG Camera Profiles (`.dcp`): the user's own camera colour profiles, read at run time from
//! files they supply (DNG specification, chapter 6 "Camera Profiles"). A `.dcp` is a TIFF-structured
//! file whose header carries the magic `RC` (0x4352) in place of 42, with the profile tags in IFD 0:
//! `UniqueCameraModel`, `ProfileName`, the colour/forward matrices and calibration illuminants, the
//! hue/saturation map, look table and tone curve. Nothing here ships a profile.
use craft_tiff::{Ifd, ParseOptions, Tiff, tags as t};
use crate::ColorData;
/// The magic number of a DNG camera profile file (`IIRC` / `MMCR`).
pub const DCP_MAGIC: u16 = 0x4352;
/// A parsed camera profile.
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct DcpProfile {
/// `ProfileName` (empty when the file has none).
pub name: String,
/// `UniqueCameraModel`: the camera the profile was made for (empty when absent).
pub camera_model: String,
/// The colour model: illuminants, matrices and the profile look (hue/sat map, look table, tone curve).
pub color: ColorData,
}
/// Errors reading a `.dcp`.
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum DcpError {
#[error("not a DNG camera profile (.dcp)")]
NotDcp,
#[error("damaged camera profile: {0}")]
Damaged(String),
#[error("the camera profile has no usable colour matrix")]
NoMatrix,
}
/// Is `bytes` a DCP file (by its header)?
pub fn is_dcp(bytes: &[u8]) -> bool {
matches!(bytes.get(0..4), Some([b'I', b'I', b'R', b'C'] | [b'M', b'M', b'C', b'R']))
}
/// The largest `.dcp` file read or parsed (real profiles are kilobytes to a few MB).
pub const MAX_DCP_BYTES: u64 = 64 << 20;
/// Read the `.dcp` file at `path`, reading at most [`MAX_DCP_BYTES`] + 1 bytes: a larger file is refused
/// before it is loaded into memory.
pub fn read_dcp(path: &std::path::Path) -> std::io::Result<Vec<u8>> {
use std::io::Read;
let mut bytes = Vec::new();
std::fs::File::open(path)?.take(MAX_DCP_BYTES.saturating_add(1)).read_to_end(&mut bytes)?;
if bytes.len() as u64 > MAX_DCP_BYTES {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "the camera profile is larger than 64 MiB"));
}
Ok(bytes)
}
/// Parse a `.dcp` file. Files above 64 MiB ([`MAX_DCP_BYTES`]) are refused.
pub fn parse_dcp(bytes: &[u8]) -> Result<DcpProfile, DcpError> {
if !is_dcp(bytes) {
return Err(DcpError::NotDcp);
}
if bytes.len() as u64 > MAX_DCP_BYTES {
return Err(DcpError::Damaged("larger than 64 MiB".into()));
}
// the TIFF reader only knows 42 / BigTIFF / raw magics: give it a header with 42 in place of RC
let mut data = bytes.to_vec();
let forty_two: [u8; 2] = if bytes.first() == Some(&b'I') { [42, 0] } else { [0, 42] };
if let Some(m) = data.get_mut(2..4) {
m.copy_from_slice(&forty_two);
}
let opts = ParseOptions { follow_children: false, max_ifds: 4, value_budget: Some(64 << 20), ..Default::default() };
let tiff = Tiff::parse_with(&data, &opts).map_err(|e| DcpError::Damaged(e.to_string()))?;
let ifd: &Ifd = tiff.ifds.first().ok_or_else(|| DcpError::Damaged("no IFD".into()))?;
let mut color = crate::dng::color_data(ifd, ifd, tiff.order);
// a profile carries no white balance or calibration of a particular shot
color.as_shot_neutral = None;
color.as_shot_white_xy = None;
color.analog_balance = None;
color.camera_calibration = [None, None];
color.baseline_sharpness = None;
color.profile.gain_table_map = None;
// only BaselineExposureOffset belongs to a profile (BaselineExposure is the camera's)
color.baseline_exposure =
ifd.f64s(t::BASELINE_EXPOSURE_OFFSET).and_then(|v| v.first().copied()).filter(|v| v.is_finite()).unwrap_or(0.0).clamp(-4.0, 4.0);
// a single-illuminant profile names one illuminant; its second matrix slot stays empty
if color.color_matrix[0].is_none() {
// the first slot is the one every profile must fill (DNG: ColorMatrix1 is required)
return Err(DcpError::NoMatrix);
}
let text = |tag: u16| ifd.string(tag).map(|s| s.trim_matches(char::from(0)).trim().to_string()).unwrap_or_default();
Ok(DcpProfile { name: text(t::PROFILE_NAME), camera_model: text(t::UNIQUE_CAMERA_MODEL), color })
}
/// Write `p` as a `.dcp` file (little-endian): its name, camera, illuminants, matrices, hue/saturation
/// map, look table, tone curve and baseline exposure offset.
pub fn write_dcp(p: &DcpProfile) -> Result<Vec<u8>, DcpError> {
use craft_tiff::writer::srational;
use craft_tiff::{ByteOrder, IfdBuilder, TiffWriter, Value};
let c = &p.color;
// ColorMatrix1 must remain usable after TIFF rational rounding, or our reader rejects it.
let first = c.color_matrix[0].as_ref().ok_or(DcpError::NoMatrix)?;
let serialized = first
.iter()
.flatten()
.map(|v| {
let (n, d) = srational(*v);
f64::from(n) / f64::from(d)
})
.collect();
if first.iter().flatten().any(|v| !v.is_finite()) || crate::dng::mat3(Some(serialized)).is_none() {
return Err(DcpError::NoMatrix);
}
let mut ifd = IfdBuilder::new();
ifd.set(t::UNIQUE_CAMERA_MODEL, Value::Ascii(p.camera_model.clone()));
ifd.set(t::PROFILE_NAME, Value::Ascii(p.name.clone()));
let mat = |m: &crate::Matrix3| Value::SRational(m.iter().flatten().map(|v| srational(*v)).collect());
for i in 0..2 {
if let Some(m) = &c.color_matrix[i] {
ifd.set([t::CALIBRATION_ILLUMINANT_1, t::CALIBRATION_ILLUMINANT_2][i], Value::Short(vec![c.illuminant[i]]));
ifd.set([t::COLOR_MATRIX_1, t::COLOR_MATRIX_2][i], mat(m));
}
if let Some(m) = &c.forward_matrix[i] {
ifd.set([t::FORWARD_MATRIX_1, t::FORWARD_MATRIX_2][i], mat(m));
}
}
let flat = |table: &crate::profile::HsvTable| Value::Float(table.data.iter().flatten().copied().collect());
let dims =
|table: &crate::profile::HsvTable| Value::Long(vec![table.hue_divisions as u32, table.sat_divisions as u32, table.val_divisions as u32]);
if let Some(a) = c.profile.hue_sat_map[0].as_ref() {
ifd.set(t::PROFILE_HUE_SAT_MAP_DIMS, dims(a));
ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_1, flat(a));
if a.srgb_value {
ifd.set(t::PROFILE_HUE_SAT_MAP_ENCODING, Value::Long(vec![1]));
}
if let Some(b) = c.profile.hue_sat_map[1]
.as_ref()
.filter(|b| (b.hue_divisions, b.sat_divisions, b.val_divisions) == (a.hue_divisions, a.sat_divisions, a.val_divisions))
{
ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_2, flat(b));
}
}
if let Some(l) = &c.profile.look_table {
ifd.set(t::PROFILE_LOOK_TABLE_DIMS, dims(l));
ifd.set(t::PROFILE_LOOK_TABLE_DATA, flat(l));
if l.srgb_value {
ifd.set(t::PROFILE_LOOK_TABLE_ENCODING, Value::Long(vec![1]));
}
}
if let Some(tc) = &c.profile.tone_curve {
ifd.set(t::PROFILE_TONE_CURVE, Value::Float(tc.points.iter().flatten().copied().collect()));
}
if c.baseline_exposure != 0.0 {
ifd.set(t::BASELINE_EXPOSURE_OFFSET, Value::SRational(vec![srational(c.baseline_exposure)]));
}
let mut out = TiffWriter::new(ByteOrder::Little, false).write(&[ifd]).map_err(|e| DcpError::Damaged(e.to_string()))?;
// the DCP magic in place of 42
if let Some(m) = out.get_mut(2..4) {
m.copy_from_slice(&DCP_MAGIC.to_le_bytes());
}
Ok(out)
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use super::*;
#[test]
fn writing_requires_the_first_color_matrix() {
let mut p = DcpProfile { name: "test".into(), camera_model: "test".into(), color: ColorData::default() };
assert_eq!(write_dcp(&p), Err(DcpError::NoMatrix));
let matrix = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]];
p.color.color_matrix[1] = Some(matrix);
assert_eq!(write_dcp(&p), Err(DcpError::NoMatrix));
p.color.color_matrix[0] = Some(matrix);
let back = parse_dcp(&write_dcp(&p).unwrap()).unwrap();
assert_eq!(back.color.color_matrix, p.color.color_matrix);
}
#[test]
fn writing_rejects_unusable_or_rounding_singular_first_matrices() {
let identity = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]];
for matrix in [
[[0.0; 3]; 3],
[[1.0, 1.0, 0.0], [1.0, 1.0 + 1e-8, 0.0], [0.0, 0.0, 1.0]],
[[f64::NAN, 0.0, 0.0], identity[1], identity[2]],
[[f64::INFINITY, 0.0, 0.0], identity[1], identity[2]],
] {
let mut p = DcpProfile { name: "test".into(), camera_model: "test".into(), color: ColorData::default() };
p.color.color_matrix[0] = Some(matrix);
assert_eq!(write_dcp(&p), Err(DcpError::NoMatrix));
}
}
#[test]
fn reading_a_profile_stops_past_the_size_limit() {
let dir = std::env::temp_dir().join(format!("lc-read-dcp-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let (fits, big) = (dir.join("fits.dcp"), dir.join("big.dcp"));
std::fs::File::create(&fits).unwrap().set_len(MAX_DCP_BYTES).unwrap();
std::fs::File::create(&big).unwrap().set_len(MAX_DCP_BYTES + 1).unwrap();
assert_eq!(read_dcp(&fits).unwrap().len() as u64, MAX_DCP_BYTES);
assert_eq!(read_dcp(&big).unwrap_err().kind(), std::io::ErrorKind::InvalidData);
let _ = std::fs::remove_dir_all(&dir);
}
/// A minimal single-illuminant profile (our own numbers) built in memory: no profile file is
/// committed to the repository.
pub(crate) fn synthetic_dcp(little: bool, with_hue_sat: bool) -> Vec<u8> {
// entries: (tag, type, count, payload bytes)
let mut entries: Vec<(u16, u16, u32, Vec<u8>)> = Vec::new();
let u16b = |v: u16| if little { v.to_le_bytes().to_vec() } else { v.to_be_bytes().to_vec() };
let u32b = |v: u32| if little { v.to_le_bytes().to_vec() } else { v.to_be_bytes().to_vec() };
let srat = |v: f64| {
let (n, d) = ((v * 10000.0).round() as i32, 10000i32);
let mut b = if little { n.to_le_bytes().to_vec() } else { n.to_be_bytes().to_vec() };
b.extend(if little { d.to_le_bytes() } else { d.to_be_bytes() });
b
};
let ascii = |s: &str| {
let mut b = s.as_bytes().to_vec();
b.push(0);
b
};
entries.push((t::UNIQUE_CAMERA_MODEL, 2, 13, ascii("Test Camera1")));
let m = [0.9, -0.3, -0.05, -0.4, 1.2, 0.2, -0.05, 0.15, 0.7];
entries.push((t::COLOR_MATRIX_1, 10, 9, m.iter().flat_map(|v| srat(*v)).collect()));
entries.push((t::CALIBRATION_ILLUMINANT_1, 3, 1, [u16b(21), vec![0, 0]].concat()));
entries.push((t::PROFILE_NAME, 2, 9, ascii("My Look")));
if with_hue_sat {
// 2 hues × 2 sats × 1 value: hue shift, sat scale, value scale per entry (FLOAT)
entries.push((t::PROFILE_HUE_SAT_MAP_DIMS, 4, 3, [u32b(2), u32b(2), u32b(1)].concat()));
let f = |v: f32| if little { v.to_le_bytes().to_vec() } else { v.to_be_bytes().to_vec() };
entries.push((
t::PROFILE_HUE_SAT_MAP_DATA_1,
11,
12,
[f(0.0), f(1.0), f(1.0), f(5.0), f(1.1), f(1.0), f(0.0), f(1.0), f(1.0), f(-5.0), f(0.9), f(1.0)].concat(),
));
}
entries.sort_by_key(|e| e.0);
let mut out = if little { b"IIRC".to_vec() } else { b"MMCR".to_vec() };
out.extend(u32b(8));
let n = entries.len();
let mut data_off = 8 + 2 + n * 12 + 4;
let mut ifd = u16b(n as u16);
let mut tail = Vec::new();
for (tag, ty, count, payload) in &entries {
ifd.extend(u16b(*tag));
ifd.extend(u16b(*ty));
ifd.extend(u32b(*count));
if payload.len() <= 4 {
let mut p = payload.clone();
p.resize(4, 0);
ifd.extend(p);
} else {
ifd.extend(u32b(data_off as u32));
tail.extend(payload);
data_off += payload.len();
if data_off % 2 == 1 {
tail.push(0);
data_off += 1;
}
}
}
ifd.extend(u32b(0));
out.extend(ifd);
out.extend(tail);
out
}
#[test]
fn reads_a_single_illuminant_profile_in_both_byte_orders() {
for little in [true, false] {
let p = parse_dcp(&synthetic_dcp(little, true)).unwrap();
assert_eq!(p.name, "My Look");
assert_eq!(p.camera_model, "Test Camera1");
assert_eq!(p.color.illuminant, [21, 0]);
let m = p.color.color_matrix[0].unwrap();
assert!((m[1][1] - 1.2).abs() < 1e-4, "{m:?}");
assert!(p.color.color_matrix[1].is_none());
assert!(p.color.profile.hue_sat_map[0].is_some());
assert!(p.color.as_shot_neutral.is_none());
}
assert!(parse_dcp(&synthetic_dcp(true, false)).unwrap().color.profile.hue_sat_map[0].is_none());
}
#[test]
fn rejects_other_files_and_damage_without_panicking() {
assert_eq!(parse_dcp(b"II*\0\x08\0\0\0"), Err(DcpError::NotDcp));
assert_eq!(parse_dcp(b""), Err(DcpError::NotDcp));
let good = synthetic_dcp(true, true);
for cut in [4, 8, 10, 30, good.len() / 2, good.len() - 1] {
let _ = parse_dcp(&good[..cut]);
}
let mut bad = good.clone();
bad[4..8].copy_from_slice(&u32::MAX.to_le_bytes());
assert!(parse_dcp(&bad).is_err());
}
/// The hue/saturation map's header is the user's file too: dimensions far past the data, an entry count
/// claiming 4 billion values, or a data offset past the end drop the map (the matrix still reads) instead of
/// allocating for the claim or reading past the file.
#[test]
fn a_hostile_hue_sat_map_header_drops_the_map() {
let good = synthetic_dcp(true, true);
let at = |needle: &[u8]| good.windows(needle.len()).position(|w| w == needle).unwrap();
let le = |v: &[u32]| v.iter().flat_map(|x| x.to_le_bytes()).collect::<Vec<u8>>();
let dims = at(&le(&[2, 2, 1]));
// the map data's directory entry: tag, FLOAT, count 12, offset
let entry = at(&[t::PROFILE_HUE_SAT_MAP_DATA_1.to_le_bytes(), 11u16.to_le_bytes()].concat());
let patched = |at: usize, bytes: &[u8]| {
let mut b = good.clone();
b[at..at + bytes.len()].copy_from_slice(bytes);
b
};
for (what, bytes) in [
("4096³ dimensions", patched(dims, &le(&[4096, 4096, 4096]))),
("dimensions past u32 products", patched(dims, &le(&[u32::MAX, u32::MAX, u32::MAX]))),
("a zero dimension", patched(dims, &le(&[0, 2, 1]))),
("4 billion values", patched(entry + 4, &le(&[u32::MAX]))),
("data past the end", patched(entry + 8, &le(&[u32::MAX - 16]))),
] {
let p = parse_dcp(&bytes).unwrap_or_else(|e| panic!("{what}: {e:?}"));
assert!(p.color.profile.hue_sat_map[0].is_none(), "{what}: the map is dropped");
assert!(p.color.color_matrix[0].is_some(), "{what}: the matrix still reads");
}
}
#[test]
fn written_profiles_read_back() {
let p = parse_dcp(&synthetic_dcp(false, true)).unwrap();
let back = parse_dcp(&write_dcp(&p).unwrap()).unwrap();
assert_eq!(back.name, p.name);
assert_eq!(back.camera_model, p.camera_model);
assert_eq!(back.color.illuminant, p.color.illuminant);
assert_eq!(back.color.profile.hue_sat_map, p.color.profile.hue_sat_map);
let (a, b) = (back.color.color_matrix[0].unwrap(), p.color.color_matrix[0].unwrap());
assert!(a.iter().flatten().zip(b.iter().flatten()).all(|(x, y)| (x - y).abs() < 1e-6));
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod boundary_tests {
use super::*;
use craft_tiff::{ByteOrder, IfdBuilder, TiffWriter, Value};
fn profile_bytes(offset: f64, matrix: bool) -> Vec<u8> {
let mut ifd = IfdBuilder::new();
if matrix {
ifd.set(t::COLOR_MATRIX_1, Value::SRational(vec![(1, 1), (0, 1), (0, 1), (0, 1), (1, 1), (0, 1), (0, 1), (0, 1), (1, 1)]));
}
ifd.set(t::BASELINE_EXPOSURE_OFFSET, Value::Double(vec![offset]));
ifd.set(t::BASELINE_EXPOSURE, Value::Double(vec![10.0]));
ifd.set(t::AS_SHOT_NEUTRAL, Value::Double(vec![0.5, 1.0, 0.7]));
let mut bytes = TiffWriter::new(ByteOrder::Little, false).write(&[ifd]).unwrap();
bytes[2..4].copy_from_slice(&DCP_MAGIC.to_le_bytes());
bytes
}
#[test]
fn profile_requires_matrix_and_never_keeps_shot_exposure_or_white_balance() {
assert_eq!(parse_dcp(&profile_bytes(0.0, false)), Err(DcpError::NoMatrix));
for (offset, expected) in [(9.0, 4.0), (-9.0, -4.0), (f64::INFINITY, 0.0), (f64::NAN, 0.0)] {
let profile = parse_dcp(&profile_bytes(offset, true)).unwrap();
assert_eq!(profile.color.baseline_exposure, expected);
assert_eq!(profile.color.as_shot_neutral, None);
assert_eq!(profile.color.camera_calibration, [None, None]);
assert_eq!(profile.color.profile.gain_table_map, None);
}
}
}
+31 -15
View File
@@ -91,7 +91,7 @@ fn permute<T: Copy + Default, F: Fn(usize, usize) -> usize>(v: &mut Vec<T>, w: u
}
/// A 3×3 matrix tag: nine finite values that can be inverted.
fn mat3(v: Option<Vec<f64>>) -> Option<Matrix3> {
pub(crate) fn mat3(v: Option<Vec<f64>>) -> Option<Matrix3> {
let v = v?;
if v.len() != 9 || v.iter().any(|x| !x.is_finite()) {
return None;
@@ -299,17 +299,11 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
deinterleave(&mut data, w, h, cpp, factor(t::ROW_INTERLEAVE_FACTOR), factor(t::COLUMN_INTERLEAVE_FACTOR))?;
}
// linearization table (integer data only)
let mut linearized = false;
// linearization table (integer data only) and opcode lists. OpcodeList1 runs on the stored
// codes first (DNG 1.7 chapter 7); see [`linearization_and_opcodes`].
let (linearized, linearization_table, opcodes) = linearization_and_opcodes(pixels, raw, &mut data);
// the largest value the table can give a sample: the white level when the file states none
let mut table_max = None;
if let (RawData::U16(v), Some(table)) = (&mut data, pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty())) {
let table: Vec<u16> = table.iter().map(|&x| x.min(65535) as u16).collect();
let last = table.len() - 1;
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
linearized = true;
table_max = table.iter().copied().max().filter(|&m| m > 0);
}
let table_max = linearization_table.as_deref().and_then(|t| t.iter().copied().max().filter(|&m| m > 0));
// a reduced copy's own levels, where it states them
let levels = |tag| if pixels.contains(tag) { pixels } else { raw };
@@ -442,10 +436,6 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
None
};
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
let opcodes =
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
let color = color_data(ifd0, raw, tiff.order);
let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2))
&& color.profile.gain_table_map.is_none();
@@ -465,6 +455,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
color,
wb_multipliers: None,
linearized,
linearization_table,
pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()),
opcodes,
camera: Camera::from_ifd0(ifd0),
@@ -477,6 +468,31 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
Ok(img)
}
/// Parse opcode lists and, for integer data, the LinearizationTable.
///
/// OpcodeList1 runs on the stored codes, then the table maps them to sensor values (DNG 1.7
/// chapter 7). Applying opcodes is the develop pipeline's work, so a non-empty list 1 leaves the
/// table unapplied (`linearized` is false) and available on the image.
fn linearization_and_opcodes(pixels: &Ifd, raw: &Ifd, data: &mut RawData) -> (bool, Option<Vec<u16>>, crate::OpcodeLists) {
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
let opcodes =
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
let linearization_table = if matches!(data, RawData::U16(_)) {
pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty()).map(|table| table.iter().map(|&x| x.min(65535) as u16).collect::<Vec<u16>>())
} else {
None
};
let mut linearized = false;
if opcodes.list1.is_empty()
&& let (RawData::U16(v), Some(table)) = (data, linearization_table.as_deref())
{
let last = table.len() - 1;
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
linearized = true;
}
(linearized, linearization_table, opcodes)
}
/// CinemaDNG `TimeCodes` (CinemaDNG Image Data Format Specification 1.0, tag 51043).
const TIME_CODES: u16 = 51043;
/// CinemaDNG `FrameRate` (tag 51044).
+10
View File
@@ -165,6 +165,14 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result<Vec<u8>> {
} else {
ifd.set(t::WHITE_LEVEL, Value::Long(white));
}
// A table still to be applied (OpcodeList1 has to run on the stored codes first) is written
// with the samples. Once the samples have been mapped, writing the tag would apply it twice
// on the next read of a file without list 1.
if !raw.linearized
&& let Some(table) = raw.linearization_table.as_deref().filter(|t| !t.is_empty())
{
ifd.set(t::LINEARIZATION_TABLE, Value::Short(table.to_vec()));
}
let a = raw.active_area;
ifd.set(t::ACTIVE_AREA, Value::Long(vec![a.y as u32, a.x as u32, (a.y + a.height) as u32, (a.x + a.width) as u32]));
ifd.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![raw.crop.x as u32, raw.crop.y as u32]));
@@ -457,6 +465,7 @@ mod tests {
color: crate::ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Default::default(),
@@ -514,6 +523,7 @@ mod tests {
color: crate::ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Default::default(),
+1
View File
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
+126 -13
View File
@@ -4,16 +4,19 @@
use crate::{RawError, Result};
/// A canonical-by-table Huffman decoder with a 12-bit lookup.
/// A canonical-by-table Huffman decoder with a 12-bit lookup that also holds the difference when code and
/// additional bits fit in 12 bits.
pub(crate) struct Huffman {
/// Indexed by the next 12 bits: (symbol, length); length 0 = invalid.
lut: Vec<(u8, u8)>,
/// Indexed by the next 12 bits. Per entry: bits 0-7 the symbol, bits 8-11 the code length (0 = invalid), bits
/// 12-15 the length of code plus additional bits when both fit in 12 bits (0 = they do not), bits 16-31 the
/// signed difference in that case.
lut: Box<[u32; 4096]>,
}
impl Huffman {
/// From `(left-aligned 12-bit code, length)` per symbol.
pub fn new(codes: &[(u16, u8)]) -> Result<Huffman> {
let mut lut = vec![(0u8, 0u8); 4096];
let mut lut = Box::new([0u32; 4096]);
for (sym, &(code, len)) in codes.iter().enumerate() {
if len == 0 {
continue;
@@ -21,10 +24,19 @@ impl Huffman {
if len > 12 || sym > 16 {
return Err(RawError::Unsupported(format!("PEF Huffman code of length {len}")));
}
let (len, s) = (len as u32, sym as u32);
let first = (code >> (12 - len) << (12 - len)) as usize;
for e in &mut lut[first..first + (1 << (12 - len))] {
if e.1 == 0 {
*e = (sym as u8, len);
for (i, e) in lut[first..first + (1 << (12 - len))].iter_mut().enumerate() {
if (*e >> 8) & 15 == 0 {
let mut entry = (len << 8) | s;
if s == 0 {
entry |= len << 12;
} else if len + s <= 12 {
let v = (((first + i) >> (12 - len - s)) & ((1 << s) - 1)) as i32;
let d = if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v };
entry |= ((len + s) << 12) | ((d as i16 as u16 as u32) << 16);
}
*e = entry;
}
}
}
@@ -85,21 +97,122 @@ impl<'a> Bits<'a> {
}
}
/// Decode one difference: Huffman category then T.81 additional bits.
/// Decode one difference: Huffman category then T.81 additional bits, in one lookup when both fit in 12 bits.
#[inline]
pub(crate) fn diff(bits: &mut Bits, h: &Huffman) -> Option<i32> {
let (sym, len) = h.lut[bits.peek(12) as usize];
let e = h.lut[(bits.peek(12) & 0xfff) as usize];
let len = (e >> 8) & 15;
if len == 0 {
return None;
}
bits.skip(len as u32);
let s = sym as u32;
if s == 0 {
return Some(0);
let total = (e >> 12) & 15;
if total > 0 {
bits.skip(total);
return Some((e as i32) >> 16);
}
bits.skip(len);
let s = e & 0xff;
if s > 16 {
return None;
}
let v = bits.get(s) as i32;
Some(if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v })
}
#[cfg(test)]
mod tests {
use super::*;
/// Canonical left-aligned 12-bit codes for symbols 0..lens.len() (shorter codes first, ties by symbol).
fn canonical(lens: &[u8]) -> Vec<(u16, u8)> {
let mut order: Vec<usize> = (0..lens.len()).collect();
order.sort_by_key(|&s| (lens[s], s));
let mut codes = vec![(0u16, 0u8); lens.len()];
let (mut code, mut prev) = (0u32, 0u8);
for s in order {
code <<= lens[s] - prev;
prev = lens[s];
codes[s] = ((code << (12 - lens[s])) as u16, lens[s]);
code += 1;
}
codes
}
fn bit(src: &[u8], i: usize) -> u32 {
src.get(i / 8).map_or(0, |b| ((b >> (7 - i % 8)) & 1) as u32)
}
/// Reference: match the code bit by bit, then read the category's additional bits (T.81 F.1.2.1).
fn reference(src: &[u8], pos: &mut usize, codes: &[(u16, u8)]) -> Option<i32> {
let window: u32 = (0..12).fold(0, |a, k| (a << 1) | bit(src, *pos + k));
let (sym, len) = codes
.iter()
.enumerate()
.find_map(|(s, &(c, l))| (l > 0 && (c as u32 >> (12 - l)) == window >> (12 - l)).then_some((s as u32, l as usize)))?;
*pos += len;
if sym == 0 {
return Some(0);
}
let v = (0..sym as usize).fold(0u32, |a, k| (a << 1) | bit(src, *pos + k)) as i32;
*pos += sym as usize;
Some(if v < 1 << (sym - 1) { v - (1 << sym) + 1 } else { v })
}
/// `diff` against the reference on random streams (invalid codes are stepped over) and on every (code, extra
/// bits) pair of every symbol.
fn check_table(lens: &[u8]) {
let codes = canonical(lens);
let h = Huffman::new(&codes).unwrap();
for (sym, &(code, len)) in codes.iter().enumerate() {
for v in 0..1u64 << sym {
let (len, n) = (len as u32, sym as u32);
let word = (((code >> (12 - len)) as u64) << n | v) << (64 - len - n);
let src = word.to_be_bytes();
let mut pos = 0;
let want = reference(&src, &mut pos, &codes);
let mut bits = Bits::new(&src);
assert_eq!(diff(&mut bits, &h), want, "symbol {sym} bits {v:#b}");
assert_eq!(bits.consumed_bits(), pos);
}
}
let mut s = 0x1234_5678_9abc_def0u64;
let (mut invalid, mut long) = (0, 0);
for _ in 0..40 {
let src: Vec<u8> = (0..400)
.map(|_| {
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
(s >> 40) as u8
})
.collect();
let mut bits = Bits::new(&src);
let mut pos = 0;
while pos < src.len() * 8 - 64 {
let before = pos;
let want = reference(&src, &mut pos, &codes);
assert_eq!(diff(&mut bits, &h), want);
if want.is_none() {
// step over the invalid code
invalid += 1;
pos += 1;
bits.skip(1);
} else {
long += (pos - before > 12) as u32;
}
assert_eq!(bits.consumed_bits(), pos);
}
}
assert!(long > 0, "{invalid} invalid codes, no symbol longer than the lookup");
}
#[test]
fn diff_matches_bitwise_reference_for_every_category() {
// categories 0..=16, code lengths up to 12 so that short codes combine with their bits and long ones do not
check_table(&[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12]);
}
#[test]
fn diff_matches_reference_when_code_and_bits_fill_the_lookup_exactly() {
// category 8 with a 4-bit code and category 7 with a 5-bit code: len + category == 12
check_table(&[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12]);
}
}
+1
View File
@@ -115,6 +115,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+35 -5
View File
@@ -10,11 +10,13 @@
//!
//! # Where this crate stops
//!
//! The samples come out as the file stores them (after a DNG `LinearizationTable`): black is not subtracted,
//! nothing is scaled, cropped, white balanced or demosaiced, opcode lists and profile tables are parsed but not
//! applied, and no colour transform is computed from the matrices. Normalisation, demosaicing, highlight
//! handling, the DNG colour model's arithmetic, lens corrections, tone and everything after belong to the
//! develop pipeline, which reads the public fields of the types here.
//! The samples come out as the file stores them (after a DNG `LinearizationTable`, except when the file
//! also has an `OpcodeList1`: that list runs on the stored codes first, and applying it is the develop
//! pipeline's work, so the table is left unapplied and carried on [`RawImage::linearization_table`]).
//! Black is not subtracted, nothing is scaled, cropped, white balanced or demosaiced, opcode lists and
//! profile tables are parsed but not applied, and no colour transform is computed from the matrices.
//! Normalisation, demosaicing, highlight handling, the DNG colour model's arithmetic, lens corrections,
//! tone and everything after belong to the develop pipeline, which reads the public fields of the types here.
//!
//! # Formats
//!
@@ -66,6 +68,9 @@ mod crw;
mod crx;
#[cfg(feature = "cr3")]
mod crx_wavelet;
/// DNG camera-profile file parsing and writing.
#[cfg(feature = "dng")]
pub mod dcp;
#[cfg(feature = "dng")]
mod dng;
#[cfg(feature = "dng")]
@@ -645,6 +650,8 @@ pub fn orientation(bytes: &[u8]) -> Option<u16> {
}
// A damaged/unsupported pixel IFD must not hide usable orientation metadata from preview
// fallback. CR3 keeps IFD0 in its CMT1 box; TIFF-based raws keep it at the file start.
// The `cr3` module exists only with the `cr3` feature: this gate is required so
// `cargo check -p craft-raw --no-default-features --features dng` builds (E0433 otherwise).
#[cfg(feature = "cr3")]
if probe(bytes) == Some(RawFormat::Cr3) {
return cr3::orientation(bytes);
@@ -1041,6 +1048,12 @@ pub struct RawImage {
pub wb_multipliers: Option<[f32; 3]>,
/// Whether a `LinearizationTable` was applied to `data`.
pub linearized: bool,
/// The file's `LinearizationTable`, when present. `data` has already been mapped through it when
/// [`Self::linearized`] is true. When `linearized` is false and this is `Some`, the table was left
/// unapplied so `OpcodeList1` can run on the stored samples first (DNG chapter 7); call
/// [`Self::apply_linearization`] after that list.
#[serde(default)]
pub linearization_table: Option<Vec<u16>>,
/// Height of a sensor pixel over its width (1.0 for square pixels). The D1X and D1H write 2.0: their frame is
/// 4028 x 1324 and the picture is 3:2 only once the rows are doubled. A developer stretches the rows by
/// this factor.
@@ -1119,6 +1132,23 @@ impl RawImage {
self.check(true)
}
/// Map integer `data` through [`Self::linearization_table`] when the table has not yet been applied.
/// No-op when [`Self::linearized`] is already true, the table is missing, or the samples are not
/// integers. Call this after `OpcodeList1`, which DNG applies to the stored codes (chapter 7).
pub fn apply_linearization(&mut self) {
if self.linearized {
return;
}
let Some(table) = self.linearization_table.as_deref().filter(|t| !t.is_empty()) else {
return;
};
if let RawData::U16(v) = &mut self.data {
let last = table.len() - 1;
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
self.linearized = true;
}
}
fn check(&self, samples: bool) -> Result<()> {
if self.width == 0 || self.height == 0 || !(1..=4).contains(&self.cpp) {
return Err(RawError::Corrupt("bad dimensions".into()));
+129 -39
View File
@@ -32,15 +32,17 @@ fn err(s: &str) -> RawError {
/// Huffman decoding table (T.81 Annex C "BITS"/"HUFFVAL").
#[derive(Clone, Debug)]
pub struct Huffman {
/// For peeks of `LOOKUP` bits: (code length, symbol); length 0 = slow path.
fast: Vec<(u8, u8)>,
/// Indexed by the next `LOOKUP` bits; length 0 = slow path. Per entry: bits 0-7 the symbol, bits 8-11 the code
/// length, bits 12-15 the length of code plus extra bits when both fit in the lookup (0 = they do not), and
/// bits 16-31 the signed difference in that case.
fast: Vec<u32>,
maxcode: [i32; 18],
valptr: [i32; 17],
mincode: [i32; 17],
values: Vec<u8>,
}
const LOOKUP: u32 = 10;
const LOOKUP: u32 = 12;
impl Huffman {
/// Build from the 16 code-length counts and the symbol list.
@@ -52,7 +54,7 @@ impl Huffman {
let mut maxcode = [-1i32; 18];
let mut valptr = [0i32; 17];
let mut mincode = [0i32; 17];
let mut fast = vec![(0u8, 0u8); 1 << LOOKUP];
let mut fast = vec![0u32; 1 << LOOKUP];
let mut code = 0i32;
let mut k = 0usize;
for len in 1..=16usize {
@@ -63,9 +65,10 @@ impl Huffman {
if len as u32 <= LOOKUP {
let shift = LOOKUP - len as u32;
let base = (code as usize) << shift;
let ssss = values[k] as u32;
for i in 0..(1usize << shift) {
if let Some(slot) = fast.get_mut(base + i) {
*slot = (len as u8, values[k]);
*slot = fast_entry(len as u32, ssss, base + i);
}
}
}
@@ -84,12 +87,41 @@ impl Huffman {
#[inline]
pub fn decode<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
let peek = br.peek(LOOKUP);
let (len, sym) = self.fast[peek as usize];
let e = self.fast[br.peek(LOOKUP) as usize];
let len = (e >> 8) & 15;
if len > 0 {
br.consume(len as u32);
return Ok(sym);
br.consume(len);
return Ok(e as u8);
}
self.decode_slow(br)
}
/// One symbol and its T.81 difference in a single lookup when code and extra bits fit in `LOOKUP` bits; the same
/// as [`Huffman::decode`] followed by [`diff_value`]. `strict`: a category above 16 is an error (subsampled
/// scans) instead of reading 16 bits.
#[inline]
pub fn decode_diff<B: Bits>(&self, br: &mut B, strict: bool) -> Result<i32, RawError> {
let e = self.fast[br.peek(LOOKUP) as usize];
let len = (e >> 8) & 15;
let total = (e >> 12) & 15;
if total > 0 {
br.consume(total);
return Ok((e as i32) >> 16);
}
let ssss = if len > 0 {
br.consume(len);
e as u8
} else {
self.decode_slow(br)?
};
if strict && ssss > 16 {
return Err(err("invalid difference category"));
}
Ok(diff_value(br, ssss))
}
#[cold]
fn decode_slow<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
let mut code = br.peek(LOOKUP + 1) as i32;
let mut len = LOOKUP as usize + 1;
while len <= 16 && code > self.maxcode[len] {
@@ -105,6 +137,21 @@ impl Huffman {
}
}
/// Lookup entry for a code of `len` bits (`len <= LOOKUP`) naming category `ssss`, at window position `index`
/// (the code is the top `len` bits of the `LOOKUP`-bit window). See [`Huffman::fast`].
fn fast_entry(len: u32, ssss: u32, index: usize) -> u32 {
let base = (len << 8) | ssss;
if ssss == 0 {
return base | (len << 12);
}
if ssss > 11 || len + ssss > LOOKUP {
return base;
}
let v = ((index >> (LOOKUP - len - ssss)) & ((1 << ssss) - 1)) as i32;
let d = if v < (1 << (ssss - 1)) { v - (1 << ssss) + 1 } else { v };
base | ((len + ssss) << 12) | ((d as i16 as u16 as u32) << 16)
}
/// A source of entropy-coded bits, first bit most significant.
pub trait Bits {
/// The next `n` (1..=32) bits without consuming them; zeros past the end.
@@ -614,11 +661,7 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
Prediction::Sequential if mx == 0 && dx == 0 && dy == 0 => row_first[c],
Prediction::Sequential => last[c],
};
let ssss = tables[c].decode(&mut br)?;
if ssss > 16 {
return Err(err("invalid difference category"));
}
let value = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16;
let value = ((pred + tables[c].decode_diff(&mut br, true)?) & 0xffff) as u16;
plane[i] = value;
last[c] = i32::from(value);
if mx == 0 && dx == 0 && dy == 0 {
@@ -642,6 +685,44 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
Ok(FrameSubsampled { width: h.width, height: h.height, vertical_subsampling: vertical, planes })
}
/// T.81 Table H.1 predictor `pred` (1..=7; 0 = none) from the left, above and above-left neighbours.
#[inline(always)]
fn predict(pred: u8, ra: i32, rb: i32, rc: i32) -> i32 {
match pred {
0 => 0,
1 => ra,
2 => rb,
3 => rc,
4 => ra + rb - rc,
5 => ra + ((rb - rc) >> 1),
6 => rb + ((ra - rc) >> 1),
_ => (ra + rb) >> 1,
}
}
/// Decode the pixels `x0..x1` (`x0 >= 1`) of one row with the fixed predictor `P`. `prev` is empty on the first
/// line, which only predicts from the left (`P = 1`).
#[inline(always)]
fn decode_run<const P: u8>(
tables: &[&Huffman],
br: &mut BitReader<'_>,
cur: &mut [u16],
prev: &[u16],
nc: usize,
x0: usize,
x1: usize,
) -> Result<(), RawError> {
for x in x0..x1 {
for (c, table) in tables.iter().enumerate() {
let i = x * nc + c;
let ra = cur[i - nc] as i32;
let p = if P <= 1 { predict(P, ra, 0, 0) } else { predict(P, ra, prev[i] as i32, prev[i - nc] as i32) };
cur[i] = ((p + table.decode_diff(br, false)?) & 0xffff) as u16;
}
}
Ok(())
}
/// Decode a lossless JPEG stream. `max_samples` bounds the allocation (hostile headers).
pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
let h = parse_header(d)?;
@@ -680,18 +761,23 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
let (prev_rows, cur_rows) = out.split_at_mut(y * row_len);
let cur = &mut cur_rows[..row_len];
let prev = if y > 0 { &prev_rows[(y - 1) * row_len..] } else { &[][..] };
for x in 0..w {
if h.restart > 0 {
if restart_left == 0 {
if !br.restart() {
return Err(err("missing restart marker"));
}
restart_left = h.restart;
first_line = true;
reset_pending = true;
let mut x = 0;
while x < w {
if h.restart > 0 && restart_left == 0 {
if !br.restart() {
return Err(err("missing restart marker"));
}
restart_left -= 1;
restart_left = h.restart;
first_line = true;
reset_pending = true;
}
// pixels up to the next restart marker or the end of the row
let mut end = w;
if h.restart > 0 {
end = w.min(x + restart_left);
restart_left -= end - x;
}
// the first pixel of the run has the special predictions (start of the interval, first column)
#[allow(clippy::needless_range_loop)] // `c` indexes the tables and the sample position alike
for c in 0..nc {
let i = x * nc + c;
@@ -702,24 +788,28 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
} else if x == 0 || pred == 0 {
if pred == 0 { 0 } else { prev[i] as i32 }
} else {
let ra = cur[i - nc] as i32;
let rb = prev[i] as i32;
let rc = prev[i - nc] as i32;
match pred {
1 => ra,
2 => rb,
3 => rc,
4 => ra + rb - rc,
5 => ra + ((rb - rc) >> 1),
6 => rb + ((ra - rc) >> 1),
_ => (ra + rb) >> 1,
}
predict(pred, cur[i - nc] as i32, prev[i] as i32, prev[i - nc] as i32)
};
let ssss = tables[c].decode(&mut br)?;
let diff = diff_value(&mut br, ssss);
cur[i] = ((p + diff) & mask) as u16;
cur[i] = ((p + tables[c].decode_diff(&mut br, false)?) & mask) as u16;
}
reset_pending = false;
// the rest of the run: the predictor is fixed, so it is chosen once here, not per sample
if first_line {
// the first line of an interval predicts from the left neighbour
decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?;
} else {
match pred {
0 => decode_run::<0>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
1 => decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
2 => decode_run::<2>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
3 => decode_run::<3>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
4 => decode_run::<4>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
5 => decode_run::<5>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
6 => decode_run::<6>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
_ => decode_run::<7>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
}
}
x = end;
}
first_line = false;
if br.overrun > 64 * 8 {
+115
View File
@@ -125,6 +125,121 @@ fn roundtrip_extremes_and_restarts() {
}
}
/// Re-label an encoded stream as `precision` bits with point transform `pt`: the encoder codes the samples at
/// `precision - pt` bits, which is what a transformed scan holds.
fn with_point_transform(mut enc: Vec<u8>, precision: u8, pt: u8) -> Vec<u8> {
enc[6] = precision; // SOF3 precision follows SOI, marker and length
let sos = enc.windows(2).position(|w| w == [0xff, 0xda]).unwrap();
let seg_len = u16::from_be_bytes([enc[sos + 2], enc[sos + 3]]) as usize;
enc[sos + 2 + seg_len - 1] = pt;
enc
}
#[test]
fn roundtrip_point_transform_all_predictors() {
for pred in 1..=7u8 {
for comps in [1usize, 2, 3] {
for (bits, pt) in [(12u8, 1u8), (14, 2), (16, 4), (8, 3)] {
let (w, h) = (11, 6);
let data = noise(w * h * comps, (bits - pt) as u32, pred as u64 * 17 + comps as u64);
let enc = with_point_transform(encode(&data, w, h, comps, bits - pt, pred, 0), bits, pt);
let f = decode(&enc, usize::MAX).unwrap();
assert_eq!((f.precision, f.point_transform, f.predictor), (bits, pt, pred));
let want: Vec<u16> = data.iter().map(|&v| v << pt).collect();
assert_eq!(f.data, want, "pred {pred} comps {comps} bits {bits} pt {pt}");
}
}
}
// with restart intervals that end mid-row
let data = noise(20 * 9 * 2, 10, 9);
for rst in [3usize, 7, 20] {
let enc = with_point_transform(encode(&data, 20, 9, 2, 10, 4, rst), 12, 2);
let want: Vec<u16> = data.iter().map(|&v| v << 2).collect();
assert_eq!(decode(&enc, usize::MAX).unwrap().data, want, "restart {rst}");
}
}
#[test]
fn roundtrip_skewed_categories_with_long_codes() {
// category k with probability 2^-(k+1): the table gets codes longer than the lookup, rare 16-bit differences
let mut s = 0x9e37_79b9_7f4a_7c15u64;
let mut last = 0u32;
let data: Vec<u16> = (0..256 * 256)
.map(|_| {
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
let r = (s >> 32) as u32;
let cat = r.trailing_zeros().min(16);
let mag = (((s >> 8) as u32) & ((1u32 << cat) - 1)) | (1 << cat) >> 1;
let d = if s & 1 == 0 { mag } else { 0u32.wrapping_sub(mag) };
last = last.wrapping_add(d) & 0xffff;
last as u16
})
.collect();
for pred in [1u8, 4, 7] {
let enc = encode(&data, 256, 256, 1, 16, pred, 0);
assert_eq!(decode(&enc, usize::MAX).unwrap().data, data, "pred {pred}");
}
}
/// Canonical decoding from the code lengths alone, bit by bit (T.81 Annex F, Figure F.16): the reference for the
/// lookup tables.
fn canonical_symbol(br: &mut BitReader, lens: &[u8]) -> Option<u8> {
let window = br.peek(16);
let (mut first, mut index) = (0u32, 0usize);
for len in 1..=16u8 {
let code = window >> (16 - len);
let n = lens.iter().filter(|&&l| l == len).count() as u32;
if code < first + n {
br.consume(len as u32);
return Some((index + (code - first) as usize) as u8);
}
index += n as usize;
first = (first + n) << 1;
}
None
}
/// `decode_diff` against canonical decoding and `diff_value`, for tables with codes inside and beyond the lookup
/// and with `len + category` equal to the lookup width.
#[test]
fn decode_diff_matches_canonical_decoding() {
let tables: [[u8; 17]; 3] = [
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12],
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 16],
[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12],
];
for lens in tables {
let mut counts = [0u8; 16];
for &l in &lens {
counts[l as usize - 1] += 1;
}
// symbols in code order: by length, ties by symbol
let mut values: Vec<u8> = (0..17).collect();
values.sort_by_key(|&v| (lens[v as usize], v));
let sorted: Vec<u8> = values.iter().map(|&v| lens[v as usize]).collect();
let t = Huffman::new(&counts, &values).unwrap();
let mut s = 77u64;
let src: Vec<u8> = (0..6000)
.map(|_| {
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
((s >> 40) as u8).min(0xfe)
})
.collect();
let (mut a, mut b) = (BitReader::new(&src), BitReader::new(&src));
for n in 0..8000 {
let want = canonical_symbol(&mut a, &sorted).map(|i| diff_value(&mut a, values[i as usize]));
let got = t.decode_diff(&mut b, false);
assert_eq!(got.ok(), want, "symbol {n}");
assert_eq!(a.peek(32), b.peek(32), "symbol {n}");
if want.is_none() {
// step past the invalid code so the walk goes on
a.consume(1);
b.consume(1);
}
}
}
}
#[test]
fn constant_image_single_symbol_table() {
let data = vec![1234u16; 16 * 4];
+1
View File
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
+1
View File
@@ -165,6 +165,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: tiff.as_ref().and_then(|t| t.ifds.first()).map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
+2
View File
@@ -329,6 +329,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect,
// Nikon's in-file distortion correction, about the optical centre (which follows the sensor shift)
opcodes: OpcodeLists {
@@ -671,6 +672,7 @@ fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::ima
// the stored colour is already white balanced (see above)
wb_multipliers: Some([1.0; 3]),
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -357,6 +357,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -238,6 +238,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -298,6 +298,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData { kelvin_presets, baseline_exposure: baseline_exposure(&h), ..Default::default() },
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists { list3: crate::raf_lens::corrections(&camera_name(bytes), &ifd, active), ..Default::default() },
camera: ifd0
+1
View File
@@ -672,6 +672,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists { list3: distortion(ifd0, active).into_iter().collect(), ..Default::default() },
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -186,6 +186,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -44,6 +44,7 @@ fn an_active_area_that_wraps_past_the_image_is_corrupt() {
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: Camera::default(),
cinema: Default::default(),
+67 -1
View File
@@ -1,7 +1,7 @@
//! Synthetic DNG round trips: every storage variant we decode is written with our own TIFF writer (and LJ92
//! encoder) and must decode bit-exactly.
use craft_raw::opcodes::{Area, Opcode};
use craft_raw::opcodes::{self, Area, Opcode};
use craft_raw::{
BlackLevel, Cfa, ColorData, DngCompression, DngWriteOptions, OpcodeLists, Orientation, RawData, RawFormat, RawImage, Rect, decode,
embedded_preview, probe, write_dng,
@@ -58,6 +58,7 @@ fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
color,
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera,
@@ -1343,12 +1344,77 @@ fn white_level_defaults_to_the_linearization_tables_maximum() {
};
let img = build(None);
assert!(img.linearized);
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
assert_eq!(img.data, RawData::U16(codes.iter().map(|&c| table[c as usize]).collect()));
assert_eq!(img.white, vec![51000.0], "255 × 200, the table's top");
// a stated level still wins
assert_eq!(build(Some(40000)).white, vec![40000.0]);
}
/// DNG chapter 7: OpcodeList1 runs on the stored codes, then the LinearizationTable maps them to
/// sensor values. When list 1 is present, decode leaves the table unapplied so the develop pipeline
/// can keep that order. Applying the table first would give a different picture.
#[test]
fn opcode_list1_runs_on_stored_samples_before_linearization() {
let (w, h) = (4, 2);
let stored = vec![2u16; w * h];
// table: 2 → 20, 7 → 30, 20 → 99 (so the two orders disagree)
let mut table = vec![0u16; 32];
table[2] = 20;
table[7] = 30;
table[20] = 99;
// MapTable: 2 → 7, 20 → 8
let mut map: Vec<u16> = (0..32).collect();
map[2] = 7;
map[20] = 8;
let area = Area { top: 0, left: 0, bottom: h as u32, right: w as u32, plane: 0, planes: 1, row_pitch: 1, col_pitch: 1 };
let list1 = opcodes::write_list(&[Opcode::MapTable { area, table: map.clone() }]);
let mut raw = base_ifd(w, h, 16, 1, true);
raw.set(t::COMPRESSION, Value::Short(vec![compression::NONE]));
raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![u16_bytes(&stored, ByteOrder::Little)] });
raw.set(t::LINEARIZATION_TABLE, Value::Short(table.clone()));
raw.set(t::OPCODE_LIST_1, Value::Undefined(list1));
raw.set(t::WHITE_LEVEL, Value::Long(vec![100]));
let img = decode(&dng(raw, ByteOrder::Little)).unwrap();
assert!(!img.linearized, "list 1 is present, so the table must wait");
assert_eq!(img.data, RawData::U16(stored.clone()));
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
assert_eq!(img.opcodes.list1.len(), 1);
let apply_map = |samples: &mut [u16]| {
for s in samples {
let i = (*s as usize).min(map.len() - 1);
*s = map[i];
}
};
let mut correct = img.clone();
if let RawData::U16(v) = &mut correct.data {
apply_map(v);
}
correct.apply_linearization();
assert!(correct.linearized);
// 2 → 7 (MapTable) → 30 (linearization)
assert_eq!(correct.data, RawData::U16(vec![30; w * h]));
let mut wrong = img.clone();
wrong.apply_linearization();
if let RawData::U16(v) = &mut wrong.data {
apply_map(v);
}
// 2 → 20 (linearization) → 8 (MapTable)
assert_eq!(wrong.data, RawData::U16(vec![8; w * h]));
assert_ne!(correct.data, wrong.data);
// the unapplied table and list 1 survive a write / decode round trip
let back = decode(&write_dng(&img, &DngWriteOptions { compression: DngCompression::Uncompressed, ..Default::default() }).unwrap()).unwrap();
assert!(!back.linearized);
assert_eq!(back.data, RawData::U16(stored));
assert_eq!(back.linearization_table.as_deref(), Some(table.as_slice()));
assert_eq!(back.opcodes.list1.len(), 1);
}
/// PhotoCraft: `AsShotWhiteXY` is a chromaticity, so both coordinates lie inside (0, 1).
#[test]
fn an_as_shot_white_outside_the_chromaticity_range_is_ignored() {
+1
View File
@@ -22,6 +22,7 @@ fn sample(comp: DngCompression) -> Vec<u8> {
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Default::default(),
+73 -1
View File
@@ -181,6 +181,12 @@ pub struct CcittRawOutput {
/// Decodes CCITT Group 3 or Group 4 fax `data` with caller-specified `params` and `max_rows`,
/// returning packed MSB-first rows, rows decoded, input bytes consumed, and any decode error.
///
/// The rows are read as the parameters say, as PostScript's `CCITTFaxDecode` filter reads them:
/// on bytes exactly when `encoded_byte_align` is set, whatever codes the data holds (unlike
/// [`decode_ccitt_within`], which reads Group 3 rows with end-of-line codes by those codes). At
/// most `rows` rows are read when it is above 0; a caller that ends the rows at the end-of-block
/// code alone passes 0.
pub fn decode_ccitt_raw(data: &[u8], p: &CcittParams, max_rows: u32) -> Result<CcittRawOutput, CodecError> {
if p.columns <= 0 || p.columns > i64::from(MAX_CCITT_COLUMNS) {
return Err(CodecError::Corrupt { filter: "CCITTFaxDecode", detail: format!("CCITT columns {} outside 1..={MAX_CCITT_COLUMNS}", p.columns) });
@@ -192,7 +198,7 @@ pub fn decode_ccitt_raw(data: &[u8], p: &CcittParams, max_rows: u32) -> Result<C
rows,
end_of_block: p.end_of_block,
end_of_line: p.end_of_line,
rows_are_byte_aligned: p.encoded_byte_align && (p.k < 0 || !starts_with_end_of_line(data)),
rows_are_byte_aligned: p.encoded_byte_align,
encoding: match p.k {
..0 => hayro_ccitt::EncodingMode::Group4,
0 => hayro_ccitt::EncodingMode::Group3_1D,
@@ -579,6 +585,72 @@ mod tests {
}
}
/// `decode_ccitt_raw` reads rows on bytes exactly when `encoded_byte_align` says, as
/// PostScript's `CCITTFaxDecode` filter (and `hayro-ccitt` itself) reads them, in every layout
/// of Group 3 rows: with or without end-of-line codes, fill before each row or before each
/// code's end, an RTC after the rows, and room for more rows than they hold. The codes in the
/// data don't make rows the parameters put on bytes read unaligned (which read an aligned RTC
/// as one more row, #778).
#[test]
fn raw_group_3_rows_follow_the_parameters() {
let mut cases = 0;
for seed in 0..24u64 {
let (columns, rows) = (8 + (seed as u32 * 37) % 200, 2 + seed as u32 % 20);
let runs = text_like(seed, columns, rows);
for k in [0i64, 1, 2] {
for (codes, fill) in [(false, Fill::None), (false, Fill::RowStart), (true, Fill::None), (true, Fill::RowStart), (true, Fill::CodeEnd)]
{
let mut data = group_3(&runs, codes, k > 0, fill);
if codes {
// RTC: six end-of-line codes (with their 1 tag bits for `k` above 0), each on a byte.
let eol: [u8; 2] = if k > 0 { [0x00, 0x18] } else { [0x00, 0x10] };
(0..6).for_each(|_| data.extend_from_slice(&eol));
}
for (aligned, end_of_line, end_of_block) in [(true, true, true), (true, false, true), (true, true, false), (false, true, true)] {
let p = CcittParams {
k,
columns: i64::from(columns),
end_of_line,
encoded_byte_align: aligned,
end_of_block,
..CcittParams::default()
};
let max_rows = rows + 8;
let raw = decode_ccitt_raw(&data, &p, max_rows).unwrap();
let settings = hayro_ccitt::DecodeSettings {
columns,
rows: max_rows,
end_of_block,
end_of_line,
rows_are_byte_aligned: aligned,
encoding: if k > 0 { hayro_ccitt::EncodingMode::Group3_2D { k: k as u32 } } else { hayro_ccitt::EncodingMode::Group3_1D },
invert_black: false,
};
let mut filter = Rows::new(columns as usize, max_rows as usize, false);
let used = hayro_ccitt::decode(&data, &mut filter, &mut hayro_ccitt::DecoderContext::new(settings));
let case = format!(
"seed {seed}, {columns} × {rows}, k {k}, codes {codes}, fill {fill:?}, aligned {aligned}, eol {end_of_line}, eob {end_of_block}"
);
assert_eq!((&raw.data, raw.rows_decoded as usize), (&filter.out, filter.done), "{case}");
if let Ok(used) = used {
assert_eq!((raw.bytes_consumed, raw.error), (used, None), "{case}");
}
cases += 1;
}
}
}
}
assert_eq!(cases, 24 * 3 * 5 * 4);
// Rows on bytes with their codes, as `fax`'s writers lay them out, read whole: the rows
// and nothing after the RTC.
let runs = text_like(7, 74, 4);
let mut data = group_3(&runs, true, false, Fill::RowStart);
(0..6).for_each(|_| data.extend_from_slice(&[0x00, 0x10]));
let p = CcittParams { k: 0, columns: 74, end_of_line: true, encoded_byte_align: true, ..CcittParams::default() };
let raw = decode_ccitt_raw(&data, &p, 9).unwrap();
assert_eq!((raw.data, raw.rows_decoded), (pack(&runs, 74), 4));
}
proptest::proptest! {
/// Any bitmap, of noise or of runs, decodes to its own bits, through an independent decoder
/// and through `decode_ccitt`.
+14 -4
View File
@@ -34,26 +34,36 @@ impl Imported {
/// above [`crate::rate::MAX_RATE`] per second and start timecodes beyond [`MAX_START_TIMECODE`]
/// frames. Every importer ends with this pass, so no format can let such a number through.
pub fn bound(&mut self, report: &mut Report) {
self.bound_with_report_limit(report, None);
}
/// Apply numeric bounds with an optional limit on distinct warning messages.
pub fn bound_with_report_limit(&mut self, report: &mut Report, limit: Option<usize>) {
for item in self.project.items.values_mut() {
let name = item.name.clone();
if let Some(seq) = item.as_sequence_mut() {
if !plausible_rate(seq.settings.frame_rate) {
report.warn(format!("sequence \"{name}\" has an implausible frame rate ({}); the default was used", seq.settings.frame_rate));
report.warn_limited(
limit,
format!("sequence \"{name}\" has an implausible frame rate ({}); the default was used", seq.settings.frame_rate),
);
seq.settings.frame_rate = FrameRate::default();
}
if seq.start_timecode.unsigned_abs() > MAX_START_TIMECODE {
report.warn(format!("sequence \"{name}\" has an out-of-range start timecode ({} frames); it starts at 0", seq.start_timecode));
report.warn_limited(
limit,
format!("sequence \"{name}\" has an out-of-range start timecode ({} frames); it starts at 0", seq.start_timecode),
);
seq.start_timecode = 0;
}
} else if let Some(media) = item.as_media_mut() {
if let Some(v) = media.info.video.as_mut()
&& !plausible_rate(v.frame_rate)
{
report.warn(format!("media \"{name}\" has an implausible frame rate ({}); the default was used", v.frame_rate));
report.warn_limited(limit, format!("media \"{name}\" has an implausible frame rate ({}); the default was used", v.frame_rate));
v.frame_rate = FrameRate::default();
}
if media.info.start_timecode.is_some_and(|tc| tc.unsigned_abs() > MAX_START_TIMECODE) {
report.warn(format!("media \"{name}\" has an out-of-range start timecode; it was ignored"));
report.warn_limited(limit, format!("media \"{name}\" has an out-of-range start timecode; it was ignored"));
media.info.start_timecode = None;
}
}
+75 -32
View File
@@ -131,7 +131,7 @@ fn color_label(s: &str) -> Label {
// ---------------------------------------------------------------------------------------------
/// `fields` with the keys of `ext` (a JSON object) added after them; a key of ours wins.
fn with_ext(mut fields: Map<String, Value>, ext: &Ext, report: &mut Report) -> Value {
fn with_ext(mut fields: Map<String, Value>, ext: &Ext, report: &mut Report, limit: Option<usize>) -> Value {
if let Some(text) = ext {
match serde_json::from_str::<Value>(text) {
Ok(Value::Object(extra)) => {
@@ -139,7 +139,7 @@ fn with_ext(mut fields: Map<String, Value>, ext: &Ext, report: &mut Report) -> V
fields.entry(k).or_insert(v);
}
}
_ => report.warn("extension data that is not a JSON object was not written to OTIO"),
_ => report.warn_limited(limit, "extension data that is not a JSON object was not written to OTIO"),
}
}
Value::Object(fields)
@@ -355,7 +355,7 @@ fn value_of(v: &Value) -> ParamValue {
const EFFECT_KEYS: [&str; 3] = ["effect", "enabled", "params"];
fn effects_json(effects: &[EffectInstance], report: &mut Report) -> Value {
fn effects_json(effects: &[EffectInstance], report: &mut Report, limit: Option<usize>) -> Value {
Value::Array(
effects
.iter()
@@ -375,7 +375,7 @@ fn effects_json(effects: &[EffectInstance], report: &mut Report) -> Value {
(k.clone(), json!({"value": value_json(&p.value), "keyframes": keys}))
})
.collect();
with_ext(obj(json!({"effect": e.effect, "enabled": e.enabled, "params": params})), &e.ext, report)
with_ext(obj(json!({"effect": e.effect, "enabled": e.enabled, "params": params})), &e.ext, report, limit)
})
.collect(),
)
@@ -475,6 +475,7 @@ struct Exp<'a, 'r> {
p: &'a Project,
opts: &'a ExportOptions<'a>,
report: &'r mut Report,
report_limit: Option<usize>,
stack: Vec<ItemId>,
/// Clips written inside nested sequences so far, and the limit for it (see [`MAX_NESTED_ITEMS`]).
nested_items: usize,
@@ -483,9 +484,19 @@ struct Exp<'a, 'r> {
/// Write `sequence` (and the media and nested sequences it uses) as an OTIO `Timeline`.
pub fn export(p: &Project, seq_id: ItemId, opts: &ExportOptions, report: &mut Report) -> Result<String> {
export_with_report_limit(p, seq_id, opts, report, None)
}
/// Write OTIO with an optional limit on distinct warnings.
pub fn export_with_report_limit(
p: &Project,
seq_id: ItemId,
opts: &ExportOptions,
report: &mut Report,
report_limit: Option<usize>,
) -> Result<String> {
let seq = p.sequence(seq_id).ok_or(Error::NoSequence(seq_id))?;
let rate = seq.settings.frame_rate;
let mut x = Exp { p, opts, report, stack: vec![seq_id], nested_items: 0, nested_limit: MAX_NESTED_ITEMS };
let mut x = Exp { p, opts, report, report_limit, stack: vec![seq_id], nested_items: 0, nested_limit: MAX_NESTED_ITEMS };
let name = opts.name.map(str::to_string).unwrap_or_else(|| p.item(seq_id).map(|i| i.name.clone()).unwrap_or_default());
let stack = x.stack_json(seq_id, &name, None);
let mut meta = Map::new();
@@ -512,13 +523,13 @@ impl Exp<'_, '_> {
fn seq_meta(&mut self, id: ItemId) -> Value {
let Some((it, s)) = self.p.item(id).and_then(|it| Some((it, it.as_sequence()?))) else { return json!({}) };
let m = obj(json!({"item": id.0, "label": it.label.name(), "settings": settings_json(&s.settings), "start_timecode": s.start_timecode}));
with_ext(m, &it.ext, self.report)
with_ext(m, &it.ext, self.report, self.report_limit)
}
/// The `Stack` for a sequence; `range` = (source in, duration) when nested in a track.
fn stack_json(&mut self, id: ItemId, name: &str, range: Option<(Tick, Tick, FrameRate)>) -> Value {
let Some(seq) = self.p.sequence(id) else {
self.report.warn("missing nested sequence skipped");
self.report.warn_limited(self.report_limit, "missing nested sequence skipped");
return json!({"OTIO_SCHEMA": "Stack.1", "name": name, "children": [], "effects": [], "markers": [], "enabled": true, "metadata": {}});
};
let rate = seq.settings.frame_rate;
@@ -564,7 +575,7 @@ impl Exp<'_, '_> {
}
pos = it.end();
}
let meta = with_ext(obj(json!({"locked": t.locked, "channels": channels_name(t.channels)})), &t.ext, self.report);
let meta = with_ext(obj(json!({"locked": t.locked, "channels": channels_name(t.channels)})), &t.ext, self.report, self.report_limit);
json!({
"OTIO_SCHEMA": "Track.1",
"name": t.name,
@@ -580,8 +591,12 @@ impl Exp<'_, '_> {
fn transition_json(&mut self, x: &Transition, cut: Tick, rate: FrameRate) -> Value {
let dissolve = matches!(x.effect.effect.as_str(), "cross_dissolve" | "constant_power" | "constant_gain" | "film_dissolve");
let effect = effects_json(std::slice::from_ref(&x.effect), self.report).as_array().and_then(|a| a.first().cloned()).unwrap_or(Value::Null);
let meta = with_ext(obj(json!({"effect": effect, "align": align_name(x.align), "reverse": x.reverse})), &x.ext, self.report);
let effect = effects_json(std::slice::from_ref(&x.effect), self.report, self.report_limit)
.as_array()
.and_then(|a| a.first().cloned())
.unwrap_or(Value::Null);
let meta =
with_ext(obj(json!({"effect": effect, "align": align_name(x.align), "reverse": x.reverse})), &x.ext, self.report, self.report_limit);
json!({
"OTIO_SCHEMA": "Transition.1",
"name": transition_name(self.opts.catalog, &x.effect),
@@ -604,7 +619,7 @@ impl Exp<'_, '_> {
fields.insert("generator".into(), generator_json(g));
}
let ext = it.and_then(|i| i.ext.clone());
let meta = with_ext(fields, &ext, self.report);
let meta = with_ext(fields, &ext, self.report, self.report_limit);
match &m.media {
MediaRef::Generator(g) => json!({
"OTIO_SCHEMA": "GeneratorReference.1",
@@ -626,11 +641,11 @@ impl Exp<'_, '_> {
fn clip_meta(&mut self, it: &TrackItem) -> Value {
let fields = obj(json!({
"label": it.label.name(), "link": it.link, "group": it.group, "effects": effects_json(&it.effects, self.report),
"label": it.label.name(), "link": it.link, "group": it.group, "effects": effects_json(&it.effects, self.report, self.report_limit),
"gain_db": it.gain_db, "scale_to_frame": it.scale_to_frame, "source_in_ticks": it.source_in.0,
"duration_ticks": it.duration.0,
}));
with_ext(fields, &it.ext, self.report)
with_ext(fields, &it.ext, self.report, self.report_limit)
}
fn item_json(&mut self, it: &TrackItem, kind: TrackKind, rate: FrameRate) -> Value {
@@ -650,17 +665,20 @@ impl Exp<'_, '_> {
let _ = kind;
if matches!(self.p.item(base).map(|i| &i.kind), Some(ItemKind::Sequence(_))) {
if self.stack.contains(&base) {
self.report.warn("recursive nested sequence skipped");
self.report.warn_limited(self.report_limit, "recursive nested sequence skipped");
return json!({"OTIO_SCHEMA": "Gap.1", "name": "", "source_range": tr(Tick::ZERO, it.duration, rate), "effects": [], "markers": [], "enabled": true, "metadata": {}});
}
if self.nested_items >= self.nested_limit {
// once: later nested sequences hit this too, and the message would repeat
if self.nested_items < usize::MAX {
self.nested_items = usize::MAX;
self.report.warn(format!(
"more than {} clips written inside nested sequences: the rest are gaps (OTIO repeats a nested sequence for every use)",
self.nested_limit
));
self.report.warn_limited(
self.report_limit,
format!(
"more than {} clips written inside nested sequences: the rest are gaps (OTIO repeats a nested sequence for every use)",
self.nested_limit
),
);
}
return json!({"OTIO_SCHEMA": "Gap.1", "name": "", "source_range": tr(Tick::ZERO, it.duration, rate), "effects": [], "markers": [], "enabled": true, "metadata": {}});
}
@@ -678,9 +696,10 @@ impl Exp<'_, '_> {
}
let _ = generator_of(self.p, base);
if let Some(clip) = crate::build::uncarried_clip(self.p, it, rate) {
self.report.warn(format!(
"{clip} has no OTIO equivalent: it is written as a clip with a missing media reference and is read back as offline media"
));
self.report.warn_limited(
self.report_limit,
format!("{clip} has no OTIO equivalent: it is written as a clip with a missing media reference and is read back as offline media"),
);
}
let mref = self.media_ref(base, rate);
// OTIO source times are in the reference's time (which starts at its available_range start).
@@ -703,7 +722,7 @@ impl Exp<'_, '_> {
let fields = obj(
json!({"kind": marker_kind_name(m.kind), "url": m.url, "label": m.color.name(), "author": m.author, "start_ticks": m.start.0, "duration_ticks": m.duration.0}),
);
let meta = with_ext(fields, &m.ext, self.report);
let meta = with_ext(fields, &m.ext, self.report, self.report_limit);
json!({
"OTIO_SCHEMA": "Marker.2",
"name": m.name,
@@ -734,12 +753,17 @@ struct Imp<'a, 'c, 'r> {
b: Builder<'c>,
opts: &'a ImportOptions<'a>,
report: &'r mut Report,
report_limit: Option<usize>,
links: HashMap<u64, u64>,
groups: HashMap<u64, u64>,
}
/// Read an OTIO document: a `Timeline`, a `SerializableCollection` of them, a `Stack` or a bare `Track`.
pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<Imported> {
import_with_report_limit(text, opts, report, None)
}
/// Read OTIO with an optional limit on distinct warnings.
pub fn import_with_report_limit(text: &str, opts: &ImportOptions, report: &mut Report, report_limit: Option<usize>) -> Result<Imported> {
let v: Value = serde_json::from_str(text).map_err(|e| Error::parse(Format::Otio, e.to_string()))?;
let mut timelines = Vec::new();
collect_timelines(&v, &mut timelines);
@@ -747,11 +771,11 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
return Err(Error::parse(Format::Otio, format!("no Timeline in a {} document", schema(&v))));
}
if has_absurd_rate(&v) {
report.warn(format!(
report.warn_limited(report_limit, format!(
"an OTIO rate above {MAX_RATE} per second is not a frame rate; the default 24 fps was used (this includes audio sample rates above it, such as DSD)"
));
}
let mut imp = Imp { b: Builder::new(&name_of(&v), opts.catalog), opts, report, links: HashMap::new(), groups: HashMap::new() };
let mut imp = Imp { b: Builder::new(&name_of(&v), opts.catalog), opts, report, report_limit, links: HashMap::new(), groups: HashMap::new() };
if imp.b.p.name.is_empty() {
imp.b.p.name = "Imported OTIO".into();
}
@@ -760,7 +784,7 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
imp.b.top.push(id);
}
let mut imported = imp.b.finish();
imported.bound(report);
imported.bound_with_report_limit(report, report_limit);
Ok(imported)
}
@@ -843,7 +867,7 @@ impl Imp<'_, '_, '_> {
let kind = if t.get("kind").and_then(Value::as_str) == Some("Audio") { TrackKind::Audio } else { TrackKind::Video };
self.track(&mut seq, t, kind);
}
other => self.report.warn(format!("OTIO {other} directly in a stack is not supported")),
other => self.report.warn_limited(self.report_limit, format!("OTIO {other} directly in a stack is not supported")),
}
}
self.b.ensure_tracks(&mut seq, TrackKind::Video, 1);
@@ -905,7 +929,7 @@ impl Imp<'_, '_, '_> {
}
}
other => {
self.report.warn(format!("OTIO {other} items are not supported and were skipped"));
self.report.warn_limited(self.report_limit, format!("OTIO {other} items are not supported and were skipped"));
}
}
}
@@ -925,7 +949,17 @@ impl Imp<'_, '_, '_> {
None => {
let n = name_of(c);
let ty = c.get("transition_type").and_then(Value::as_str).unwrap_or("SMPTE_Dissolve");
transition_effect(self.b.catalog, if n.is_empty() { ty } else { &n }, kind == TrackKind::Audio, self.report)
{
let mut temporary = Report::default();
let effect =
transition_effect(self.b.catalog, if n.is_empty() { ty } else { &n }, kind == TrackKind::Audio, &mut temporary);
for e in temporary.entries {
for _ in 0..e.count {
self.report.warn_limited(self.report_limit, &e.message);
}
}
effect
}
}
};
let align = align_of(meta.and_then(|m| m.get("align"))).unwrap_or(if inn == Tick::ZERO {
@@ -972,7 +1006,7 @@ impl Imp<'_, '_, '_> {
_ => {
if meta.and_then(|m| m.get("effects")).is_none() {
let n = e.get("effect_name").and_then(Value::as_str).unwrap_or("effect");
self.report.warn(format!("OTIO effect \"{n}\" is not supported"));
self.report.warn_limited(self.report_limit, format!("OTIO effect \"{n}\" is not supported"));
}
}
}
@@ -1107,7 +1141,8 @@ impl Imp<'_, '_, '_> {
other => match self.b.catalog.generator_by_name(other) {
Some(g) => g,
None => {
self.report.warn(format!("OTIO generator \"{other}\" is not supported; imported as Black Video"));
self.report
.warn_limited(self.report_limit, format!("OTIO generator \"{other}\" is not supported; imported as Black Video"));
Generator::BlackVideo
}
},
@@ -1127,7 +1162,7 @@ impl Imp<'_, '_, '_> {
}
_ => {
if schema_name(r) != "MissingReference" && !r.is_null() {
self.report.warn(format!("OTIO media reference {} is not supported; imported offline", schema(r)));
self.report.warn_limited(self.report_limit, format!("OTIO media reference {} is not supported; imported offline", schema(r)));
}
let key = format!("missing:{name}");
let existed = self.b.find_media(&key);
@@ -1279,7 +1314,15 @@ mod tests {
let opts = xopts();
let write = |limit: usize| {
let mut report = Report::default();
let mut x = Exp { p: &imported.project, opts: &opts, report: &mut report, stack: vec![seq], nested_items: 0, nested_limit: limit };
let mut x = Exp {
p: &imported.project,
opts: &opts,
report: &mut report,
report_limit: None,
stack: vec![seq],
nested_items: 0,
nested_limit: limit,
};
let stacks = x.stack_json(seq, "t", None).to_string().matches("\"Stack.1\"").count();
(stacks, report)
};
+14
View File
@@ -35,6 +35,20 @@ impl Report {
self.entries.push(ReportEntry { level, message, count: 1 });
}
}
/// Add a warning while bounding distinct diagnostics. `limit` counts every distinct entry already
/// in the report, informational ones included, exactly like FilmCraft's `MAX_REPORT_ENTRIES`
/// budget this replaces; past it a new message is counted under "further diagnostics omitted".
/// Existing messages continue to be counted.
pub fn warn_limited(&mut self, limit: Option<usize>, message: impl Into<String>) {
let mut message = message.into();
if let Some(limit) = limit
&& self.entries.len() >= limit
&& !self.entries.iter().any(|e| e.level == Level::Warning && e.message == message)
{
message = "further diagnostics omitted".into();
}
self.push(Level::Warning, message);
}
pub fn warn(&mut self, message: impl Into<String>) {
self.push(Level::Warning, message);
}
+14
View File
@@ -570,3 +570,17 @@ fn relative_target_urls_are_percent_decoded() {
let clip = &p.sequence(imp.sequences[0]).unwrap().video_tracks[0].items[0];
assert_eq!(media_key(p, clip.item), "/proj/media/Clip A.mov");
}
#[test]
fn optional_report_limit_bounds_distinct_otio_diagnostics_without_changing_default() {
let children: Vec<_> = (0..10).map(|i| serde_json::json!({"OTIO_SCHEMA": format!("Unsupported{i}.1")})).collect();
let doc = serde_json::json!({"OTIO_SCHEMA":"Timeline.1", "tracks":{"OTIO_SCHEMA":"Stack.1", "children":[{"OTIO_SCHEMA":"Track.1", "kind":"Video", "children":children}]}}).to_string();
let opts = iopts(None);
let mut bounded = craft_timeline::Report::default();
assert!(craft_timeline::otio::import_with_report_limit(&doc, &opts, &mut bounded, Some(2)).is_ok());
assert_eq!(bounded.entries.len(), 3);
assert!(bounded.mentions("further diagnostics omitted"));
let mut unlimited = craft_timeline::Report::default();
assert!(craft_timeline::otio::import(&doc, &opts, &mut unlimited).is_ok());
assert_eq!(unlimited.entries.len(), 10);
}
+1 -1
View File
@@ -2,6 +2,6 @@
Bounded wire primitives for USD Crate, independent of application scene and material models.
Provides bounded integer-delta array decoding and Crate-framed LZ4 decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and export remain outside this extraction. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation.
Provides bounded integer-delta array decoding and Crate-framed LZ4 compression/decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and full scene export remain outside this extraction. Compression uses a caller cap on worst-case reserved output and fallible allocation. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation.
Adapted from TextureCraft `crates/formats/src/mesh_import/usd/codec.rs`, introduced by bflatastic in `bdbb52c9b66b221a55b370105b42f485a76e6fc7`. The LZ4 backend is MIT-licensed `lz4_flex`.
+76
View File
@@ -0,0 +1,76 @@
//! Bounded single-block USD Crate LZ4 writing.
/// Compression admission, allocation or backend failure.
#[derive(Debug, thiserror::Error)]
pub enum Error {
/// Input or worst-case output exceeds the supported/caller budget.
#[error("USD Crate LZ4 compression exceeds budget")]
Limit,
/// The bounded output reservation failed.
#[error("cannot allocate USD Crate LZ4 output")]
Allocation,
/// The backend could not encode into the preflighted output buffer.
#[error("USD Crate LZ4 compression: {0}")]
Encode(#[from] lz4_flex::block::CompressError),
}
fn capacity(input: usize, max_output_bytes: usize) -> Result<usize, Error> {
// Keep inputs below the LZ4 single-block limit and preflight the backend's
// 20 + input*110/100 sizing arithmetic before calling it, including the count byte.
if input > 0x7e00_0000 {
return Err(Error::Limit);
}
input
.checked_mul(110)
.map(|n| n / 100)
.and_then(|n| n.checked_add(21))
.filter(|n| *n <= max_output_bytes && *n <= isize::MAX as usize)
.ok_or(Error::Limit)
}
/// Write one Crate-framed LZ4 block, beginning with a zero chunk-count byte.
///
/// `max_output_bytes` bounds the reserved output including framing. Admission uses
/// the backend's worst-case size, so even highly compressible input must fit that bound.
/// Allocation is fallible; backend scratch memory is not included in the output cap.
/// Uses TextureCraft's original LZ4 backend/configuration and produces the same block bytes.
pub fn compress(input: &[u8], max_output_bytes: usize) -> Result<Vec<u8>, Error> {
let capacity = capacity(input.len(), max_output_bytes)?;
let mut output = Vec::new();
output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?;
output.resize(capacity, 0);
let target = output.get_mut(1..).ok_or(Error::Limit)?;
let written = lz4_flex::block::compress_into(input, target)?;
output.truncate(written.checked_add(1).ok_or(Error::Limit)?);
Ok(output)
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use super::*;
#[test]
fn original_wire_bytes_and_roundtrip() {
for size in [0, 1, 15, 16, 255, 4096, 65536] {
for repeated in [false, true] {
let input: Vec<u8> = (0..size).map(|i| if repeated { 17 } else { (i % 251) as u8 }).collect();
let mut old = vec![0];
old.extend(lz4_flex::block::compress(&input));
let encoded = compress(&input, 1 << 20).unwrap();
assert_eq!(encoded, old);
assert_eq!(crate::decompress(&encoded, size).unwrap(), input);
}
}
}
#[test]
fn checks_worst_case_before_allocation() {
let input = [0; 100];
assert!(matches!(compress(&input, 130), Err(Error::Limit)));
assert!(compress(&input, 131).is_ok());
assert!(capacity(usize::MAX, usize::MAX).is_err());
assert!(capacity(0x7e00_0001, usize::MAX).is_err());
assert_eq!(capacity(0, 21).unwrap(), 21);
}
}
+144
View File
@@ -80,6 +80,72 @@ pub fn decode(data: &[u8], count: usize, width: Width, max_output_bytes: usize)
Ok(output)
}
/// Encode integers with their original signed wrapping and largest-delta tie-break policy.
///
/// `max_bytes` bounds the combined worst-case output reservation and sorted delta scratch.
/// Both reservations are fallible. Sorting uses no extra heap storage; input storage and
/// allocator bookkeeping are outside this budget. Empty input emits no bytes.
pub fn encode(values: &[i64], width: Width, max_bytes: usize) -> Result<Vec<u8>, Error> {
if values.is_empty() {
return Ok(Vec::new());
}
let size = width.bytes();
let codes_len = values.len().div_ceil(4);
let capacity = values.len().checked_mul(size).and_then(|n| n.checked_add(size)).and_then(|n| n.checked_add(codes_len)).ok_or(Error::Limit)?;
let scratch = values.len().checked_mul(8).ok_or(Error::Limit)?;
if capacity > isize::MAX as usize || scratch > isize::MAX as usize || capacity.checked_add(scratch).is_none_or(|n| n > max_bytes) {
return Err(Error::Limit);
}
let wrap = |v: i64| match width {
Width::I32 => i64::from(v as i32),
Width::I64 => v,
};
let mut deltas = Vec::new();
deltas.try_reserve_exact(values.len()).map_err(|_| Error::Allocation)?;
let mut previous = 0i64;
for &value in values {
deltas.push(wrap(value.wrapping_sub(previous)));
previous = value;
}
deltas.sort_unstable();
let mut best = (0usize, 0i64);
let mut at = 0;
while at < deltas.len() {
let value = deltas[at];
let start = at;
while at < deltas.len() && deltas[at] == value {
at += 1;
}
best = best.max((at - start, value));
}
let common = best.1;
let mut output = Vec::new();
output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?;
output.extend_from_slice(&common.to_le_bytes()[..size]);
output.resize(size + codes_len, 0);
previous = 0;
for (i, &value) in values.iter().enumerate() {
let delta = wrap(value.wrapping_sub(previous));
previous = value;
let small = |bits: u32| delta >= -(1i64 << (bits - 1)) && delta < (1i64 << (bits - 1));
let code: u8 = if delta == common {
0
} else if small(size as u32 * 2) {
1
} else if small(size as u32 * 4) {
2
} else {
3
};
if code != 0 {
let bytes = (size >> (3 - usize::from(code))).max(1);
output.extend_from_slice(&delta.to_le_bytes()[..bytes]);
}
output[size + i / 4] |= code << (2 * (i % 4));
}
Ok(output)
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
@@ -125,3 +191,81 @@ mod tests {
}
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod encode_tests {
use super::*;
fn oracle(values: &[i64], width: usize) -> Vec<u8> {
if values.is_empty() {
return Vec::new();
}
let wrap = |v: i64| if width == 4 { i64::from(v as i32) } else { v };
let mut prev = 0i64;
let deltas: Vec<i64> = values
.iter()
.map(|&v| {
let d = wrap(v.wrapping_sub(prev));
prev = v;
d
})
.collect();
let mut counts = std::collections::HashMap::new();
for d in &deltas {
*counts.entry(*d).or_insert(0usize) += 1;
}
let common = counts.into_iter().max_by_key(|&(v, c)| (c, v)).map_or(0, |(v, _)| v);
let mut out = Vec::with_capacity(width + values.len().div_ceil(4) + values.len() * width);
out.extend_from_slice(&common.to_le_bytes()[..width]);
let codes_at = out.len();
out.resize(codes_at + values.len().div_ceil(4), 0);
for (i, &d) in deltas.iter().enumerate() {
let small = |bits: u32| d >= -(1i64 << (bits - 1)) && d < (1i64 << (bits - 1));
let code: u8 = if d == common {
0
} else if small(width as u32 * 2) {
1
} else if small(width as u32 * 4) {
2
} else {
3
};
if code > 0 {
let w = (width >> (3 - code as usize)).max(1);
out.extend_from_slice(&d.to_le_bytes()[..w]);
}
if let Some(b) = out.get_mut(codes_at + i / 4) {
*b |= code << (2 * (i % 4));
}
}
out
}
#[test]
fn matches_original_bytes_and_wrapping() {
for width in [Width::I32, Width::I64] {
for n in [0, 1, 2, 3, 4, 5, 127, 1024] {
let mut seed = 17u64;
let values: Vec<i64> = (0..n)
.map(|_| {
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
seed as i64
})
.collect();
let bytes = encode(&values, width, 32768).unwrap();
assert_eq!(bytes, oracle(&values, width.bytes()));
let expected: Vec<i64> = values.iter().map(|&v| if width == Width::I32 { i64::from(v as i32) } else { v }).collect();
assert_eq!(decode(&bytes, n, width, 32768).unwrap(), expected);
}
for values in [vec![1, 3, 4, 6], vec![i64::MAX, i64::MIN, 0], vec![7; 32]] {
assert_eq!(encode(&values, width, 4096).unwrap(), oracle(&values, width.bytes()));
}
}
}
#[test]
fn combined_reservation_budget_is_enforced() {
assert_eq!(encode(&[], Width::I64, 0).unwrap(), Vec::<u8>::new());
assert!(matches!(encode(&[1], Width::I32, 16), Err(Error::Limit)));
assert!(encode(&[1], Width::I32, 17).is_ok());
}
}
+3
View File
@@ -5,6 +5,9 @@
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
/// Bounded single-block Crate LZ4 compression.
pub mod compress;
/// Bounded Crate integer delta arrays.
pub mod integers;
+3 -3
View File
@@ -46,7 +46,7 @@ assert_eq!((rows, sheet.map(|s| s.children.len())), (3, Ok(0)));
| `max_input` | 256 MiB | `TooLarge` |
| `max_depth` | 256 levels (the root is 1) | `TooDeep` |
| `max_nodes` | 2,000,000 nodes kept in the tree: elements, text stretches, kept CDATA, instructions and comments | `TooManyNodes` |
| `max_attrs` | 1024 on one start tag, namespace declarations and malformed ones included; with canonical names, also the declarations in scope at once | `TooManyAttrs` |
| `max_attrs` | 1024 on one start tag, namespace declarations and malformed ones included; with canonical names or strict reads, also the declarations in scope at once | `TooManyAttrs` |
| `max_attrs_total` | 4,000,000 attributes kept in the tree, over all its elements | `TooManyAttrsTotal` |
A limit exceeded is an error, never a panic and never a truncated tree. `parse_streaming_partial`
@@ -73,9 +73,9 @@ A lenient read keeps what a damaged document had:
- reading stops when the root element ends: nothing after it is looked at;
- a malformed attribute is skipped; bytes that are not UTF-8 become U+FFFD.
A strict read makes each of those an error, and also a second root element and text outside the
A strict read rejects duplicate raw or expanded attribute names (while allowing undeclared prefixes in fragments). It makes each of those an error, and also a second root element and text outside the
root, in the pull reader as in the tree. Markup the tokenizer cannot read (`<a b='1`, an unterminated comment) is an error either
way. A repeated attribute is kept each time in both.
way. A lenient read keeps repeated attributes.
### Text
+53 -2
View File
@@ -1,6 +1,7 @@
//! Reading: the pull reader over `quick-xml`, the tree builder on top of it, and streaming.
use std::borrow::Cow;
use std::collections::BTreeSet;
use std::ops::Range;
use quick_xml::Reader;
@@ -12,6 +13,10 @@ use crate::write::is_xml_char;
/// The namespace the `xml` prefix is always bound to.
pub const XML_NS: &str = "http://www.w3.org/XML/1998/namespace";
/// [`Events::scope_ids`] id of the XML namespace, bound to `xml:` without a declaration.
const XML_NS_ID: usize = usize::MAX;
/// Expanded-name id of an attribute in no namespace (or with an undeclared prefix).
const NO_NS_ID: usize = usize::MAX - 1;
/// What the reader refuses to exceed.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -469,6 +474,10 @@ pub struct Events<'o, 'i> {
end_names: bool,
open: Vec<Open>,
scope: Vec<(String, String)>,
/// For each binding in `scope`, an id shared by every binding of an equal URI (the index of
/// the first such binding, or [`XML_NS_ID`]), with the URI's hash. Strict reads compare
/// expanded attribute names by this id instead of copying the URI for each attribute.
scope_ids: Vec<(u64, usize)>,
/// The end of an empty element, due after its start.
pending: bool,
/// Whether a root element has started.
@@ -500,7 +509,7 @@ impl<'o, 'i> Events<'o, 'i> {
cfg.trim_text_start = false;
cfg.trim_text_end = false;
cfg.expand_empty_elements = false;
Events { reader, opts, end_names, open: Vec::new(), scope: Vec::new(), pending: false, rooted: false, span: 0..0 }
Events { reader, opts, end_names, open: Vec::new(), scope: Vec::new(), scope_ids: Vec::new(), pending: false, rooted: false, span: 0..0 }
}
/// The bytes of the input the last event was read from. Both events of an empty element
@@ -614,6 +623,7 @@ impl<'o, 'i> Events<'o, 'i> {
fn end(&mut self) -> Event {
let (scope, name) = self.open.pop().map_or((0, String::new()), |o| (o.scope, o.name));
self.scope.truncate(scope);
self.scope_ids.truncate(scope);
Event::End { name }
}
@@ -669,6 +679,7 @@ impl<'o, 'i> Events<'o, 'i> {
if is_ns_decl(key) {
let prefix = self.decode(key.get(6..).unwrap_or_default())?.into_owned();
let uri = self.attr_value(&a.value)?;
self.scope_ids.push(self.uri_id(&uri));
self.scope.push((prefix, uri));
}
}
@@ -680,6 +691,20 @@ impl<'o, 'i> Events<'o, 'i> {
Ok(())
}
/// The id an expanded name uses for `uri`: one hash and at most one full comparison per
/// declaration, so a long URI costs its length once however many attributes use it.
fn uri_id(&self, uri: &str) -> (u64, usize) {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
uri.hash(&mut h);
let hash = h.finish();
if uri == XML_NS {
return (hash, XML_NS_ID);
}
let first = self.scope_ids.iter().zip(&self.scope).position(|((h, _), (_, u))| *h == hash && u == uri);
(hash, first.map_or(self.scope.len(), |i| self.scope_ids.get(i).map_or(i, |(_, id)| *id)))
}
/// In a strict read: is the reader before or after the root element?
fn outside_root(&self) -> bool {
self.opts.strict && self.open.is_empty()
@@ -699,7 +724,7 @@ impl<'o, 'i> Events<'o, 'i> {
_ => None,
};
let scope = self.scope.len();
if canonical.is_some() {
if canonical.is_some() || self.opts.strict {
self.declare(e)?;
}
let raw = e.name();
@@ -707,6 +732,8 @@ impl<'o, 'i> Events<'o, 'i> {
let mut out = Vec::new();
let mut attrs = e.attributes();
attrs.with_checks(false);
let mut raw_names = BTreeSet::new();
let mut expanded_names = BTreeSet::new();
for (seen, a) in attrs.enumerate() {
if seen >= limits.max_attrs {
return Err(Error::TooManyAttrs(limits.max_attrs));
@@ -718,6 +745,30 @@ impl<'o, 'i> Events<'o, 'i> {
};
let key = a.key.as_ref();
let decl = is_ns_decl(key);
if self.opts.strict {
let raw_key = self.decode(key)?;
if !raw_names.insert(raw_key.to_string()) {
return self.syntax(format!("duplicate attribute `{raw_key}`"));
}
if !decl {
// Default namespaces do not apply to attributes. Keep undeclared prefixes
// distinct so standalone fragments retain the reader's existing behaviour.
let expanded = match raw_key.split_once(':') {
Some(("xml", local)) => (XML_NS_ID, local),
Some((prefix, local)) => self
.scope
.iter()
.zip(&self.scope_ids)
.rev()
.find(|((p, _), _)| p == prefix)
.map_or((NO_NS_ID, raw_key.as_ref()), |(_, (_, id))| (*id, local)),
None => (NO_NS_ID, raw_key.as_ref()),
};
if !expanded_names.insert((expanded.0, expanded.1.to_string())) {
return self.syntax(format!("duplicate expanded attribute `{raw_key}`"));
}
}
}
if decl && !self.opts.ns_decls {
continue;
}
+43 -1
View File
@@ -117,7 +117,11 @@ fn an_attribute_flood_is_refused_at_the_limit() {
let opts = opts.with_limits(limits);
for (k, kind) in kinds.iter().enumerate() {
let _ = opts.parse(tag(16, kind).as_bytes());
assert_eq!(opts.parse(tag(17, kind).as_bytes()).err().map(|e| e.is_limit() || (opts.strict && k == 3)), Some(true), "kind {k}");
assert_eq!(
opts.parse(tag(17, kind).as_bytes()).err().map(|e| e.is_limit() || (opts.strict && (k == 1 || k == 3))),
Some(true),
"kind {k}"
);
assert!(opts.first_start(tag(17, kind).as_bytes(), "a").is_none());
}
assert_eq!(opts.parse(tag(17, kinds[0]).as_bytes()), Err(Error::TooManyAttrs(16)));
@@ -524,3 +528,41 @@ fn a_strict_pull_reader_refuses_what_is_outside_the_root() {
// white space, comments and instructions may stand outside the root
assert!(read(strict, b"<?xml version='1.0'?>\r\n <!-- c -->\t<a>x</a>\n<?pi?> ").is_ok());
}
#[test]
fn strict_attributes_are_unique_before_name_rewriting() {
use craft_xml::{Names, Options};
for names in [Names::AsWritten, Names::Local] {
let strict = Options { names, ..Options::default() }.strict();
for xml in [
"<r a='1' a='2'/>",
"<r xmlns:p='urn:x' xmlns:q='urn:x' p:a='1' q:a='2'/>",
"<r xml:a='1' xmlns:p='http://www.w3.org/XML/1998/namespace' p:a='2'/>",
"<r xmlns:p='urn:x'><c xmlns:q='urn:x' p:a='1' q:a='2'/></r>",
] {
assert!(strict.parse(xml.as_bytes()).is_err(), "{xml}");
let mut events = strict.events(xml.as_bytes());
assert!(
loop {
match events.next() {
Ok(Some(_)) => {}
Ok(None) => break false,
Err(_) => break true,
}
},
"{xml}"
);
}
for xml in [
"<r xmlns='urn:x' xmlns:p='urn:x' a='1' p:a='2'/>",
"<r p:a='1' q:a='2'/>",
"<r xmlns:p='urn:x'><c xmlns:p='urn:y' xmlns:q='urn:x' p:a='1' q:a='2'/></r>",
] {
assert!(strict.parse(xml.as_bytes()).is_ok(), "{xml}");
}
assert!(Options { names, ..Options::default() }.parse(b"<r a='1' a='2'/>").is_ok());
}
let strict = Options { limits: craft_xml::Limits { max_attrs: 2, ..craft_xml::Limits::DEFAULT }, ..Options::default() }.strict();
assert!(strict.parse(b"<r xmlns:a='a' xmlns:b='b'><c xmlns:c='c'/></r>").is_err());
assert!(strict.parse(b"<r a='1' b='2' c='3'/>").is_err());
}