mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:42:27 +00:00
Merge remote-tracking branch 'origin/main' into w1692
This commit is contained in:
commit
eb5eff868a
67 files changed
+3538
-167
No files matched your search
@@ -23,5 +23,5 @@ jobs:
|
||||
# The codec and format code behind features that are off by default. Without these the
|
||||
# coverage build never compiles it, and new code there reads as untested.
|
||||
cargo-features: >-
|
||||
craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write
|
||||
craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write,craft-image-codecs/sgi
|
||||
secrets: inherit
|
||||
Generated
+13
@@ -1868,6 +1868,15 @@ dependencies = [
|
||||
"image",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-gridcraft-format"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-gzip"
|
||||
version = "0.1.0"
|
||||
@@ -1903,6 +1912,10 @@ dependencies = [
|
||||
"proptest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-http-url"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "craft-i18n"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -11,6 +11,7 @@ rust-version = "1.95"
|
||||
repository = "https://github.com/storytold/craft-libs"
|
||||
|
||||
[workspace.dependencies]
|
||||
craft-http-url = { path = "crates/http-url" }
|
||||
craft-usd = { path = "crates/usd" }
|
||||
craft-smime = { path = "crates/smime" }
|
||||
craft-ber = { path = "crates/ber" }
|
||||
@@ -40,6 +41,7 @@ bytemuck = "1"
|
||||
kurbo = "0.13"
|
||||
thiserror = "2"
|
||||
craft-gzip = { path = "crates/gzip" }
|
||||
craft-gridcraft-format = { path = "crates/gridcraft-format" }
|
||||
craft-aaf = { path = "crates/aaf" }
|
||||
craft-atomic = { path = "crates/atomic" }
|
||||
craft-captions = { path = "crates/captions" }
|
||||
|
||||
@@ -176,6 +176,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
every frame's position, delay, disposal and transparent colour, the loop count and the first
|
||||
frame's colour table, without decompressing pixels. Tells an animation from a still and whether
|
||||
a file can hold the picture it declares before a decoder runs.
|
||||
- [`craft-gridcraft-format`](crates/gridcraft-format/README.md): GridCraft's native JSON workbook
|
||||
(`.json`, `.scjson`): the extensions, the reader, the pretty writer, and newer entries kept
|
||||
unread and written back into a list. The workbook model and what follows a read stay in GridCraft.
|
||||
- [`craft-hdr`](crates/hdr/src/lib.rs): Radiance HDR (RGBE) picture reader, from FilmCraft: flat,
|
||||
old and new run-length scan lines and all eight axis orders, to linear float RGB. Scan lines are
|
||||
validated before the picture is allocated.
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
//! Compatibility WAVE and AIFF wrappers for embedded PCM essence in interchange files.
|
||||
//! PCM inside the model is interleaved little-endian signed integers (8-bit unsigned for WAVE).
|
||||
|
||||
pub use crate::{pcm_aiff_header as aiff_header, pcm_wav_file as wav_file, pcm_wav_header as wav_header};
|
||||
|
||||
/// PCM, channels, sample rate and bits of a WAVE file (integer PCM only).
|
||||
pub fn parse_wav(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
|
||||
if b.len() < 12 || &b[..4] != b"RIFF" || &b[8..12] != b"WAVE" {
|
||||
return None;
|
||||
}
|
||||
let mut at = 12;
|
||||
let mut fmt = None;
|
||||
while at + 8 <= b.len() {
|
||||
let id = &b[at..at + 4];
|
||||
let len = u32::from_le_bytes(b[at + 4..at + 8].try_into().ok()?) as usize;
|
||||
let body = &b[at + 8..(at + 8).saturating_add(len).min(b.len())];
|
||||
if id == b"fmt " && body.len() >= 16 {
|
||||
let tag = u16::from_le_bytes([body[0], body[1]]);
|
||||
let ch = u16::from_le_bytes([body[2], body[3]]);
|
||||
let sr = u32::from_le_bytes(body[4..8].try_into().ok()?);
|
||||
let bits = u16::from_le_bytes([body[14], body[15]]);
|
||||
if tag != 1 && tag != 0xFFFE {
|
||||
return None;
|
||||
}
|
||||
if tag == 0xFFFE {
|
||||
// The first two bytes of the sub-format GUID carry the real format code.
|
||||
let sub = body.get(24..26)?;
|
||||
if u16::from_le_bytes([sub[0], sub[1]]) != 1 {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
fmt = Some((ch, sr, bits));
|
||||
} else if id == b"data" {
|
||||
let (ch, sr, bits) = fmt?;
|
||||
return Some((body.to_vec(), ch, sr, bits));
|
||||
}
|
||||
at = at.checked_add(8usize.checked_add(len)?.checked_add(len & 1)?)?;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// A complete AIFF file (big-endian PCM) from little-endian PCM.
|
||||
pub fn aiff_file(pcm: &[u8], channels: u16, sample_rate: u32, bits: u16) -> Vec<u8> {
|
||||
let bps = bits.div_ceil(8) as usize;
|
||||
let frames = pcm.len() / (bps * channels.max(1) as usize).max(1);
|
||||
let mut v = aiff_header(frames as u32, channels, sample_rate, bits);
|
||||
for s in pcm.chunks_exact(bps.max(1)) {
|
||||
if bps == 1 {
|
||||
v.push(s[0] ^ 0x80); // WAVE 8-bit is unsigned, AIFF signed
|
||||
} else {
|
||||
v.extend(s.iter().rev());
|
||||
}
|
||||
}
|
||||
if (v.len() - 8) & 1 == 1 {
|
||||
v.push(0);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// PCM (converted to little-endian), channels, sample rate and bits of an AIFF / AIFF-C (`NONE`
|
||||
/// / `twos` / `sowt`) file.
|
||||
pub fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
|
||||
if b.len() < 12 || &b[..4] != b"FORM" || !(&b[8..12] == b"AIFF" || &b[8..12] == b"AIFC") {
|
||||
return None;
|
||||
}
|
||||
let mut at = 12;
|
||||
let mut comm = None;
|
||||
while at + 8 <= b.len() {
|
||||
let id = &b[at..at + 4];
|
||||
let len = u32::from_be_bytes(b[at + 4..at + 8].try_into().ok()?) as usize;
|
||||
let body = &b[at + 8..(at + 8).saturating_add(len).min(b.len())];
|
||||
if id == b"COMM" && body.len() >= 18 {
|
||||
let ch = u16::from_be_bytes([body[0], body[1]]);
|
||||
let bits = u16::from_be_bytes([body[6], body[7]]);
|
||||
let sr = from_extended(body[8..18].try_into().ok()?).round() as u32;
|
||||
let little = body.len() >= 22 && &body[18..22] == b"sowt";
|
||||
comm = Some((ch, sr, bits, little));
|
||||
} else if id == b"SSND" && body.len() >= 8 {
|
||||
let (ch, sr, bits, little) = comm?;
|
||||
let off = u32::from_be_bytes(body[..4].try_into().ok()?) as usize;
|
||||
let data = body.get(8usize.checked_add(off)?..)?;
|
||||
let bps = bits.div_ceil(8) as usize;
|
||||
let mut pcm = Vec::with_capacity(data.len());
|
||||
for s in data.chunks_exact(bps.max(1)) {
|
||||
if bps == 1 {
|
||||
pcm.push(s[0] ^ 0x80);
|
||||
} else if little {
|
||||
pcm.extend_from_slice(s);
|
||||
} else {
|
||||
pcm.extend(s.iter().rev());
|
||||
}
|
||||
}
|
||||
return Some((pcm, ch, sr, bits));
|
||||
}
|
||||
at = at.checked_add(8usize.checked_add(len)?.checked_add(len & 1)?)?;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn from_extended(b: [u8; 10]) -> f64 {
|
||||
let e = (u16::from_be_bytes([b[0], b[1]]) & 0x7FFF) as i32;
|
||||
let m = u64::from_be_bytes(b[2..10].try_into().unwrap_or([0; 8]));
|
||||
if e == 0 && m == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
m as f64 / (1u64 << 63) as f64 * 2f64.powi(e - 16383)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
/// A mutated EDL whose clip ran for days produced an OMF descriptor for more than 4 GiB of
|
||||
/// PCM: the RIFF size field overflowed ("attempt to add with overflow").
|
||||
#[test]
|
||||
fn huge_wav_header_saturates() {
|
||||
let h = super::wav_header(u32::MAX, 2, u32::MAX, 24);
|
||||
assert_eq!(&h[4..8], &u32::MAX.to_le_bytes());
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn accepts_truncated_essence_summaries() {
|
||||
let mut wav = wav_file(&[1, 2, 3, 4], 1, 48_000, 16);
|
||||
wav.truncate(46);
|
||||
assert_eq!(parse_wav(&wav), Some((vec![1, 2], 1, 48_000, 16)));
|
||||
let mut aiff = aiff_file(&[1, 2, 3, 4], 1, 48_000, 16);
|
||||
aiff.truncate(56);
|
||||
assert_eq!(parse_aiff(&aiff), Some((vec![1, 2], 1, 48_000, 16)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wav_and_aiff_round_trip() {
|
||||
let pcm: Vec<u8> = (0..48u8).collect(); // 8 stereo 24-bit frames
|
||||
let w = wav_file(&pcm, 2, 48_000, 24);
|
||||
assert_eq!(parse_wav(&w), Some((pcm.clone(), 2, 48_000, 24)));
|
||||
let a = aiff_file(&pcm, 2, 44_100, 24);
|
||||
assert_eq!(parse_aiff(&a), Some((pcm.clone(), 2, 44_100, 24)));
|
||||
let p16: Vec<u8> = (0..32u8).collect();
|
||||
assert_eq!(parse_aiff(&aiff_file(&p16, 1, 96_000, 16)), Some((p16, 1, 96_000, 16)));
|
||||
assert!(parse_wav(b"RIFF").is_none());
|
||||
assert!(parse_aiff(&a[..30]).is_none());
|
||||
for cut in 0..w.len() {
|
||||
let _ = parse_wav(&w[..cut]);
|
||||
let _ = parse_aiff(&a[..cut.min(a.len())]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,6 +76,9 @@ pub mod wav_markers;
|
||||
))]
|
||||
mod write;
|
||||
|
||||
/// PCM WAVE and AIFF wrappers for interchange essence.
|
||||
#[cfg(feature = "pcm-encode")]
|
||||
pub mod essence_pcm;
|
||||
/// Interleaved PCM packet decoding.
|
||||
pub mod pcm_interleaved;
|
||||
/// Range-backed PCM WAVE access and a 16-bit PCM writer.
|
||||
|
||||
@@ -31,6 +31,7 @@ pub mod math;
|
||||
pub mod named;
|
||||
pub mod pipeline;
|
||||
pub mod profile;
|
||||
pub mod sony_lut;
|
||||
pub mod spi1d;
|
||||
pub mod synth;
|
||||
pub mod transform;
|
||||
|
||||
+171
-42
@@ -150,62 +150,142 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
|
||||
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
|
||||
let mut title = String::new();
|
||||
let (mut size3, mut size1) = (0usize, 0usize);
|
||||
let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]);
|
||||
let mut rows: Vec<f32> = Vec::new();
|
||||
for line in text.lines() {
|
||||
// Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an
|
||||
// unquoted title's comment must not turn that comment into part of the title.
|
||||
if let Some(quoted) = line
|
||||
.trim_start()
|
||||
.strip_prefix("TITLE")
|
||||
.filter(|rest| rest.starts_with(char::is_whitespace))
|
||||
.and_then(|rest| rest.trim_start().strip_prefix('"'))
|
||||
/// A bounded, three-dimensional Cube profile with RGB rows in file order.
|
||||
/// Unlike [`parse_cube`], this preserves the optional title and does not resample 1D curves.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Cube3d {
|
||||
pub title: Option<String>,
|
||||
pub size: usize,
|
||||
pub data: Vec<[f32; 3]>,
|
||||
pub domain_min: [f32; 3],
|
||||
pub domain_max: [f32; 3],
|
||||
}
|
||||
|
||||
/// Read the simple 3D profile dialect used by camera-profile consumers.
|
||||
///
|
||||
/// `max_size` must be in 2..=256; the row budget is checked before each allocation.
|
||||
/// Whole-line comments and literal titles are preserved. Unknown alphabetic directives
|
||||
/// are ignored; 1D tables are refused. Finite domains are returned unchanged so the
|
||||
/// consumer retains its existing policy for equal or reversed endpoints.
|
||||
/// Numeric fields must consist of exactly three finite numbers (never silently skip tokens).
|
||||
pub fn parse_cube_3d(text: &str, max_size: usize) -> Result<Cube3d, LutError> {
|
||||
let p = read_cube(text, Some(max_size))?;
|
||||
if p.size3 == 0 {
|
||||
return err("not a 3D .cube LUT (no LUT_3D_SIZE)");
|
||||
}
|
||||
let expected = p.size3.checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
|
||||
if p.rows.len() != expected {
|
||||
return err(format!("expected {expected} entries, found {}", p.rows.len()));
|
||||
}
|
||||
Ok(Cube3d { title: p.title, size: p.size3, data: p.rows, domain_min: p.min, domain_max: p.max })
|
||||
}
|
||||
|
||||
struct CubeRows {
|
||||
title: Option<String>,
|
||||
size3: usize,
|
||||
size1: usize,
|
||||
min: [f32; 3],
|
||||
max: [f32; 3],
|
||||
rows: Vec<[f32; 3]>,
|
||||
}
|
||||
|
||||
fn cube_triplet(text: &str) -> Result<[f32; 3], LutError> {
|
||||
let mut it = text.split_whitespace();
|
||||
let mut row = [0.0; 3];
|
||||
for value in &mut row {
|
||||
*value = it.next().and_then(|s| s.parse::<f32>().ok()).filter(|v| v.is_finite()).ok_or_else(|| LutError(format!("bad line `{text}`")))?;
|
||||
}
|
||||
if it.next().is_some() {
|
||||
return err(format!("bad line `{text}`"));
|
||||
}
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
fn read_cube(text: &str, profile_max: Option<usize>) -> Result<CubeRows, LutError> {
|
||||
if profile_max.is_some_and(|n| !(2..=256).contains(&n)) {
|
||||
return err("3D profile size limit must be in 2..=256");
|
||||
}
|
||||
let mut p = CubeRows { title: None, size3: 0, size1: 0, min: [0.0; 3], max: [1.0; 3], rows: Vec::new() };
|
||||
let row_limit = profile_max.unwrap_or(256).checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
|
||||
for raw in text.lines() {
|
||||
let profile = profile_max.is_some();
|
||||
let line = if profile { raw.trim() } else { raw.split('#').next().unwrap_or("").trim() };
|
||||
// The standard dialect protects '#' inside quoted titles; profile titles are literal.
|
||||
if !profile
|
||||
&& let Some(quoted) =
|
||||
raw.trim_start().strip_prefix("TITLE").filter(|r| r.starts_with(char::is_whitespace)).and_then(|r| r.trim_start().strip_prefix('"'))
|
||||
{
|
||||
let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
|
||||
title = value.to_string();
|
||||
let Some((title, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
|
||||
p.title = Some(title.to_string());
|
||||
continue;
|
||||
}
|
||||
let line = line.split('#').next().unwrap_or("").trim();
|
||||
if line.is_empty() {
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let mut it = line.split_whitespace();
|
||||
let key = it.next().unwrap_or("");
|
||||
let nums = |it: std::str::SplitWhitespace<'_>| -> Result<[f32; 3], LutError> {
|
||||
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
|
||||
if v.len() == 3 { Ok([v[0], v[1], v[2]]) } else { err(format!("bad line `{line}`")) }
|
||||
let (key, rest) = if profile {
|
||||
["TITLE", "LUT_3D_SIZE", "LUT_1D_SIZE", "DOMAIN_MIN", "DOMAIN_MAX"]
|
||||
.into_iter()
|
||||
.find_map(|key| line.strip_prefix(key).map(|rest| (key, rest.trim())))
|
||||
.unwrap_or_else(|| line.split_once(char::is_whitespace).unwrap_or((line, "")))
|
||||
} else {
|
||||
line.split_once(char::is_whitespace).map_or((line, ""), |(key, rest)| (key, rest.trim()))
|
||||
};
|
||||
match key {
|
||||
"TITLE" => title = line[5..].trim().trim_matches('"').to_string(),
|
||||
"LUT_3D_SIZE" => size3 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_3D_SIZE".into()))?,
|
||||
"LUT_1D_SIZE" => size1 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?,
|
||||
"DOMAIN_MIN" => dmin = nums(it)?,
|
||||
"DOMAIN_MAX" => dmax = nums(it)?,
|
||||
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" => {
|
||||
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
|
||||
if v.len() == 2 {
|
||||
dmin = [v[0]; 3];
|
||||
dmax = [v[1]; 3];
|
||||
"TITLE" => p.title = Some(rest.trim_matches('"').to_string()),
|
||||
"LUT_3D_SIZE" => {
|
||||
let value = if profile { rest } else { rest.split_whitespace().next().unwrap_or("") };
|
||||
p.size3 = value.parse().map_err(|_| LutError(format!("bad LUT_3D_SIZE `{value}`")))?;
|
||||
let limit = profile_max.unwrap_or(MAX_SIZE);
|
||||
if !(2..=limit).contains(&p.size3) {
|
||||
return err(format!("LUT_3D_SIZE {} out of range", p.size3));
|
||||
}
|
||||
}
|
||||
k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => {
|
||||
let mut v = vec![k.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?];
|
||||
for value in it {
|
||||
v.push(value.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?);
|
||||
"LUT_1D_SIZE" => {
|
||||
if profile {
|
||||
return err("1D LUTs are not supported (only 3D .cube files)");
|
||||
}
|
||||
if v.len() != 3 {
|
||||
return err(format!("bad data line `{line}`"));
|
||||
p.size1 = rest.split_whitespace().next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?;
|
||||
if p.size1 > row_limit {
|
||||
return err("1D LUT exceeds row budget");
|
||||
}
|
||||
}
|
||||
"DOMAIN_MIN" => p.min = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MIN".into()))?,
|
||||
"DOMAIN_MAX" => p.max = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MAX".into()))?,
|
||||
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" if !profile => {
|
||||
let mut it = rest.split_whitespace();
|
||||
let min = it.next().and_then(|s| s.parse::<f32>().ok());
|
||||
let max = it.next().and_then(|s| s.parse::<f32>().ok());
|
||||
if let (Some(min), Some(max), None) = (min, max, it.next()) {
|
||||
p.min = [min; 3];
|
||||
p.max = [max; 3];
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let numeric = key.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.');
|
||||
let nonfinite = key.parse::<f32>().is_ok();
|
||||
if numeric || nonfinite || (profile && !key.starts_with(|c: char| c.is_ascii_alphabetic())) {
|
||||
let row = cube_triplet(line)?;
|
||||
if p.rows.len() >= row_limit {
|
||||
return err("LUT exceeds row budget");
|
||||
}
|
||||
// Explicit fallible growth prevents capacity growth past the configured budget.
|
||||
if p.rows.len() == p.rows.capacity() {
|
||||
let grow = (p.rows.capacity().max(1)).min(row_limit - p.rows.len());
|
||||
p.rows.try_reserve_exact(grow).map_err(|_| LutError("LUT allocation failed".into()))?;
|
||||
}
|
||||
p.rows.push(row);
|
||||
}
|
||||
rows.extend(v);
|
||||
}
|
||||
// Unknown keywords (e.g. Resolve's LUT_IN_VIDEO_RANGE) are ignored, as the spec allows.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
|
||||
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
|
||||
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
|
||||
let p = read_cube(text, None)?;
|
||||
let (title, size3, size1, dmin, dmax) = (p.title.unwrap_or_default(), p.size3, p.size1, p.min, p.max);
|
||||
let rows: Vec<f32> = p.rows.into_iter().flatten().collect();
|
||||
let domain_ok = (0..3).all(|k| dmin[k] < dmax[k] && dmin[k].is_finite() && dmax[k].is_finite());
|
||||
if (size3 > 0 || size1 >= 2) && !domain_ok {
|
||||
return err(format!("DOMAIN_MIN {:?} must lie below DOMAIN_MAX {:?}", dmin, dmax));
|
||||
@@ -1386,3 +1466,52 @@ LUT_3D_SIZE 2
|
||||
assert_eq!(back.data, [0.0, 0.25, 1.0].repeat(8));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod profile_cube_tests {
|
||||
use super::*;
|
||||
|
||||
fn cube(prefix: &str) -> String {
|
||||
format!("{prefix}\nLUT_3D_SIZE 2\n{}", "0.25 0.5 0.75\n".repeat(8))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_preserves_optional_literal_title_and_domains() {
|
||||
let no_title = parse_cube_3d(&cube(""), 256).unwrap();
|
||||
assert_eq!(no_title.title, None);
|
||||
let value = parse_cube_3d(&cube("TITLE \"look # literal\"\nDOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1"), 256).unwrap();
|
||||
assert_eq!(value.title.as_deref(), Some("look # literal"));
|
||||
assert_eq!(value.domain_min, [1.0, 0.0, 2.0]);
|
||||
assert_eq!(value.domain_max, [0.0, 0.0, 1.0]);
|
||||
assert_eq!(value.data, vec![[0.25, 0.5, 0.75]; 8]);
|
||||
assert_eq!(parse_cube_3d(&cube("TITLE \"\""), 256).unwrap().title.as_deref(), Some(""));
|
||||
assert!(parse_cube(&cube("DOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_limits_are_checked_without_large_allocations() {
|
||||
assert!(parse_cube_3d("LUT_3D_SIZE 256", 256).unwrap_err().0.contains("16777216"));
|
||||
assert!(parse_cube_3d("LUT_3D_SIZE 257", 256).unwrap_err().0.contains("out of range"));
|
||||
assert!(parse_cube_3d("LUT_3D_SIZE 256", 129).unwrap_err().0.contains("out of range"));
|
||||
assert!(parse_cube_3d(&cube(""), usize::MAX).is_err());
|
||||
assert!(parse_cube_3d(&cube(""), 1).is_err());
|
||||
assert!(parse_cube_3d(&format!("{}0 0 0\n", cube("")), 2).unwrap_err().0.contains("budget"));
|
||||
assert!(parse_cube_3d("LUT_1D_SIZE 2\n0 0 0\n1 1 1", 256).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_numeric_token_must_be_valid_and_finite() {
|
||||
for row in ["0 ignored 0 0", "0 0 0 ignored", "NaN 0 0", "inf 0 0", "0 0 -inf", "1e39 0 0", "0 0", "0 0 0 0"] {
|
||||
assert!(parse_cube_3d(&format!("LUT_3D_SIZE 2\n{row}\n{}", "0 0 0\n".repeat(7)), 256).is_err(), "{row}");
|
||||
assert!(parse_cube_3d(&cube(&format!("DOMAIN_MIN {row}")), 256).is_err(), "{row}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_reader_keeps_quoted_comments_and_1d_expansion() {
|
||||
assert_eq!(parse_cube(&cube("TITLE \"look # preserved\" # comment")).unwrap().title, "look # preserved");
|
||||
let value = parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1").unwrap();
|
||||
assert_eq!(value.size, 33);
|
||||
assert!(parse_cube_3d(&cube("DOMAIN_MIN 0 0 0 # not a whole-line comment"), 256).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//! Sony Imageworks SPI3D and SPIMTX text readers, extracted from EffectCraft.
|
||||
//!
|
||||
//! These preserve the source reader's format tolerance. SPI3D sample values are not
|
||||
//! clamped or sanitized; rendering policy belongs to the caller. Duplicate coordinates
|
||||
//! retain their last value, and the total row count must match the lattice volume.
|
||||
|
||||
/// Maximum lattice side accepted by the source reader.
|
||||
pub const MAX_SIDE: usize = 129;
|
||||
|
||||
/// Row-major homogeneous transform with a separate normalized code-value offset.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Matrix {
|
||||
pub matrix: [f64; 16],
|
||||
pub offset: [f64; 4],
|
||||
}
|
||||
|
||||
/// A cubic RGB table, red coordinate varying fastest.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Lattice {
|
||||
pub size: usize,
|
||||
pub values: Vec<[f32; 3]>,
|
||||
}
|
||||
|
||||
/// Read exactly twelve finite SPIMTX numbers, dividing the offset column by 65535.
|
||||
/// Parsing uses a fixed array, not storage proportional to an oversized input.
|
||||
pub fn parse_spimtx(text: &str) -> Option<Matrix> {
|
||||
let mut words = text.split_whitespace();
|
||||
let mut v = [0.0f64; 12];
|
||||
for value in &mut v {
|
||||
*value = words.next()?.parse().ok()?;
|
||||
if !value.is_finite() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
if words.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let mut matrix = [0.0; 16];
|
||||
let mut offset = [0.0; 4];
|
||||
for (r, row) in v.as_chunks::<4>().0.iter().enumerate() {
|
||||
matrix[r * 4..r * 4 + 3].copy_from_slice(&row[..3]);
|
||||
offset[r] = row[3] / 65535.0;
|
||||
}
|
||||
matrix[15] = 1.0;
|
||||
Some(Matrix { matrix, offset })
|
||||
}
|
||||
|
||||
/// Read a SPI3D table with a bounded 2..=129 cubic lattice and exactly six fields per row.
|
||||
/// A `SPILUT` prefix is accepted and the second header line is ignored, as in EffectCraft.
|
||||
pub fn parse_spi3d(text: &str) -> Option<Lattice> {
|
||||
let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty());
|
||||
if !lines.next()?.starts_with("SPILUT") {
|
||||
return None;
|
||||
}
|
||||
lines.next()?;
|
||||
let mut sizes = lines.next()?.split_whitespace();
|
||||
let n: usize = sizes.next()?.parse().ok()?;
|
||||
let g: usize = sizes.next()?.parse().ok()?;
|
||||
let b: usize = sizes.next()?.parse().ok()?;
|
||||
if sizes.next().is_some() || g != n || b != n || !(2..=MAX_SIDE).contains(&n) {
|
||||
return None;
|
||||
}
|
||||
let count = n * n * n;
|
||||
let mut values = vec![[0.0f32; 3]; count];
|
||||
let mut seen = 0usize;
|
||||
for line in lines {
|
||||
if seen == count {
|
||||
return None;
|
||||
}
|
||||
let mut fields = line.split_whitespace();
|
||||
let r: usize = fields.next()?.parse().ok()?;
|
||||
let g: usize = fields.next()?.parse().ok()?;
|
||||
let b: usize = fields.next()?.parse().ok()?;
|
||||
let color = [fields.next()?.parse().ok()?, fields.next()?.parse().ok()?, fields.next()?.parse().ok()?];
|
||||
if fields.next().is_some() || r >= n || g >= n || b >= n {
|
||||
return None;
|
||||
}
|
||||
*values.get_mut(r + g * n + b * n * n)? = color;
|
||||
seen += 1;
|
||||
}
|
||||
(seen == count).then_some(Lattice { size: n, values })
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
use craft_cms::sony_lut::{MAX_SIDE, parse_spi3d, parse_spimtx};
|
||||
|
||||
#[test]
|
||||
fn spimtx_is_finite_exact_and_scales_offsets() {
|
||||
let m = parse_spimtx("2 0 0 6553.5\n0 3 0 -65535\n0 0 4 0").unwrap();
|
||||
assert_eq!(m.matrix, [2.0, 0.0, 0.0, 0.0, 0.0, 3.0, 0.0, 0.0, 0.0, 0.0, 4.0, 0.0, 0.0, 0.0, 0.0, 1.0]);
|
||||
assert_eq!(m.offset, [0.1, -1.0, 0.0, 0.0]);
|
||||
for bad in ["", "1 0 0 0", "NaN 0 0 0 0 1 0 0 0 0 1 0", "inf 0 0 0 0 1 0 0 0 0 1 0", "1 0 0 0 0 1 0 0 0 0 1 0 9"] {
|
||||
assert!(parse_spimtx(bad).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
fn table() -> String {
|
||||
let mut text = String::from("SPILUT 1.0\n3 3\n2 2 2\n");
|
||||
// Reverse row order to prove coordinates, rather than row order, determine storage.
|
||||
for i in (0..8).rev() {
|
||||
let (r, g, b) = (i & 1, (i >> 1) & 1, i >> 2);
|
||||
text.push_str(&format!("{r} {g} {b} {r} {g} {b}\n"));
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spi3d_red_fast_coordinates_and_limits() {
|
||||
let text = table();
|
||||
let lut = parse_spi3d(&text).unwrap();
|
||||
assert_eq!(lut.size, 2);
|
||||
for (i, row) in lut.values.iter().enumerate() {
|
||||
assert_eq!(*row, [(i & 1) as f32, ((i >> 1) & 1) as f32, (i >> 2) as f32]);
|
||||
}
|
||||
assert!(parse_spi3d(&format!("{text}0 0 0 0 0 0\n")).is_none());
|
||||
assert!(parse_spi3d("SPILUT 1.0\n3 3\n2 2 2\n0 0 0 0 0 0\n").is_none());
|
||||
assert!(parse_spi3d(&format!("SPILUT 1.0\n3 3\n{0} {0} {0}\n", MAX_SIDE + 1)).is_none());
|
||||
assert!(parse_spi3d(&text.replace("2 2 2", "2 2 3")).is_none());
|
||||
assert!(parse_spi3d(&text.replace("0 0 0 0 0 0", "2 0 0 0 0 0")).is_none());
|
||||
assert!(parse_spi3d(&text.replace("0 0 0 0 0 0", "0 0 0 0 0 0 extra")).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inherited_tolerance_is_explicit() {
|
||||
let text = "SPILUT alternative\nignored\n2 2 2\n".to_owned() + &"0 0 0 -1 2 NaN\n".repeat(8);
|
||||
let lut = parse_spi3d(&text).unwrap();
|
||||
assert_eq!(&lut.values[0][..2], &[-1.0, 2.0]);
|
||||
assert!(lut.values[0][2].is_nan());
|
||||
assert_eq!(lut.values[1], [0.0; 3]);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "craft-gridcraft-format"
|
||||
description = "GridCraft's native JSON workbook file: its extensions, reader and pretty writer"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
@@ -0,0 +1,17 @@
|
||||
# craft-gridcraft-format
|
||||
|
||||
GridCraft's native workbook file is one pretty-printed UTF-8 JSON object, with a two-space indent
|
||||
and no newline at the end. It is saved as `.json`, or as `.scjson` by older builds.
|
||||
|
||||
The workbook is a type parameter. GridCraft owns the model, and after a read it rebuilds the
|
||||
model's indexes, drops cells outside the grid and reads the macros. This crate owns the bytes:
|
||||
|
||||
- `EXTENSIONS` and `is_extension` name the format.
|
||||
- `read(bytes)` reads a workbook. A failure reads as "not a GridCraft JSON workbook: …".
|
||||
- `write(workbook)` writes one with `serde_json`'s pretty printer.
|
||||
- `write_with_kept(workbook, field, kept)` appends entries that a newer version wrote, and this
|
||||
build kept without reading, to the top-level list `field`, as the file held them. With nothing
|
||||
kept, the output is exactly `write`'s. Otherwise the workbook goes through a `serde_json::Value`
|
||||
first, so its object keys come out in map order, as GridCraft has always written them.
|
||||
|
||||
There is no `unsafe` and no panics, and it builds for `wasm32-unknown-unknown`.
|
||||
@@ -0,0 +1,59 @@
|
||||
//! GridCraft's native workbook file: the workbook as one pretty-printed UTF-8 JSON object,
|
||||
//! saved as `.json` (or `.scjson`, its older name).
|
||||
//!
|
||||
//! GridCraft owns the workbook model and what happens after a read: rebuilding its indexes,
|
||||
//! dropping cells outside the grid, and reading its macros. This crate owns the bytes: which
|
||||
//! extensions name the format, how the JSON is read and written, and how entries kept from a
|
||||
//! newer version are written back into a list as the file held them.
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::Value;
|
||||
|
||||
/// The extensions of the format, lower case. `scjson` is the name older builds saved.
|
||||
pub const EXTENSIONS: [&str; 2] = ["json", "scjson"];
|
||||
|
||||
/// Whether `extension` (without the dot, any case) names the format.
|
||||
pub fn is_extension(extension: &str) -> bool {
|
||||
EXTENSIONS.iter().any(|e| e.eq_ignore_ascii_case(extension))
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
/// The bytes are not a workbook: not JSON, or JSON of another shape.
|
||||
#[error("not a GridCraft JSON workbook: {0}")]
|
||||
Read(serde_json::Error),
|
||||
/// The workbook could not be written as JSON.
|
||||
#[error("{0}")]
|
||||
Write(serde_json::Error),
|
||||
}
|
||||
|
||||
/// Read a workbook. Unknown fields are left to the model's own serde rules.
|
||||
pub fn read<T: DeserializeOwned>(bytes: &[u8]) -> Result<T, Error> {
|
||||
serde_json::from_slice(bytes).map_err(Error::Read)
|
||||
}
|
||||
|
||||
/// Write a workbook as pretty-printed JSON (two-space indent, no newline at the end).
|
||||
pub fn write<T: Serialize>(workbook: &T) -> Result<Vec<u8>, Error> {
|
||||
serde_json::to_vec_pretty(workbook).map_err(Error::Write)
|
||||
}
|
||||
|
||||
/// [`write`], with `kept` appended to the top-level list `field` (made if absent): entries a
|
||||
/// newer version wrote that this build kept without reading. With nothing kept this is exactly
|
||||
/// [`write`]. Otherwise the workbook goes through a [`Value`] first, so its object keys are
|
||||
/// written in `serde_json`'s map order.
|
||||
pub fn write_with_kept<T: Serialize>(workbook: &T, field: &str, kept: &[Value]) -> Result<Vec<u8>, Error> {
|
||||
if kept.is_empty() {
|
||||
return write(workbook);
|
||||
}
|
||||
let mut v = serde_json::to_value(workbook).map_err(Error::Write)?;
|
||||
if let Some(list) = v.as_object_mut().map(|o| o.entry(field).or_insert_with(|| Value::Array(vec![]))).and_then(Value::as_array_mut) {
|
||||
list.extend(kept.iter().cloned());
|
||||
}
|
||||
serde_json::to_vec_pretty(&v).map_err(Error::Write)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,87 @@
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A stand-in for an app's workbook: fields in declaration order, one of them a list.
|
||||
#[derive(Debug, PartialEq, Serialize, Deserialize)]
|
||||
struct Book {
|
||||
sheets: Vec<String>,
|
||||
active: u32,
|
||||
#[serde(default)]
|
||||
macros: Vec<Value>,
|
||||
}
|
||||
|
||||
fn book() -> Book {
|
||||
Book { sheets: vec!["Sheet1".into(), "Été ✓".into()], active: 1, macros: vec![json!({"v": 1, "name": "Fill"})] }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extensions_are_json_and_scjson_in_any_case() {
|
||||
for yes in ["json", "scjson", "JSON", "ScJson"] {
|
||||
assert!(is_extension(yes), "{yes}");
|
||||
}
|
||||
for no in ["", "xlsx", "jsonl", ".json", "scjs"] {
|
||||
assert!(!is_extension(no), "{no}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_workbook_is_written_pretty_in_field_order() {
|
||||
let bytes = write(&book()).unwrap();
|
||||
let text = String::from_utf8(bytes).unwrap();
|
||||
assert_eq!(
|
||||
text,
|
||||
"{\n \"sheets\": [\n \"Sheet1\",\n \"Été ✓\"\n ],\n \"active\": 1,\n \"macros\": [\n {\n \"name\": \"Fill\",\n \"v\": 1\n }\n ]\n}"
|
||||
);
|
||||
assert_eq!(read::<Book>(text.as_bytes()).unwrap(), book());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kept_entries_are_appended_as_the_file_held_them() {
|
||||
let kept = [json!({"v": 9, "name": "Future", "extra": [1, 2]})];
|
||||
let bytes = write_with_kept(&book(), "macros", &kept).unwrap();
|
||||
let back: Book = read(&bytes).unwrap();
|
||||
assert_eq!(back.macros, [json!({"v": 1, "name": "Fill"}), kept[0].clone()]);
|
||||
// Through a JSON value, keys are in map order: `active` before `macros` before `sheets`.
|
||||
let text = String::from_utf8(bytes).unwrap();
|
||||
assert!(text.find("\"active\"") < text.find("\"macros\"") && text.find("\"macros\"") < text.find("\"sheets\""), "{text}");
|
||||
// With nothing kept, the plain writer's bytes.
|
||||
assert_eq!(write_with_kept(&book(), "macros", &[]).unwrap(), write(&book()).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kept_entries_make_the_list_when_it_is_absent() {
|
||||
#[derive(Serialize)]
|
||||
struct Bare {
|
||||
sheets: Vec<String>,
|
||||
}
|
||||
let bytes = write_with_kept(&Bare { sheets: vec![] }, "macros", &[json!(1)]).unwrap();
|
||||
assert_eq!(serde_json::from_slice::<Value>(&bytes).unwrap(), json!({"sheets": [], "macros": [1]}));
|
||||
// A field that is not a list is left as it is.
|
||||
let bytes = write_with_kept(&json!({"macros": "text"}), "macros", &[json!(1)]).unwrap();
|
||||
assert_eq!(serde_json::from_slice::<Value>(&bytes).unwrap(), json!({"macros": "text"}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_not_a_workbook_says_so() {
|
||||
for bad in [&b""[..], b"{", b"[]", b"{\"sheets\": 7, \"active\": 0}", b"\xff\xfe"] {
|
||||
let err = read::<Book>(bad).unwrap_err();
|
||||
assert!(matches!(err, Error::Read(_)));
|
||||
assert!(err.to_string().starts_with("not a GridCraft JSON workbook: "), "{err}");
|
||||
}
|
||||
assert_eq!(
|
||||
read::<Book>(b"{\"sheets\": 7}").unwrap_err().to_string(),
|
||||
"not a GridCraft JSON workbook: invalid type: integer `7`, expected a sequence at line 1 column 12"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_value_json_cannot_hold_is_a_write_error() {
|
||||
use std::collections::BTreeMap;
|
||||
let bad: BTreeMap<Vec<u8>, u8> = [(vec![1], 1)].into();
|
||||
assert!(matches!(write(&bad), Err(Error::Write(_))));
|
||||
assert!(matches!(write_with_kept(&bad, "macros", &[json!(1)]), Err(Error::Write(_))));
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
[package]
|
||||
name = "craft-http-url"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license = "Apache-2.0"
|
||||
repository.workspace = true
|
||||
description = "Bounded HTTP URL parsing and redirect resolution without network access"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,25 @@
|
||||
# craft-http-url
|
||||
|
||||
HTTP/HTTPS URL parsing, dot-segment removal, relative redirects, HTTPS downgrade rejection,
|
||||
and query-redacted display. No dependencies or network operations. Parsing rejects inputs
|
||||
larger than 16 KiB and redirects are bounded before concatenation.
|
||||
|
||||
This is not an SSRF filter, DNS resolver or HTTP client. Local/private hosts are allowed.
|
||||
Applications keep destination permission, authentication, TLS, request/response limits and
|
||||
redirect-count policy. Public fields preserve the original API; manually constructed values
|
||||
must not be treated as validated URLs.
|
||||
|
||||
## Attribution
|
||||
|
||||
Extracted from LightCraft `crates/fetch/src/http.rs` at
|
||||
`2dcb258ba40738230a07484a95eb27dc84ed66fe` (Apache-2.0).
|
||||
Original parser: Daniel Voshart, `7fa7506db46d69d609e85e39b338f33786ed0293`.
|
||||
HTTPS downgrade protection: MarekSpodBiedry, `4a91a1c218bf3d0ac49cc73ebb60621a21ae55e2`.
|
||||
URL normalization, scheme, fragment and authority fixes: NetterSilver, including
|
||||
`edc369565eef92161b56b8f1e39ef53629c05489`,
|
||||
`78f0e6e650444886069f4d4ae4e6593f221b98d4`,
|
||||
`cee93496587d31c514aaeced3e6b8027cfdb41a0`,
|
||||
`dee5f7910ad8c24cf28bab528f8d007550af2b23`,
|
||||
`c3a5d9b7f1f837dc14c01942e3d0293ecbf36345` and
|
||||
`13849307a9343504339d68b086f1a32cdcbbb457`.
|
||||
Source-author commits are retained separately in this extraction.
|
||||
@@ -0,0 +1,277 @@
|
||||
//! Bounded HTTP/HTTPS URL parsing and redirect resolution, extracted from LightCraft.
|
||||
//!
|
||||
//! This is syntax handling, not a network permission or SSRF policy. It permits local/private
|
||||
//! hosts and performs no DNS, socket or TLS operations. Callers own destination allowlists,
|
||||
//! credential forwarding, redirect counts, connection deadlines and response limits.
|
||||
//! Only parsed values have validated fields; constructing public fields bypasses parsing.
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
/// A URL or redirect cannot be represented by this bounded HTTP URL syntax.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UrlError(pub String);
|
||||
|
||||
impl std::fmt::Display for UrlError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for UrlError {}
|
||||
|
||||
/// A parsed `http://` or `https://` URL.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Url {
|
||||
pub tls: bool,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
/// Path and query, starting with `/`.
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
/// Remove `.` and `..` segments from the path part of `path` (RFC 3986 section 5.2.4); the query is kept as is.
|
||||
fn remove_dot_segments(path: &str) -> String {
|
||||
let (p, query) = match path.find('?') {
|
||||
Some(i) => (path.get(..i).unwrap_or_default(), path.get(i..).unwrap_or_default()),
|
||||
None => (path, ""),
|
||||
};
|
||||
let mut out: Vec<&str> = Vec::new();
|
||||
let mut segments = p.split('/').skip(1).peekable();
|
||||
let mut trailing_slash = false;
|
||||
while let Some(seg) = segments.next() {
|
||||
let last = segments.peek().is_none();
|
||||
match seg {
|
||||
"." => trailing_slash = last,
|
||||
".." => {
|
||||
out.pop();
|
||||
trailing_slash = last;
|
||||
}
|
||||
_ => {
|
||||
out.push(seg);
|
||||
trailing_slash = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut result = format!("/{}", out.join("/"));
|
||||
if trailing_slash && !result.ends_with('/') {
|
||||
result.push('/');
|
||||
}
|
||||
result.push_str(query);
|
||||
result
|
||||
}
|
||||
|
||||
/// `s` without the leading `prefix`, matched ASCII case-insensitively (RFC 3986 §3.1).
|
||||
fn strip_scheme<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
|
||||
let head = s.get(..prefix.len())?;
|
||||
head.eq_ignore_ascii_case(prefix).then(|| s.get(prefix.len()..).unwrap_or_default())
|
||||
}
|
||||
|
||||
impl Url {
|
||||
pub fn parse(s: &str) -> Result<Url, UrlError> {
|
||||
let bad = |why: &str| UrlError(why.to_string());
|
||||
if s.len() > 16 * 1024 {
|
||||
return Err(bad("URL exceeds 16 KiB"));
|
||||
}
|
||||
let s = s.trim();
|
||||
if s.chars().any(|c| c.is_control() || c == ' ') {
|
||||
return Err(bad("contains spaces or control characters"));
|
||||
}
|
||||
let (tls, rest) = if let Some(r) = strip_scheme(s, "https://") {
|
||||
(true, r)
|
||||
} else if let Some(r) = strip_scheme(s, "http://") {
|
||||
(false, r)
|
||||
} else {
|
||||
return Err(bad("only http:// and https:// are supported"));
|
||||
};
|
||||
let rest = rest.split('#').next().unwrap_or_default();
|
||||
let (authority, path) = match rest.find(['/', '?']) {
|
||||
Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()),
|
||||
None => (rest, ""),
|
||||
};
|
||||
let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") };
|
||||
let path = remove_dot_segments(&path);
|
||||
if authority.contains('@') {
|
||||
return Err(bad("user names and passwords in URLs are not supported"));
|
||||
}
|
||||
let (host, port) = if let Some(v6) = authority.strip_prefix('[') {
|
||||
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
|
||||
let host = v6.get(..end).unwrap_or_default();
|
||||
let after = v6.get(end + 1..).unwrap_or_default();
|
||||
if !after.is_empty() && !after.starts_with(':') {
|
||||
return Err(bad("unexpected text after IPv6 address"));
|
||||
}
|
||||
(host.to_string(), after.strip_prefix(':'))
|
||||
} else {
|
||||
match authority.rsplit_once(':') {
|
||||
Some((h, p)) => (h.to_string(), Some(p)),
|
||||
None => (authority.to_string(), None),
|
||||
}
|
||||
};
|
||||
let port = match port {
|
||||
Some(p) if p.bytes().all(|b| b.is_ascii_digit()) => p.parse::<u16>().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?,
|
||||
Some(_) => return Err(bad("bad port")),
|
||||
None if tls => 443,
|
||||
None => 80,
|
||||
};
|
||||
if host.is_empty() || !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | ':' | '_')) {
|
||||
return Err(bad("bad host name"));
|
||||
}
|
||||
Ok(Url { tls, host: host.to_ascii_lowercase(), port, path })
|
||||
}
|
||||
|
||||
/// `Location` of a redirect, resolved against this URL.
|
||||
pub fn join(&self, location: &str) -> Result<Url, UrlError> {
|
||||
// Reject before concatenating a hostile Location with the base URL.
|
||||
if location.len() > 16 * 1024 || self.host.len().saturating_add(self.path.len()) > 16 * 1024 {
|
||||
return Err(UrlError("URL exceeds 16 KiB".into()));
|
||||
}
|
||||
let l = location.trim();
|
||||
if strip_scheme(l, "http://").is_some() || strip_scheme(l, "https://").is_some() {
|
||||
let url = Url::parse(l)?;
|
||||
if self.tls && !url.tls {
|
||||
return Err(UrlError("redirect from https to http refused".into()));
|
||||
}
|
||||
return Ok(url);
|
||||
}
|
||||
// another scheme (`ftp:`, `file:`, `data:`…)
|
||||
if let Some((scheme, _)) = l.split_once(':')
|
||||
&& !scheme.is_empty()
|
||||
&& scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
|
||||
{
|
||||
return Err(UrlError(format!("redirect to an unsupported location `{scheme}:`")));
|
||||
}
|
||||
let scheme = if self.tls { "https" } else { "http" };
|
||||
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
|
||||
let base = format!("{scheme}://{host}:{}", self.port);
|
||||
if let Some(rest) = l.strip_prefix("//") {
|
||||
return Url::parse(&format!("{scheme}://{rest}"));
|
||||
}
|
||||
if l.starts_with('/') {
|
||||
return Url::parse(&format!("{base}{l}"));
|
||||
}
|
||||
// empty or fragment-only: the same document
|
||||
if l.is_empty() || l.starts_with('#') {
|
||||
return Ok(self.clone());
|
||||
}
|
||||
// relative to the current directory
|
||||
let dir = self.path.split('?').next().unwrap_or("/");
|
||||
// a query-only reference keeps the whole path, file name included (RFC 3986 §5.2.2)
|
||||
if l.starts_with('?') {
|
||||
return Url::parse(&format!("{base}{dir}{l}"));
|
||||
}
|
||||
let dir = dir.rsplit_once('/').map_or("", |(d, _)| d);
|
||||
Url::parse(&format!("{base}{dir}/{l}"))
|
||||
}
|
||||
|
||||
/// Authority formatted for an HTTP Host field.
|
||||
pub fn host_header(&self) -> String {
|
||||
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
|
||||
let default = if self.tls { 443 } else { 80 };
|
||||
if self.port == default { host } else { format!("{host}:{}", self.port) }
|
||||
}
|
||||
|
||||
/// The URL without its query (for messages: a query may hold a signed token).
|
||||
pub fn display(&self) -> String {
|
||||
let scheme = if self.tls { "https" } else { "http" };
|
||||
format!("{scheme}://{}{}", self.host_header(), self.path.split('?').next().unwrap_or("/"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::{Url, UrlError};
|
||||
|
||||
#[test]
|
||||
fn fragment_only_or_empty_location_keeps_file_name_and_query() {
|
||||
let base = Url::parse("https://a.example/x/y/file?q").unwrap();
|
||||
for l in ["#top", "", " "] {
|
||||
let u = base.join(l).unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/x/y/file?q"), "{l:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_oversized_and_control_character_urls() {
|
||||
let base = Url::parse("https://example.com/path").unwrap();
|
||||
assert!(base.join(&format!("#{}", "x".repeat(16 * 1024))).is_err());
|
||||
assert!(base.join(&"x".repeat(16 * 1024 + 1)).is_err());
|
||||
assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err());
|
||||
assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_query_only_keeps_file_name() {
|
||||
let u = Url::parse("https://a.example/x/y/file?q=1").unwrap();
|
||||
assert_eq!(u.join("?page=2").unwrap().path, "/x/y/file?page=2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_removes_dot_segments() {
|
||||
let base = Url::parse("https://a.example/x/y/file").unwrap();
|
||||
for (loc, want) in
|
||||
[("../z", "/x/z"), ("./z", "/x/y/z"), ("../../../z", "/z"), ("/a/./b/../c", "/a/c"), ("../z?q=1", "/x/z?q=1"), ("..", "/x/")]
|
||||
{
|
||||
assert_eq!(base.join(loc).unwrap().path, want, "{loc}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_scheme_is_matched_case_insensitively() {
|
||||
let u = Url::parse("HTTPS://A.example/z").unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/z"));
|
||||
assert!(Url::parse("Http://a.example/").is_ok());
|
||||
let j = Url::parse("https://a.example/start").unwrap().join("HTTPS://cdn.example/file").unwrap();
|
||||
assert_eq!((j.tls, j.host.as_str(), j.path.as_str()), (true, "cdn.example", "/file"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_urls() {
|
||||
let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "cdn.example.com", 443, "/models/sam3/model.safetensors?sig=1"));
|
||||
assert_eq!(u.display(), "https://cdn.example.com/models/sam3/model.safetensors");
|
||||
let u = Url::parse("http://127.0.0.1:8080").unwrap();
|
||||
assert_eq!((u.tls, u.port, u.path.as_str()), (false, 8080, "/"));
|
||||
let u = Url::parse("http://[::1]:9/a").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 9));
|
||||
for bad in ["ftp://x/y", "https://", "https://a b/", "https://u:p@h/", "http://h:0/", "http://h:99999/", "https://h/\r\nX: y", "file:///etc"]
|
||||
{
|
||||
assert!(Url::parse(bad).is_err(), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_signed_ports() {
|
||||
for bad in ["http://example.com:+80/x", "http://example.com:-80/x", "http://[::1]:+80/x"] {
|
||||
assert!(Url::parse(bad).is_err(), "{bad}");
|
||||
}
|
||||
assert!(Url::parse("http://a.test/").unwrap().join("http://example.com:+80/x").is_err());
|
||||
assert_eq!(Url::parse("http://example.com:80/x").unwrap().port, 80);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_text_after_a_bracketed_ipv6_host() {
|
||||
for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] {
|
||||
assert!(matches!(Url::parse(bad), Err(UrlError(_))), "{bad}");
|
||||
}
|
||||
let u = Url::parse("http://[::1]:8080/x").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 8080));
|
||||
let u = Url::parse("http://[::1]/x").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 80));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_redirects() {
|
||||
let u = Url::parse("https://a.example/x/y/file?q").unwrap();
|
||||
assert_eq!(u.join("https://b.example/z").unwrap().host, "b.example");
|
||||
assert_eq!(u.join("/root").unwrap().path, "/root");
|
||||
assert_eq!(u.join("other").unwrap().path, "/x/y/other");
|
||||
assert_eq!(u.join("//c.example/p").unwrap().host, "c.example");
|
||||
assert!(u.join("ftp://nope").is_err());
|
||||
// no downgrade from https to http, however the redirect is written
|
||||
assert!(u.join("http://a.example/z").is_err());
|
||||
let plain = Url::parse("http://127.0.0.1:8080/a").unwrap();
|
||||
assert!(!plain.join("http://127.0.0.1:9/b").unwrap().tls);
|
||||
assert!(plain.join("https://b.example/z").unwrap().tls);
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ workspace = true
|
||||
[features]
|
||||
corpus = []
|
||||
default = []
|
||||
sgi = []
|
||||
avif = ["image/avif", "dep:ravif", "dep:rav1e", "dep:avif-serialize"]
|
||||
heif = ["dep:craft-heif"]
|
||||
# AVIF decode: craft-video's clean-room pure-Rust AV1 decoder. Native targets only (on wasm32 a
|
||||
|
||||
@@ -57,6 +57,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "<
|
||||
| Cineon | yes | yes | U16 (10-bit code values read and written exactly through U16) | Gray, RGB | no | no | no | no | no | no | no | built in |
|
||||
| DICOM | yes | **no** | U8, U16 | Gray, RGB | no | no | no | no | no | no | n/a | built in |
|
||||
| Amiga IFF (ILBM/PBM) | yes | yes (24-bit ILBM) | U8 | RGB, RGBA | yes (1-bit mask plane on write) | no | no | no | no | no | no | built in |
|
||||
| SGI (`.sgi`, `.rgb`, `.bw`) | with feature `sgi` | with feature `sgi` (RLE) | U8, U16 | Gray, Gray+A, RGB, RGBA | yes | no | no | no | no | no | no | `sgi` (off by default) |
|
||||
| JPEG-LS (`.jls`) | yes (any NEAR; one scan per component) | yes (lossless) | U8, U16 | Gray, RGB | no | no | no | no | no | no | no | built in |
|
||||
| AVIF | with feature `avif-decode` (native only) | only with feature `avif` | U8, U16 (10/12-bit decodes to U16; PQ/HLG to linear F32) | RGB, RGBA | yes (auxiliary alpha) | read (`colr` prof/rICC) | read | read | no | no | yes | `craft-video` AV1 decoder; `image` + `ravif` to write |
|
||||
| HEIF/HEIC | with feature `heif` | **no** | U8, U16 (10/12-bit decodes to U16) | RGB, RGBA | yes (auxiliary alpha) | yes (`colr` prof) | yes | yes | no | no | n/a | `heic-rs` |
|
||||
|
||||
@@ -20,6 +20,8 @@ pub(crate) mod jxl;
|
||||
pub(crate) mod pcx;
|
||||
pub(crate) mod png;
|
||||
pub(crate) mod pnm;
|
||||
#[cfg(feature = "sgi")]
|
||||
pub(crate) mod sgi;
|
||||
pub(crate) mod tga;
|
||||
pub(crate) mod tiff;
|
||||
pub(crate) mod tiff_exif;
|
||||
|
||||
@@ -842,6 +842,10 @@ fn read_metadata(c: &Container) -> (Option<Vec<u8>>, Option<String>) {
|
||||
/// Decodes the primary image. `keep_orientation` leaves the container's `clap`, `irot` and
|
||||
/// `imir` unapplied (the stored pixels), as for HEIF.
|
||||
pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<Image> {
|
||||
decode_with_depth(bytes, limits, keep_orientation).map(|(image, _)| image)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_with_depth(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<(Image, u8)> {
|
||||
let c = parse(bytes)?;
|
||||
let id = c.primary;
|
||||
let (w, h) = item_size(&c, id)?;
|
||||
@@ -890,9 +894,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> R
|
||||
// PQ or HLG: linear light in 32-bit float, as for HEIF.
|
||||
if cicp.is_hdr() {
|
||||
limits.check(w, h, layout, SampleType::F32)?;
|
||||
return crate::cicp::linearize(img, cicp);
|
||||
return crate::cicp::linearize(img, cicp).map(|image| (image, bits));
|
||||
}
|
||||
Ok(img)
|
||||
Ok((img, bits))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1301,6 +1305,9 @@ mod tests {
|
||||
let src = Image::from_u8(w as u32, h as u32, ChannelLayout::Rgb, card(w, h, 3)).unwrap();
|
||||
let file = encode_avif_with_speed(&src, 95, 10).unwrap();
|
||||
let img = crate::decode(&file).unwrap();
|
||||
let (with_depth, depth) = crate::decode_avif_with_depth(&file, &crate::DecodeOptions::default()).unwrap();
|
||||
assert_eq!(depth, 10);
|
||||
assert_eq!(with_depth, img);
|
||||
assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (48, 32, ChannelLayout::Rgb, SampleType::U16));
|
||||
let diff = mean_abs_diff(&rgb8(&img), &card(w, h, 3));
|
||||
assert!(diff < 2.5, "mean difference {diff}");
|
||||
|
||||
@@ -0,0 +1,395 @@
|
||||
//! SGI image files (`.sgi`, `.rgb`, `.rgba`, `.bw`): 8- and 16-bit gray, gray + alpha, RGB and
|
||||
//! RGBA, verbatim or run-length encoded.
|
||||
//!
|
||||
//! Read and write follow the published SGI image file format description: a 512-byte big-endian
|
||||
//! header, then one plane per channel with the rows stored bottom to top. Palette-mapped files
|
||||
//! (colormap ID other than 0) are rejected. The writer uses the run-length encoding.
|
||||
|
||||
use crate::error::CodecError;
|
||||
use crate::fidelity::Plan;
|
||||
use crate::format::Format;
|
||||
use crate::image::{ChannelLayout, Image, SampleType};
|
||||
use crate::options::Limits;
|
||||
|
||||
const F: Format = Format::Sgi;
|
||||
const HEADER: usize = 512;
|
||||
const MAGIC: u16 = 474;
|
||||
|
||||
fn bad(reason: impl std::fmt::Display) -> CodecError {
|
||||
CodecError::malformed(F, reason)
|
||||
}
|
||||
|
||||
fn be16(b: &[u8], i: usize) -> u16 {
|
||||
u16::from_be_bytes([b[i], b[i + 1]])
|
||||
}
|
||||
|
||||
fn be32(b: &[u8], i: usize) -> u32 {
|
||||
u32::from_be_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]])
|
||||
}
|
||||
|
||||
/// The fields of the header that decoding needs, already checked for plausibility.
|
||||
struct Header {
|
||||
rle: bool,
|
||||
wide: bool,
|
||||
width: u32,
|
||||
height: u32,
|
||||
planes: usize,
|
||||
}
|
||||
|
||||
/// Parse the 512-byte header. `None` when the bytes are not a plausible SGI header.
|
||||
fn header(b: &[u8]) -> Option<Header> {
|
||||
if b.len() < HEADER || be16(b, 0) != MAGIC {
|
||||
return None;
|
||||
}
|
||||
let (storage, bpc, dimension) = (b[2], b[3], be16(b, 4));
|
||||
let (width, height, planes) = (u32::from(be16(b, 6)), u32::from(be16(b, 8)), usize::from(be16(b, 10)));
|
||||
if storage > 1 || !(1..=2).contains(&bpc) || !(1..=3).contains(&dimension) {
|
||||
return None;
|
||||
}
|
||||
// A one-dimensional image has one row, a two-dimensional one a single plane.
|
||||
let (height, planes) = match dimension {
|
||||
1 => (1, 1),
|
||||
2 => (height, 1),
|
||||
_ => (height, planes),
|
||||
};
|
||||
if width == 0 || height == 0 || !(1..=4).contains(&planes) {
|
||||
return None;
|
||||
}
|
||||
Some(Header { rle: storage == 1, wide: bpc == 2, width, height, planes })
|
||||
}
|
||||
|
||||
/// Whether `b` starts with a plausible SGI header (the magic number plus consistent fields).
|
||||
pub(crate) fn looks_like_sgi(b: &[u8]) -> bool {
|
||||
header(b).is_some()
|
||||
}
|
||||
|
||||
fn layout_of(planes: usize) -> ChannelLayout {
|
||||
match planes {
|
||||
1 => ChannelLayout::Gray,
|
||||
2 => ChannelLayout::GrayA,
|
||||
3 => ChannelLayout::Rgb,
|
||||
_ => ChannelLayout::Rgba,
|
||||
}
|
||||
}
|
||||
|
||||
/// Expand one run-length encoded row into exactly `out.len()` samples.
|
||||
fn read_rle_row(src: &[u8], wide: bool, out: &mut [u16]) -> Result<(), CodecError> {
|
||||
let step = if wide { 2 } else { 1 };
|
||||
let sample = |pos: usize| -> Result<u16, CodecError> {
|
||||
let s = src.get(pos..pos + step).ok_or_else(|| bad("truncated RLE row"))?;
|
||||
Ok(if wide { u16::from_be_bytes([s[0], s[1]]) } else { u16::from(s[0]) })
|
||||
};
|
||||
let (mut pos, mut n) = (0usize, 0usize);
|
||||
while n < out.len() {
|
||||
let code = sample(pos)?;
|
||||
pos += step;
|
||||
let count = usize::from(code & 0x7f);
|
||||
if count == 0 {
|
||||
return Err(bad("RLE row ends before the image width"));
|
||||
}
|
||||
if count > out.len() - n {
|
||||
return Err(bad("RLE packet runs past the end of the row"));
|
||||
}
|
||||
if code & 0x80 != 0 {
|
||||
for slot in &mut out[n..n + count] {
|
||||
*slot = sample(pos)?;
|
||||
pos += step;
|
||||
}
|
||||
} else {
|
||||
let value = sample(pos)?;
|
||||
pos += step;
|
||||
out[n..n + count].fill(value);
|
||||
}
|
||||
n += count;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result<Image, CodecError> {
|
||||
let h = header(b).ok_or_else(|| bad("invalid SGI header"))?;
|
||||
if be32(b, 104) != 0 {
|
||||
return Err(CodecError::unsupported(F, "palette-mapped SGI files are not supported"));
|
||||
}
|
||||
let layout = layout_of(h.planes);
|
||||
let sample = if h.wide { SampleType::U16 } else { SampleType::U8 };
|
||||
limits.check(h.width, h.height, layout, sample)?;
|
||||
let (w, ht, planes) = (h.width as usize, h.height as usize, h.planes);
|
||||
let rows = ht.checked_mul(planes).ok_or_else(|| bad("image size overflow"))?;
|
||||
let bps = if h.wide { 2 } else { 1 };
|
||||
let total = w.checked_mul(rows).ok_or_else(|| bad("image size overflow"))?;
|
||||
// Reject inputs that cannot hold the declared pixels before allocating anything for them.
|
||||
let table = if h.rle { rows.checked_mul(8).ok_or_else(|| bad("image size overflow"))? } else { 0 };
|
||||
let min_len = if h.rle {
|
||||
// Both offset tables must fit; rows may share compressed data, so the data itself is
|
||||
// checked row by row against the offsets below.
|
||||
HEADER + table
|
||||
} else {
|
||||
HEADER + total.checked_mul(bps).ok_or_else(|| bad("image size overflow"))?
|
||||
};
|
||||
if b.len() < min_len {
|
||||
return Err(bad("the file is too short for the declared image"));
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
out.try_reserve_exact(total).map_err(|_| bad("pixel buffer is too large"))?;
|
||||
out.resize(total, 0u16);
|
||||
let mut row = vec![0u16; w];
|
||||
for z in 0..planes {
|
||||
for y in 0..ht {
|
||||
let index = y + z * ht;
|
||||
if h.rle {
|
||||
let start = be32(b, HEADER + index * 4) as usize;
|
||||
let len = be32(b, HEADER + rows * 4 + index * 4) as usize;
|
||||
let end = start
|
||||
.checked_add(len)
|
||||
.filter(|&e| e <= b.len() && start >= HEADER + table)
|
||||
.ok_or_else(|| bad("RLE row lies outside the file"))?;
|
||||
read_rle_row(&b[start..end], h.wide, &mut row)?;
|
||||
} else {
|
||||
let start = HEADER + index * w * bps;
|
||||
let s = &b[start..start + w * bps];
|
||||
for (x, v) in row.iter_mut().enumerate() {
|
||||
*v = if h.wide { u16::from_be_bytes([s[2 * x], s[2 * x + 1]]) } else { u16::from(s[x]) };
|
||||
}
|
||||
}
|
||||
// Rows are stored bottom to top; planes are separate.
|
||||
let top_y = ht - 1 - y;
|
||||
for (x, &v) in row.iter().enumerate() {
|
||||
out[(top_y * w + x) * planes + z] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
if h.wide {
|
||||
Image::from_u16(h.width, h.height, layout, &out)
|
||||
} else {
|
||||
Image::from_u8(h.width, h.height, layout, out.iter().map(|&v| v as u8).collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// Append one run-length encoded row (one packet list and a terminator).
|
||||
fn write_rle_row(row: &[u16], wide: bool, out: &mut Vec<u8>) {
|
||||
let put = |out: &mut Vec<u8>, v: u16| {
|
||||
if wide {
|
||||
out.extend_from_slice(&v.to_be_bytes());
|
||||
} else {
|
||||
out.push(v as u8);
|
||||
}
|
||||
};
|
||||
let mut i = 0;
|
||||
while i < row.len() {
|
||||
let mut run = 1;
|
||||
while i + run < row.len() && run < 127 && row[i + run] == row[i] {
|
||||
run += 1;
|
||||
}
|
||||
if run >= 2 {
|
||||
put(out, run as u16);
|
||||
put(out, row[i]);
|
||||
i += run;
|
||||
} else {
|
||||
// A literal packet extends until the next run of two or more.
|
||||
let mut end = i + 1;
|
||||
while end < row.len() && end - i < 127 && !(end + 1 < row.len() && row[end] == row[end + 1]) {
|
||||
end += 1;
|
||||
}
|
||||
put(out, 0x80 | (end - i) as u16);
|
||||
for &v in &row[i..end] {
|
||||
put(out, v);
|
||||
}
|
||||
i = end;
|
||||
}
|
||||
}
|
||||
put(out, 0);
|
||||
}
|
||||
|
||||
/// Writes a run-length encoded SGI file: 8 or 16 bits per channel, gray, gray + alpha, RGB or RGBA.
|
||||
pub(crate) fn encode(src: &Image, plan: Plan) -> Result<Vec<u8>, CodecError> {
|
||||
let enc = |msg: String| CodecError::encode(F, msg);
|
||||
let img = src.converted(plan.layout, plan.sample);
|
||||
let wide = match img.sample_type() {
|
||||
SampleType::U8 => false,
|
||||
SampleType::U16 => true,
|
||||
s => return Err(enc(format!("SGI stores 8 or 16 bits per channel, not {s:?}"))),
|
||||
};
|
||||
let planes = img.layout().channels();
|
||||
if !matches!(img.layout(), ChannelLayout::Gray | ChannelLayout::GrayA | ChannelLayout::Rgb | ChannelLayout::Rgba) {
|
||||
return Err(enc(format!("no SGI layout for {:?}", img.layout())));
|
||||
}
|
||||
let (w, ht) = img.dimensions();
|
||||
let (Ok(w16), Ok(h16)) = (u16::try_from(w), u16::try_from(ht)) else {
|
||||
return Err(enc(format!("dimensions exceed 65535x65535: {w}x{ht}")));
|
||||
};
|
||||
let (w, ht) = (w as usize, ht as usize);
|
||||
let bps = if wide { 2 } else { 1 };
|
||||
let data = img.data();
|
||||
if data.len() != w * ht * planes * bps {
|
||||
return Err(enc("the image has the wrong number of samples".into()));
|
||||
}
|
||||
let sample_at = |i: usize| -> u16 { if wide { u16::from_ne_bytes([data[2 * i], data[2 * i + 1]]) } else { u16::from(data[i]) } };
|
||||
let rows = ht * planes;
|
||||
let table = HEADER;
|
||||
let mut starts = Vec::with_capacity(rows);
|
||||
let mut lens = Vec::with_capacity(rows);
|
||||
let mut body = Vec::new();
|
||||
let mut row = vec![0u16; w];
|
||||
let base = u32::try_from(HEADER + rows * 8).map_err(|_| enc("image too large".into()))?;
|
||||
for z in 0..planes {
|
||||
for y in 0..ht {
|
||||
let src_y = ht - 1 - y; // stored bottom to top
|
||||
for (x, v) in row.iter_mut().enumerate() {
|
||||
*v = sample_at((src_y * w + x) * planes + z);
|
||||
}
|
||||
let before = body.len();
|
||||
write_rle_row(&row, wide, &mut body);
|
||||
let start = u32::try_from(base as usize + before).map_err(|_| enc("image too large".into()))?;
|
||||
starts.push(start);
|
||||
lens.push((body.len() - before) as u32);
|
||||
}
|
||||
}
|
||||
let mut out = Vec::with_capacity(table + rows * 8 + body.len());
|
||||
out.extend_from_slice(&MAGIC.to_be_bytes());
|
||||
out.push(1); // RLE
|
||||
out.push(bps as u8);
|
||||
out.extend_from_slice(&(if planes > 1 { 3u16 } else { 2u16 }).to_be_bytes());
|
||||
out.extend_from_slice(&w16.to_be_bytes());
|
||||
out.extend_from_slice(&h16.to_be_bytes());
|
||||
out.extend_from_slice(&(planes as u16).to_be_bytes());
|
||||
out.extend_from_slice(&0u32.to_be_bytes()); // pixmin
|
||||
out.extend_from_slice(&(if wide { 65535u32 } else { 255 }).to_be_bytes()); // pixmax
|
||||
out.resize(HEADER, 0); // dummy, image name, colormap 0, padding
|
||||
for s in &starts {
|
||||
out.extend_from_slice(&s.to_be_bytes());
|
||||
}
|
||||
for l in &lens {
|
||||
out.extend_from_slice(&l.to_be_bytes());
|
||||
}
|
||||
out.extend_from_slice(&body);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn gradient(w: u32, h: u32, layout: ChannelLayout) -> Image {
|
||||
let ch = layout.channels();
|
||||
let data = (0..w as usize * h as usize * ch).map(|i| ((i * 7 + i / 5) % 251) as u8).collect();
|
||||
Image::from_u8(w, h, layout, data).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rle_round_trips_every_layout_in_8_bits() {
|
||||
for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] {
|
||||
let img = gradient(37, 11, layout);
|
||||
let plan = Plan { layout, sample: SampleType::U8 };
|
||||
let bytes = encode(&img, plan).unwrap();
|
||||
assert!(looks_like_sgi(&bytes));
|
||||
let back = decode(&bytes, &Limits::default()).unwrap();
|
||||
assert_eq!(back.layout(), layout);
|
||||
assert_eq!(back.data(), img.data());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rle_round_trips_16_bits_with_long_runs() {
|
||||
let (w, h) = (300u32, 4u32);
|
||||
let samples: Vec<u16> = (0..w * h * 3).map(|i| if (i / 3) % 200 < 150 { 4242 } else { (i * 31) as u16 }).collect();
|
||||
let img = Image::from_u16(w, h, ChannelLayout::Rgb, &samples).unwrap();
|
||||
let bytes = encode(&img, Plan { layout: ChannelLayout::Rgb, sample: SampleType::U16 }).unwrap();
|
||||
let back = decode(&bytes, &Limits::default()).unwrap();
|
||||
assert_eq!(back.sample_type(), SampleType::U16);
|
||||
assert_eq!(back.data(), img.data());
|
||||
}
|
||||
|
||||
/// A hand-built verbatim file: two rows, one plane, stored bottom to top.
|
||||
#[test]
|
||||
fn verbatim_files_read_bottom_to_top() {
|
||||
let mut f = vec![0u8; HEADER];
|
||||
f[0..2].copy_from_slice(&MAGIC.to_be_bytes());
|
||||
f[2] = 0; // verbatim
|
||||
f[3] = 1;
|
||||
f[4..6].copy_from_slice(&2u16.to_be_bytes());
|
||||
f[6..8].copy_from_slice(&3u16.to_be_bytes()); // x
|
||||
f[8..10].copy_from_slice(&2u16.to_be_bytes()); // y
|
||||
f[10..12].copy_from_slice(&1u16.to_be_bytes()); // z
|
||||
f.extend_from_slice(&[10, 11, 12, 20, 21, 22]); // bottom row first
|
||||
let img = decode(&f, &Limits::default()).unwrap();
|
||||
assert_eq!(img.layout(), ChannelLayout::Gray);
|
||||
assert_eq!(img.data(), &[20, 21, 22, 10, 11, 12]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_input_is_an_error_not_a_panic() {
|
||||
let img = gradient(16, 8, ChannelLayout::Rgb);
|
||||
let good = encode(&img, Plan { layout: ChannelLayout::Rgb, sample: SampleType::U8 }).unwrap();
|
||||
// Every truncation fails cleanly.
|
||||
for len in 0..good.len() {
|
||||
let _ = decode(&good[..len], &Limits::default());
|
||||
}
|
||||
// A row table pointing outside the file, and a packet that overruns its row.
|
||||
let mut bad_table = good.clone();
|
||||
bad_table[HEADER..HEADER + 4].copy_from_slice(&u32::MAX.to_be_bytes());
|
||||
assert!(decode(&bad_table, &Limits::default()).is_err());
|
||||
let mut overrun = good.clone();
|
||||
let start = be32(&overrun, HEADER) as usize;
|
||||
overrun[start] = 127; // a run longer than the 16-pixel row
|
||||
assert!(decode(&overrun, &Limits::default()).is_err());
|
||||
// A huge declared size is stopped by the limits or the length check, before allocating.
|
||||
let mut huge = good;
|
||||
huge[6..8].copy_from_slice(&u16::MAX.to_be_bytes());
|
||||
huge[8..10].copy_from_slice(&u16::MAX.to_be_bytes());
|
||||
assert!(decode(&huge, &Limits::default()).is_err());
|
||||
}
|
||||
|
||||
/// A 1-plane, 8-bit RLE file whose `rows` all start where `starts[i]` says, each `len` bytes long.
|
||||
fn rle_file(w: u16, starts: &[u32], len: u32, payload: &[u8]) -> Vec<u8> {
|
||||
let mut f = vec![0u8; HEADER];
|
||||
f[0..2].copy_from_slice(&MAGIC.to_be_bytes());
|
||||
f[2] = 1;
|
||||
f[3] = 1;
|
||||
f[4..6].copy_from_slice(&2u16.to_be_bytes());
|
||||
f[6..8].copy_from_slice(&w.to_be_bytes());
|
||||
f[8..10].copy_from_slice(&(starts.len() as u16).to_be_bytes());
|
||||
f[10..12].copy_from_slice(&1u16.to_be_bytes());
|
||||
for s in starts {
|
||||
f.extend_from_slice(&s.to_be_bytes());
|
||||
}
|
||||
for _ in starts {
|
||||
f.extend_from_slice(&len.to_be_bytes());
|
||||
}
|
||||
f.extend_from_slice(payload);
|
||||
f
|
||||
}
|
||||
|
||||
/// The format lets table entries share one compressed row: eight rows, one payload.
|
||||
#[test]
|
||||
fn rows_may_share_compressed_data() {
|
||||
let table = HEADER + 8 * 8;
|
||||
let f = rle_file(1, &[table as u32; 8], 3, &[0x81, 255, 0]);
|
||||
assert_eq!(f.len(), table + 3);
|
||||
let img = decode(&f, &Limits::default()).unwrap();
|
||||
assert_eq!((img.dimensions(), img.data()), ((1, 8), &[255u8; 8][..]));
|
||||
}
|
||||
|
||||
/// Compressed rows start after both tables: an offset into a table is refused.
|
||||
#[test]
|
||||
fn rows_may_not_start_inside_the_offset_tables() {
|
||||
// 3x1: the tables end at 520; 515 reads the table's own bytes as a packet list.
|
||||
let f = rle_file(3, &[515], 3, &[0, 0, 0, 0, 0]);
|
||||
assert!(decode(&f, &Limits::default()).is_err());
|
||||
}
|
||||
|
||||
/// FFmpeg's decoder, run as an oracle, also reads a row that ends with its last packet and no
|
||||
/// zero terminator; so does this one.
|
||||
#[test]
|
||||
fn a_row_without_its_terminator_still_reads() {
|
||||
let f = rle_file(1, &[520], 2, &[0x81, 42]);
|
||||
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[42]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn palette_mapped_files_are_unsupported() {
|
||||
let img = gradient(4, 4, ChannelLayout::Gray);
|
||||
let mut f = encode(&img, Plan { layout: ChannelLayout::Gray, sample: SampleType::U8 }).unwrap();
|
||||
f[104..108].copy_from_slice(&1u32.to_be_bytes());
|
||||
assert!(matches!(decode(&f, &Limits::default()), Err(CodecError::Unsupported { .. })));
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,8 @@ pub enum Format {
|
||||
Iff,
|
||||
/// JPEG 2000 (JP2 container or raw J2K codestream): read-only; see [`ASYMMETRIC_EXCEPTIONS`].
|
||||
Jp2,
|
||||
/// SGI image files (`.sgi`, `.rgb`, `.bw`): 8- and 16-bit gray, gray + alpha, RGB and RGBA.
|
||||
Sgi,
|
||||
/// JPEG-LS (`.jls`): lossless and near-lossless gray and RGB.
|
||||
JpegLs,
|
||||
}
|
||||
@@ -118,7 +120,7 @@ const TIFF_LAYOUTS: &[ChannelLayout] = &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L::
|
||||
|
||||
impl Format {
|
||||
/// Every format known to the crate (enabled or not).
|
||||
pub const ALL: [Format; 25] = [
|
||||
pub const ALL: [Format; 26] = [
|
||||
Format::Png,
|
||||
Format::Jpeg,
|
||||
Format::Jp2,
|
||||
@@ -143,6 +145,7 @@ impl Format {
|
||||
Format::Cineon,
|
||||
Format::Dicom,
|
||||
Format::Iff,
|
||||
Format::Sgi,
|
||||
Format::JpegLs,
|
||||
];
|
||||
|
||||
@@ -182,6 +185,7 @@ impl Format {
|
||||
Format::Dicom => "DICOM",
|
||||
Format::Iff => "Amiga IFF",
|
||||
Format::Jp2 => "JPEG 2000",
|
||||
Format::Sgi => "SGI",
|
||||
Format::JpegLs => "JPEG-LS",
|
||||
}
|
||||
}
|
||||
@@ -214,6 +218,7 @@ impl Format {
|
||||
Format::Dicom => &["dcm", "dicom", "dc3"],
|
||||
Format::Iff => &["iff", "lbm", "ilbm"],
|
||||
Format::Jp2 => &["jp2", "j2k", "j2c", "jpf", "jpx"],
|
||||
Format::Sgi => &["sgi", "rgb", "rgba", "bw"],
|
||||
Format::JpegLs => &["jls"],
|
||||
}
|
||||
}
|
||||
@@ -244,6 +249,7 @@ impl Format {
|
||||
Format::Dicom => "application/dicom",
|
||||
Format::Iff => "image/x-iff",
|
||||
Format::Jp2 => "image/jp2",
|
||||
Format::Sgi => "image/x-sgi",
|
||||
Format::JpegLs => "image/jls",
|
||||
}
|
||||
}
|
||||
@@ -251,7 +257,7 @@ impl Format {
|
||||
/// Maximum encodable dimensions, if the format has a hard limit.
|
||||
pub fn max_dimensions(self) -> Option<(u32, u32)> {
|
||||
match self {
|
||||
Format::Jpeg | Format::Gif | Format::Iff | Format::JpegLs => Some((65535, 65535)),
|
||||
Format::Jpeg | Format::Gif | Format::Iff | Format::Sgi | Format::JpegLs => Some((65535, 65535)),
|
||||
Format::WebP => Some((16384, 16384)),
|
||||
Format::Ico => Some((256, 256)),
|
||||
_ => None,
|
||||
@@ -321,6 +327,9 @@ pub fn caps(format: Format) -> FormatCaps {
|
||||
Format::Cineon => FormatCaps { depths: &[S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
|
||||
Format::Dicom => FormatCaps { read: true, write: false, depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
|
||||
Format::Iff => FormatCaps { depths: &[S::U8], layouts: &[L::Rgb, L::Rgba], alpha: true, ..base },
|
||||
Format::Sgi => {
|
||||
FormatCaps { read: cfg!(feature = "sgi"), write: cfg!(feature = "sgi"), depths: &[S::U8, S::U16], layouts: RGB_GRAY, alpha: true, ..base }
|
||||
}
|
||||
Format::JpegLs => FormatCaps { depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
|
||||
Format::Jp2 => FormatCaps { read: true, write: false, layouts: &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L::Cmyk, L::CmykA], icc: true, ..base },
|
||||
}
|
||||
@@ -403,6 +412,10 @@ pub fn detect(bytes: &[u8]) -> Option<Format> {
|
||||
if b.starts_with(b"/* XPM */") || b.starts_with(b"! XPM2\n") || b.starts_with(b"! XPM2\r\n") {
|
||||
return Some(Format::Xpm);
|
||||
}
|
||||
#[cfg(feature = "sgi")]
|
||||
if crate::codecs::sgi::looks_like_sgi(b) {
|
||||
return Some(Format::Sgi);
|
||||
}
|
||||
if crate::codecs::pcx::looks_like_pcx(b) {
|
||||
return Some(Format::Pcx);
|
||||
}
|
||||
|
||||
@@ -67,6 +67,8 @@ pub use crate::resolution::{exif_resolution, export_exif, export_xmp, photoshop_
|
||||
pub use crate::source_settings::{WebpEncoding, jpeg_quality, webp_encoding};
|
||||
pub use half::f16;
|
||||
|
||||
#[cfg(feature = "sgi")]
|
||||
use crate::codecs::sgi;
|
||||
use crate::codecs::{dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm};
|
||||
|
||||
/// Detect the format and decode with default [`Limits`].
|
||||
@@ -106,6 +108,16 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
|
||||
Format::Cineon => dpx::decode_cineon(bytes, l),
|
||||
Format::Dicom => dicom::decode(bytes, l),
|
||||
Format::Iff => iff::decode(bytes, l),
|
||||
Format::Sgi => {
|
||||
#[cfg(feature = "sgi")]
|
||||
{
|
||||
sgi::decode(bytes, l)
|
||||
}
|
||||
#[cfg(not(feature = "sgi"))]
|
||||
{
|
||||
Err(CodecError::unsupported(format, "SGI support requires the sgi feature"))
|
||||
}
|
||||
}
|
||||
Format::JpegLs => jpegls::decode(bytes, l),
|
||||
Format::OpenExr => exr::decode(bytes, l),
|
||||
Format::Heif => heif::decode(bytes, l, opts.keep_orientation),
|
||||
@@ -120,6 +132,14 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
|
||||
finish_format(format, bytes, img, opts)
|
||||
}
|
||||
|
||||
/// Decode AVIF pixels and report their original AV1 bit depth (8, 10 or 12).
|
||||
/// The returned image may use a wider sample type, particularly for linear HDR.
|
||||
#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))]
|
||||
pub fn decode_avif_with_depth(bytes: &[u8], opts: &DecodeOptions) -> Result<(Image, u8), CodecError> {
|
||||
let (image, depth) = codecs::avif_decode::decode_with_depth(bytes, &opts.limits, opts.keep_orientation)?;
|
||||
Ok((finish_format(Format::Avif, bytes, image, opts)?, depth))
|
||||
}
|
||||
|
||||
/// What [`decode_as_with`] does with a PNG decoded in steps by [`PngDecodeRows`] (`img` is its
|
||||
/// output): the resolution from the metadata, the upright orientation and the final limit check,
|
||||
/// so the result equals [`decode_as_with`]'s on the same file.
|
||||
@@ -473,6 +493,16 @@ pub fn encode(image: &Image, format: Format, opts: &EncodeOptions) -> Result<Vec
|
||||
Format::Cineon => dpx::encode_cineon(image, plan),
|
||||
Format::Dicom => Err(CodecError::unsupported(format, "DICOM is import-only; save as TIFF or PNG instead")),
|
||||
Format::Iff => iff::encode(image, plan),
|
||||
Format::Sgi => {
|
||||
#[cfg(feature = "sgi")]
|
||||
{
|
||||
sgi::encode(image, plan)
|
||||
}
|
||||
#[cfg(not(feature = "sgi"))]
|
||||
{
|
||||
Err(CodecError::unsupported(format, "SGI support requires the sgi feature"))
|
||||
}
|
||||
}
|
||||
Format::JpegLs => jpegls::encode(image, plan),
|
||||
Format::OpenExr => exr::encode(image, plan, opts),
|
||||
Format::Heif => Err(CodecError::unsupported(format, "encoding is not available in this build")),
|
||||
|
||||
@@ -62,6 +62,8 @@ fn default_build_formats_all_symmetric() {
|
||||
Format::Dpx,
|
||||
Format::Cineon,
|
||||
Format::Iff,
|
||||
#[cfg(feature = "sgi")]
|
||||
Format::Sgi,
|
||||
Format::JpegLs,
|
||||
];
|
||||
for f in expected {
|
||||
@@ -224,6 +226,23 @@ fn detect_magic_iff() {
|
||||
assert_eq!(detect(b"FORM\0\0\0\x14AIFF"), None);
|
||||
assert!(decode_as(Format::Iff, b"FORM\0\0\0\x14ILBM").is_err());
|
||||
}
|
||||
#[cfg(feature = "sgi")]
|
||||
#[test]
|
||||
fn detect_magic_sgi() {
|
||||
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
|
||||
let img = Image::from_u8(5, 3, ChannelLayout::Rgba, (0..60).collect()).unwrap();
|
||||
let bytes = encode(&img, Format::Sgi, &EncodeOptions::default()).unwrap();
|
||||
assert_eq!(detect(&bytes), Some(Format::Sgi));
|
||||
assert_eq!(from_extension("scan.RGB"), Some(Format::Sgi));
|
||||
assert_eq!(from_extension("scan.bw"), Some(Format::Sgi));
|
||||
assert_eq!(decode(&bytes).unwrap().data(), img.data());
|
||||
assert!(!ASYMMETRIC_EXCEPTIONS.iter().any(|(f, _)| *f == Format::Sgi));
|
||||
assert_eq!(Format::Sgi.max_dimensions(), Some((65535, 65535)));
|
||||
// The magic number alone, without a full header, is not SGI.
|
||||
assert_eq!(detect(&[0x01, 0xda, 0, 1, 0, 2]), None);
|
||||
assert!(decode_as(Format::Sgi, &[0x01, 0xda]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_magic_jpeg_ls() {
|
||||
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
|
||||
@@ -328,3 +347,16 @@ fn extensions_unique_across_formats() {
|
||||
all.dedup();
|
||||
assert_eq!(all.len(), n);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "sgi"))]
|
||||
#[test]
|
||||
fn sgi_requires_explicit_feature() {
|
||||
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
|
||||
assert!(!caps(Format::Sgi).read && !caps(Format::Sgi).write);
|
||||
assert!(decode_as(Format::Sgi, &[]).is_err());
|
||||
let image = Image::from_u8(1, 1, ChannelLayout::Gray, vec![0]).unwrap();
|
||||
assert!(encode(&image, Format::Sgi, &EncodeOptions::default()).is_err());
|
||||
let mut header = vec![0; 512];
|
||||
header[..12].copy_from_slice(&[1, 218, 0, 1, 0, 2, 0, 1, 0, 1, 0, 1]);
|
||||
assert_eq!(detect(&header), None);
|
||||
}
|
||||
@@ -899,7 +899,7 @@ struct Layer {
|
||||
impl Layer {
|
||||
/// The layer and its flat array of `width` numbers per entry.
|
||||
fn of(geom: &Record, element: &str, data: &str, index: &[&str], width: usize) -> Option<(Layer, Vec<f64>)> {
|
||||
let e = geom.child(element)?;
|
||||
let e = geom.children_named(element).min_by_key(|e| e.props.first().and_then(Prop::as_i64) != Some(0))?;
|
||||
let mapping = match e.child("MappingInformationType").and_then(|m| m.str_at(0)).unwrap_or("ByPolygonVertex") {
|
||||
"ByPolygonVertex" => Mapping::PolygonVertex,
|
||||
"ByVertex" | "ByVertice" | "ByControlPoint" => Mapping::Vertex,
|
||||
@@ -908,7 +908,7 @@ impl Layer {
|
||||
_ => return None,
|
||||
};
|
||||
let reference = e.child("ReferenceInformationType").and_then(|m| m.str_at(0)).unwrap_or("Direct");
|
||||
let index = (reference != "Direct").then(|| index.iter().find_map(|n| e.child(n)).map(Record::ints)).flatten();
|
||||
let index = if matches!(reference, "IndexToDirect" | "Index") { Some(index.iter().find_map(|n| e.child(n))?.ints()) } else { None };
|
||||
let data = e.child(data)?.floats();
|
||||
Some((Layer { mapping, index, len: data.len() / width }, data))
|
||||
}
|
||||
|
||||
@@ -628,6 +628,16 @@ fn fbx6_model_holds_its_mesh() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layer_zero_takes_precedence_and_an_indexed_layer_needs_indices() {
|
||||
let layer_one = "LayerElementUV: 1 { MappingInformationType: \"ByPolygonVertex\" ReferenceInformationType: \"Direct\" UV: 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25, 0.25,0.25 }\n";
|
||||
let text = ASCII.replace("LayerElementUV: 0 {", &format!("{layer_one} LayerElementUV: 0 {{"));
|
||||
let scene = parse(text.as_bytes()).unwrap();
|
||||
assert_eq!(scene.meshes[0].uv_sets[0][0], [0.0, 1.0]);
|
||||
let missing = ASCII.replace("UVIndex: *7", "UnusedIndex: *7");
|
||||
assert!(parse(missing.as_bytes()).unwrap().meshes[0].uv_sets.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optional_inactive_pre_rotation_keeps_legacy_placements() {
|
||||
let text = "FBXHeaderExtension: { FBXVersion: 7400 } Objects: { Model: 1, \"Model::A\", \"Null\" { Properties70: { P: \"PreRotation\", \"\", \"\", \"\", 0,0,90 } } }";
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
//! Callers map their page and color models to these neutral records. This module owns PDF object
|
||||
//! layout, image streams, ICC/Separation resources, page labels, marks, xref and trailer syntax.
|
||||
|
||||
/// Multi-image 8/16-bit print pages with per-image ICC resources and explicit byte budgets.
|
||||
#[path = "raster_print.rs"]
|
||||
pub mod print;
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Write as _;
|
||||
use std::io::Write as _;
|
||||
|
||||
@@ -0,0 +1,658 @@
|
||||
//! A small PDF writer for print output: pages of a given size in points, each with raster images
|
||||
//! placed on it, tagged with their ICC profile (`ICCBased`) so the numbers mean the same colour in
|
||||
//! every reader and RIP, Flate-compressed (lossless) at 8 or 16 bits per component, or carried as
|
||||
//! JPEG (`DCTDecode`) where small files matter more than exact samples. Optionally an output intent:
|
||||
//! the printing condition (the printer / paper profile) the file was made for, as PDF/X readers
|
||||
//! expect it in the catalog. Profiles shared by several images are written once.
|
||||
//!
|
||||
//! The file is PDF 1.6 (16-bit images need 1.5): classic cross-reference table, no object streams,
|
||||
//! deterministic (no dates or IDs), so the same pages always give the same bytes.
|
||||
//!
|
||||
//! Encoded JPEG and Flate payloads are copied, not decoded or verified against declared
|
||||
//! dimensions. Callers own their encoding validity and ICC/profile-channel compatibility.
|
||||
//! An output intent describes a printing condition; this API does not certify PDF/X.
|
||||
//! Byte limits bound accounted input and emitted output, not peak process memory or
|
||||
//! caller-owned image construction. This is not the streaming slideshow writer.
|
||||
//!
|
||||
//! Sources: ISO 32000-1:2008 (§7.3 objects, §7.5 file structure, §8.6.5.5 ICCBased colour spaces,
|
||||
//! §8.9 images, §14.11.5 output intents). Own design.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Write as _;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Pages per document: a print job beyond this is a mistake, not a book.
|
||||
pub const MAX_PAGES: usize = 1000;
|
||||
/// Images per page.
|
||||
pub const MAX_IMAGES_PER_PAGE: usize = 256;
|
||||
/// Largest page side in points (200 inches): more than any printer's roll.
|
||||
pub const MAX_PAGE_POINTS: f64 = 14_400.0;
|
||||
/// Largest image, in pixels per side.
|
||||
pub const MAX_IMAGE_SIDE: u32 = 65_535;
|
||||
|
||||
/// Serialization budgets; input records already belong to the caller. These are not a
|
||||
/// process-wide peak-memory guarantee. Repeated references count toward the input budget.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Limits {
|
||||
pub input_bytes: usize,
|
||||
pub output_bytes: usize,
|
||||
}
|
||||
impl Default for Limits {
|
||||
fn default() -> Self {
|
||||
Self { input_bytes: 512 * 1024 * 1024, output_bytes: 512 * 1024 * 1024 }
|
||||
}
|
||||
}
|
||||
|
||||
/// How an image's samples are stored.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ImageData {
|
||||
/// Interleaved samples, `bits` (8 or 16) per component, 16-bit values big-endian: compressed
|
||||
/// losslessly in the file.
|
||||
Raw { bits: u8, samples: Vec<u8> },
|
||||
/// A baseline JPEG stream, copied into the file as it is.
|
||||
Jpeg(Vec<u8>),
|
||||
/// Samples already compressed with zlib (see [`PdfImage::deflated`]), `bits` per component.
|
||||
Flate { bits: u8, data: Vec<u8> },
|
||||
}
|
||||
|
||||
/// A raster image and the colour space its samples are in.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PdfImage {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Components per pixel: 1 (grey), 3 (RGB) or 4 (CMYK).
|
||||
pub channels: u8,
|
||||
pub data: ImageData,
|
||||
/// The ICC profile the samples are in; without one the image is device colour.
|
||||
pub icc: Option<Arc<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl PdfImage {
|
||||
/// An image from interleaved 16-bit samples.
|
||||
pub fn from_u16(width: u32, height: u32, channels: u8, samples: &[u16], icc: Option<Arc<Vec<u8>>>) -> PdfImage {
|
||||
let bytes = samples.iter().flat_map(|v| v.to_be_bytes()).collect();
|
||||
PdfImage { width, height, channels, data: ImageData::Raw { bits: 16, samples: bytes }, icc }
|
||||
}
|
||||
|
||||
/// The same image with its samples compressed now (checked first), so a long print doesn't
|
||||
/// hold every page's uncompressed samples until the file is written.
|
||||
pub fn deflated(self) -> Result<PdfImage, String> {
|
||||
self.check()?;
|
||||
if matches!(&self.data, ImageData::Raw { samples, .. } if samples.len() > Limits::default().input_bytes) {
|
||||
return Err("PDF input byte limit exceeded".into());
|
||||
}
|
||||
let data = match self.data {
|
||||
ImageData::Raw { bits, samples } => ImageData::Flate { bits, data: deflate(&samples, Limits::default().output_bytes)? },
|
||||
other => other,
|
||||
};
|
||||
Ok(PdfImage { data, ..self })
|
||||
}
|
||||
|
||||
fn check(&self) -> Result<(), String> {
|
||||
if self.width == 0 || self.height == 0 || self.width > MAX_IMAGE_SIDE || self.height > MAX_IMAGE_SIDE {
|
||||
return Err(format!("an image of {} × {} pixels can't be written", self.width, self.height));
|
||||
}
|
||||
if !matches!(self.channels, 1 | 3 | 4) {
|
||||
return Err(format!("images have 1, 3 or 4 components, not {}", self.channels));
|
||||
}
|
||||
match &self.data {
|
||||
ImageData::Raw { bits, samples } => {
|
||||
if !matches!(bits, 8 | 16) {
|
||||
return Err(format!("images have 8 or 16 bits per component, not {bits}"));
|
||||
}
|
||||
let want = u64::from(self.width) * u64::from(self.height) * u64::from(self.channels) * u64::from(bits / 8);
|
||||
if samples.len() as u64 != want {
|
||||
return Err(format!("the image needs {want} bytes of samples, not {}", samples.len()));
|
||||
}
|
||||
}
|
||||
ImageData::Jpeg(data) => {
|
||||
if !data.starts_with(&[0xFF, 0xD8]) {
|
||||
return Err("the image isn't a JPEG stream".into());
|
||||
}
|
||||
}
|
||||
ImageData::Flate { bits, data } => {
|
||||
if !matches!(bits, 8 | 16) || data.is_empty() {
|
||||
return Err("a compressed image has 8 or 16 bits per component and some data".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Where an image goes on its page, in points from the page's bottom-left corner.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Placement {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
}
|
||||
|
||||
/// One page: its size in points and the images on it, drawn in order (later ones on top).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PdfPage {
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
pub images: Vec<(PdfImage, Placement)>,
|
||||
}
|
||||
|
||||
/// The printing condition the document is made for (PDF/X-style output intent).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct OutputIntent {
|
||||
/// The printer / paper profile.
|
||||
pub icc: Arc<Vec<u8>>,
|
||||
/// Its components: 1, 3 or 4.
|
||||
pub channels: u8,
|
||||
/// A name for the condition (the profile's description).
|
||||
pub condition: String,
|
||||
}
|
||||
|
||||
/// Document-level settings.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct PdfInfo {
|
||||
pub title: Option<String>,
|
||||
pub output_intent: Option<OutputIntent>,
|
||||
/// Application name, not a PDF/X conformance claim.
|
||||
pub producer: String,
|
||||
}
|
||||
|
||||
fn finite(v: f64) -> bool {
|
||||
v.is_finite()
|
||||
}
|
||||
|
||||
/// A number as PDF writes it: up to four decimals, no exponent, no trailing zeros.
|
||||
fn num(v: f64) -> String {
|
||||
let s = format!("{:.4}", if finite(v) { v } else { 0.0 });
|
||||
let s = s.trim_end_matches('0').trim_end_matches('.');
|
||||
if s == "-0" || s.is_empty() { "0".into() } else { s.into() }
|
||||
}
|
||||
|
||||
/// A text string: plain ASCII as a literal string, anything else as UTF-16BE with a byte order mark.
|
||||
fn text(s: &str) -> String {
|
||||
if s.bytes().all(|b| (0x20..0x7f).contains(&b)) {
|
||||
let mut out = String::with_capacity(s.len() + 2);
|
||||
out.push('(');
|
||||
for c in s.chars() {
|
||||
if matches!(c, '(' | ')' | '\\') {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out.push(')');
|
||||
out
|
||||
} else {
|
||||
super::pdf_unicode_string(s)
|
||||
}
|
||||
}
|
||||
|
||||
struct Writer {
|
||||
out: Vec<u8>,
|
||||
offsets: Vec<usize>,
|
||||
limit: usize,
|
||||
}
|
||||
impl Writer {
|
||||
fn put(&mut self, bytes: &[u8]) -> Result<(), String> {
|
||||
let end = self.out.len().checked_add(bytes.len()).ok_or("PDF output size overflow")?;
|
||||
if end > self.limit {
|
||||
return Err("PDF output byte limit exceeded".into());
|
||||
}
|
||||
self.out.try_reserve(bytes.len()).map_err(|_| "PDF output allocation failed")?;
|
||||
self.out.extend_from_slice(bytes);
|
||||
Ok(())
|
||||
}
|
||||
fn reserve(&mut self) -> Result<usize, String> {
|
||||
self.offsets.try_reserve(1).map_err(|_| "PDF object allocation failed")?;
|
||||
self.offsets.push(0);
|
||||
Ok(self.offsets.len())
|
||||
}
|
||||
fn object(&mut self, id: usize, dict: &str, stream: Option<&[u8]>) -> Result<(), String> {
|
||||
let pos = self.out.len();
|
||||
*self.offsets.get_mut(id.checked_sub(1).ok_or("invalid PDF object")?).ok_or("invalid PDF object")? = pos;
|
||||
self.put(format!("{id} 0 obj\n{dict}").as_bytes())?;
|
||||
if let Some(data) = stream {
|
||||
self.put(b"\nstream\n")?;
|
||||
self.put(data)?;
|
||||
self.put(b"\nendstream")?;
|
||||
}
|
||||
self.put(b"\nendobj\n")
|
||||
}
|
||||
}
|
||||
|
||||
fn deflate(data: &[u8], max: usize) -> Result<Vec<u8>, String> {
|
||||
struct Sink {
|
||||
bytes: Vec<u8>,
|
||||
max: usize,
|
||||
}
|
||||
impl std::io::Write for Sink {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
if bytes.len() > self.max.saturating_sub(self.bytes.len()) {
|
||||
return Err(std::io::Error::other("PDF compressed byte limit exceeded"));
|
||||
}
|
||||
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut encoder = flate2::write::ZlibEncoder::new(Sink { bytes: Vec::new(), max }, flate2::Compression::new(6));
|
||||
encoder.write_all(data).map_err(|e| e.to_string())?;
|
||||
Ok(encoder.finish().map_err(|e| e.to_string())?.bytes)
|
||||
}
|
||||
|
||||
/// Write a document. Errors name the first page or image that can't be written.
|
||||
pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result<Vec<u8>, String> {
|
||||
write_with_limits(pages, info, Limits::default())
|
||||
}
|
||||
|
||||
/// Write with explicit total input and serialized output budgets. ICC bytes are opaque:
|
||||
/// callers validate their profile semantics; resources are keyed by bytes AND channels.
|
||||
pub fn write_with_limits(pages: &[PdfPage], info: &PdfInfo, limits: Limits) -> Result<Vec<u8>, String> {
|
||||
let mut input = 0usize;
|
||||
let mut account = |n: usize| -> Result<(), String> {
|
||||
input = input.checked_add(n).ok_or("PDF input size overflow")?;
|
||||
if input > limits.input_bytes {
|
||||
return Err("PDF input byte limit exceeded".into());
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
account(info.producer.len())?;
|
||||
account(info.title.as_ref().map_or(0, String::len))?;
|
||||
if let Some(intent) = &info.output_intent {
|
||||
account(intent.icc.len())?;
|
||||
account(intent.condition.len())?;
|
||||
}
|
||||
|
||||
if pages.is_empty() {
|
||||
return Err("there are no pages to write".into());
|
||||
}
|
||||
if pages.len() > MAX_PAGES {
|
||||
return Err(format!("a document holds at most {MAX_PAGES} pages"));
|
||||
}
|
||||
for (n, page) in pages.iter().enumerate() {
|
||||
let n = n + 1;
|
||||
if !(finite(page.width) && finite(page.height))
|
||||
|| page.width <= 0.0
|
||||
|| page.height <= 0.0
|
||||
|| page.width > MAX_PAGE_POINTS
|
||||
|| page.height > MAX_PAGE_POINTS
|
||||
{
|
||||
return Err(format!("page {n} has no usable size"));
|
||||
}
|
||||
if page.images.len() > MAX_IMAGES_PER_PAGE {
|
||||
return Err(format!("page {n} has more than {MAX_IMAGES_PER_PAGE} images"));
|
||||
}
|
||||
for (image, at) in &page.images {
|
||||
account(match &image.data {
|
||||
ImageData::Raw { samples, .. } => samples.len(),
|
||||
ImageData::Jpeg(data) | ImageData::Flate { data, .. } => data.len(),
|
||||
})?;
|
||||
if let Some(icc) = &image.icc {
|
||||
account(icc.len())?;
|
||||
}
|
||||
image.check().map_err(|e| format!("page {n}: {e}"))?;
|
||||
if ![at.x, at.y, at.width, at.height].into_iter().all(finite) || at.width <= 0.0 || at.height <= 0.0 {
|
||||
return Err(format!("page {n}: an image has no usable placement"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(intent) = &info.output_intent
|
||||
&& !matches!(intent.channels, 1 | 3 | 4)
|
||||
{
|
||||
return Err("the output intent's profile has 1, 3 or 4 components".into());
|
||||
}
|
||||
|
||||
let mut w = Writer { out: Vec::new(), offsets: Vec::new(), limit: limits.output_bytes };
|
||||
w.put(b"%PDF-1.6\n%\xE2\xE3\xCF\xD3\n")?;
|
||||
let catalog = w.reserve()?;
|
||||
let pages_id = w.reserve()?;
|
||||
let info_id = w.reserve()?;
|
||||
|
||||
// ICC profiles, each written once (keyed by content) with its component count
|
||||
let mut profiles: HashMap<(Arc<Vec<u8>>, u8), usize> = HashMap::new();
|
||||
let mut profile = |w: &mut Writer, icc: &Arc<Vec<u8>>, n: u8| -> Result<usize, String> {
|
||||
if let Some(&id) = profiles.get(&(Arc::clone(icc), n)) {
|
||||
return Ok(id);
|
||||
}
|
||||
let id = w.reserve()?;
|
||||
let data = deflate(icc, limits.output_bytes)?;
|
||||
let alternate = match n {
|
||||
1 => "/DeviceGray",
|
||||
4 => "/DeviceCMYK",
|
||||
_ => "/DeviceRGB",
|
||||
};
|
||||
w.object(id, &format!("<< /N {n} /Alternate {alternate} /Filter /FlateDecode /Length {} >>", data.len()), Some(&data))?;
|
||||
profiles.try_reserve(1).map_err(|_| "PDF profile allocation failed")?;
|
||||
profiles.insert((Arc::clone(icc), n), id);
|
||||
Ok(id)
|
||||
};
|
||||
|
||||
let intent_id = info.output_intent.as_ref().map(|intent| profile(&mut w, &intent.icc, intent.channels)).transpose()?;
|
||||
|
||||
let mut kids = Vec::with_capacity(pages.len());
|
||||
for page in pages {
|
||||
let page_id = w.reserve()?;
|
||||
let mut xobjects = String::new();
|
||||
let mut content = String::new();
|
||||
for (k, (image, at)) in page.images.iter().enumerate() {
|
||||
let space = match &image.icc {
|
||||
Some(icc) => format!("[/ICCBased {} 0 R]", profile(&mut w, icc, image.channels)?),
|
||||
None => match image.channels {
|
||||
1 => "/DeviceGray".into(),
|
||||
4 => "/DeviceCMYK".into(),
|
||||
_ => "/DeviceRGB".into(),
|
||||
},
|
||||
};
|
||||
let id = w.reserve()?;
|
||||
let (bits, filter, data): (u8, &str, std::borrow::Cow<[u8]>) = match &image.data {
|
||||
ImageData::Raw { bits, samples } => (*bits, "/FlateDecode", deflate(samples, limits.output_bytes)?.into()),
|
||||
ImageData::Flate { bits, data } => (*bits, "/FlateDecode", data.as_slice().into()),
|
||||
ImageData::Jpeg(jpeg) => (8, "/DCTDecode", jpeg.as_slice().into()),
|
||||
};
|
||||
w.object(
|
||||
id,
|
||||
&format!(
|
||||
"<< /Type /XObject /Subtype /Image /Width {} /Height {} /ColorSpace {space} /BitsPerComponent {bits} /Filter {filter} /Length {} >>",
|
||||
image.width,
|
||||
image.height,
|
||||
data.len()
|
||||
),
|
||||
Some(&data),
|
||||
)?;
|
||||
xobjects.push_str(&format!("/Im{k} {id} 0 R "));
|
||||
content.push_str(&format!("q {} 0 0 {} {} {} cm /Im{k} Do Q\n", num(at.width), num(at.height), num(at.x), num(at.y)));
|
||||
}
|
||||
let content_id = w.reserve()?;
|
||||
let stream = deflate(content.as_bytes(), limits.output_bytes)?;
|
||||
w.object(content_id, &format!("<< /Filter /FlateDecode /Length {} >>", stream.len()), Some(&stream))?;
|
||||
w.object(
|
||||
page_id,
|
||||
&format!(
|
||||
"<< /Type /Page /Parent {pages_id} 0 R /MediaBox [0 0 {} {}] /Resources << /XObject << {xobjects}>> >> /Contents {content_id} 0 R >>",
|
||||
num(page.width),
|
||||
num(page.height)
|
||||
),
|
||||
None,
|
||||
)?;
|
||||
kids.push(format!("{page_id} 0 R"));
|
||||
}
|
||||
w.object(pages_id, &format!("<< /Type /Pages /Count {} /Kids [{}] >>", kids.len(), kids.join(" ")), None)?;
|
||||
|
||||
let intents = match (&info.output_intent, intent_id) {
|
||||
(Some(intent), Some(id)) => format!(
|
||||
" /OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX /OutputConditionIdentifier (Custom) /Info {} /DestOutputProfile {id} 0 R >>]",
|
||||
text(&intent.condition)
|
||||
),
|
||||
_ => String::new(),
|
||||
};
|
||||
w.object(catalog, &format!("<< /Type /Catalog /Pages {pages_id} 0 R{intents} >>"), None)?;
|
||||
let title = info.title.as_deref().map(|t| format!(" /Title {}", text(t))).unwrap_or_default();
|
||||
w.object(info_id, &format!("<< /Producer {}{title} >>", text(&info.producer)), None)?;
|
||||
|
||||
let xref = w.out.len();
|
||||
let mut table = format!("xref\n0 {}\n0000000000 65535 f \n", w.offsets.len() + 1);
|
||||
for offset in &w.offsets {
|
||||
table.push_str(&format!("{offset:010} 00000 n \n"));
|
||||
}
|
||||
table.push_str(&format!("trailer\n<< /Size {} /Root {catalog} 0 R /Info {info_id} 0 R >>\nstartxref\n{xref}\n%%EOF\n", w.offsets.len() + 1));
|
||||
w.put(table.as_bytes())?;
|
||||
Ok(w.out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The body of object `id` (dictionary and raw stream bytes), found through the xref table.
|
||||
fn object(pdf: &[u8], id: usize) -> (String, Vec<u8>) {
|
||||
// byte offsets: the streams are binary, so search the bytes, not a lossy string
|
||||
let start = pdf.windows(10).rposition(|w| w == b"startxref\n").unwrap() + 10;
|
||||
let tail = std::str::from_utf8(&pdf[start..]).unwrap();
|
||||
let xref: usize = tail.lines().next().unwrap().trim().parse().unwrap();
|
||||
let table = std::str::from_utf8(&pdf[xref..]).unwrap();
|
||||
let line = table.lines().nth(2 + id).unwrap();
|
||||
let offset: usize = line[..10].parse().unwrap();
|
||||
let head = format!("{id} 0 obj\n");
|
||||
assert_eq!(&pdf[offset..offset + head.len()], head.as_bytes(), "xref entry for {id}");
|
||||
let rest = &pdf[offset + head.len()..];
|
||||
let end = rest.windows(7).position(|w| w == b"\nendobj").unwrap();
|
||||
let body = &rest[..end];
|
||||
match body.windows(8).position(|w| w == b"\nstream\n") {
|
||||
Some(p) => {
|
||||
let data = &body[p + 8..body.len() - "\nendstream".len()];
|
||||
(String::from_utf8_lossy(&body[..p]).into_owned(), data.to_vec())
|
||||
}
|
||||
None => (String::from_utf8_lossy(body).into_owned(), vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
fn inflate(data: &[u8]) -> Vec<u8> {
|
||||
{
|
||||
use std::io::Read;
|
||||
let mut out = Vec::new();
|
||||
flate2::read::ZlibDecoder::new(data).read_to_end(&mut out).unwrap();
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Object ids of every image XObject, in file order.
|
||||
fn images(pdf: &[u8]) -> Vec<usize> {
|
||||
let s = String::from_utf8_lossy(pdf);
|
||||
s.match_indices(" 0 obj\n<< /Type /XObject /Subtype /Image").map(|(i, _)| s[..i].rsplit('\n').next().unwrap().parse().unwrap()).collect()
|
||||
}
|
||||
|
||||
fn reference(dict: &str, key: &str) -> usize {
|
||||
let at = dict.find(key).unwrap() + key.len();
|
||||
dict[at..].trim_start().split(' ').next().unwrap().parse().unwrap()
|
||||
}
|
||||
|
||||
fn srgb() -> Arc<Vec<u8>> {
|
||||
Arc::new(b"opaque RGB profile fixture".to_vec())
|
||||
}
|
||||
|
||||
fn page(images: Vec<(PdfImage, Placement)>) -> PdfPage {
|
||||
PdfPage { width: 612.0, height: 792.0, images }
|
||||
}
|
||||
|
||||
fn full(w: f64, h: f64) -> Placement {
|
||||
Placement { x: 0.0, y: 0.0, width: w, height: h }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_16_bit_icc_image_round_trips() {
|
||||
let samples: Vec<u16> = (0..2 * 3 * 3).map(|i| (i * 3851) as u16).collect();
|
||||
let img = PdfImage::from_u16(3, 2, 3, &samples, Some(srgb()));
|
||||
let pdf = write(&[page(vec![(img, full(612.0, 792.0))])], &PdfInfo::default()).unwrap();
|
||||
assert!(pdf.starts_with(b"%PDF-1.6\n"));
|
||||
assert!(pdf.ends_with(b"%%EOF\n"));
|
||||
let ids = images(&pdf);
|
||||
assert_eq!(ids.len(), 1);
|
||||
let (dict, data) = object(&pdf, ids[0]);
|
||||
assert!(dict.contains("/Width 3 /Height 2") && dict.contains("/BitsPerComponent 16") && dict.contains("/FlateDecode"), "{dict}");
|
||||
let back: Vec<u16> = inflate(&data).chunks(2).map(|b| u16::from_be_bytes([b[0], b[1]])).collect();
|
||||
assert_eq!(back, samples);
|
||||
let (profile, icc) = object(&pdf, reference(&dict, "/ICCBased"));
|
||||
assert!(profile.contains("/N 3") && profile.contains("/Alternate /DeviceRGB"), "{profile}");
|
||||
assert_eq!(inflate(&icc), *srgb());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pages_sizes_placements_and_a_shared_profile() {
|
||||
let px = |v: u16| PdfImage::from_u16(1, 1, 3, &[v, v, v], Some(srgb()));
|
||||
let at = Placement { x: 36.0, y: 72.5, width: 288.0, height: 216.0 };
|
||||
let pages = [page(vec![(px(1), at), (px(2), full(612.0, 792.0))]), PdfPage { width: 841.89, height: 595.28, images: vec![(px(3), at)] }];
|
||||
let pdf = write(&pages, &PdfInfo::default()).unwrap();
|
||||
let s = String::from_utf8_lossy(&pdf);
|
||||
assert!(s.contains("/Type /Pages /Count 2"), "two pages");
|
||||
assert!(s.contains("/MediaBox [0 0 612 792]") && s.contains("/MediaBox [0 0 841.89 595.28]"));
|
||||
// the sRGB profile is written once for all three images
|
||||
assert_eq!(s.matches("/N 3 /Alternate").count(), 1);
|
||||
let first: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
|
||||
assert!(first.windows(2).all(|w| w[0] == w[1]), "{first:?}");
|
||||
// placements reach the content streams
|
||||
let contents: Vec<String> = s
|
||||
.match_indices("/Contents ")
|
||||
.map(|(i, _)| s[i + 10..].split(' ').next().unwrap().parse::<usize>().unwrap())
|
||||
.map(|id| String::from_utf8(inflate(&object(&pdf, id).1)).unwrap())
|
||||
.collect();
|
||||
assert_eq!(contents[0], "q 288 0 0 216 36 72.5 cm /Im0 Do Q\nq 612 0 0 792 0 0 cm /Im1 Do Q\n");
|
||||
assert_eq!(write(&pages, &PdfInfo::default()).unwrap(), pdf, "deterministic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_intent_and_title() {
|
||||
let cmyk_like = srgb();
|
||||
let info = PdfInfo {
|
||||
title: Some("Garden – prints (2)".into()),
|
||||
producer: "Fixture".into(),
|
||||
output_intent: Some(OutputIntent { icc: cmyk_like.clone(), channels: 3, condition: "Glossy (Photo) \\ test".into() }),
|
||||
};
|
||||
let img = PdfImage::from_u16(1, 1, 3, &[0, 0, 0], Some(cmyk_like));
|
||||
let pdf = write(&[page(vec![(img, full(10.0, 10.0))])], &info).unwrap();
|
||||
let (catalog, _) = object(&pdf, 1);
|
||||
assert!(catalog.contains("/OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX"), "{catalog}");
|
||||
assert!(catalog.contains("/Info (Glossy \\(Photo\\) \\\\ test)"), "{catalog}");
|
||||
// the intent and the image share one profile object
|
||||
let dest = reference(&catalog, "/DestOutputProfile");
|
||||
assert_eq!(reference(&object(&pdf, images(&pdf)[0]).0, "/ICCBased"), dest);
|
||||
let (info_dict, _) = object(&pdf, 3);
|
||||
assert!(info_dict.contains("/Title <FEFF"), "non-ASCII titles are UTF-16: {info_dict}");
|
||||
assert!(info_dict.contains("2013"), "the en dash: {info_dict}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmyk_grey_jpeg_and_device_images() {
|
||||
let jpeg = vec![0xFF, 0xD8, 0xFF, 0xD9];
|
||||
let imgs = vec![
|
||||
(
|
||||
PdfImage { width: 1, height: 1, channels: 4, data: ImageData::Raw { bits: 8, samples: vec![0, 10, 20, 30] }, icc: None },
|
||||
full(1.0, 1.0),
|
||||
),
|
||||
(PdfImage { width: 2, height: 1, channels: 1, data: ImageData::Raw { bits: 8, samples: vec![0, 255] }, icc: None }, full(1.0, 1.0)),
|
||||
(PdfImage { width: 4, height: 4, channels: 3, data: ImageData::Jpeg(jpeg.clone()), icc: None }, full(1.0, 1.0)),
|
||||
];
|
||||
let pdf = write(&[page(imgs)], &PdfInfo::default()).unwrap();
|
||||
let ids = images(&pdf);
|
||||
let dicts: Vec<_> = ids.iter().map(|&id| object(&pdf, id)).collect();
|
||||
assert!(dicts[0].0.contains("/ColorSpace /DeviceCMYK") && dicts[0].0.contains("/BitsPerComponent 8"));
|
||||
assert!(dicts[1].0.contains("/ColorSpace /DeviceGray"));
|
||||
assert!(dicts[2].0.contains("/DCTDecode") && dicts[2].0.contains("/DeviceRGB"));
|
||||
assert_eq!(dicts[2].1, jpeg, "JPEG streams are copied as they are");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressing_early_writes_the_same_file() {
|
||||
let samples: Vec<u16> = (0..4 * 3 * 3).map(|i| (i * 977) as u16).collect();
|
||||
let raw = PdfImage::from_u16(4, 3, 3, &samples, Some(srgb()));
|
||||
let early = raw.clone().deflated().unwrap();
|
||||
assert!(matches!(early.data, ImageData::Flate { bits: 16, .. }));
|
||||
let at = full(100.0, 75.0);
|
||||
let a = write(&[page(vec![(raw, at)])], &PdfInfo::default()).unwrap();
|
||||
let b = write(&[page(vec![(early, at)])], &PdfInfo::default()).unwrap();
|
||||
assert_eq!(a, b);
|
||||
let mut short = PdfImage::from_u16(4, 3, 3, &samples, None);
|
||||
short.width = 5;
|
||||
assert!(short.deflated().is_err(), "checked before compressing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_documents_are_errors() {
|
||||
let ok = || PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
|
||||
let at = full(10.0, 10.0);
|
||||
let bad = |pages: Vec<PdfPage>| write(&pages, &PdfInfo::default()).is_err();
|
||||
assert!(bad(vec![]));
|
||||
assert!(bad(vec![PdfPage { width: f64::NAN, height: 10.0, images: vec![] }]));
|
||||
assert!(bad(vec![PdfPage { width: -1.0, height: 10.0, images: vec![] }]));
|
||||
assert!(bad(vec![PdfPage { width: 1e9, height: 10.0, images: vec![] }]));
|
||||
assert!(bad(vec![page(vec![(ok(), Placement { x: f64::INFINITY, ..at })])]));
|
||||
assert!(bad(vec![page(vec![(ok(), Placement { width: 0.0, ..at })])]));
|
||||
let mut short = ok();
|
||||
short.data = ImageData::Raw { bits: 16, samples: vec![0; 5] };
|
||||
assert!(bad(vec![page(vec![(short, at)])]));
|
||||
let mut empty = ok();
|
||||
empty.width = 0;
|
||||
assert!(bad(vec![page(vec![(empty, at)])]));
|
||||
let mut odd = ok();
|
||||
odd.channels = 2;
|
||||
assert!(bad(vec![page(vec![(odd, at)])]));
|
||||
let mut not_jpeg = ok();
|
||||
not_jpeg.data = ImageData::Jpeg(vec![1, 2, 3]);
|
||||
assert!(bad(vec![page(vec![(not_jpeg, at)])]));
|
||||
let too_many = (0..MAX_PAGES + 1).map(|_| page(vec![])).collect();
|
||||
assert!(bad(too_many));
|
||||
let info = PdfInfo {
|
||||
producer: String::new(),
|
||||
title: None,
|
||||
output_intent: Some(OutputIntent { icc: srgb(), channels: 2, condition: String::new() }),
|
||||
};
|
||||
assert!(write(&[page(vec![])], &info).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbers_and_strings() {
|
||||
assert_eq!(num(612.0), "612");
|
||||
assert_eq!(num(72.5), "72.5");
|
||||
assert_eq!(num(-0.00001), "0");
|
||||
assert_eq!(num(f64::NAN), "0");
|
||||
assert_eq!(num(1.0 / 3.0), "0.3333");
|
||||
assert_eq!(text("a(b)c\\"), "(a\\(b\\)c\\\\)");
|
||||
assert_eq!(text("é"), "<FEFF00E9>");
|
||||
}
|
||||
#[test]
|
||||
fn distinct_profiles_and_component_counts_are_not_conflated() {
|
||||
let common = Arc::new(b"opaque profile A".to_vec());
|
||||
let different = Arc::new(b"opaque profile B".to_vec());
|
||||
let image = |channels, profile| PdfImage {
|
||||
width: 1,
|
||||
height: 1,
|
||||
channels,
|
||||
data: ImageData::Raw { bits: 8, samples: vec![7; channels as usize] },
|
||||
icc: Some(profile),
|
||||
};
|
||||
let pages = [page(vec![
|
||||
(image(3, common.clone()), full(1.0, 1.0)),
|
||||
(image(4, common.clone()), full(1.0, 1.0)),
|
||||
(image(3, different.clone()), full(1.0, 1.0)),
|
||||
(image(3, common.clone()), full(1.0, 1.0)),
|
||||
])];
|
||||
let pdf = write(&pages, &PdfInfo::default()).unwrap();
|
||||
let refs: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
|
||||
assert_ne!(refs[0], refs[1]);
|
||||
assert_ne!(refs[0], refs[2]);
|
||||
assert_eq!(refs[0], refs[3]);
|
||||
for (i, channels, bytes) in [(0, 3, common.as_slice()), (1, 4, common.as_slice()), (2, 3, different.as_slice())] {
|
||||
let (dict, data) = object(&pdf, refs[i]);
|
||||
assert!(dict.contains(&format!("/N {channels}")));
|
||||
assert_eq!(inflate(&data), bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gray_rgb_cmyk_raw_depths_preserve_samples() {
|
||||
for channels in [1, 3, 4] {
|
||||
for bits in [8, 16] {
|
||||
let bytes: Vec<u8> = (0..channels * (bits / 8)).collect();
|
||||
let image = PdfImage { width: 1, height: 1, channels, data: ImageData::Raw { bits, samples: bytes.clone() }, icc: None };
|
||||
let pdf = write(&[page(vec![(image, full(1.0, 1.0))])], &PdfInfo::default()).unwrap();
|
||||
let (dict, data) = object(&pdf, images(&pdf)[0]);
|
||||
assert!(dict.contains(&format!("/BitsPerComponent {bits}")));
|
||||
assert_eq!(inflate(&data), bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_and_output_budgets_fail_without_partial_success() {
|
||||
let image = PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
|
||||
let pages = [page(vec![(image, full(1.0, 1.0))])];
|
||||
let info = PdfInfo::default();
|
||||
let expected = write(&pages, &info).unwrap();
|
||||
assert_eq!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() }).unwrap(), expected);
|
||||
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 5, output_bytes: expected.len() }).is_err());
|
||||
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() - 1 }).is_err());
|
||||
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: 0 }).is_err());
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,7 @@ The format layer ends where arithmetic on the picture begins.
|
||||
|---|---|
|
||||
| Recognising containers (`probe`), walking TIFF/IFD structure on `craft-tiff` | Black subtraction, white scaling, cropping to the active area |
|
||||
| Decompressing the samples: unpacking, lossless JPEG, lossy JPEG, Deflate, JPEG XL | Demosaicing, binned previews, hot-pixel and highlight handling |
|
||||
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
|
||||
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values; left unapplied when `OpcodeList1` is present, so the develop pipeline can run that list on the stored codes first) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
|
||||
| Black and white levels, CFA layout, active area, default crop, orientation, pixel aspect | The DNG colour model's arithmetic (illuminant interpolation, camera to working space) |
|
||||
| Colour tags as data: matrices, calibration illuminants, as-shot neutral / white | White balance, Kelvin scales, spectral or fitted camera colour |
|
||||
| Opcode lists parsed and written (`opcodes`), profile tables and gain table maps as data | Applying opcodes (gain maps, warps, bad pixels), profile look tables, tone curves |
|
||||
|
||||
@@ -852,6 +852,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
// makes a neutral surface magenta. WB edits remain relative to this as-shot RGB.
|
||||
wb_multipliers: if linear_rgb { Some([1.0; 3]) } else { wb },
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
// only validated components become opcodes (see `arw_lens`, and `arw_lens_rm6` for the ILCE-7RM6)
|
||||
opcodes: OpcodeLists { list3: warp.into_iter().chain(rm6).collect(), ..Default::default() },
|
||||
|
||||
@@ -311,6 +311,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData { kelvin_presets, ..Default::default() },
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -388,6 +388,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode, limits: &
|
||||
// the camera's white balance is already applied, as in Sony's linear YCbCr
|
||||
wb_multipliers: Some([1.0; 3]),
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -269,6 +269,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: levels.wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: ifd0.map(Camera::from_ifd0).unwrap_or_default(),
|
||||
@@ -824,6 +825,7 @@ mod tests {
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Camera::default(),
|
||||
|
||||
@@ -59,6 +59,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, caller: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: h.wb_multipliers,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera,
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
//! DNG Camera Profiles (`.dcp`): the user's own camera colour profiles, read at run time from
|
||||
//! files they supply (DNG specification, chapter 6 "Camera Profiles"). A `.dcp` is a TIFF-structured
|
||||
//! file whose header carries the magic `RC` (0x4352) in place of 42, with the profile tags in IFD 0:
|
||||
//! `UniqueCameraModel`, `ProfileName`, the colour/forward matrices and calibration illuminants, the
|
||||
//! hue/saturation map, look table and tone curve. Nothing here ships a profile.
|
||||
|
||||
use craft_tiff::{Ifd, ParseOptions, Tiff, tags as t};
|
||||
|
||||
use crate::ColorData;
|
||||
|
||||
/// The magic number of a DNG camera profile file (`IIRC` / `MMCR`).
|
||||
pub const DCP_MAGIC: u16 = 0x4352;
|
||||
|
||||
/// A parsed camera profile.
|
||||
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct DcpProfile {
|
||||
/// `ProfileName` (empty when the file has none).
|
||||
pub name: String,
|
||||
/// `UniqueCameraModel`: the camera the profile was made for (empty when absent).
|
||||
pub camera_model: String,
|
||||
/// The colour model: illuminants, matrices and the profile look (hue/sat map, look table, tone curve).
|
||||
pub color: ColorData,
|
||||
}
|
||||
|
||||
/// Errors reading a `.dcp`.
|
||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||
pub enum DcpError {
|
||||
#[error("not a DNG camera profile (.dcp)")]
|
||||
NotDcp,
|
||||
#[error("damaged camera profile: {0}")]
|
||||
Damaged(String),
|
||||
#[error("the camera profile has no usable colour matrix")]
|
||||
NoMatrix,
|
||||
}
|
||||
|
||||
/// Is `bytes` a DCP file (by its header)?
|
||||
pub fn is_dcp(bytes: &[u8]) -> bool {
|
||||
matches!(bytes.get(0..4), Some([b'I', b'I', b'R', b'C'] | [b'M', b'M', b'C', b'R']))
|
||||
}
|
||||
|
||||
/// The largest `.dcp` file read or parsed (real profiles are kilobytes to a few MB).
|
||||
pub const MAX_DCP_BYTES: u64 = 64 << 20;
|
||||
|
||||
/// Read the `.dcp` file at `path`, reading at most [`MAX_DCP_BYTES`] + 1 bytes: a larger file is refused
|
||||
/// before it is loaded into memory.
|
||||
pub fn read_dcp(path: &std::path::Path) -> std::io::Result<Vec<u8>> {
|
||||
use std::io::Read;
|
||||
let mut bytes = Vec::new();
|
||||
std::fs::File::open(path)?.take(MAX_DCP_BYTES.saturating_add(1)).read_to_end(&mut bytes)?;
|
||||
if bytes.len() as u64 > MAX_DCP_BYTES {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "the camera profile is larger than 64 MiB"));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// Parse a `.dcp` file. Files above 64 MiB ([`MAX_DCP_BYTES`]) are refused.
|
||||
pub fn parse_dcp(bytes: &[u8]) -> Result<DcpProfile, DcpError> {
|
||||
if !is_dcp(bytes) {
|
||||
return Err(DcpError::NotDcp);
|
||||
}
|
||||
if bytes.len() as u64 > MAX_DCP_BYTES {
|
||||
return Err(DcpError::Damaged("larger than 64 MiB".into()));
|
||||
}
|
||||
// the TIFF reader only knows 42 / BigTIFF / raw magics: give it a header with 42 in place of RC
|
||||
let mut data = bytes.to_vec();
|
||||
let forty_two: [u8; 2] = if bytes.first() == Some(&b'I') { [42, 0] } else { [0, 42] };
|
||||
if let Some(m) = data.get_mut(2..4) {
|
||||
m.copy_from_slice(&forty_two);
|
||||
}
|
||||
let opts = ParseOptions { follow_children: false, max_ifds: 4, value_budget: Some(64 << 20), ..Default::default() };
|
||||
let tiff = Tiff::parse_with(&data, &opts).map_err(|e| DcpError::Damaged(e.to_string()))?;
|
||||
let ifd: &Ifd = tiff.ifds.first().ok_or_else(|| DcpError::Damaged("no IFD".into()))?;
|
||||
let mut color = crate::dng::color_data(ifd, ifd, tiff.order);
|
||||
// a profile carries no white balance or calibration of a particular shot
|
||||
color.as_shot_neutral = None;
|
||||
color.as_shot_white_xy = None;
|
||||
color.analog_balance = None;
|
||||
color.camera_calibration = [None, None];
|
||||
color.baseline_sharpness = None;
|
||||
color.profile.gain_table_map = None;
|
||||
// only BaselineExposureOffset belongs to a profile (BaselineExposure is the camera's)
|
||||
color.baseline_exposure =
|
||||
ifd.f64s(t::BASELINE_EXPOSURE_OFFSET).and_then(|v| v.first().copied()).filter(|v| v.is_finite()).unwrap_or(0.0).clamp(-4.0, 4.0);
|
||||
// a single-illuminant profile names one illuminant; its second matrix slot stays empty
|
||||
if color.color_matrix[0].is_none() {
|
||||
// the first slot is the one every profile must fill (DNG: ColorMatrix1 is required)
|
||||
return Err(DcpError::NoMatrix);
|
||||
}
|
||||
let text = |tag: u16| ifd.string(tag).map(|s| s.trim_matches(char::from(0)).trim().to_string()).unwrap_or_default();
|
||||
Ok(DcpProfile { name: text(t::PROFILE_NAME), camera_model: text(t::UNIQUE_CAMERA_MODEL), color })
|
||||
}
|
||||
|
||||
/// Write `p` as a `.dcp` file (little-endian): its name, camera, illuminants, matrices, hue/saturation
|
||||
/// map, look table, tone curve and baseline exposure offset.
|
||||
pub fn write_dcp(p: &DcpProfile) -> Result<Vec<u8>, DcpError> {
|
||||
use craft_tiff::writer::srational;
|
||||
use craft_tiff::{ByteOrder, IfdBuilder, TiffWriter, Value};
|
||||
let c = &p.color;
|
||||
// ColorMatrix1 must remain usable after TIFF rational rounding, or our reader rejects it.
|
||||
let first = c.color_matrix[0].as_ref().ok_or(DcpError::NoMatrix)?;
|
||||
let serialized = first
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|v| {
|
||||
let (n, d) = srational(*v);
|
||||
f64::from(n) / f64::from(d)
|
||||
})
|
||||
.collect();
|
||||
if first.iter().flatten().any(|v| !v.is_finite()) || crate::dng::mat3(Some(serialized)).is_none() {
|
||||
return Err(DcpError::NoMatrix);
|
||||
}
|
||||
let mut ifd = IfdBuilder::new();
|
||||
ifd.set(t::UNIQUE_CAMERA_MODEL, Value::Ascii(p.camera_model.clone()));
|
||||
ifd.set(t::PROFILE_NAME, Value::Ascii(p.name.clone()));
|
||||
let mat = |m: &crate::Matrix3| Value::SRational(m.iter().flatten().map(|v| srational(*v)).collect());
|
||||
for i in 0..2 {
|
||||
if let Some(m) = &c.color_matrix[i] {
|
||||
ifd.set([t::CALIBRATION_ILLUMINANT_1, t::CALIBRATION_ILLUMINANT_2][i], Value::Short(vec![c.illuminant[i]]));
|
||||
ifd.set([t::COLOR_MATRIX_1, t::COLOR_MATRIX_2][i], mat(m));
|
||||
}
|
||||
if let Some(m) = &c.forward_matrix[i] {
|
||||
ifd.set([t::FORWARD_MATRIX_1, t::FORWARD_MATRIX_2][i], mat(m));
|
||||
}
|
||||
}
|
||||
let flat = |table: &crate::profile::HsvTable| Value::Float(table.data.iter().flatten().copied().collect());
|
||||
let dims =
|
||||
|table: &crate::profile::HsvTable| Value::Long(vec![table.hue_divisions as u32, table.sat_divisions as u32, table.val_divisions as u32]);
|
||||
if let Some(a) = c.profile.hue_sat_map[0].as_ref() {
|
||||
ifd.set(t::PROFILE_HUE_SAT_MAP_DIMS, dims(a));
|
||||
ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_1, flat(a));
|
||||
if a.srgb_value {
|
||||
ifd.set(t::PROFILE_HUE_SAT_MAP_ENCODING, Value::Long(vec![1]));
|
||||
}
|
||||
if let Some(b) = c.profile.hue_sat_map[1]
|
||||
.as_ref()
|
||||
.filter(|b| (b.hue_divisions, b.sat_divisions, b.val_divisions) == (a.hue_divisions, a.sat_divisions, a.val_divisions))
|
||||
{
|
||||
ifd.set(t::PROFILE_HUE_SAT_MAP_DATA_2, flat(b));
|
||||
}
|
||||
}
|
||||
if let Some(l) = &c.profile.look_table {
|
||||
ifd.set(t::PROFILE_LOOK_TABLE_DIMS, dims(l));
|
||||
ifd.set(t::PROFILE_LOOK_TABLE_DATA, flat(l));
|
||||
if l.srgb_value {
|
||||
ifd.set(t::PROFILE_LOOK_TABLE_ENCODING, Value::Long(vec![1]));
|
||||
}
|
||||
}
|
||||
if let Some(tc) = &c.profile.tone_curve {
|
||||
ifd.set(t::PROFILE_TONE_CURVE, Value::Float(tc.points.iter().flatten().copied().collect()));
|
||||
}
|
||||
if c.baseline_exposure != 0.0 {
|
||||
ifd.set(t::BASELINE_EXPOSURE_OFFSET, Value::SRational(vec![srational(c.baseline_exposure)]));
|
||||
}
|
||||
let mut out = TiffWriter::new(ByteOrder::Little, false).write(&[ifd]).map_err(|e| DcpError::Damaged(e.to_string()))?;
|
||||
// the DCP magic in place of 42
|
||||
if let Some(m) = out.get_mut(2..4) {
|
||||
m.copy_from_slice(&DCP_MAGIC.to_le_bytes());
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn writing_requires_the_first_color_matrix() {
|
||||
let mut p = DcpProfile { name: "test".into(), camera_model: "test".into(), color: ColorData::default() };
|
||||
assert_eq!(write_dcp(&p), Err(DcpError::NoMatrix));
|
||||
let matrix = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]];
|
||||
p.color.color_matrix[1] = Some(matrix);
|
||||
assert_eq!(write_dcp(&p), Err(DcpError::NoMatrix));
|
||||
p.color.color_matrix[0] = Some(matrix);
|
||||
let back = parse_dcp(&write_dcp(&p).unwrap()).unwrap();
|
||||
assert_eq!(back.color.color_matrix, p.color.color_matrix);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writing_rejects_unusable_or_rounding_singular_first_matrices() {
|
||||
let identity = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]];
|
||||
for matrix in [
|
||||
[[0.0; 3]; 3],
|
||||
[[1.0, 1.0, 0.0], [1.0, 1.0 + 1e-8, 0.0], [0.0, 0.0, 1.0]],
|
||||
[[f64::NAN, 0.0, 0.0], identity[1], identity[2]],
|
||||
[[f64::INFINITY, 0.0, 0.0], identity[1], identity[2]],
|
||||
] {
|
||||
let mut p = DcpProfile { name: "test".into(), camera_model: "test".into(), color: ColorData::default() };
|
||||
p.color.color_matrix[0] = Some(matrix);
|
||||
assert_eq!(write_dcp(&p), Err(DcpError::NoMatrix));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reading_a_profile_stops_past_the_size_limit() {
|
||||
let dir = std::env::temp_dir().join(format!("lc-read-dcp-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let (fits, big) = (dir.join("fits.dcp"), dir.join("big.dcp"));
|
||||
std::fs::File::create(&fits).unwrap().set_len(MAX_DCP_BYTES).unwrap();
|
||||
std::fs::File::create(&big).unwrap().set_len(MAX_DCP_BYTES + 1).unwrap();
|
||||
assert_eq!(read_dcp(&fits).unwrap().len() as u64, MAX_DCP_BYTES);
|
||||
assert_eq!(read_dcp(&big).unwrap_err().kind(), std::io::ErrorKind::InvalidData);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A minimal single-illuminant profile (our own numbers) built in memory: no profile file is
|
||||
/// committed to the repository.
|
||||
pub(crate) fn synthetic_dcp(little: bool, with_hue_sat: bool) -> Vec<u8> {
|
||||
// entries: (tag, type, count, payload bytes)
|
||||
let mut entries: Vec<(u16, u16, u32, Vec<u8>)> = Vec::new();
|
||||
let u16b = |v: u16| if little { v.to_le_bytes().to_vec() } else { v.to_be_bytes().to_vec() };
|
||||
let u32b = |v: u32| if little { v.to_le_bytes().to_vec() } else { v.to_be_bytes().to_vec() };
|
||||
let srat = |v: f64| {
|
||||
let (n, d) = ((v * 10000.0).round() as i32, 10000i32);
|
||||
let mut b = if little { n.to_le_bytes().to_vec() } else { n.to_be_bytes().to_vec() };
|
||||
b.extend(if little { d.to_le_bytes() } else { d.to_be_bytes() });
|
||||
b
|
||||
};
|
||||
let ascii = |s: &str| {
|
||||
let mut b = s.as_bytes().to_vec();
|
||||
b.push(0);
|
||||
b
|
||||
};
|
||||
entries.push((t::UNIQUE_CAMERA_MODEL, 2, 13, ascii("Test Camera1")));
|
||||
let m = [0.9, -0.3, -0.05, -0.4, 1.2, 0.2, -0.05, 0.15, 0.7];
|
||||
entries.push((t::COLOR_MATRIX_1, 10, 9, m.iter().flat_map(|v| srat(*v)).collect()));
|
||||
entries.push((t::CALIBRATION_ILLUMINANT_1, 3, 1, [u16b(21), vec![0, 0]].concat()));
|
||||
entries.push((t::PROFILE_NAME, 2, 9, ascii("My Look")));
|
||||
if with_hue_sat {
|
||||
// 2 hues × 2 sats × 1 value: hue shift, sat scale, value scale per entry (FLOAT)
|
||||
entries.push((t::PROFILE_HUE_SAT_MAP_DIMS, 4, 3, [u32b(2), u32b(2), u32b(1)].concat()));
|
||||
let f = |v: f32| if little { v.to_le_bytes().to_vec() } else { v.to_be_bytes().to_vec() };
|
||||
entries.push((
|
||||
t::PROFILE_HUE_SAT_MAP_DATA_1,
|
||||
11,
|
||||
12,
|
||||
[f(0.0), f(1.0), f(1.0), f(5.0), f(1.1), f(1.0), f(0.0), f(1.0), f(1.0), f(-5.0), f(0.9), f(1.0)].concat(),
|
||||
));
|
||||
}
|
||||
entries.sort_by_key(|e| e.0);
|
||||
let mut out = if little { b"IIRC".to_vec() } else { b"MMCR".to_vec() };
|
||||
out.extend(u32b(8));
|
||||
let n = entries.len();
|
||||
let mut data_off = 8 + 2 + n * 12 + 4;
|
||||
let mut ifd = u16b(n as u16);
|
||||
let mut tail = Vec::new();
|
||||
for (tag, ty, count, payload) in &entries {
|
||||
ifd.extend(u16b(*tag));
|
||||
ifd.extend(u16b(*ty));
|
||||
ifd.extend(u32b(*count));
|
||||
if payload.len() <= 4 {
|
||||
let mut p = payload.clone();
|
||||
p.resize(4, 0);
|
||||
ifd.extend(p);
|
||||
} else {
|
||||
ifd.extend(u32b(data_off as u32));
|
||||
tail.extend(payload);
|
||||
data_off += payload.len();
|
||||
if data_off % 2 == 1 {
|
||||
tail.push(0);
|
||||
data_off += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
ifd.extend(u32b(0));
|
||||
out.extend(ifd);
|
||||
out.extend(tail);
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_a_single_illuminant_profile_in_both_byte_orders() {
|
||||
for little in [true, false] {
|
||||
let p = parse_dcp(&synthetic_dcp(little, true)).unwrap();
|
||||
assert_eq!(p.name, "My Look");
|
||||
assert_eq!(p.camera_model, "Test Camera1");
|
||||
assert_eq!(p.color.illuminant, [21, 0]);
|
||||
let m = p.color.color_matrix[0].unwrap();
|
||||
assert!((m[1][1] - 1.2).abs() < 1e-4, "{m:?}");
|
||||
assert!(p.color.color_matrix[1].is_none());
|
||||
assert!(p.color.profile.hue_sat_map[0].is_some());
|
||||
assert!(p.color.as_shot_neutral.is_none());
|
||||
}
|
||||
assert!(parse_dcp(&synthetic_dcp(true, false)).unwrap().color.profile.hue_sat_map[0].is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_other_files_and_damage_without_panicking() {
|
||||
assert_eq!(parse_dcp(b"II*\0\x08\0\0\0"), Err(DcpError::NotDcp));
|
||||
assert_eq!(parse_dcp(b""), Err(DcpError::NotDcp));
|
||||
let good = synthetic_dcp(true, true);
|
||||
for cut in [4, 8, 10, 30, good.len() / 2, good.len() - 1] {
|
||||
let _ = parse_dcp(&good[..cut]);
|
||||
}
|
||||
let mut bad = good.clone();
|
||||
bad[4..8].copy_from_slice(&u32::MAX.to_le_bytes());
|
||||
assert!(parse_dcp(&bad).is_err());
|
||||
}
|
||||
|
||||
/// The hue/saturation map's header is the user's file too: dimensions far past the data, an entry count
|
||||
/// claiming 4 billion values, or a data offset past the end drop the map (the matrix still reads) instead of
|
||||
/// allocating for the claim or reading past the file.
|
||||
#[test]
|
||||
fn a_hostile_hue_sat_map_header_drops_the_map() {
|
||||
let good = synthetic_dcp(true, true);
|
||||
let at = |needle: &[u8]| good.windows(needle.len()).position(|w| w == needle).unwrap();
|
||||
let le = |v: &[u32]| v.iter().flat_map(|x| x.to_le_bytes()).collect::<Vec<u8>>();
|
||||
let dims = at(&le(&[2, 2, 1]));
|
||||
// the map data's directory entry: tag, FLOAT, count 12, offset
|
||||
let entry = at(&[t::PROFILE_HUE_SAT_MAP_DATA_1.to_le_bytes(), 11u16.to_le_bytes()].concat());
|
||||
let patched = |at: usize, bytes: &[u8]| {
|
||||
let mut b = good.clone();
|
||||
b[at..at + bytes.len()].copy_from_slice(bytes);
|
||||
b
|
||||
};
|
||||
for (what, bytes) in [
|
||||
("4096³ dimensions", patched(dims, &le(&[4096, 4096, 4096]))),
|
||||
("dimensions past u32 products", patched(dims, &le(&[u32::MAX, u32::MAX, u32::MAX]))),
|
||||
("a zero dimension", patched(dims, &le(&[0, 2, 1]))),
|
||||
("4 billion values", patched(entry + 4, &le(&[u32::MAX]))),
|
||||
("data past the end", patched(entry + 8, &le(&[u32::MAX - 16]))),
|
||||
] {
|
||||
let p = parse_dcp(&bytes).unwrap_or_else(|e| panic!("{what}: {e:?}"));
|
||||
assert!(p.color.profile.hue_sat_map[0].is_none(), "{what}: the map is dropped");
|
||||
assert!(p.color.color_matrix[0].is_some(), "{what}: the matrix still reads");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn written_profiles_read_back() {
|
||||
let p = parse_dcp(&synthetic_dcp(false, true)).unwrap();
|
||||
let back = parse_dcp(&write_dcp(&p).unwrap()).unwrap();
|
||||
assert_eq!(back.name, p.name);
|
||||
assert_eq!(back.camera_model, p.camera_model);
|
||||
assert_eq!(back.color.illuminant, p.color.illuminant);
|
||||
assert_eq!(back.color.profile.hue_sat_map, p.color.profile.hue_sat_map);
|
||||
let (a, b) = (back.color.color_matrix[0].unwrap(), p.color.color_matrix[0].unwrap());
|
||||
assert!(a.iter().flatten().zip(b.iter().flatten()).all(|(x, y)| (x - y).abs() < 1e-6));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod boundary_tests {
|
||||
use super::*;
|
||||
use craft_tiff::{ByteOrder, IfdBuilder, TiffWriter, Value};
|
||||
|
||||
fn profile_bytes(offset: f64, matrix: bool) -> Vec<u8> {
|
||||
let mut ifd = IfdBuilder::new();
|
||||
if matrix {
|
||||
ifd.set(t::COLOR_MATRIX_1, Value::SRational(vec![(1, 1), (0, 1), (0, 1), (0, 1), (1, 1), (0, 1), (0, 1), (0, 1), (1, 1)]));
|
||||
}
|
||||
ifd.set(t::BASELINE_EXPOSURE_OFFSET, Value::Double(vec![offset]));
|
||||
ifd.set(t::BASELINE_EXPOSURE, Value::Double(vec![10.0]));
|
||||
ifd.set(t::AS_SHOT_NEUTRAL, Value::Double(vec![0.5, 1.0, 0.7]));
|
||||
let mut bytes = TiffWriter::new(ByteOrder::Little, false).write(&[ifd]).unwrap();
|
||||
bytes[2..4].copy_from_slice(&DCP_MAGIC.to_le_bytes());
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_requires_matrix_and_never_keeps_shot_exposure_or_white_balance() {
|
||||
assert_eq!(parse_dcp(&profile_bytes(0.0, false)), Err(DcpError::NoMatrix));
|
||||
for (offset, expected) in [(9.0, 4.0), (-9.0, -4.0), (f64::INFINITY, 0.0), (f64::NAN, 0.0)] {
|
||||
let profile = parse_dcp(&profile_bytes(offset, true)).unwrap();
|
||||
assert_eq!(profile.color.baseline_exposure, expected);
|
||||
assert_eq!(profile.color.as_shot_neutral, None);
|
||||
assert_eq!(profile.color.camera_calibration, [None, None]);
|
||||
assert_eq!(profile.color.profile.gain_table_map, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
+31
-15
@@ -91,7 +91,7 @@ fn permute<T: Copy + Default, F: Fn(usize, usize) -> usize>(v: &mut Vec<T>, w: u
|
||||
}
|
||||
|
||||
/// A 3×3 matrix tag: nine finite values that can be inverted.
|
||||
fn mat3(v: Option<Vec<f64>>) -> Option<Matrix3> {
|
||||
pub(crate) fn mat3(v: Option<Vec<f64>>) -> Option<Matrix3> {
|
||||
let v = v?;
|
||||
if v.len() != 9 || v.iter().any(|x| !x.is_finite()) {
|
||||
return None;
|
||||
@@ -299,17 +299,11 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
deinterleave(&mut data, w, h, cpp, factor(t::ROW_INTERLEAVE_FACTOR), factor(t::COLUMN_INTERLEAVE_FACTOR))?;
|
||||
}
|
||||
|
||||
// linearization table (integer data only)
|
||||
let mut linearized = false;
|
||||
// linearization table (integer data only) and opcode lists. OpcodeList1 runs on the stored
|
||||
// codes first (DNG 1.7 chapter 7); see [`linearization_and_opcodes`].
|
||||
let (linearized, linearization_table, opcodes) = linearization_and_opcodes(pixels, raw, &mut data);
|
||||
// the largest value the table can give a sample: the white level when the file states none
|
||||
let mut table_max = None;
|
||||
if let (RawData::U16(v), Some(table)) = (&mut data, pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty())) {
|
||||
let table: Vec<u16> = table.iter().map(|&x| x.min(65535) as u16).collect();
|
||||
let last = table.len() - 1;
|
||||
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
|
||||
linearized = true;
|
||||
table_max = table.iter().copied().max().filter(|&m| m > 0);
|
||||
}
|
||||
let table_max = linearization_table.as_deref().and_then(|t| t.iter().copied().max().filter(|&m| m > 0));
|
||||
|
||||
// a reduced copy's own levels, where it states them
|
||||
let levels = |tag| if pixels.contains(tag) { pixels } else { raw };
|
||||
@@ -442,10 +436,6 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
None
|
||||
};
|
||||
|
||||
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
|
||||
let opcodes =
|
||||
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
|
||||
|
||||
let color = color_data(ifd0, raw, tiff.order);
|
||||
let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2))
|
||||
&& color.profile.gain_table_map.is_none();
|
||||
@@ -465,6 +455,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
color,
|
||||
wb_multipliers: None,
|
||||
linearized,
|
||||
linearization_table,
|
||||
pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()),
|
||||
opcodes,
|
||||
camera: Camera::from_ifd0(ifd0),
|
||||
@@ -477,6 +468,31 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
/// Parse opcode lists and, for integer data, the LinearizationTable.
|
||||
///
|
||||
/// OpcodeList1 runs on the stored codes, then the table maps them to sensor values (DNG 1.7
|
||||
/// chapter 7). Applying opcodes is the develop pipeline's work, so a non-empty list 1 leaves the
|
||||
/// table unapplied (`linearized` is false) and available on the image.
|
||||
fn linearization_and_opcodes(pixels: &Ifd, raw: &Ifd, data: &mut RawData) -> (bool, Option<Vec<u16>>, crate::OpcodeLists) {
|
||||
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
|
||||
let opcodes =
|
||||
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
|
||||
let linearization_table = if matches!(data, RawData::U16(_)) {
|
||||
pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty()).map(|table| table.iter().map(|&x| x.min(65535) as u16).collect::<Vec<u16>>())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut linearized = false;
|
||||
if opcodes.list1.is_empty()
|
||||
&& let (RawData::U16(v), Some(table)) = (data, linearization_table.as_deref())
|
||||
{
|
||||
let last = table.len() - 1;
|
||||
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
|
||||
linearized = true;
|
||||
}
|
||||
(linearized, linearization_table, opcodes)
|
||||
}
|
||||
|
||||
/// CinemaDNG `TimeCodes` (CinemaDNG Image Data Format Specification 1.0, tag 51043).
|
||||
const TIME_CODES: u16 = 51043;
|
||||
/// CinemaDNG `FrameRate` (tag 51044).
|
||||
|
||||
@@ -165,6 +165,14 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result<Vec<u8>> {
|
||||
} else {
|
||||
ifd.set(t::WHITE_LEVEL, Value::Long(white));
|
||||
}
|
||||
// A table still to be applied (OpcodeList1 has to run on the stored codes first) is written
|
||||
// with the samples. Once the samples have been mapped, writing the tag would apply it twice
|
||||
// on the next read of a file without list 1.
|
||||
if !raw.linearized
|
||||
&& let Some(table) = raw.linearization_table.as_deref().filter(|t| !t.is_empty())
|
||||
{
|
||||
ifd.set(t::LINEARIZATION_TABLE, Value::Short(table.to_vec()));
|
||||
}
|
||||
let a = raw.active_area;
|
||||
ifd.set(t::ACTIVE_AREA, Value::Long(vec![a.y as u32, a.x as u32, (a.y + a.height) as u32, (a.x + a.width) as u32]));
|
||||
ifd.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![raw.crop.x as u32, raw.crop.y as u32]));
|
||||
@@ -457,6 +465,7 @@ mod tests {
|
||||
color: crate::ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Default::default(),
|
||||
@@ -514,6 +523,7 @@ mod tests {
|
||||
color: crate::ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Default::default(),
|
||||
|
||||
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
|
||||
+126
-13
@@ -4,16 +4,19 @@
|
||||
|
||||
use crate::{RawError, Result};
|
||||
|
||||
/// A canonical-by-table Huffman decoder with a 12-bit lookup.
|
||||
/// A canonical-by-table Huffman decoder with a 12-bit lookup that also holds the difference when code and
|
||||
/// additional bits fit in 12 bits.
|
||||
pub(crate) struct Huffman {
|
||||
/// Indexed by the next 12 bits: (symbol, length); length 0 = invalid.
|
||||
lut: Vec<(u8, u8)>,
|
||||
/// Indexed by the next 12 bits. Per entry: bits 0-7 the symbol, bits 8-11 the code length (0 = invalid), bits
|
||||
/// 12-15 the length of code plus additional bits when both fit in 12 bits (0 = they do not), bits 16-31 the
|
||||
/// signed difference in that case.
|
||||
lut: Box<[u32; 4096]>,
|
||||
}
|
||||
|
||||
impl Huffman {
|
||||
/// From `(left-aligned 12-bit code, length)` per symbol.
|
||||
pub fn new(codes: &[(u16, u8)]) -> Result<Huffman> {
|
||||
let mut lut = vec![(0u8, 0u8); 4096];
|
||||
let mut lut = Box::new([0u32; 4096]);
|
||||
for (sym, &(code, len)) in codes.iter().enumerate() {
|
||||
if len == 0 {
|
||||
continue;
|
||||
@@ -21,10 +24,19 @@ impl Huffman {
|
||||
if len > 12 || sym > 16 {
|
||||
return Err(RawError::Unsupported(format!("PEF Huffman code of length {len}")));
|
||||
}
|
||||
let (len, s) = (len as u32, sym as u32);
|
||||
let first = (code >> (12 - len) << (12 - len)) as usize;
|
||||
for e in &mut lut[first..first + (1 << (12 - len))] {
|
||||
if e.1 == 0 {
|
||||
*e = (sym as u8, len);
|
||||
for (i, e) in lut[first..first + (1 << (12 - len))].iter_mut().enumerate() {
|
||||
if (*e >> 8) & 15 == 0 {
|
||||
let mut entry = (len << 8) | s;
|
||||
if s == 0 {
|
||||
entry |= len << 12;
|
||||
} else if len + s <= 12 {
|
||||
let v = (((first + i) >> (12 - len - s)) & ((1 << s) - 1)) as i32;
|
||||
let d = if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v };
|
||||
entry |= ((len + s) << 12) | ((d as i16 as u16 as u32) << 16);
|
||||
}
|
||||
*e = entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -85,21 +97,122 @@ impl<'a> Bits<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode one difference: Huffman category then T.81 additional bits.
|
||||
/// Decode one difference: Huffman category then T.81 additional bits, in one lookup when both fit in 12 bits.
|
||||
#[inline]
|
||||
pub(crate) fn diff(bits: &mut Bits, h: &Huffman) -> Option<i32> {
|
||||
let (sym, len) = h.lut[bits.peek(12) as usize];
|
||||
let e = h.lut[(bits.peek(12) & 0xfff) as usize];
|
||||
let len = (e >> 8) & 15;
|
||||
if len == 0 {
|
||||
return None;
|
||||
}
|
||||
bits.skip(len as u32);
|
||||
let s = sym as u32;
|
||||
if s == 0 {
|
||||
return Some(0);
|
||||
let total = (e >> 12) & 15;
|
||||
if total > 0 {
|
||||
bits.skip(total);
|
||||
return Some((e as i32) >> 16);
|
||||
}
|
||||
bits.skip(len);
|
||||
let s = e & 0xff;
|
||||
if s > 16 {
|
||||
return None;
|
||||
}
|
||||
let v = bits.get(s) as i32;
|
||||
Some(if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Canonical left-aligned 12-bit codes for symbols 0..lens.len() (shorter codes first, ties by symbol).
|
||||
fn canonical(lens: &[u8]) -> Vec<(u16, u8)> {
|
||||
let mut order: Vec<usize> = (0..lens.len()).collect();
|
||||
order.sort_by_key(|&s| (lens[s], s));
|
||||
let mut codes = vec![(0u16, 0u8); lens.len()];
|
||||
let (mut code, mut prev) = (0u32, 0u8);
|
||||
for s in order {
|
||||
code <<= lens[s] - prev;
|
||||
prev = lens[s];
|
||||
codes[s] = ((code << (12 - lens[s])) as u16, lens[s]);
|
||||
code += 1;
|
||||
}
|
||||
codes
|
||||
}
|
||||
|
||||
fn bit(src: &[u8], i: usize) -> u32 {
|
||||
src.get(i / 8).map_or(0, |b| ((b >> (7 - i % 8)) & 1) as u32)
|
||||
}
|
||||
|
||||
/// Reference: match the code bit by bit, then read the category's additional bits (T.81 F.1.2.1).
|
||||
fn reference(src: &[u8], pos: &mut usize, codes: &[(u16, u8)]) -> Option<i32> {
|
||||
let window: u32 = (0..12).fold(0, |a, k| (a << 1) | bit(src, *pos + k));
|
||||
let (sym, len) = codes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(s, &(c, l))| (l > 0 && (c as u32 >> (12 - l)) == window >> (12 - l)).then_some((s as u32, l as usize)))?;
|
||||
*pos += len;
|
||||
if sym == 0 {
|
||||
return Some(0);
|
||||
}
|
||||
let v = (0..sym as usize).fold(0u32, |a, k| (a << 1) | bit(src, *pos + k)) as i32;
|
||||
*pos += sym as usize;
|
||||
Some(if v < 1 << (sym - 1) { v - (1 << sym) + 1 } else { v })
|
||||
}
|
||||
|
||||
/// `diff` against the reference on random streams (invalid codes are stepped over) and on every (code, extra
|
||||
/// bits) pair of every symbol.
|
||||
fn check_table(lens: &[u8]) {
|
||||
let codes = canonical(lens);
|
||||
let h = Huffman::new(&codes).unwrap();
|
||||
for (sym, &(code, len)) in codes.iter().enumerate() {
|
||||
for v in 0..1u64 << sym {
|
||||
let (len, n) = (len as u32, sym as u32);
|
||||
let word = (((code >> (12 - len)) as u64) << n | v) << (64 - len - n);
|
||||
let src = word.to_be_bytes();
|
||||
let mut pos = 0;
|
||||
let want = reference(&src, &mut pos, &codes);
|
||||
let mut bits = Bits::new(&src);
|
||||
assert_eq!(diff(&mut bits, &h), want, "symbol {sym} bits {v:#b}");
|
||||
assert_eq!(bits.consumed_bits(), pos);
|
||||
}
|
||||
}
|
||||
let mut s = 0x1234_5678_9abc_def0u64;
|
||||
let (mut invalid, mut long) = (0, 0);
|
||||
for _ in 0..40 {
|
||||
let src: Vec<u8> = (0..400)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
(s >> 40) as u8
|
||||
})
|
||||
.collect();
|
||||
let mut bits = Bits::new(&src);
|
||||
let mut pos = 0;
|
||||
while pos < src.len() * 8 - 64 {
|
||||
let before = pos;
|
||||
let want = reference(&src, &mut pos, &codes);
|
||||
assert_eq!(diff(&mut bits, &h), want);
|
||||
if want.is_none() {
|
||||
// step over the invalid code
|
||||
invalid += 1;
|
||||
pos += 1;
|
||||
bits.skip(1);
|
||||
} else {
|
||||
long += (pos - before > 12) as u32;
|
||||
}
|
||||
assert_eq!(bits.consumed_bits(), pos);
|
||||
}
|
||||
}
|
||||
assert!(long > 0, "{invalid} invalid codes, no symbol longer than the lookup");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_matches_bitwise_reference_for_every_category() {
|
||||
// categories 0..=16, code lengths up to 12 so that short codes combine with their bits and long ones do not
|
||||
check_table(&[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_matches_reference_when_code_and_bits_fill_the_lookup_exactly() {
|
||||
// category 8 with a 4-bit code and category 7 with a 5-bit code: len + category == 12
|
||||
check_table(&[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12]);
|
||||
}
|
||||
}
|
||||
@@ -115,6 +115,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
+35
-5
@@ -10,11 +10,13 @@
|
||||
//!
|
||||
//! # Where this crate stops
|
||||
//!
|
||||
//! The samples come out as the file stores them (after a DNG `LinearizationTable`): black is not subtracted,
|
||||
//! nothing is scaled, cropped, white balanced or demosaiced, opcode lists and profile tables are parsed but not
|
||||
//! applied, and no colour transform is computed from the matrices. Normalisation, demosaicing, highlight
|
||||
//! handling, the DNG colour model's arithmetic, lens corrections, tone and everything after belong to the
|
||||
//! develop pipeline, which reads the public fields of the types here.
|
||||
//! The samples come out as the file stores them (after a DNG `LinearizationTable`, except when the file
|
||||
//! also has an `OpcodeList1`: that list runs on the stored codes first, and applying it is the develop
|
||||
//! pipeline's work, so the table is left unapplied and carried on [`RawImage::linearization_table`]).
|
||||
//! Black is not subtracted, nothing is scaled, cropped, white balanced or demosaiced, opcode lists and
|
||||
//! profile tables are parsed but not applied, and no colour transform is computed from the matrices.
|
||||
//! Normalisation, demosaicing, highlight handling, the DNG colour model's arithmetic, lens corrections,
|
||||
//! tone and everything after belong to the develop pipeline, which reads the public fields of the types here.
|
||||
//!
|
||||
//! # Formats
|
||||
//!
|
||||
@@ -66,6 +68,9 @@ mod crw;
|
||||
mod crx;
|
||||
#[cfg(feature = "cr3")]
|
||||
mod crx_wavelet;
|
||||
/// DNG camera-profile file parsing and writing.
|
||||
#[cfg(feature = "dng")]
|
||||
pub mod dcp;
|
||||
#[cfg(feature = "dng")]
|
||||
mod dng;
|
||||
#[cfg(feature = "dng")]
|
||||
@@ -645,6 +650,8 @@ pub fn orientation(bytes: &[u8]) -> Option<u16> {
|
||||
}
|
||||
// A damaged/unsupported pixel IFD must not hide usable orientation metadata from preview
|
||||
// fallback. CR3 keeps IFD0 in its CMT1 box; TIFF-based raws keep it at the file start.
|
||||
// The `cr3` module exists only with the `cr3` feature: this gate is required so
|
||||
// `cargo check -p craft-raw --no-default-features --features dng` builds (E0433 otherwise).
|
||||
#[cfg(feature = "cr3")]
|
||||
if probe(bytes) == Some(RawFormat::Cr3) {
|
||||
return cr3::orientation(bytes);
|
||||
@@ -1041,6 +1048,12 @@ pub struct RawImage {
|
||||
pub wb_multipliers: Option<[f32; 3]>,
|
||||
/// Whether a `LinearizationTable` was applied to `data`.
|
||||
pub linearized: bool,
|
||||
/// The file's `LinearizationTable`, when present. `data` has already been mapped through it when
|
||||
/// [`Self::linearized`] is true. When `linearized` is false and this is `Some`, the table was left
|
||||
/// unapplied so `OpcodeList1` can run on the stored samples first (DNG chapter 7); call
|
||||
/// [`Self::apply_linearization`] after that list.
|
||||
#[serde(default)]
|
||||
pub linearization_table: Option<Vec<u16>>,
|
||||
/// Height of a sensor pixel over its width (1.0 for square pixels). The D1X and D1H write 2.0: their frame is
|
||||
/// 4028 x 1324 and the picture is 3:2 only once the rows are doubled. A developer stretches the rows by
|
||||
/// this factor.
|
||||
@@ -1119,6 +1132,23 @@ impl RawImage {
|
||||
self.check(true)
|
||||
}
|
||||
|
||||
/// Map integer `data` through [`Self::linearization_table`] when the table has not yet been applied.
|
||||
/// No-op when [`Self::linearized`] is already true, the table is missing, or the samples are not
|
||||
/// integers. Call this after `OpcodeList1`, which DNG applies to the stored codes (chapter 7).
|
||||
pub fn apply_linearization(&mut self) {
|
||||
if self.linearized {
|
||||
return;
|
||||
}
|
||||
let Some(table) = self.linearization_table.as_deref().filter(|t| !t.is_empty()) else {
|
||||
return;
|
||||
};
|
||||
if let RawData::U16(v) = &mut self.data {
|
||||
let last = table.len() - 1;
|
||||
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
|
||||
self.linearized = true;
|
||||
}
|
||||
}
|
||||
|
||||
fn check(&self, samples: bool) -> Result<()> {
|
||||
if self.width == 0 || self.height == 0 || !(1..=4).contains(&self.cpp) {
|
||||
return Err(RawError::Corrupt("bad dimensions".into()));
|
||||
|
||||
+129
-39
@@ -32,15 +32,17 @@ fn err(s: &str) -> RawError {
|
||||
/// Huffman decoding table (T.81 Annex C "BITS"/"HUFFVAL").
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Huffman {
|
||||
/// For peeks of `LOOKUP` bits: (code length, symbol); length 0 = slow path.
|
||||
fast: Vec<(u8, u8)>,
|
||||
/// Indexed by the next `LOOKUP` bits; length 0 = slow path. Per entry: bits 0-7 the symbol, bits 8-11 the code
|
||||
/// length, bits 12-15 the length of code plus extra bits when both fit in the lookup (0 = they do not), and
|
||||
/// bits 16-31 the signed difference in that case.
|
||||
fast: Vec<u32>,
|
||||
maxcode: [i32; 18],
|
||||
valptr: [i32; 17],
|
||||
mincode: [i32; 17],
|
||||
values: Vec<u8>,
|
||||
}
|
||||
|
||||
const LOOKUP: u32 = 10;
|
||||
const LOOKUP: u32 = 12;
|
||||
|
||||
impl Huffman {
|
||||
/// Build from the 16 code-length counts and the symbol list.
|
||||
@@ -52,7 +54,7 @@ impl Huffman {
|
||||
let mut maxcode = [-1i32; 18];
|
||||
let mut valptr = [0i32; 17];
|
||||
let mut mincode = [0i32; 17];
|
||||
let mut fast = vec![(0u8, 0u8); 1 << LOOKUP];
|
||||
let mut fast = vec![0u32; 1 << LOOKUP];
|
||||
let mut code = 0i32;
|
||||
let mut k = 0usize;
|
||||
for len in 1..=16usize {
|
||||
@@ -63,9 +65,10 @@ impl Huffman {
|
||||
if len as u32 <= LOOKUP {
|
||||
let shift = LOOKUP - len as u32;
|
||||
let base = (code as usize) << shift;
|
||||
let ssss = values[k] as u32;
|
||||
for i in 0..(1usize << shift) {
|
||||
if let Some(slot) = fast.get_mut(base + i) {
|
||||
*slot = (len as u8, values[k]);
|
||||
*slot = fast_entry(len as u32, ssss, base + i);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -84,12 +87,41 @@ impl Huffman {
|
||||
|
||||
#[inline]
|
||||
pub fn decode<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
|
||||
let peek = br.peek(LOOKUP);
|
||||
let (len, sym) = self.fast[peek as usize];
|
||||
let e = self.fast[br.peek(LOOKUP) as usize];
|
||||
let len = (e >> 8) & 15;
|
||||
if len > 0 {
|
||||
br.consume(len as u32);
|
||||
return Ok(sym);
|
||||
br.consume(len);
|
||||
return Ok(e as u8);
|
||||
}
|
||||
self.decode_slow(br)
|
||||
}
|
||||
|
||||
/// One symbol and its T.81 difference in a single lookup when code and extra bits fit in `LOOKUP` bits; the same
|
||||
/// as [`Huffman::decode`] followed by [`diff_value`]. `strict`: a category above 16 is an error (subsampled
|
||||
/// scans) instead of reading 16 bits.
|
||||
#[inline]
|
||||
pub fn decode_diff<B: Bits>(&self, br: &mut B, strict: bool) -> Result<i32, RawError> {
|
||||
let e = self.fast[br.peek(LOOKUP) as usize];
|
||||
let len = (e >> 8) & 15;
|
||||
let total = (e >> 12) & 15;
|
||||
if total > 0 {
|
||||
br.consume(total);
|
||||
return Ok((e as i32) >> 16);
|
||||
}
|
||||
let ssss = if len > 0 {
|
||||
br.consume(len);
|
||||
e as u8
|
||||
} else {
|
||||
self.decode_slow(br)?
|
||||
};
|
||||
if strict && ssss > 16 {
|
||||
return Err(err("invalid difference category"));
|
||||
}
|
||||
Ok(diff_value(br, ssss))
|
||||
}
|
||||
|
||||
#[cold]
|
||||
fn decode_slow<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
|
||||
let mut code = br.peek(LOOKUP + 1) as i32;
|
||||
let mut len = LOOKUP as usize + 1;
|
||||
while len <= 16 && code > self.maxcode[len] {
|
||||
@@ -105,6 +137,21 @@ impl Huffman {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lookup entry for a code of `len` bits (`len <= LOOKUP`) naming category `ssss`, at window position `index`
|
||||
/// (the code is the top `len` bits of the `LOOKUP`-bit window). See [`Huffman::fast`].
|
||||
fn fast_entry(len: u32, ssss: u32, index: usize) -> u32 {
|
||||
let base = (len << 8) | ssss;
|
||||
if ssss == 0 {
|
||||
return base | (len << 12);
|
||||
}
|
||||
if ssss > 11 || len + ssss > LOOKUP {
|
||||
return base;
|
||||
}
|
||||
let v = ((index >> (LOOKUP - len - ssss)) & ((1 << ssss) - 1)) as i32;
|
||||
let d = if v < (1 << (ssss - 1)) { v - (1 << ssss) + 1 } else { v };
|
||||
base | ((len + ssss) << 12) | ((d as i16 as u16 as u32) << 16)
|
||||
}
|
||||
|
||||
/// A source of entropy-coded bits, first bit most significant.
|
||||
pub trait Bits {
|
||||
/// The next `n` (1..=32) bits without consuming them; zeros past the end.
|
||||
@@ -614,11 +661,7 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
|
||||
Prediction::Sequential if mx == 0 && dx == 0 && dy == 0 => row_first[c],
|
||||
Prediction::Sequential => last[c],
|
||||
};
|
||||
let ssss = tables[c].decode(&mut br)?;
|
||||
if ssss > 16 {
|
||||
return Err(err("invalid difference category"));
|
||||
}
|
||||
let value = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16;
|
||||
let value = ((pred + tables[c].decode_diff(&mut br, true)?) & 0xffff) as u16;
|
||||
plane[i] = value;
|
||||
last[c] = i32::from(value);
|
||||
if mx == 0 && dx == 0 && dy == 0 {
|
||||
@@ -642,6 +685,44 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
|
||||
Ok(FrameSubsampled { width: h.width, height: h.height, vertical_subsampling: vertical, planes })
|
||||
}
|
||||
|
||||
/// T.81 Table H.1 predictor `pred` (1..=7; 0 = none) from the left, above and above-left neighbours.
|
||||
#[inline(always)]
|
||||
fn predict(pred: u8, ra: i32, rb: i32, rc: i32) -> i32 {
|
||||
match pred {
|
||||
0 => 0,
|
||||
1 => ra,
|
||||
2 => rb,
|
||||
3 => rc,
|
||||
4 => ra + rb - rc,
|
||||
5 => ra + ((rb - rc) >> 1),
|
||||
6 => rb + ((ra - rc) >> 1),
|
||||
_ => (ra + rb) >> 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode the pixels `x0..x1` (`x0 >= 1`) of one row with the fixed predictor `P`. `prev` is empty on the first
|
||||
/// line, which only predicts from the left (`P = 1`).
|
||||
#[inline(always)]
|
||||
fn decode_run<const P: u8>(
|
||||
tables: &[&Huffman],
|
||||
br: &mut BitReader<'_>,
|
||||
cur: &mut [u16],
|
||||
prev: &[u16],
|
||||
nc: usize,
|
||||
x0: usize,
|
||||
x1: usize,
|
||||
) -> Result<(), RawError> {
|
||||
for x in x0..x1 {
|
||||
for (c, table) in tables.iter().enumerate() {
|
||||
let i = x * nc + c;
|
||||
let ra = cur[i - nc] as i32;
|
||||
let p = if P <= 1 { predict(P, ra, 0, 0) } else { predict(P, ra, prev[i] as i32, prev[i - nc] as i32) };
|
||||
cur[i] = ((p + table.decode_diff(br, false)?) & 0xffff) as u16;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decode a lossless JPEG stream. `max_samples` bounds the allocation (hostile headers).
|
||||
pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
|
||||
let h = parse_header(d)?;
|
||||
@@ -680,18 +761,23 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
|
||||
let (prev_rows, cur_rows) = out.split_at_mut(y * row_len);
|
||||
let cur = &mut cur_rows[..row_len];
|
||||
let prev = if y > 0 { &prev_rows[(y - 1) * row_len..] } else { &[][..] };
|
||||
for x in 0..w {
|
||||
if h.restart > 0 {
|
||||
if restart_left == 0 {
|
||||
if !br.restart() {
|
||||
return Err(err("missing restart marker"));
|
||||
}
|
||||
restart_left = h.restart;
|
||||
first_line = true;
|
||||
reset_pending = true;
|
||||
let mut x = 0;
|
||||
while x < w {
|
||||
if h.restart > 0 && restart_left == 0 {
|
||||
if !br.restart() {
|
||||
return Err(err("missing restart marker"));
|
||||
}
|
||||
restart_left -= 1;
|
||||
restart_left = h.restart;
|
||||
first_line = true;
|
||||
reset_pending = true;
|
||||
}
|
||||
// pixels up to the next restart marker or the end of the row
|
||||
let mut end = w;
|
||||
if h.restart > 0 {
|
||||
end = w.min(x + restart_left);
|
||||
restart_left -= end - x;
|
||||
}
|
||||
// the first pixel of the run has the special predictions (start of the interval, first column)
|
||||
#[allow(clippy::needless_range_loop)] // `c` indexes the tables and the sample position alike
|
||||
for c in 0..nc {
|
||||
let i = x * nc + c;
|
||||
@@ -702,24 +788,28 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
|
||||
} else if x == 0 || pred == 0 {
|
||||
if pred == 0 { 0 } else { prev[i] as i32 }
|
||||
} else {
|
||||
let ra = cur[i - nc] as i32;
|
||||
let rb = prev[i] as i32;
|
||||
let rc = prev[i - nc] as i32;
|
||||
match pred {
|
||||
1 => ra,
|
||||
2 => rb,
|
||||
3 => rc,
|
||||
4 => ra + rb - rc,
|
||||
5 => ra + ((rb - rc) >> 1),
|
||||
6 => rb + ((ra - rc) >> 1),
|
||||
_ => (ra + rb) >> 1,
|
||||
}
|
||||
predict(pred, cur[i - nc] as i32, prev[i] as i32, prev[i - nc] as i32)
|
||||
};
|
||||
let ssss = tables[c].decode(&mut br)?;
|
||||
let diff = diff_value(&mut br, ssss);
|
||||
cur[i] = ((p + diff) & mask) as u16;
|
||||
cur[i] = ((p + tables[c].decode_diff(&mut br, false)?) & mask) as u16;
|
||||
}
|
||||
reset_pending = false;
|
||||
// the rest of the run: the predictor is fixed, so it is chosen once here, not per sample
|
||||
if first_line {
|
||||
// the first line of an interval predicts from the left neighbour
|
||||
decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?;
|
||||
} else {
|
||||
match pred {
|
||||
0 => decode_run::<0>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
1 => decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
2 => decode_run::<2>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
3 => decode_run::<3>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
4 => decode_run::<4>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
5 => decode_run::<5>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
6 => decode_run::<6>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
_ => decode_run::<7>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
}
|
||||
}
|
||||
x = end;
|
||||
}
|
||||
first_line = false;
|
||||
if br.overrun > 64 * 8 {
|
||||
|
||||
@@ -125,6 +125,121 @@ fn roundtrip_extremes_and_restarts() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-label an encoded stream as `precision` bits with point transform `pt`: the encoder codes the samples at
|
||||
/// `precision - pt` bits, which is what a transformed scan holds.
|
||||
fn with_point_transform(mut enc: Vec<u8>, precision: u8, pt: u8) -> Vec<u8> {
|
||||
enc[6] = precision; // SOF3 precision follows SOI, marker and length
|
||||
let sos = enc.windows(2).position(|w| w == [0xff, 0xda]).unwrap();
|
||||
let seg_len = u16::from_be_bytes([enc[sos + 2], enc[sos + 3]]) as usize;
|
||||
enc[sos + 2 + seg_len - 1] = pt;
|
||||
enc
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_point_transform_all_predictors() {
|
||||
for pred in 1..=7u8 {
|
||||
for comps in [1usize, 2, 3] {
|
||||
for (bits, pt) in [(12u8, 1u8), (14, 2), (16, 4), (8, 3)] {
|
||||
let (w, h) = (11, 6);
|
||||
let data = noise(w * h * comps, (bits - pt) as u32, pred as u64 * 17 + comps as u64);
|
||||
let enc = with_point_transform(encode(&data, w, h, comps, bits - pt, pred, 0), bits, pt);
|
||||
let f = decode(&enc, usize::MAX).unwrap();
|
||||
assert_eq!((f.precision, f.point_transform, f.predictor), (bits, pt, pred));
|
||||
let want: Vec<u16> = data.iter().map(|&v| v << pt).collect();
|
||||
assert_eq!(f.data, want, "pred {pred} comps {comps} bits {bits} pt {pt}");
|
||||
}
|
||||
}
|
||||
}
|
||||
// with restart intervals that end mid-row
|
||||
let data = noise(20 * 9 * 2, 10, 9);
|
||||
for rst in [3usize, 7, 20] {
|
||||
let enc = with_point_transform(encode(&data, 20, 9, 2, 10, 4, rst), 12, 2);
|
||||
let want: Vec<u16> = data.iter().map(|&v| v << 2).collect();
|
||||
assert_eq!(decode(&enc, usize::MAX).unwrap().data, want, "restart {rst}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_skewed_categories_with_long_codes() {
|
||||
// category k with probability 2^-(k+1): the table gets codes longer than the lookup, rare 16-bit differences
|
||||
let mut s = 0x9e37_79b9_7f4a_7c15u64;
|
||||
let mut last = 0u32;
|
||||
let data: Vec<u16> = (0..256 * 256)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
let r = (s >> 32) as u32;
|
||||
let cat = r.trailing_zeros().min(16);
|
||||
let mag = (((s >> 8) as u32) & ((1u32 << cat) - 1)) | (1 << cat) >> 1;
|
||||
let d = if s & 1 == 0 { mag } else { 0u32.wrapping_sub(mag) };
|
||||
last = last.wrapping_add(d) & 0xffff;
|
||||
last as u16
|
||||
})
|
||||
.collect();
|
||||
for pred in [1u8, 4, 7] {
|
||||
let enc = encode(&data, 256, 256, 1, 16, pred, 0);
|
||||
assert_eq!(decode(&enc, usize::MAX).unwrap().data, data, "pred {pred}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical decoding from the code lengths alone, bit by bit (T.81 Annex F, Figure F.16): the reference for the
|
||||
/// lookup tables.
|
||||
fn canonical_symbol(br: &mut BitReader, lens: &[u8]) -> Option<u8> {
|
||||
let window = br.peek(16);
|
||||
let (mut first, mut index) = (0u32, 0usize);
|
||||
for len in 1..=16u8 {
|
||||
let code = window >> (16 - len);
|
||||
let n = lens.iter().filter(|&&l| l == len).count() as u32;
|
||||
if code < first + n {
|
||||
br.consume(len as u32);
|
||||
return Some((index + (code - first) as usize) as u8);
|
||||
}
|
||||
index += n as usize;
|
||||
first = (first + n) << 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// `decode_diff` against canonical decoding and `diff_value`, for tables with codes inside and beyond the lookup
|
||||
/// and with `len + category` equal to the lookup width.
|
||||
#[test]
|
||||
fn decode_diff_matches_canonical_decoding() {
|
||||
let tables: [[u8; 17]; 3] = [
|
||||
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12],
|
||||
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 16],
|
||||
[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12],
|
||||
];
|
||||
for lens in tables {
|
||||
let mut counts = [0u8; 16];
|
||||
for &l in &lens {
|
||||
counts[l as usize - 1] += 1;
|
||||
}
|
||||
// symbols in code order: by length, ties by symbol
|
||||
let mut values: Vec<u8> = (0..17).collect();
|
||||
values.sort_by_key(|&v| (lens[v as usize], v));
|
||||
let sorted: Vec<u8> = values.iter().map(|&v| lens[v as usize]).collect();
|
||||
let t = Huffman::new(&counts, &values).unwrap();
|
||||
let mut s = 77u64;
|
||||
let src: Vec<u8> = (0..6000)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
((s >> 40) as u8).min(0xfe)
|
||||
})
|
||||
.collect();
|
||||
let (mut a, mut b) = (BitReader::new(&src), BitReader::new(&src));
|
||||
for n in 0..8000 {
|
||||
let want = canonical_symbol(&mut a, &sorted).map(|i| diff_value(&mut a, values[i as usize]));
|
||||
let got = t.decode_diff(&mut b, false);
|
||||
assert_eq!(got.ok(), want, "symbol {n}");
|
||||
assert_eq!(a.peek(32), b.peek(32), "symbol {n}");
|
||||
if want.is_none() {
|
||||
// step past the invalid code so the walk goes on
|
||||
a.consume(1);
|
||||
b.consume(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constant_image_single_symbol_table() {
|
||||
let data = vec![1234u16; 16 * 4];
|
||||
|
||||
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
|
||||
@@ -165,6 +165,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.as_ref().and_then(|t| t.ifds.first()).map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
|
||||
@@ -329,6 +329,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect,
|
||||
// Nikon's in-file distortion correction, about the optical centre (which follows the sensor shift)
|
||||
opcodes: OpcodeLists {
|
||||
@@ -671,6 +672,7 @@ fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::ima
|
||||
// the stored colour is already white balanced (see above)
|
||||
wb_multipliers: Some([1.0; 3]),
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -357,6 +357,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -238,6 +238,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -298,6 +298,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData { kelvin_presets, baseline_exposure: baseline_exposure(&h), ..Default::default() },
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists { list3: crate::raf_lens::corrections(&camera_name(bytes), &ifd, active), ..Default::default() },
|
||||
camera: ifd0
|
||||
|
||||
@@ -672,6 +672,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists { list3: distortion(ifd0, active).into_iter().collect(), ..Default::default() },
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -186,6 +186,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -44,6 +44,7 @@ fn an_active_area_that_wraps_past_the_image_is_corrupt() {
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Camera::default(),
|
||||
cinema: Default::default(),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Synthetic DNG round trips: every storage variant we decode is written with our own TIFF writer (and LJ92
|
||||
//! encoder) and must decode bit-exactly.
|
||||
|
||||
use craft_raw::opcodes::{Area, Opcode};
|
||||
use craft_raw::opcodes::{self, Area, Opcode};
|
||||
use craft_raw::{
|
||||
BlackLevel, Cfa, ColorData, DngCompression, DngWriteOptions, OpcodeLists, Orientation, RawData, RawFormat, RawImage, Rect, decode,
|
||||
embedded_preview, probe, write_dng,
|
||||
@@ -58,6 +58,7 @@ fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
|
||||
color,
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera,
|
||||
@@ -1343,12 +1344,77 @@ fn white_level_defaults_to_the_linearization_tables_maximum() {
|
||||
};
|
||||
let img = build(None);
|
||||
assert!(img.linearized);
|
||||
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
|
||||
assert_eq!(img.data, RawData::U16(codes.iter().map(|&c| table[c as usize]).collect()));
|
||||
assert_eq!(img.white, vec![51000.0], "255 × 200, the table's top");
|
||||
// a stated level still wins
|
||||
assert_eq!(build(Some(40000)).white, vec![40000.0]);
|
||||
}
|
||||
|
||||
/// DNG chapter 7: OpcodeList1 runs on the stored codes, then the LinearizationTable maps them to
|
||||
/// sensor values. When list 1 is present, decode leaves the table unapplied so the develop pipeline
|
||||
/// can keep that order. Applying the table first would give a different picture.
|
||||
#[test]
|
||||
fn opcode_list1_runs_on_stored_samples_before_linearization() {
|
||||
let (w, h) = (4, 2);
|
||||
let stored = vec![2u16; w * h];
|
||||
// table: 2 → 20, 7 → 30, 20 → 99 (so the two orders disagree)
|
||||
let mut table = vec![0u16; 32];
|
||||
table[2] = 20;
|
||||
table[7] = 30;
|
||||
table[20] = 99;
|
||||
// MapTable: 2 → 7, 20 → 8
|
||||
let mut map: Vec<u16> = (0..32).collect();
|
||||
map[2] = 7;
|
||||
map[20] = 8;
|
||||
let area = Area { top: 0, left: 0, bottom: h as u32, right: w as u32, plane: 0, planes: 1, row_pitch: 1, col_pitch: 1 };
|
||||
let list1 = opcodes::write_list(&[Opcode::MapTable { area, table: map.clone() }]);
|
||||
|
||||
let mut raw = base_ifd(w, h, 16, 1, true);
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![compression::NONE]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![u16_bytes(&stored, ByteOrder::Little)] });
|
||||
raw.set(t::LINEARIZATION_TABLE, Value::Short(table.clone()));
|
||||
raw.set(t::OPCODE_LIST_1, Value::Undefined(list1));
|
||||
raw.set(t::WHITE_LEVEL, Value::Long(vec![100]));
|
||||
let img = decode(&dng(raw, ByteOrder::Little)).unwrap();
|
||||
assert!(!img.linearized, "list 1 is present, so the table must wait");
|
||||
assert_eq!(img.data, RawData::U16(stored.clone()));
|
||||
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
|
||||
assert_eq!(img.opcodes.list1.len(), 1);
|
||||
|
||||
let apply_map = |samples: &mut [u16]| {
|
||||
for s in samples {
|
||||
let i = (*s as usize).min(map.len() - 1);
|
||||
*s = map[i];
|
||||
}
|
||||
};
|
||||
|
||||
let mut correct = img.clone();
|
||||
if let RawData::U16(v) = &mut correct.data {
|
||||
apply_map(v);
|
||||
}
|
||||
correct.apply_linearization();
|
||||
assert!(correct.linearized);
|
||||
// 2 → 7 (MapTable) → 30 (linearization)
|
||||
assert_eq!(correct.data, RawData::U16(vec![30; w * h]));
|
||||
|
||||
let mut wrong = img.clone();
|
||||
wrong.apply_linearization();
|
||||
if let RawData::U16(v) = &mut wrong.data {
|
||||
apply_map(v);
|
||||
}
|
||||
// 2 → 20 (linearization) → 8 (MapTable)
|
||||
assert_eq!(wrong.data, RawData::U16(vec![8; w * h]));
|
||||
assert_ne!(correct.data, wrong.data);
|
||||
|
||||
// the unapplied table and list 1 survive a write / decode round trip
|
||||
let back = decode(&write_dng(&img, &DngWriteOptions { compression: DngCompression::Uncompressed, ..Default::default() }).unwrap()).unwrap();
|
||||
assert!(!back.linearized);
|
||||
assert_eq!(back.data, RawData::U16(stored));
|
||||
assert_eq!(back.linearization_table.as_deref(), Some(table.as_slice()));
|
||||
assert_eq!(back.opcodes.list1.len(), 1);
|
||||
}
|
||||
|
||||
/// PhotoCraft: `AsShotWhiteXY` is a chromaticity, so both coordinates lie inside (0, 1).
|
||||
#[test]
|
||||
fn an_as_shot_white_outside_the_chromaticity_range_is_ignored() {
|
||||
|
||||
@@ -22,6 +22,7 @@ fn sample(comp: DngCompression) -> Vec<u8> {
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Default::default(),
|
||||
|
||||
@@ -181,6 +181,12 @@ pub struct CcittRawOutput {
|
||||
|
||||
/// Decodes CCITT Group 3 or Group 4 fax `data` with caller-specified `params` and `max_rows`,
|
||||
/// returning packed MSB-first rows, rows decoded, input bytes consumed, and any decode error.
|
||||
///
|
||||
/// The rows are read as the parameters say, as PostScript's `CCITTFaxDecode` filter reads them:
|
||||
/// on bytes exactly when `encoded_byte_align` is set, whatever codes the data holds (unlike
|
||||
/// [`decode_ccitt_within`], which reads Group 3 rows with end-of-line codes by those codes). At
|
||||
/// most `rows` rows are read when it is above 0; a caller that ends the rows at the end-of-block
|
||||
/// code alone passes 0.
|
||||
pub fn decode_ccitt_raw(data: &[u8], p: &CcittParams, max_rows: u32) -> Result<CcittRawOutput, CodecError> {
|
||||
if p.columns <= 0 || p.columns > i64::from(MAX_CCITT_COLUMNS) {
|
||||
return Err(CodecError::Corrupt { filter: "CCITTFaxDecode", detail: format!("CCITT columns {} outside 1..={MAX_CCITT_COLUMNS}", p.columns) });
|
||||
@@ -192,7 +198,7 @@ pub fn decode_ccitt_raw(data: &[u8], p: &CcittParams, max_rows: u32) -> Result<C
|
||||
rows,
|
||||
end_of_block: p.end_of_block,
|
||||
end_of_line: p.end_of_line,
|
||||
rows_are_byte_aligned: p.encoded_byte_align && (p.k < 0 || !starts_with_end_of_line(data)),
|
||||
rows_are_byte_aligned: p.encoded_byte_align,
|
||||
encoding: match p.k {
|
||||
..0 => hayro_ccitt::EncodingMode::Group4,
|
||||
0 => hayro_ccitt::EncodingMode::Group3_1D,
|
||||
@@ -579,6 +585,72 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// `decode_ccitt_raw` reads rows on bytes exactly when `encoded_byte_align` says, as
|
||||
/// PostScript's `CCITTFaxDecode` filter (and `hayro-ccitt` itself) reads them, in every layout
|
||||
/// of Group 3 rows: with or without end-of-line codes, fill before each row or before each
|
||||
/// code's end, an RTC after the rows, and room for more rows than they hold. The codes in the
|
||||
/// data don't make rows the parameters put on bytes read unaligned (which read an aligned RTC
|
||||
/// as one more row, #778).
|
||||
#[test]
|
||||
fn raw_group_3_rows_follow_the_parameters() {
|
||||
let mut cases = 0;
|
||||
for seed in 0..24u64 {
|
||||
let (columns, rows) = (8 + (seed as u32 * 37) % 200, 2 + seed as u32 % 20);
|
||||
let runs = text_like(seed, columns, rows);
|
||||
for k in [0i64, 1, 2] {
|
||||
for (codes, fill) in [(false, Fill::None), (false, Fill::RowStart), (true, Fill::None), (true, Fill::RowStart), (true, Fill::CodeEnd)]
|
||||
{
|
||||
let mut data = group_3(&runs, codes, k > 0, fill);
|
||||
if codes {
|
||||
// RTC: six end-of-line codes (with their 1 tag bits for `k` above 0), each on a byte.
|
||||
let eol: [u8; 2] = if k > 0 { [0x00, 0x18] } else { [0x00, 0x10] };
|
||||
(0..6).for_each(|_| data.extend_from_slice(&eol));
|
||||
}
|
||||
for (aligned, end_of_line, end_of_block) in [(true, true, true), (true, false, true), (true, true, false), (false, true, true)] {
|
||||
let p = CcittParams {
|
||||
k,
|
||||
columns: i64::from(columns),
|
||||
end_of_line,
|
||||
encoded_byte_align: aligned,
|
||||
end_of_block,
|
||||
..CcittParams::default()
|
||||
};
|
||||
let max_rows = rows + 8;
|
||||
let raw = decode_ccitt_raw(&data, &p, max_rows).unwrap();
|
||||
let settings = hayro_ccitt::DecodeSettings {
|
||||
columns,
|
||||
rows: max_rows,
|
||||
end_of_block,
|
||||
end_of_line,
|
||||
rows_are_byte_aligned: aligned,
|
||||
encoding: if k > 0 { hayro_ccitt::EncodingMode::Group3_2D { k: k as u32 } } else { hayro_ccitt::EncodingMode::Group3_1D },
|
||||
invert_black: false,
|
||||
};
|
||||
let mut filter = Rows::new(columns as usize, max_rows as usize, false);
|
||||
let used = hayro_ccitt::decode(&data, &mut filter, &mut hayro_ccitt::DecoderContext::new(settings));
|
||||
let case = format!(
|
||||
"seed {seed}, {columns} × {rows}, k {k}, codes {codes}, fill {fill:?}, aligned {aligned}, eol {end_of_line}, eob {end_of_block}"
|
||||
);
|
||||
assert_eq!((&raw.data, raw.rows_decoded as usize), (&filter.out, filter.done), "{case}");
|
||||
if let Ok(used) = used {
|
||||
assert_eq!((raw.bytes_consumed, raw.error), (used, None), "{case}");
|
||||
}
|
||||
cases += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(cases, 24 * 3 * 5 * 4);
|
||||
// Rows on bytes with their codes, as `fax`'s writers lay them out, read whole: the rows
|
||||
// and nothing after the RTC.
|
||||
let runs = text_like(7, 74, 4);
|
||||
let mut data = group_3(&runs, true, false, Fill::RowStart);
|
||||
(0..6).for_each(|_| data.extend_from_slice(&[0x00, 0x10]));
|
||||
let p = CcittParams { k: 0, columns: 74, end_of_line: true, encoded_byte_align: true, ..CcittParams::default() };
|
||||
let raw = decode_ccitt_raw(&data, &p, 9).unwrap();
|
||||
assert_eq!((raw.data, raw.rows_decoded), (pack(&runs, 74), 4));
|
||||
}
|
||||
|
||||
proptest::proptest! {
|
||||
/// Any bitmap, of noise or of runs, decodes to its own bits, through an independent decoder
|
||||
/// and through `decode_ccitt`.
|
||||
|
||||
@@ -34,26 +34,36 @@ impl Imported {
|
||||
/// above [`crate::rate::MAX_RATE`] per second and start timecodes beyond [`MAX_START_TIMECODE`]
|
||||
/// frames. Every importer ends with this pass, so no format can let such a number through.
|
||||
pub fn bound(&mut self, report: &mut Report) {
|
||||
self.bound_with_report_limit(report, None);
|
||||
}
|
||||
/// Apply numeric bounds with an optional limit on distinct warning messages.
|
||||
pub fn bound_with_report_limit(&mut self, report: &mut Report, limit: Option<usize>) {
|
||||
for item in self.project.items.values_mut() {
|
||||
let name = item.name.clone();
|
||||
if let Some(seq) = item.as_sequence_mut() {
|
||||
if !plausible_rate(seq.settings.frame_rate) {
|
||||
report.warn(format!("sequence \"{name}\" has an implausible frame rate ({}); the default was used", seq.settings.frame_rate));
|
||||
report.warn_limited(
|
||||
limit,
|
||||
format!("sequence \"{name}\" has an implausible frame rate ({}); the default was used", seq.settings.frame_rate),
|
||||
);
|
||||
seq.settings.frame_rate = FrameRate::default();
|
||||
}
|
||||
if seq.start_timecode.unsigned_abs() > MAX_START_TIMECODE {
|
||||
report.warn(format!("sequence \"{name}\" has an out-of-range start timecode ({} frames); it starts at 0", seq.start_timecode));
|
||||
report.warn_limited(
|
||||
limit,
|
||||
format!("sequence \"{name}\" has an out-of-range start timecode ({} frames); it starts at 0", seq.start_timecode),
|
||||
);
|
||||
seq.start_timecode = 0;
|
||||
}
|
||||
} else if let Some(media) = item.as_media_mut() {
|
||||
if let Some(v) = media.info.video.as_mut()
|
||||
&& !plausible_rate(v.frame_rate)
|
||||
{
|
||||
report.warn(format!("media \"{name}\" has an implausible frame rate ({}); the default was used", v.frame_rate));
|
||||
report.warn_limited(limit, format!("media \"{name}\" has an implausible frame rate ({}); the default was used", v.frame_rate));
|
||||
v.frame_rate = FrameRate::default();
|
||||
}
|
||||
if media.info.start_timecode.is_some_and(|tc| tc.unsigned_abs() > MAX_START_TIMECODE) {
|
||||
report.warn(format!("media \"{name}\" has an out-of-range start timecode; it was ignored"));
|
||||
report.warn_limited(limit, format!("media \"{name}\" has an out-of-range start timecode; it was ignored"));
|
||||
media.info.start_timecode = None;
|
||||
}
|
||||
}
|
||||
|
||||
+75
-32
@@ -131,7 +131,7 @@ fn color_label(s: &str) -> Label {
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
/// `fields` with the keys of `ext` (a JSON object) added after them; a key of ours wins.
|
||||
fn with_ext(mut fields: Map<String, Value>, ext: &Ext, report: &mut Report) -> Value {
|
||||
fn with_ext(mut fields: Map<String, Value>, ext: &Ext, report: &mut Report, limit: Option<usize>) -> Value {
|
||||
if let Some(text) = ext {
|
||||
match serde_json::from_str::<Value>(text) {
|
||||
Ok(Value::Object(extra)) => {
|
||||
@@ -139,7 +139,7 @@ fn with_ext(mut fields: Map<String, Value>, ext: &Ext, report: &mut Report) -> V
|
||||
fields.entry(k).or_insert(v);
|
||||
}
|
||||
}
|
||||
_ => report.warn("extension data that is not a JSON object was not written to OTIO"),
|
||||
_ => report.warn_limited(limit, "extension data that is not a JSON object was not written to OTIO"),
|
||||
}
|
||||
}
|
||||
Value::Object(fields)
|
||||
@@ -355,7 +355,7 @@ fn value_of(v: &Value) -> ParamValue {
|
||||
|
||||
const EFFECT_KEYS: [&str; 3] = ["effect", "enabled", "params"];
|
||||
|
||||
fn effects_json(effects: &[EffectInstance], report: &mut Report) -> Value {
|
||||
fn effects_json(effects: &[EffectInstance], report: &mut Report, limit: Option<usize>) -> Value {
|
||||
Value::Array(
|
||||
effects
|
||||
.iter()
|
||||
@@ -375,7 +375,7 @@ fn effects_json(effects: &[EffectInstance], report: &mut Report) -> Value {
|
||||
(k.clone(), json!({"value": value_json(&p.value), "keyframes": keys}))
|
||||
})
|
||||
.collect();
|
||||
with_ext(obj(json!({"effect": e.effect, "enabled": e.enabled, "params": params})), &e.ext, report)
|
||||
with_ext(obj(json!({"effect": e.effect, "enabled": e.enabled, "params": params})), &e.ext, report, limit)
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
@@ -475,6 +475,7 @@ struct Exp<'a, 'r> {
|
||||
p: &'a Project,
|
||||
opts: &'a ExportOptions<'a>,
|
||||
report: &'r mut Report,
|
||||
report_limit: Option<usize>,
|
||||
stack: Vec<ItemId>,
|
||||
/// Clips written inside nested sequences so far, and the limit for it (see [`MAX_NESTED_ITEMS`]).
|
||||
nested_items: usize,
|
||||
@@ -483,9 +484,19 @@ struct Exp<'a, 'r> {
|
||||
|
||||
/// Write `sequence` (and the media and nested sequences it uses) as an OTIO `Timeline`.
|
||||
pub fn export(p: &Project, seq_id: ItemId, opts: &ExportOptions, report: &mut Report) -> Result<String> {
|
||||
export_with_report_limit(p, seq_id, opts, report, None)
|
||||
}
|
||||
/// Write OTIO with an optional limit on distinct warnings.
|
||||
pub fn export_with_report_limit(
|
||||
p: &Project,
|
||||
seq_id: ItemId,
|
||||
opts: &ExportOptions,
|
||||
report: &mut Report,
|
||||
report_limit: Option<usize>,
|
||||
) -> Result<String> {
|
||||
let seq = p.sequence(seq_id).ok_or(Error::NoSequence(seq_id))?;
|
||||
let rate = seq.settings.frame_rate;
|
||||
let mut x = Exp { p, opts, report, stack: vec![seq_id], nested_items: 0, nested_limit: MAX_NESTED_ITEMS };
|
||||
let mut x = Exp { p, opts, report, report_limit, stack: vec![seq_id], nested_items: 0, nested_limit: MAX_NESTED_ITEMS };
|
||||
let name = opts.name.map(str::to_string).unwrap_or_else(|| p.item(seq_id).map(|i| i.name.clone()).unwrap_or_default());
|
||||
let stack = x.stack_json(seq_id, &name, None);
|
||||
let mut meta = Map::new();
|
||||
@@ -512,13 +523,13 @@ impl Exp<'_, '_> {
|
||||
fn seq_meta(&mut self, id: ItemId) -> Value {
|
||||
let Some((it, s)) = self.p.item(id).and_then(|it| Some((it, it.as_sequence()?))) else { return json!({}) };
|
||||
let m = obj(json!({"item": id.0, "label": it.label.name(), "settings": settings_json(&s.settings), "start_timecode": s.start_timecode}));
|
||||
with_ext(m, &it.ext, self.report)
|
||||
with_ext(m, &it.ext, self.report, self.report_limit)
|
||||
}
|
||||
|
||||
/// The `Stack` for a sequence; `range` = (source in, duration) when nested in a track.
|
||||
fn stack_json(&mut self, id: ItemId, name: &str, range: Option<(Tick, Tick, FrameRate)>) -> Value {
|
||||
let Some(seq) = self.p.sequence(id) else {
|
||||
self.report.warn("missing nested sequence skipped");
|
||||
self.report.warn_limited(self.report_limit, "missing nested sequence skipped");
|
||||
return json!({"OTIO_SCHEMA": "Stack.1", "name": name, "children": [], "effects": [], "markers": [], "enabled": true, "metadata": {}});
|
||||
};
|
||||
let rate = seq.settings.frame_rate;
|
||||
@@ -564,7 +575,7 @@ impl Exp<'_, '_> {
|
||||
}
|
||||
pos = it.end();
|
||||
}
|
||||
let meta = with_ext(obj(json!({"locked": t.locked, "channels": channels_name(t.channels)})), &t.ext, self.report);
|
||||
let meta = with_ext(obj(json!({"locked": t.locked, "channels": channels_name(t.channels)})), &t.ext, self.report, self.report_limit);
|
||||
json!({
|
||||
"OTIO_SCHEMA": "Track.1",
|
||||
"name": t.name,
|
||||
@@ -580,8 +591,12 @@ impl Exp<'_, '_> {
|
||||
|
||||
fn transition_json(&mut self, x: &Transition, cut: Tick, rate: FrameRate) -> Value {
|
||||
let dissolve = matches!(x.effect.effect.as_str(), "cross_dissolve" | "constant_power" | "constant_gain" | "film_dissolve");
|
||||
let effect = effects_json(std::slice::from_ref(&x.effect), self.report).as_array().and_then(|a| a.first().cloned()).unwrap_or(Value::Null);
|
||||
let meta = with_ext(obj(json!({"effect": effect, "align": align_name(x.align), "reverse": x.reverse})), &x.ext, self.report);
|
||||
let effect = effects_json(std::slice::from_ref(&x.effect), self.report, self.report_limit)
|
||||
.as_array()
|
||||
.and_then(|a| a.first().cloned())
|
||||
.unwrap_or(Value::Null);
|
||||
let meta =
|
||||
with_ext(obj(json!({"effect": effect, "align": align_name(x.align), "reverse": x.reverse})), &x.ext, self.report, self.report_limit);
|
||||
json!({
|
||||
"OTIO_SCHEMA": "Transition.1",
|
||||
"name": transition_name(self.opts.catalog, &x.effect),
|
||||
@@ -604,7 +619,7 @@ impl Exp<'_, '_> {
|
||||
fields.insert("generator".into(), generator_json(g));
|
||||
}
|
||||
let ext = it.and_then(|i| i.ext.clone());
|
||||
let meta = with_ext(fields, &ext, self.report);
|
||||
let meta = with_ext(fields, &ext, self.report, self.report_limit);
|
||||
match &m.media {
|
||||
MediaRef::Generator(g) => json!({
|
||||
"OTIO_SCHEMA": "GeneratorReference.1",
|
||||
@@ -626,11 +641,11 @@ impl Exp<'_, '_> {
|
||||
|
||||
fn clip_meta(&mut self, it: &TrackItem) -> Value {
|
||||
let fields = obj(json!({
|
||||
"label": it.label.name(), "link": it.link, "group": it.group, "effects": effects_json(&it.effects, self.report),
|
||||
"label": it.label.name(), "link": it.link, "group": it.group, "effects": effects_json(&it.effects, self.report, self.report_limit),
|
||||
"gain_db": it.gain_db, "scale_to_frame": it.scale_to_frame, "source_in_ticks": it.source_in.0,
|
||||
"duration_ticks": it.duration.0,
|
||||
}));
|
||||
with_ext(fields, &it.ext, self.report)
|
||||
with_ext(fields, &it.ext, self.report, self.report_limit)
|
||||
}
|
||||
|
||||
fn item_json(&mut self, it: &TrackItem, kind: TrackKind, rate: FrameRate) -> Value {
|
||||
@@ -650,17 +665,20 @@ impl Exp<'_, '_> {
|
||||
let _ = kind;
|
||||
if matches!(self.p.item(base).map(|i| &i.kind), Some(ItemKind::Sequence(_))) {
|
||||
if self.stack.contains(&base) {
|
||||
self.report.warn("recursive nested sequence skipped");
|
||||
self.report.warn_limited(self.report_limit, "recursive nested sequence skipped");
|
||||
return json!({"OTIO_SCHEMA": "Gap.1", "name": "", "source_range": tr(Tick::ZERO, it.duration, rate), "effects": [], "markers": [], "enabled": true, "metadata": {}});
|
||||
}
|
||||
if self.nested_items >= self.nested_limit {
|
||||
// once: later nested sequences hit this too, and the message would repeat
|
||||
if self.nested_items < usize::MAX {
|
||||
self.nested_items = usize::MAX;
|
||||
self.report.warn(format!(
|
||||
"more than {} clips written inside nested sequences: the rest are gaps (OTIO repeats a nested sequence for every use)",
|
||||
self.nested_limit
|
||||
));
|
||||
self.report.warn_limited(
|
||||
self.report_limit,
|
||||
format!(
|
||||
"more than {} clips written inside nested sequences: the rest are gaps (OTIO repeats a nested sequence for every use)",
|
||||
self.nested_limit
|
||||
),
|
||||
);
|
||||
}
|
||||
return json!({"OTIO_SCHEMA": "Gap.1", "name": "", "source_range": tr(Tick::ZERO, it.duration, rate), "effects": [], "markers": [], "enabled": true, "metadata": {}});
|
||||
}
|
||||
@@ -678,9 +696,10 @@ impl Exp<'_, '_> {
|
||||
}
|
||||
let _ = generator_of(self.p, base);
|
||||
if let Some(clip) = crate::build::uncarried_clip(self.p, it, rate) {
|
||||
self.report.warn(format!(
|
||||
"{clip} has no OTIO equivalent: it is written as a clip with a missing media reference and is read back as offline media"
|
||||
));
|
||||
self.report.warn_limited(
|
||||
self.report_limit,
|
||||
format!("{clip} has no OTIO equivalent: it is written as a clip with a missing media reference and is read back as offline media"),
|
||||
);
|
||||
}
|
||||
let mref = self.media_ref(base, rate);
|
||||
// OTIO source times are in the reference's time (which starts at its available_range start).
|
||||
@@ -703,7 +722,7 @@ impl Exp<'_, '_> {
|
||||
let fields = obj(
|
||||
json!({"kind": marker_kind_name(m.kind), "url": m.url, "label": m.color.name(), "author": m.author, "start_ticks": m.start.0, "duration_ticks": m.duration.0}),
|
||||
);
|
||||
let meta = with_ext(fields, &m.ext, self.report);
|
||||
let meta = with_ext(fields, &m.ext, self.report, self.report_limit);
|
||||
json!({
|
||||
"OTIO_SCHEMA": "Marker.2",
|
||||
"name": m.name,
|
||||
@@ -734,12 +753,17 @@ struct Imp<'a, 'c, 'r> {
|
||||
b: Builder<'c>,
|
||||
opts: &'a ImportOptions<'a>,
|
||||
report: &'r mut Report,
|
||||
report_limit: Option<usize>,
|
||||
links: HashMap<u64, u64>,
|
||||
groups: HashMap<u64, u64>,
|
||||
}
|
||||
|
||||
/// Read an OTIO document: a `Timeline`, a `SerializableCollection` of them, a `Stack` or a bare `Track`.
|
||||
pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<Imported> {
|
||||
import_with_report_limit(text, opts, report, None)
|
||||
}
|
||||
/// Read OTIO with an optional limit on distinct warnings.
|
||||
pub fn import_with_report_limit(text: &str, opts: &ImportOptions, report: &mut Report, report_limit: Option<usize>) -> Result<Imported> {
|
||||
let v: Value = serde_json::from_str(text).map_err(|e| Error::parse(Format::Otio, e.to_string()))?;
|
||||
let mut timelines = Vec::new();
|
||||
collect_timelines(&v, &mut timelines);
|
||||
@@ -747,11 +771,11 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
|
||||
return Err(Error::parse(Format::Otio, format!("no Timeline in a {} document", schema(&v))));
|
||||
}
|
||||
if has_absurd_rate(&v) {
|
||||
report.warn(format!(
|
||||
report.warn_limited(report_limit, format!(
|
||||
"an OTIO rate above {MAX_RATE} per second is not a frame rate; the default 24 fps was used (this includes audio sample rates above it, such as DSD)"
|
||||
));
|
||||
}
|
||||
let mut imp = Imp { b: Builder::new(&name_of(&v), opts.catalog), opts, report, links: HashMap::new(), groups: HashMap::new() };
|
||||
let mut imp = Imp { b: Builder::new(&name_of(&v), opts.catalog), opts, report, report_limit, links: HashMap::new(), groups: HashMap::new() };
|
||||
if imp.b.p.name.is_empty() {
|
||||
imp.b.p.name = "Imported OTIO".into();
|
||||
}
|
||||
@@ -760,7 +784,7 @@ pub fn import(text: &str, opts: &ImportOptions, report: &mut Report) -> Result<I
|
||||
imp.b.top.push(id);
|
||||
}
|
||||
let mut imported = imp.b.finish();
|
||||
imported.bound(report);
|
||||
imported.bound_with_report_limit(report, report_limit);
|
||||
Ok(imported)
|
||||
}
|
||||
|
||||
@@ -843,7 +867,7 @@ impl Imp<'_, '_, '_> {
|
||||
let kind = if t.get("kind").and_then(Value::as_str) == Some("Audio") { TrackKind::Audio } else { TrackKind::Video };
|
||||
self.track(&mut seq, t, kind);
|
||||
}
|
||||
other => self.report.warn(format!("OTIO {other} directly in a stack is not supported")),
|
||||
other => self.report.warn_limited(self.report_limit, format!("OTIO {other} directly in a stack is not supported")),
|
||||
}
|
||||
}
|
||||
self.b.ensure_tracks(&mut seq, TrackKind::Video, 1);
|
||||
@@ -905,7 +929,7 @@ impl Imp<'_, '_, '_> {
|
||||
}
|
||||
}
|
||||
other => {
|
||||
self.report.warn(format!("OTIO {other} items are not supported and were skipped"));
|
||||
self.report.warn_limited(self.report_limit, format!("OTIO {other} items are not supported and were skipped"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -925,7 +949,17 @@ impl Imp<'_, '_, '_> {
|
||||
None => {
|
||||
let n = name_of(c);
|
||||
let ty = c.get("transition_type").and_then(Value::as_str).unwrap_or("SMPTE_Dissolve");
|
||||
transition_effect(self.b.catalog, if n.is_empty() { ty } else { &n }, kind == TrackKind::Audio, self.report)
|
||||
{
|
||||
let mut temporary = Report::default();
|
||||
let effect =
|
||||
transition_effect(self.b.catalog, if n.is_empty() { ty } else { &n }, kind == TrackKind::Audio, &mut temporary);
|
||||
for e in temporary.entries {
|
||||
for _ in 0..e.count {
|
||||
self.report.warn_limited(self.report_limit, &e.message);
|
||||
}
|
||||
}
|
||||
effect
|
||||
}
|
||||
}
|
||||
};
|
||||
let align = align_of(meta.and_then(|m| m.get("align"))).unwrap_or(if inn == Tick::ZERO {
|
||||
@@ -972,7 +1006,7 @@ impl Imp<'_, '_, '_> {
|
||||
_ => {
|
||||
if meta.and_then(|m| m.get("effects")).is_none() {
|
||||
let n = e.get("effect_name").and_then(Value::as_str).unwrap_or("effect");
|
||||
self.report.warn(format!("OTIO effect \"{n}\" is not supported"));
|
||||
self.report.warn_limited(self.report_limit, format!("OTIO effect \"{n}\" is not supported"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1107,7 +1141,8 @@ impl Imp<'_, '_, '_> {
|
||||
other => match self.b.catalog.generator_by_name(other) {
|
||||
Some(g) => g,
|
||||
None => {
|
||||
self.report.warn(format!("OTIO generator \"{other}\" is not supported; imported as Black Video"));
|
||||
self.report
|
||||
.warn_limited(self.report_limit, format!("OTIO generator \"{other}\" is not supported; imported as Black Video"));
|
||||
Generator::BlackVideo
|
||||
}
|
||||
},
|
||||
@@ -1127,7 +1162,7 @@ impl Imp<'_, '_, '_> {
|
||||
}
|
||||
_ => {
|
||||
if schema_name(r) != "MissingReference" && !r.is_null() {
|
||||
self.report.warn(format!("OTIO media reference {} is not supported; imported offline", schema(r)));
|
||||
self.report.warn_limited(self.report_limit, format!("OTIO media reference {} is not supported; imported offline", schema(r)));
|
||||
}
|
||||
let key = format!("missing:{name}");
|
||||
let existed = self.b.find_media(&key);
|
||||
@@ -1279,7 +1314,15 @@ mod tests {
|
||||
let opts = xopts();
|
||||
let write = |limit: usize| {
|
||||
let mut report = Report::default();
|
||||
let mut x = Exp { p: &imported.project, opts: &opts, report: &mut report, stack: vec![seq], nested_items: 0, nested_limit: limit };
|
||||
let mut x = Exp {
|
||||
p: &imported.project,
|
||||
opts: &opts,
|
||||
report: &mut report,
|
||||
report_limit: None,
|
||||
stack: vec![seq],
|
||||
nested_items: 0,
|
||||
nested_limit: limit,
|
||||
};
|
||||
let stacks = x.stack_json(seq, "t", None).to_string().matches("\"Stack.1\"").count();
|
||||
(stacks, report)
|
||||
};
|
||||
|
||||
@@ -35,6 +35,20 @@ impl Report {
|
||||
self.entries.push(ReportEntry { level, message, count: 1 });
|
||||
}
|
||||
}
|
||||
/// Add a warning while bounding distinct diagnostics. `limit` counts every distinct entry already
|
||||
/// in the report, informational ones included, exactly like FilmCraft's `MAX_REPORT_ENTRIES`
|
||||
/// budget this replaces; past it a new message is counted under "further diagnostics omitted".
|
||||
/// Existing messages continue to be counted.
|
||||
pub fn warn_limited(&mut self, limit: Option<usize>, message: impl Into<String>) {
|
||||
let mut message = message.into();
|
||||
if let Some(limit) = limit
|
||||
&& self.entries.len() >= limit
|
||||
&& !self.entries.iter().any(|e| e.level == Level::Warning && e.message == message)
|
||||
{
|
||||
message = "further diagnostics omitted".into();
|
||||
}
|
||||
self.push(Level::Warning, message);
|
||||
}
|
||||
pub fn warn(&mut self, message: impl Into<String>) {
|
||||
self.push(Level::Warning, message);
|
||||
}
|
||||
|
||||
@@ -570,3 +570,17 @@ fn relative_target_urls_are_percent_decoded() {
|
||||
let clip = &p.sequence(imp.sequences[0]).unwrap().video_tracks[0].items[0];
|
||||
assert_eq!(media_key(p, clip.item), "/proj/media/Clip A.mov");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optional_report_limit_bounds_distinct_otio_diagnostics_without_changing_default() {
|
||||
let children: Vec<_> = (0..10).map(|i| serde_json::json!({"OTIO_SCHEMA": format!("Unsupported{i}.1")})).collect();
|
||||
let doc = serde_json::json!({"OTIO_SCHEMA":"Timeline.1", "tracks":{"OTIO_SCHEMA":"Stack.1", "children":[{"OTIO_SCHEMA":"Track.1", "kind":"Video", "children":children}]}}).to_string();
|
||||
let opts = iopts(None);
|
||||
let mut bounded = craft_timeline::Report::default();
|
||||
assert!(craft_timeline::otio::import_with_report_limit(&doc, &opts, &mut bounded, Some(2)).is_ok());
|
||||
assert_eq!(bounded.entries.len(), 3);
|
||||
assert!(bounded.mentions("further diagnostics omitted"));
|
||||
let mut unlimited = craft_timeline::Report::default();
|
||||
assert!(craft_timeline::otio::import(&doc, &opts, &mut unlimited).is_ok());
|
||||
assert_eq!(unlimited.entries.len(), 10);
|
||||
}
|
||||
@@ -2,6 +2,6 @@
|
||||
|
||||
Bounded wire primitives for USD Crate, independent of application scene and material models.
|
||||
|
||||
Provides bounded integer-delta array decoding and Crate-framed LZ4 decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and export remain outside this extraction. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation.
|
||||
Provides bounded integer-delta array decoding and Crate-framed LZ4 compression/decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and full scene export remain outside this extraction. Compression uses a caller cap on worst-case reserved output and fallible allocation. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation.
|
||||
|
||||
Adapted from TextureCraft `crates/formats/src/mesh_import/usd/codec.rs`, introduced by bflatastic in `bdbb52c9b66b221a55b370105b42f485a76e6fc7`. The LZ4 backend is MIT-licensed `lz4_flex`.
|
||||
@@ -0,0 +1,76 @@
|
||||
//! Bounded single-block USD Crate LZ4 writing.
|
||||
|
||||
/// Compression admission, allocation or backend failure.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
/// Input or worst-case output exceeds the supported/caller budget.
|
||||
#[error("USD Crate LZ4 compression exceeds budget")]
|
||||
Limit,
|
||||
/// The bounded output reservation failed.
|
||||
#[error("cannot allocate USD Crate LZ4 output")]
|
||||
Allocation,
|
||||
/// The backend could not encode into the preflighted output buffer.
|
||||
#[error("USD Crate LZ4 compression: {0}")]
|
||||
Encode(#[from] lz4_flex::block::CompressError),
|
||||
}
|
||||
|
||||
fn capacity(input: usize, max_output_bytes: usize) -> Result<usize, Error> {
|
||||
// Keep inputs below the LZ4 single-block limit and preflight the backend's
|
||||
// 20 + input*110/100 sizing arithmetic before calling it, including the count byte.
|
||||
if input > 0x7e00_0000 {
|
||||
return Err(Error::Limit);
|
||||
}
|
||||
input
|
||||
.checked_mul(110)
|
||||
.map(|n| n / 100)
|
||||
.and_then(|n| n.checked_add(21))
|
||||
.filter(|n| *n <= max_output_bytes && *n <= isize::MAX as usize)
|
||||
.ok_or(Error::Limit)
|
||||
}
|
||||
|
||||
/// Write one Crate-framed LZ4 block, beginning with a zero chunk-count byte.
|
||||
///
|
||||
/// `max_output_bytes` bounds the reserved output including framing. Admission uses
|
||||
/// the backend's worst-case size, so even highly compressible input must fit that bound.
|
||||
/// Allocation is fallible; backend scratch memory is not included in the output cap.
|
||||
/// Uses TextureCraft's original LZ4 backend/configuration and produces the same block bytes.
|
||||
pub fn compress(input: &[u8], max_output_bytes: usize) -> Result<Vec<u8>, Error> {
|
||||
let capacity = capacity(input.len(), max_output_bytes)?;
|
||||
let mut output = Vec::new();
|
||||
output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?;
|
||||
output.resize(capacity, 0);
|
||||
let target = output.get_mut(1..).ok_or(Error::Limit)?;
|
||||
let written = lz4_flex::block::compress_into(input, target)?;
|
||||
output.truncate(written.checked_add(1).ok_or(Error::Limit)?);
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn original_wire_bytes_and_roundtrip() {
|
||||
for size in [0, 1, 15, 16, 255, 4096, 65536] {
|
||||
for repeated in [false, true] {
|
||||
let input: Vec<u8> = (0..size).map(|i| if repeated { 17 } else { (i % 251) as u8 }).collect();
|
||||
let mut old = vec![0];
|
||||
old.extend(lz4_flex::block::compress(&input));
|
||||
let encoded = compress(&input, 1 << 20).unwrap();
|
||||
assert_eq!(encoded, old);
|
||||
assert_eq!(crate::decompress(&encoded, size).unwrap(), input);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checks_worst_case_before_allocation() {
|
||||
let input = [0; 100];
|
||||
assert!(matches!(compress(&input, 130), Err(Error::Limit)));
|
||||
assert!(compress(&input, 131).is_ok());
|
||||
assert!(capacity(usize::MAX, usize::MAX).is_err());
|
||||
assert!(capacity(0x7e00_0001, usize::MAX).is_err());
|
||||
assert_eq!(capacity(0, 21).unwrap(), 21);
|
||||
}
|
||||
}
|
||||
@@ -80,6 +80,72 @@ pub fn decode(data: &[u8], count: usize, width: Width, max_output_bytes: usize)
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// Encode integers with their original signed wrapping and largest-delta tie-break policy.
|
||||
///
|
||||
/// `max_bytes` bounds the combined worst-case output reservation and sorted delta scratch.
|
||||
/// Both reservations are fallible. Sorting uses no extra heap storage; input storage and
|
||||
/// allocator bookkeeping are outside this budget. Empty input emits no bytes.
|
||||
pub fn encode(values: &[i64], width: Width, max_bytes: usize) -> Result<Vec<u8>, Error> {
|
||||
if values.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let size = width.bytes();
|
||||
let codes_len = values.len().div_ceil(4);
|
||||
let capacity = values.len().checked_mul(size).and_then(|n| n.checked_add(size)).and_then(|n| n.checked_add(codes_len)).ok_or(Error::Limit)?;
|
||||
let scratch = values.len().checked_mul(8).ok_or(Error::Limit)?;
|
||||
if capacity > isize::MAX as usize || scratch > isize::MAX as usize || capacity.checked_add(scratch).is_none_or(|n| n > max_bytes) {
|
||||
return Err(Error::Limit);
|
||||
}
|
||||
let wrap = |v: i64| match width {
|
||||
Width::I32 => i64::from(v as i32),
|
||||
Width::I64 => v,
|
||||
};
|
||||
let mut deltas = Vec::new();
|
||||
deltas.try_reserve_exact(values.len()).map_err(|_| Error::Allocation)?;
|
||||
let mut previous = 0i64;
|
||||
for &value in values {
|
||||
deltas.push(wrap(value.wrapping_sub(previous)));
|
||||
previous = value;
|
||||
}
|
||||
deltas.sort_unstable();
|
||||
let mut best = (0usize, 0i64);
|
||||
let mut at = 0;
|
||||
while at < deltas.len() {
|
||||
let value = deltas[at];
|
||||
let start = at;
|
||||
while at < deltas.len() && deltas[at] == value {
|
||||
at += 1;
|
||||
}
|
||||
best = best.max((at - start, value));
|
||||
}
|
||||
let common = best.1;
|
||||
let mut output = Vec::new();
|
||||
output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?;
|
||||
output.extend_from_slice(&common.to_le_bytes()[..size]);
|
||||
output.resize(size + codes_len, 0);
|
||||
previous = 0;
|
||||
for (i, &value) in values.iter().enumerate() {
|
||||
let delta = wrap(value.wrapping_sub(previous));
|
||||
previous = value;
|
||||
let small = |bits: u32| delta >= -(1i64 << (bits - 1)) && delta < (1i64 << (bits - 1));
|
||||
let code: u8 = if delta == common {
|
||||
0
|
||||
} else if small(size as u32 * 2) {
|
||||
1
|
||||
} else if small(size as u32 * 4) {
|
||||
2
|
||||
} else {
|
||||
3
|
||||
};
|
||||
if code != 0 {
|
||||
let bytes = (size >> (3 - usize::from(code))).max(1);
|
||||
output.extend_from_slice(&delta.to_le_bytes()[..bytes]);
|
||||
}
|
||||
output[size + i / 4] |= code << (2 * (i % 4));
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
@@ -125,3 +191,81 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod encode_tests {
|
||||
use super::*;
|
||||
fn oracle(values: &[i64], width: usize) -> Vec<u8> {
|
||||
if values.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let wrap = |v: i64| if width == 4 { i64::from(v as i32) } else { v };
|
||||
let mut prev = 0i64;
|
||||
let deltas: Vec<i64> = values
|
||||
.iter()
|
||||
.map(|&v| {
|
||||
let d = wrap(v.wrapping_sub(prev));
|
||||
prev = v;
|
||||
d
|
||||
})
|
||||
.collect();
|
||||
let mut counts = std::collections::HashMap::new();
|
||||
for d in &deltas {
|
||||
*counts.entry(*d).or_insert(0usize) += 1;
|
||||
}
|
||||
let common = counts.into_iter().max_by_key(|&(v, c)| (c, v)).map_or(0, |(v, _)| v);
|
||||
let mut out = Vec::with_capacity(width + values.len().div_ceil(4) + values.len() * width);
|
||||
out.extend_from_slice(&common.to_le_bytes()[..width]);
|
||||
let codes_at = out.len();
|
||||
out.resize(codes_at + values.len().div_ceil(4), 0);
|
||||
for (i, &d) in deltas.iter().enumerate() {
|
||||
let small = |bits: u32| d >= -(1i64 << (bits - 1)) && d < (1i64 << (bits - 1));
|
||||
let code: u8 = if d == common {
|
||||
0
|
||||
} else if small(width as u32 * 2) {
|
||||
1
|
||||
} else if small(width as u32 * 4) {
|
||||
2
|
||||
} else {
|
||||
3
|
||||
};
|
||||
if code > 0 {
|
||||
let w = (width >> (3 - code as usize)).max(1);
|
||||
out.extend_from_slice(&d.to_le_bytes()[..w]);
|
||||
}
|
||||
if let Some(b) = out.get_mut(codes_at + i / 4) {
|
||||
*b |= code << (2 * (i % 4));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_original_bytes_and_wrapping() {
|
||||
for width in [Width::I32, Width::I64] {
|
||||
for n in [0, 1, 2, 3, 4, 5, 127, 1024] {
|
||||
let mut seed = 17u64;
|
||||
let values: Vec<i64> = (0..n)
|
||||
.map(|_| {
|
||||
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
|
||||
seed as i64
|
||||
})
|
||||
.collect();
|
||||
let bytes = encode(&values, width, 32768).unwrap();
|
||||
assert_eq!(bytes, oracle(&values, width.bytes()));
|
||||
let expected: Vec<i64> = values.iter().map(|&v| if width == Width::I32 { i64::from(v as i32) } else { v }).collect();
|
||||
assert_eq!(decode(&bytes, n, width, 32768).unwrap(), expected);
|
||||
}
|
||||
for values in [vec![1, 3, 4, 6], vec![i64::MAX, i64::MIN, 0], vec![7; 32]] {
|
||||
assert_eq!(encode(&values, width, 4096).unwrap(), oracle(&values, width.bytes()));
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn combined_reservation_budget_is_enforced() {
|
||||
assert_eq!(encode(&[], Width::I64, 0).unwrap(), Vec::<u8>::new());
|
||||
assert!(matches!(encode(&[1], Width::I32, 16), Err(Error::Limit)));
|
||||
assert!(encode(&[1], Width::I32, 17).is_ok());
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,9 @@
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
/// Bounded single-block Crate LZ4 compression.
|
||||
pub mod compress;
|
||||
|
||||
/// Bounded Crate integer delta arrays.
|
||||
pub mod integers;
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ assert_eq!((rows, sheet.map(|s| s.children.len())), (3, Ok(0)));
|
||||
| `max_input` | 256 MiB | `TooLarge` |
|
||||
| `max_depth` | 256 levels (the root is 1) | `TooDeep` |
|
||||
| `max_nodes` | 2,000,000 nodes kept in the tree: elements, text stretches, kept CDATA, instructions and comments | `TooManyNodes` |
|
||||
| `max_attrs` | 1024 on one start tag, namespace declarations and malformed ones included; with canonical names, also the declarations in scope at once | `TooManyAttrs` |
|
||||
| `max_attrs` | 1024 on one start tag, namespace declarations and malformed ones included; with canonical names or strict reads, also the declarations in scope at once | `TooManyAttrs` |
|
||||
| `max_attrs_total` | 4,000,000 attributes kept in the tree, over all its elements | `TooManyAttrsTotal` |
|
||||
|
||||
A limit exceeded is an error, never a panic and never a truncated tree. `parse_streaming_partial`
|
||||
@@ -73,9 +73,9 @@ A lenient read keeps what a damaged document had:
|
||||
- reading stops when the root element ends: nothing after it is looked at;
|
||||
- a malformed attribute is skipped; bytes that are not UTF-8 become U+FFFD.
|
||||
|
||||
A strict read makes each of those an error, and also a second root element and text outside the
|
||||
A strict read rejects duplicate raw or expanded attribute names (while allowing undeclared prefixes in fragments). It makes each of those an error, and also a second root element and text outside the
|
||||
root, in the pull reader as in the tree. Markup the tokenizer cannot read (`<a b='1`, an unterminated comment) is an error either
|
||||
way. A repeated attribute is kept each time in both.
|
||||
way. A lenient read keeps repeated attributes.
|
||||
|
||||
### Text
|
||||
|
||||
|
||||
+53
-2
@@ -1,6 +1,7 @@
|
||||
//! Reading: the pull reader over `quick-xml`, the tree builder on top of it, and streaming.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::BTreeSet;
|
||||
use std::ops::Range;
|
||||
|
||||
use quick_xml::Reader;
|
||||
@@ -12,6 +13,10 @@ use crate::write::is_xml_char;
|
||||
|
||||
/// The namespace the `xml` prefix is always bound to.
|
||||
pub const XML_NS: &str = "http://www.w3.org/XML/1998/namespace";
|
||||
/// [`Events::scope_ids`] id of the XML namespace, bound to `xml:` without a declaration.
|
||||
const XML_NS_ID: usize = usize::MAX;
|
||||
/// Expanded-name id of an attribute in no namespace (or with an undeclared prefix).
|
||||
const NO_NS_ID: usize = usize::MAX - 1;
|
||||
|
||||
/// What the reader refuses to exceed.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -469,6 +474,10 @@ pub struct Events<'o, 'i> {
|
||||
end_names: bool,
|
||||
open: Vec<Open>,
|
||||
scope: Vec<(String, String)>,
|
||||
/// For each binding in `scope`, an id shared by every binding of an equal URI (the index of
|
||||
/// the first such binding, or [`XML_NS_ID`]), with the URI's hash. Strict reads compare
|
||||
/// expanded attribute names by this id instead of copying the URI for each attribute.
|
||||
scope_ids: Vec<(u64, usize)>,
|
||||
/// The end of an empty element, due after its start.
|
||||
pending: bool,
|
||||
/// Whether a root element has started.
|
||||
@@ -500,7 +509,7 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
cfg.trim_text_start = false;
|
||||
cfg.trim_text_end = false;
|
||||
cfg.expand_empty_elements = false;
|
||||
Events { reader, opts, end_names, open: Vec::new(), scope: Vec::new(), pending: false, rooted: false, span: 0..0 }
|
||||
Events { reader, opts, end_names, open: Vec::new(), scope: Vec::new(), scope_ids: Vec::new(), pending: false, rooted: false, span: 0..0 }
|
||||
}
|
||||
|
||||
/// The bytes of the input the last event was read from. Both events of an empty element
|
||||
@@ -614,6 +623,7 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
fn end(&mut self) -> Event {
|
||||
let (scope, name) = self.open.pop().map_or((0, String::new()), |o| (o.scope, o.name));
|
||||
self.scope.truncate(scope);
|
||||
self.scope_ids.truncate(scope);
|
||||
Event::End { name }
|
||||
}
|
||||
|
||||
@@ -669,6 +679,7 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
if is_ns_decl(key) {
|
||||
let prefix = self.decode(key.get(6..).unwrap_or_default())?.into_owned();
|
||||
let uri = self.attr_value(&a.value)?;
|
||||
self.scope_ids.push(self.uri_id(&uri));
|
||||
self.scope.push((prefix, uri));
|
||||
}
|
||||
}
|
||||
@@ -680,6 +691,20 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The id an expanded name uses for `uri`: one hash and at most one full comparison per
|
||||
/// declaration, so a long URI costs its length once however many attributes use it.
|
||||
fn uri_id(&self, uri: &str) -> (u64, usize) {
|
||||
use std::hash::{Hash, Hasher};
|
||||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
||||
uri.hash(&mut h);
|
||||
let hash = h.finish();
|
||||
if uri == XML_NS {
|
||||
return (hash, XML_NS_ID);
|
||||
}
|
||||
let first = self.scope_ids.iter().zip(&self.scope).position(|((h, _), (_, u))| *h == hash && u == uri);
|
||||
(hash, first.map_or(self.scope.len(), |i| self.scope_ids.get(i).map_or(i, |(_, id)| *id)))
|
||||
}
|
||||
|
||||
/// In a strict read: is the reader before or after the root element?
|
||||
fn outside_root(&self) -> bool {
|
||||
self.opts.strict && self.open.is_empty()
|
||||
@@ -699,7 +724,7 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
_ => None,
|
||||
};
|
||||
let scope = self.scope.len();
|
||||
if canonical.is_some() {
|
||||
if canonical.is_some() || self.opts.strict {
|
||||
self.declare(e)?;
|
||||
}
|
||||
let raw = e.name();
|
||||
@@ -707,6 +732,8 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
let mut out = Vec::new();
|
||||
let mut attrs = e.attributes();
|
||||
attrs.with_checks(false);
|
||||
let mut raw_names = BTreeSet::new();
|
||||
let mut expanded_names = BTreeSet::new();
|
||||
for (seen, a) in attrs.enumerate() {
|
||||
if seen >= limits.max_attrs {
|
||||
return Err(Error::TooManyAttrs(limits.max_attrs));
|
||||
@@ -718,6 +745,30 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
};
|
||||
let key = a.key.as_ref();
|
||||
let decl = is_ns_decl(key);
|
||||
if self.opts.strict {
|
||||
let raw_key = self.decode(key)?;
|
||||
if !raw_names.insert(raw_key.to_string()) {
|
||||
return self.syntax(format!("duplicate attribute `{raw_key}`"));
|
||||
}
|
||||
if !decl {
|
||||
// Default namespaces do not apply to attributes. Keep undeclared prefixes
|
||||
// distinct so standalone fragments retain the reader's existing behaviour.
|
||||
let expanded = match raw_key.split_once(':') {
|
||||
Some(("xml", local)) => (XML_NS_ID, local),
|
||||
Some((prefix, local)) => self
|
||||
.scope
|
||||
.iter()
|
||||
.zip(&self.scope_ids)
|
||||
.rev()
|
||||
.find(|((p, _), _)| p == prefix)
|
||||
.map_or((NO_NS_ID, raw_key.as_ref()), |(_, (_, id))| (*id, local)),
|
||||
None => (NO_NS_ID, raw_key.as_ref()),
|
||||
};
|
||||
if !expanded_names.insert((expanded.0, expanded.1.to_string())) {
|
||||
return self.syntax(format!("duplicate expanded attribute `{raw_key}`"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if decl && !self.opts.ns_decls {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -117,7 +117,11 @@ fn an_attribute_flood_is_refused_at_the_limit() {
|
||||
let opts = opts.with_limits(limits);
|
||||
for (k, kind) in kinds.iter().enumerate() {
|
||||
let _ = opts.parse(tag(16, kind).as_bytes());
|
||||
assert_eq!(opts.parse(tag(17, kind).as_bytes()).err().map(|e| e.is_limit() || (opts.strict && k == 3)), Some(true), "kind {k}");
|
||||
assert_eq!(
|
||||
opts.parse(tag(17, kind).as_bytes()).err().map(|e| e.is_limit() || (opts.strict && (k == 1 || k == 3))),
|
||||
Some(true),
|
||||
"kind {k}"
|
||||
);
|
||||
assert!(opts.first_start(tag(17, kind).as_bytes(), "a").is_none());
|
||||
}
|
||||
assert_eq!(opts.parse(tag(17, kinds[0]).as_bytes()), Err(Error::TooManyAttrs(16)));
|
||||
@@ -524,3 +528,41 @@ fn a_strict_pull_reader_refuses_what_is_outside_the_root() {
|
||||
// white space, comments and instructions may stand outside the root
|
||||
assert!(read(strict, b"<?xml version='1.0'?>\r\n <!-- c -->\t<a>x</a>\n<?pi?> ").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_attributes_are_unique_before_name_rewriting() {
|
||||
use craft_xml::{Names, Options};
|
||||
for names in [Names::AsWritten, Names::Local] {
|
||||
let strict = Options { names, ..Options::default() }.strict();
|
||||
for xml in [
|
||||
"<r a='1' a='2'/>",
|
||||
"<r xmlns:p='urn:x' xmlns:q='urn:x' p:a='1' q:a='2'/>",
|
||||
"<r xml:a='1' xmlns:p='http://www.w3.org/XML/1998/namespace' p:a='2'/>",
|
||||
"<r xmlns:p='urn:x'><c xmlns:q='urn:x' p:a='1' q:a='2'/></r>",
|
||||
] {
|
||||
assert!(strict.parse(xml.as_bytes()).is_err(), "{xml}");
|
||||
let mut events = strict.events(xml.as_bytes());
|
||||
assert!(
|
||||
loop {
|
||||
match events.next() {
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break false,
|
||||
Err(_) => break true,
|
||||
}
|
||||
},
|
||||
"{xml}"
|
||||
);
|
||||
}
|
||||
for xml in [
|
||||
"<r xmlns='urn:x' xmlns:p='urn:x' a='1' p:a='2'/>",
|
||||
"<r p:a='1' q:a='2'/>",
|
||||
"<r xmlns:p='urn:x'><c xmlns:p='urn:y' xmlns:q='urn:x' p:a='1' q:a='2'/></r>",
|
||||
] {
|
||||
assert!(strict.parse(xml.as_bytes()).is_ok(), "{xml}");
|
||||
}
|
||||
assert!(Options { names, ..Options::default() }.parse(b"<r a='1' a='2'/>").is_ok());
|
||||
}
|
||||
let strict = Options { limits: craft_xml::Limits { max_attrs: 2, ..craft_xml::Limits::DEFAULT }, ..Options::default() }.strict();
|
||||
assert!(strict.parse(b"<r xmlns:a='a' xmlns:b='b'><c xmlns:c='c'/></r>").is_err());
|
||||
assert!(strict.parse(b"<r a='1' b='2' c='3'/>").is_err());
|
||||
}
|
||||
Reference in new issue
Block a user