mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:42:27 +00:00
Merge remote-tracking branch 'origin/main' into w1616
This commit is contained in:
commit
b6d9bd3b37
17 files changed
+1638
-102
No files matched your search
Generated
+4
@@ -1903,6 +1903,10 @@ dependencies = [
|
||||
"proptest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-http-url"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "craft-i18n"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -11,6 +11,7 @@ rust-version = "1.95"
|
||||
repository = "https://github.com/storytold/craft-libs"
|
||||
|
||||
[workspace.dependencies]
|
||||
craft-http-url = { path = "crates/http-url" }
|
||||
craft-usd = { path = "crates/usd" }
|
||||
craft-smime = { path = "crates/smime" }
|
||||
craft-ber = { path = "crates/ber" }
|
||||
|
||||
+171
-42
@@ -150,62 +150,142 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
|
||||
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
|
||||
let mut title = String::new();
|
||||
let (mut size3, mut size1) = (0usize, 0usize);
|
||||
let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]);
|
||||
let mut rows: Vec<f32> = Vec::new();
|
||||
for line in text.lines() {
|
||||
// Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an
|
||||
// unquoted title's comment must not turn that comment into part of the title.
|
||||
if let Some(quoted) = line
|
||||
.trim_start()
|
||||
.strip_prefix("TITLE")
|
||||
.filter(|rest| rest.starts_with(char::is_whitespace))
|
||||
.and_then(|rest| rest.trim_start().strip_prefix('"'))
|
||||
/// A bounded, three-dimensional Cube profile with RGB rows in file order.
|
||||
/// Unlike [`parse_cube`], this preserves the optional title and does not resample 1D curves.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Cube3d {
|
||||
pub title: Option<String>,
|
||||
pub size: usize,
|
||||
pub data: Vec<[f32; 3]>,
|
||||
pub domain_min: [f32; 3],
|
||||
pub domain_max: [f32; 3],
|
||||
}
|
||||
|
||||
/// Read the simple 3D profile dialect used by camera-profile consumers.
|
||||
///
|
||||
/// `max_size` must be in 2..=256; the row budget is checked before each allocation.
|
||||
/// Whole-line comments and literal titles are preserved. Unknown alphabetic directives
|
||||
/// are ignored; 1D tables are refused. Finite domains are returned unchanged so the
|
||||
/// consumer retains its existing policy for equal or reversed endpoints.
|
||||
/// Numeric fields must consist of exactly three finite numbers (never silently skip tokens).
|
||||
pub fn parse_cube_3d(text: &str, max_size: usize) -> Result<Cube3d, LutError> {
|
||||
let p = read_cube(text, Some(max_size))?;
|
||||
if p.size3 == 0 {
|
||||
return err("not a 3D .cube LUT (no LUT_3D_SIZE)");
|
||||
}
|
||||
let expected = p.size3.checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
|
||||
if p.rows.len() != expected {
|
||||
return err(format!("expected {expected} entries, found {}", p.rows.len()));
|
||||
}
|
||||
Ok(Cube3d { title: p.title, size: p.size3, data: p.rows, domain_min: p.min, domain_max: p.max })
|
||||
}
|
||||
|
||||
struct CubeRows {
|
||||
title: Option<String>,
|
||||
size3: usize,
|
||||
size1: usize,
|
||||
min: [f32; 3],
|
||||
max: [f32; 3],
|
||||
rows: Vec<[f32; 3]>,
|
||||
}
|
||||
|
||||
fn cube_triplet(text: &str) -> Result<[f32; 3], LutError> {
|
||||
let mut it = text.split_whitespace();
|
||||
let mut row = [0.0; 3];
|
||||
for value in &mut row {
|
||||
*value = it.next().and_then(|s| s.parse::<f32>().ok()).filter(|v| v.is_finite()).ok_or_else(|| LutError(format!("bad line `{text}`")))?;
|
||||
}
|
||||
if it.next().is_some() {
|
||||
return err(format!("bad line `{text}`"));
|
||||
}
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
fn read_cube(text: &str, profile_max: Option<usize>) -> Result<CubeRows, LutError> {
|
||||
if profile_max.is_some_and(|n| !(2..=256).contains(&n)) {
|
||||
return err("3D profile size limit must be in 2..=256");
|
||||
}
|
||||
let mut p = CubeRows { title: None, size3: 0, size1: 0, min: [0.0; 3], max: [1.0; 3], rows: Vec::new() };
|
||||
let row_limit = profile_max.unwrap_or(256).checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
|
||||
for raw in text.lines() {
|
||||
let profile = profile_max.is_some();
|
||||
let line = if profile { raw.trim() } else { raw.split('#').next().unwrap_or("").trim() };
|
||||
// The standard dialect protects '#' inside quoted titles; profile titles are literal.
|
||||
if !profile
|
||||
&& let Some(quoted) =
|
||||
raw.trim_start().strip_prefix("TITLE").filter(|r| r.starts_with(char::is_whitespace)).and_then(|r| r.trim_start().strip_prefix('"'))
|
||||
{
|
||||
let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
|
||||
title = value.to_string();
|
||||
let Some((title, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
|
||||
p.title = Some(title.to_string());
|
||||
continue;
|
||||
}
|
||||
let line = line.split('#').next().unwrap_or("").trim();
|
||||
if line.is_empty() {
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let mut it = line.split_whitespace();
|
||||
let key = it.next().unwrap_or("");
|
||||
let nums = |it: std::str::SplitWhitespace<'_>| -> Result<[f32; 3], LutError> {
|
||||
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
|
||||
if v.len() == 3 { Ok([v[0], v[1], v[2]]) } else { err(format!("bad line `{line}`")) }
|
||||
let (key, rest) = if profile {
|
||||
["TITLE", "LUT_3D_SIZE", "LUT_1D_SIZE", "DOMAIN_MIN", "DOMAIN_MAX"]
|
||||
.into_iter()
|
||||
.find_map(|key| line.strip_prefix(key).map(|rest| (key, rest.trim())))
|
||||
.unwrap_or_else(|| line.split_once(char::is_whitespace).unwrap_or((line, "")))
|
||||
} else {
|
||||
line.split_once(char::is_whitespace).map_or((line, ""), |(key, rest)| (key, rest.trim()))
|
||||
};
|
||||
match key {
|
||||
"TITLE" => title = line[5..].trim().trim_matches('"').to_string(),
|
||||
"LUT_3D_SIZE" => size3 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_3D_SIZE".into()))?,
|
||||
"LUT_1D_SIZE" => size1 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?,
|
||||
"DOMAIN_MIN" => dmin = nums(it)?,
|
||||
"DOMAIN_MAX" => dmax = nums(it)?,
|
||||
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" => {
|
||||
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
|
||||
if v.len() == 2 {
|
||||
dmin = [v[0]; 3];
|
||||
dmax = [v[1]; 3];
|
||||
"TITLE" => p.title = Some(rest.trim_matches('"').to_string()),
|
||||
"LUT_3D_SIZE" => {
|
||||
let value = if profile { rest } else { rest.split_whitespace().next().unwrap_or("") };
|
||||
p.size3 = value.parse().map_err(|_| LutError(format!("bad LUT_3D_SIZE `{value}`")))?;
|
||||
let limit = profile_max.unwrap_or(MAX_SIZE);
|
||||
if !(2..=limit).contains(&p.size3) {
|
||||
return err(format!("LUT_3D_SIZE {} out of range", p.size3));
|
||||
}
|
||||
}
|
||||
k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => {
|
||||
let mut v = vec![k.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?];
|
||||
for value in it {
|
||||
v.push(value.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?);
|
||||
"LUT_1D_SIZE" => {
|
||||
if profile {
|
||||
return err("1D LUTs are not supported (only 3D .cube files)");
|
||||
}
|
||||
if v.len() != 3 {
|
||||
return err(format!("bad data line `{line}`"));
|
||||
p.size1 = rest.split_whitespace().next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?;
|
||||
if p.size1 > row_limit {
|
||||
return err("1D LUT exceeds row budget");
|
||||
}
|
||||
}
|
||||
"DOMAIN_MIN" => p.min = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MIN".into()))?,
|
||||
"DOMAIN_MAX" => p.max = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MAX".into()))?,
|
||||
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" if !profile => {
|
||||
let mut it = rest.split_whitespace();
|
||||
let min = it.next().and_then(|s| s.parse::<f32>().ok());
|
||||
let max = it.next().and_then(|s| s.parse::<f32>().ok());
|
||||
if let (Some(min), Some(max), None) = (min, max, it.next()) {
|
||||
p.min = [min; 3];
|
||||
p.max = [max; 3];
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let numeric = key.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.');
|
||||
let nonfinite = key.parse::<f32>().is_ok();
|
||||
if numeric || nonfinite || (profile && !key.starts_with(|c: char| c.is_ascii_alphabetic())) {
|
||||
let row = cube_triplet(line)?;
|
||||
if p.rows.len() >= row_limit {
|
||||
return err("LUT exceeds row budget");
|
||||
}
|
||||
// Explicit fallible growth prevents capacity growth past the configured budget.
|
||||
if p.rows.len() == p.rows.capacity() {
|
||||
let grow = (p.rows.capacity().max(1)).min(row_limit - p.rows.len());
|
||||
p.rows.try_reserve_exact(grow).map_err(|_| LutError("LUT allocation failed".into()))?;
|
||||
}
|
||||
p.rows.push(row);
|
||||
}
|
||||
rows.extend(v);
|
||||
}
|
||||
// Unknown keywords (e.g. Resolve's LUT_IN_VIDEO_RANGE) are ignored, as the spec allows.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
|
||||
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
|
||||
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
|
||||
let p = read_cube(text, None)?;
|
||||
let (title, size3, size1, dmin, dmax) = (p.title.unwrap_or_default(), p.size3, p.size1, p.min, p.max);
|
||||
let rows: Vec<f32> = p.rows.into_iter().flatten().collect();
|
||||
let domain_ok = (0..3).all(|k| dmin[k] < dmax[k] && dmin[k].is_finite() && dmax[k].is_finite());
|
||||
if (size3 > 0 || size1 >= 2) && !domain_ok {
|
||||
return err(format!("DOMAIN_MIN {:?} must lie below DOMAIN_MAX {:?}", dmin, dmax));
|
||||
@@ -1386,3 +1466,52 @@ LUT_3D_SIZE 2
|
||||
assert_eq!(back.data, [0.0, 0.25, 1.0].repeat(8));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod profile_cube_tests {
|
||||
use super::*;
|
||||
|
||||
fn cube(prefix: &str) -> String {
|
||||
format!("{prefix}\nLUT_3D_SIZE 2\n{}", "0.25 0.5 0.75\n".repeat(8))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_preserves_optional_literal_title_and_domains() {
|
||||
let no_title = parse_cube_3d(&cube(""), 256).unwrap();
|
||||
assert_eq!(no_title.title, None);
|
||||
let value = parse_cube_3d(&cube("TITLE \"look # literal\"\nDOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1"), 256).unwrap();
|
||||
assert_eq!(value.title.as_deref(), Some("look # literal"));
|
||||
assert_eq!(value.domain_min, [1.0, 0.0, 2.0]);
|
||||
assert_eq!(value.domain_max, [0.0, 0.0, 1.0]);
|
||||
assert_eq!(value.data, vec![[0.25, 0.5, 0.75]; 8]);
|
||||
assert_eq!(parse_cube_3d(&cube("TITLE \"\""), 256).unwrap().title.as_deref(), Some(""));
|
||||
assert!(parse_cube(&cube("DOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_limits_are_checked_without_large_allocations() {
|
||||
assert!(parse_cube_3d("LUT_3D_SIZE 256", 256).unwrap_err().0.contains("16777216"));
|
||||
assert!(parse_cube_3d("LUT_3D_SIZE 257", 256).unwrap_err().0.contains("out of range"));
|
||||
assert!(parse_cube_3d("LUT_3D_SIZE 256", 129).unwrap_err().0.contains("out of range"));
|
||||
assert!(parse_cube_3d(&cube(""), usize::MAX).is_err());
|
||||
assert!(parse_cube_3d(&cube(""), 1).is_err());
|
||||
assert!(parse_cube_3d(&format!("{}0 0 0\n", cube("")), 2).unwrap_err().0.contains("budget"));
|
||||
assert!(parse_cube_3d("LUT_1D_SIZE 2\n0 0 0\n1 1 1", 256).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_numeric_token_must_be_valid_and_finite() {
|
||||
for row in ["0 ignored 0 0", "0 0 0 ignored", "NaN 0 0", "inf 0 0", "0 0 -inf", "1e39 0 0", "0 0", "0 0 0 0"] {
|
||||
assert!(parse_cube_3d(&format!("LUT_3D_SIZE 2\n{row}\n{}", "0 0 0\n".repeat(7)), 256).is_err(), "{row}");
|
||||
assert!(parse_cube_3d(&cube(&format!("DOMAIN_MIN {row}")), 256).is_err(), "{row}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_reader_keeps_quoted_comments_and_1d_expansion() {
|
||||
assert_eq!(parse_cube(&cube("TITLE \"look # preserved\" # comment")).unwrap().title, "look # preserved");
|
||||
let value = parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1").unwrap();
|
||||
assert_eq!(value.size, 33);
|
||||
assert!(parse_cube_3d(&cube("DOMAIN_MIN 0 0 0 # not a whole-line comment"), 256).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
[package]
|
||||
name = "craft-http-url"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license = "Apache-2.0"
|
||||
repository.workspace = true
|
||||
description = "Bounded HTTP URL parsing and redirect resolution without network access"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,25 @@
|
||||
# craft-http-url
|
||||
|
||||
HTTP/HTTPS URL parsing, dot-segment removal, relative redirects, HTTPS downgrade rejection,
|
||||
and query-redacted display. No dependencies or network operations. Parsing rejects inputs
|
||||
larger than 16 KiB and redirects are bounded before concatenation.
|
||||
|
||||
This is not an SSRF filter, DNS resolver or HTTP client. Local/private hosts are allowed.
|
||||
Applications keep destination permission, authentication, TLS, request/response limits and
|
||||
redirect-count policy. Public fields preserve the original API; manually constructed values
|
||||
must not be treated as validated URLs.
|
||||
|
||||
## Attribution
|
||||
|
||||
Extracted from LightCraft `crates/fetch/src/http.rs` at
|
||||
`2dcb258ba40738230a07484a95eb27dc84ed66fe` (Apache-2.0).
|
||||
Original parser: Daniel Voshart, `7fa7506db46d69d609e85e39b338f33786ed0293`.
|
||||
HTTPS downgrade protection: MarekSpodBiedry, `4a91a1c218bf3d0ac49cc73ebb60621a21ae55e2`.
|
||||
URL normalization, scheme, fragment and authority fixes: NetterSilver, including
|
||||
`edc369565eef92161b56b8f1e39ef53629c05489`,
|
||||
`78f0e6e650444886069f4d4ae4e6593f221b98d4`,
|
||||
`cee93496587d31c514aaeced3e6b8027cfdb41a0`,
|
||||
`dee5f7910ad8c24cf28bab528f8d007550af2b23`,
|
||||
`c3a5d9b7f1f837dc14c01942e3d0293ecbf36345` and
|
||||
`13849307a9343504339d68b086f1a32cdcbbb457`.
|
||||
Source-author commits are retained separately in this extraction.
|
||||
@@ -0,0 +1,277 @@
|
||||
//! Bounded HTTP/HTTPS URL parsing and redirect resolution, extracted from LightCraft.
|
||||
//!
|
||||
//! This is syntax handling, not a network permission or SSRF policy. It permits local/private
|
||||
//! hosts and performs no DNS, socket or TLS operations. Callers own destination allowlists,
|
||||
//! credential forwarding, redirect counts, connection deadlines and response limits.
|
||||
//! Only parsed values have validated fields; constructing public fields bypasses parsing.
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
/// A URL or redirect cannot be represented by this bounded HTTP URL syntax.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UrlError(pub String);
|
||||
|
||||
impl std::fmt::Display for UrlError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for UrlError {}
|
||||
|
||||
/// A parsed `http://` or `https://` URL.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Url {
|
||||
pub tls: bool,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
/// Path and query, starting with `/`.
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
/// Remove `.` and `..` segments from the path part of `path` (RFC 3986 section 5.2.4); the query is kept as is.
|
||||
fn remove_dot_segments(path: &str) -> String {
|
||||
let (p, query) = match path.find('?') {
|
||||
Some(i) => (path.get(..i).unwrap_or_default(), path.get(i..).unwrap_or_default()),
|
||||
None => (path, ""),
|
||||
};
|
||||
let mut out: Vec<&str> = Vec::new();
|
||||
let mut segments = p.split('/').skip(1).peekable();
|
||||
let mut trailing_slash = false;
|
||||
while let Some(seg) = segments.next() {
|
||||
let last = segments.peek().is_none();
|
||||
match seg {
|
||||
"." => trailing_slash = last,
|
||||
".." => {
|
||||
out.pop();
|
||||
trailing_slash = last;
|
||||
}
|
||||
_ => {
|
||||
out.push(seg);
|
||||
trailing_slash = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut result = format!("/{}", out.join("/"));
|
||||
if trailing_slash && !result.ends_with('/') {
|
||||
result.push('/');
|
||||
}
|
||||
result.push_str(query);
|
||||
result
|
||||
}
|
||||
|
||||
/// `s` without the leading `prefix`, matched ASCII case-insensitively (RFC 3986 §3.1).
|
||||
fn strip_scheme<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
|
||||
let head = s.get(..prefix.len())?;
|
||||
head.eq_ignore_ascii_case(prefix).then(|| s.get(prefix.len()..).unwrap_or_default())
|
||||
}
|
||||
|
||||
impl Url {
|
||||
pub fn parse(s: &str) -> Result<Url, UrlError> {
|
||||
let bad = |why: &str| UrlError(why.to_string());
|
||||
if s.len() > 16 * 1024 {
|
||||
return Err(bad("URL exceeds 16 KiB"));
|
||||
}
|
||||
let s = s.trim();
|
||||
if s.chars().any(|c| c.is_control() || c == ' ') {
|
||||
return Err(bad("contains spaces or control characters"));
|
||||
}
|
||||
let (tls, rest) = if let Some(r) = strip_scheme(s, "https://") {
|
||||
(true, r)
|
||||
} else if let Some(r) = strip_scheme(s, "http://") {
|
||||
(false, r)
|
||||
} else {
|
||||
return Err(bad("only http:// and https:// are supported"));
|
||||
};
|
||||
let rest = rest.split('#').next().unwrap_or_default();
|
||||
let (authority, path) = match rest.find(['/', '?']) {
|
||||
Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()),
|
||||
None => (rest, ""),
|
||||
};
|
||||
let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") };
|
||||
let path = remove_dot_segments(&path);
|
||||
if authority.contains('@') {
|
||||
return Err(bad("user names and passwords in URLs are not supported"));
|
||||
}
|
||||
let (host, port) = if let Some(v6) = authority.strip_prefix('[') {
|
||||
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
|
||||
let host = v6.get(..end).unwrap_or_default();
|
||||
let after = v6.get(end + 1..).unwrap_or_default();
|
||||
if !after.is_empty() && !after.starts_with(':') {
|
||||
return Err(bad("unexpected text after IPv6 address"));
|
||||
}
|
||||
(host.to_string(), after.strip_prefix(':'))
|
||||
} else {
|
||||
match authority.rsplit_once(':') {
|
||||
Some((h, p)) => (h.to_string(), Some(p)),
|
||||
None => (authority.to_string(), None),
|
||||
}
|
||||
};
|
||||
let port = match port {
|
||||
Some(p) if p.bytes().all(|b| b.is_ascii_digit()) => p.parse::<u16>().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?,
|
||||
Some(_) => return Err(bad("bad port")),
|
||||
None if tls => 443,
|
||||
None => 80,
|
||||
};
|
||||
if host.is_empty() || !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | ':' | '_')) {
|
||||
return Err(bad("bad host name"));
|
||||
}
|
||||
Ok(Url { tls, host: host.to_ascii_lowercase(), port, path })
|
||||
}
|
||||
|
||||
/// `Location` of a redirect, resolved against this URL.
|
||||
pub fn join(&self, location: &str) -> Result<Url, UrlError> {
|
||||
// Reject before concatenating a hostile Location with the base URL.
|
||||
if location.len() > 16 * 1024 || self.host.len().saturating_add(self.path.len()) > 16 * 1024 {
|
||||
return Err(UrlError("URL exceeds 16 KiB".into()));
|
||||
}
|
||||
let l = location.trim();
|
||||
if strip_scheme(l, "http://").is_some() || strip_scheme(l, "https://").is_some() {
|
||||
let url = Url::parse(l)?;
|
||||
if self.tls && !url.tls {
|
||||
return Err(UrlError("redirect from https to http refused".into()));
|
||||
}
|
||||
return Ok(url);
|
||||
}
|
||||
// another scheme (`ftp:`, `file:`, `data:`…)
|
||||
if let Some((scheme, _)) = l.split_once(':')
|
||||
&& !scheme.is_empty()
|
||||
&& scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
|
||||
{
|
||||
return Err(UrlError(format!("redirect to an unsupported location `{scheme}:`")));
|
||||
}
|
||||
let scheme = if self.tls { "https" } else { "http" };
|
||||
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
|
||||
let base = format!("{scheme}://{host}:{}", self.port);
|
||||
if let Some(rest) = l.strip_prefix("//") {
|
||||
return Url::parse(&format!("{scheme}://{rest}"));
|
||||
}
|
||||
if l.starts_with('/') {
|
||||
return Url::parse(&format!("{base}{l}"));
|
||||
}
|
||||
// empty or fragment-only: the same document
|
||||
if l.is_empty() || l.starts_with('#') {
|
||||
return Ok(self.clone());
|
||||
}
|
||||
// relative to the current directory
|
||||
let dir = self.path.split('?').next().unwrap_or("/");
|
||||
// a query-only reference keeps the whole path, file name included (RFC 3986 §5.2.2)
|
||||
if l.starts_with('?') {
|
||||
return Url::parse(&format!("{base}{dir}{l}"));
|
||||
}
|
||||
let dir = dir.rsplit_once('/').map_or("", |(d, _)| d);
|
||||
Url::parse(&format!("{base}{dir}/{l}"))
|
||||
}
|
||||
|
||||
/// Authority formatted for an HTTP Host field.
|
||||
pub fn host_header(&self) -> String {
|
||||
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
|
||||
let default = if self.tls { 443 } else { 80 };
|
||||
if self.port == default { host } else { format!("{host}:{}", self.port) }
|
||||
}
|
||||
|
||||
/// The URL without its query (for messages: a query may hold a signed token).
|
||||
pub fn display(&self) -> String {
|
||||
let scheme = if self.tls { "https" } else { "http" };
|
||||
format!("{scheme}://{}{}", self.host_header(), self.path.split('?').next().unwrap_or("/"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::{Url, UrlError};
|
||||
|
||||
#[test]
|
||||
fn fragment_only_or_empty_location_keeps_file_name_and_query() {
|
||||
let base = Url::parse("https://a.example/x/y/file?q").unwrap();
|
||||
for l in ["#top", "", " "] {
|
||||
let u = base.join(l).unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/x/y/file?q"), "{l:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_oversized_and_control_character_urls() {
|
||||
let base = Url::parse("https://example.com/path").unwrap();
|
||||
assert!(base.join(&format!("#{}", "x".repeat(16 * 1024))).is_err());
|
||||
assert!(base.join(&"x".repeat(16 * 1024 + 1)).is_err());
|
||||
assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err());
|
||||
assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_query_only_keeps_file_name() {
|
||||
let u = Url::parse("https://a.example/x/y/file?q=1").unwrap();
|
||||
assert_eq!(u.join("?page=2").unwrap().path, "/x/y/file?page=2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_removes_dot_segments() {
|
||||
let base = Url::parse("https://a.example/x/y/file").unwrap();
|
||||
for (loc, want) in
|
||||
[("../z", "/x/z"), ("./z", "/x/y/z"), ("../../../z", "/z"), ("/a/./b/../c", "/a/c"), ("../z?q=1", "/x/z?q=1"), ("..", "/x/")]
|
||||
{
|
||||
assert_eq!(base.join(loc).unwrap().path, want, "{loc}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_scheme_is_matched_case_insensitively() {
|
||||
let u = Url::parse("HTTPS://A.example/z").unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/z"));
|
||||
assert!(Url::parse("Http://a.example/").is_ok());
|
||||
let j = Url::parse("https://a.example/start").unwrap().join("HTTPS://cdn.example/file").unwrap();
|
||||
assert_eq!((j.tls, j.host.as_str(), j.path.as_str()), (true, "cdn.example", "/file"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_urls() {
|
||||
let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "cdn.example.com", 443, "/models/sam3/model.safetensors?sig=1"));
|
||||
assert_eq!(u.display(), "https://cdn.example.com/models/sam3/model.safetensors");
|
||||
let u = Url::parse("http://127.0.0.1:8080").unwrap();
|
||||
assert_eq!((u.tls, u.port, u.path.as_str()), (false, 8080, "/"));
|
||||
let u = Url::parse("http://[::1]:9/a").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 9));
|
||||
for bad in ["ftp://x/y", "https://", "https://a b/", "https://u:p@h/", "http://h:0/", "http://h:99999/", "https://h/\r\nX: y", "file:///etc"]
|
||||
{
|
||||
assert!(Url::parse(bad).is_err(), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_signed_ports() {
|
||||
for bad in ["http://example.com:+80/x", "http://example.com:-80/x", "http://[::1]:+80/x"] {
|
||||
assert!(Url::parse(bad).is_err(), "{bad}");
|
||||
}
|
||||
assert!(Url::parse("http://a.test/").unwrap().join("http://example.com:+80/x").is_err());
|
||||
assert_eq!(Url::parse("http://example.com:80/x").unwrap().port, 80);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_text_after_a_bracketed_ipv6_host() {
|
||||
for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] {
|
||||
assert!(matches!(Url::parse(bad), Err(UrlError(_))), "{bad}");
|
||||
}
|
||||
let u = Url::parse("http://[::1]:8080/x").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 8080));
|
||||
let u = Url::parse("http://[::1]/x").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 80));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_redirects() {
|
||||
let u = Url::parse("https://a.example/x/y/file?q").unwrap();
|
||||
assert_eq!(u.join("https://b.example/z").unwrap().host, "b.example");
|
||||
assert_eq!(u.join("/root").unwrap().path, "/root");
|
||||
assert_eq!(u.join("other").unwrap().path, "/x/y/other");
|
||||
assert_eq!(u.join("//c.example/p").unwrap().host, "c.example");
|
||||
assert!(u.join("ftp://nope").is_err());
|
||||
// no downgrade from https to http, however the redirect is written
|
||||
assert!(u.join("http://a.example/z").is_err());
|
||||
let plain = Url::parse("http://127.0.0.1:8080/a").unwrap();
|
||||
assert!(!plain.join("http://127.0.0.1:9/b").unwrap().tls);
|
||||
assert!(plain.join("https://b.example/z").unwrap().tls);
|
||||
}
|
||||
}
|
||||
@@ -842,6 +842,10 @@ fn read_metadata(c: &Container) -> (Option<Vec<u8>>, Option<String>) {
|
||||
/// Decodes the primary image. `keep_orientation` leaves the container's `clap`, `irot` and
|
||||
/// `imir` unapplied (the stored pixels), as for HEIF.
|
||||
pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<Image> {
|
||||
decode_with_depth(bytes, limits, keep_orientation).map(|(image, _)| image)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_with_depth(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<(Image, u8)> {
|
||||
let c = parse(bytes)?;
|
||||
let id = c.primary;
|
||||
let (w, h) = item_size(&c, id)?;
|
||||
@@ -890,9 +894,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> R
|
||||
// PQ or HLG: linear light in 32-bit float, as for HEIF.
|
||||
if cicp.is_hdr() {
|
||||
limits.check(w, h, layout, SampleType::F32)?;
|
||||
return crate::cicp::linearize(img, cicp);
|
||||
return crate::cicp::linearize(img, cicp).map(|image| (image, bits));
|
||||
}
|
||||
Ok(img)
|
||||
Ok((img, bits))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1301,6 +1305,9 @@ mod tests {
|
||||
let src = Image::from_u8(w as u32, h as u32, ChannelLayout::Rgb, card(w, h, 3)).unwrap();
|
||||
let file = encode_avif_with_speed(&src, 95, 10).unwrap();
|
||||
let img = crate::decode(&file).unwrap();
|
||||
let (with_depth, depth) = crate::decode_avif_with_depth(&file, &crate::DecodeOptions::default()).unwrap();
|
||||
assert_eq!(depth, 10);
|
||||
assert_eq!(with_depth, img);
|
||||
assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (48, 32, ChannelLayout::Rgb, SampleType::U16));
|
||||
let diff = mean_abs_diff(&rgb8(&img), &card(w, h, 3));
|
||||
assert!(diff < 2.5, "mean difference {diff}");
|
||||
|
||||
@@ -120,6 +120,14 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
|
||||
finish_format(format, bytes, img, opts)
|
||||
}
|
||||
|
||||
/// Decode AVIF pixels and report their original AV1 bit depth (8, 10 or 12).
|
||||
/// The returned image may use a wider sample type, particularly for linear HDR.
|
||||
#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))]
|
||||
pub fn decode_avif_with_depth(bytes: &[u8], opts: &DecodeOptions) -> Result<(Image, u8), CodecError> {
|
||||
let (image, depth) = codecs::avif_decode::decode_with_depth(bytes, &opts.limits, opts.keep_orientation)?;
|
||||
Ok((finish_format(Format::Avif, bytes, image, opts)?, depth))
|
||||
}
|
||||
|
||||
/// What [`decode_as_with`] does with a PNG decoded in steps by [`PngDecodeRows`] (`img` is its
|
||||
/// output): the resolution from the metadata, the upright orientation and the final limit check,
|
||||
/// so the result equals [`decode_as_with`]'s on the same file.
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
//! Callers map their page and color models to these neutral records. This module owns PDF object
|
||||
//! layout, image streams, ICC/Separation resources, page labels, marks, xref and trailer syntax.
|
||||
|
||||
/// Multi-image 8/16-bit print pages with per-image ICC resources and explicit byte budgets.
|
||||
#[path = "raster_print.rs"]
|
||||
pub mod print;
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Write as _;
|
||||
use std::io::Write as _;
|
||||
|
||||
@@ -0,0 +1,658 @@
|
||||
//! A small PDF writer for print output: pages of a given size in points, each with raster images
|
||||
//! placed on it, tagged with their ICC profile (`ICCBased`) so the numbers mean the same colour in
|
||||
//! every reader and RIP, Flate-compressed (lossless) at 8 or 16 bits per component, or carried as
|
||||
//! JPEG (`DCTDecode`) where small files matter more than exact samples. Optionally an output intent:
|
||||
//! the printing condition (the printer / paper profile) the file was made for, as PDF/X readers
|
||||
//! expect it in the catalog. Profiles shared by several images are written once.
|
||||
//!
|
||||
//! The file is PDF 1.6 (16-bit images need 1.5): classic cross-reference table, no object streams,
|
||||
//! deterministic (no dates or IDs), so the same pages always give the same bytes.
|
||||
//!
|
||||
//! Encoded JPEG and Flate payloads are copied, not decoded or verified against declared
|
||||
//! dimensions. Callers own their encoding validity and ICC/profile-channel compatibility.
|
||||
//! An output intent describes a printing condition; this API does not certify PDF/X.
|
||||
//! Byte limits bound accounted input and emitted output, not peak process memory or
|
||||
//! caller-owned image construction. This is not the streaming slideshow writer.
|
||||
//!
|
||||
//! Sources: ISO 32000-1:2008 (§7.3 objects, §7.5 file structure, §8.6.5.5 ICCBased colour spaces,
|
||||
//! §8.9 images, §14.11.5 output intents). Own design.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Write as _;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Pages per document: a print job beyond this is a mistake, not a book.
|
||||
pub const MAX_PAGES: usize = 1000;
|
||||
/// Images per page.
|
||||
pub const MAX_IMAGES_PER_PAGE: usize = 256;
|
||||
/// Largest page side in points (200 inches): more than any printer's roll.
|
||||
pub const MAX_PAGE_POINTS: f64 = 14_400.0;
|
||||
/// Largest image, in pixels per side.
|
||||
pub const MAX_IMAGE_SIDE: u32 = 65_535;
|
||||
|
||||
/// Serialization budgets; input records already belong to the caller. These are not a
|
||||
/// process-wide peak-memory guarantee. Repeated references count toward the input budget.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Limits {
|
||||
pub input_bytes: usize,
|
||||
pub output_bytes: usize,
|
||||
}
|
||||
impl Default for Limits {
|
||||
fn default() -> Self {
|
||||
Self { input_bytes: 512 * 1024 * 1024, output_bytes: 512 * 1024 * 1024 }
|
||||
}
|
||||
}
|
||||
|
||||
/// How an image's samples are stored.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ImageData {
|
||||
/// Interleaved samples, `bits` (8 or 16) per component, 16-bit values big-endian: compressed
|
||||
/// losslessly in the file.
|
||||
Raw { bits: u8, samples: Vec<u8> },
|
||||
/// A baseline JPEG stream, copied into the file as it is.
|
||||
Jpeg(Vec<u8>),
|
||||
/// Samples already compressed with zlib (see [`PdfImage::deflated`]), `bits` per component.
|
||||
Flate { bits: u8, data: Vec<u8> },
|
||||
}
|
||||
|
||||
/// A raster image and the colour space its samples are in.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PdfImage {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Components per pixel: 1 (grey), 3 (RGB) or 4 (CMYK).
|
||||
pub channels: u8,
|
||||
pub data: ImageData,
|
||||
/// The ICC profile the samples are in; without one the image is device colour.
|
||||
pub icc: Option<Arc<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl PdfImage {
|
||||
/// An image from interleaved 16-bit samples.
|
||||
pub fn from_u16(width: u32, height: u32, channels: u8, samples: &[u16], icc: Option<Arc<Vec<u8>>>) -> PdfImage {
|
||||
let bytes = samples.iter().flat_map(|v| v.to_be_bytes()).collect();
|
||||
PdfImage { width, height, channels, data: ImageData::Raw { bits: 16, samples: bytes }, icc }
|
||||
}
|
||||
|
||||
/// The same image with its samples compressed now (checked first), so a long print doesn't
|
||||
/// hold every page's uncompressed samples until the file is written.
|
||||
pub fn deflated(self) -> Result<PdfImage, String> {
|
||||
self.check()?;
|
||||
if matches!(&self.data, ImageData::Raw { samples, .. } if samples.len() > Limits::default().input_bytes) {
|
||||
return Err("PDF input byte limit exceeded".into());
|
||||
}
|
||||
let data = match self.data {
|
||||
ImageData::Raw { bits, samples } => ImageData::Flate { bits, data: deflate(&samples, Limits::default().output_bytes)? },
|
||||
other => other,
|
||||
};
|
||||
Ok(PdfImage { data, ..self })
|
||||
}
|
||||
|
||||
fn check(&self) -> Result<(), String> {
|
||||
if self.width == 0 || self.height == 0 || self.width > MAX_IMAGE_SIDE || self.height > MAX_IMAGE_SIDE {
|
||||
return Err(format!("an image of {} × {} pixels can't be written", self.width, self.height));
|
||||
}
|
||||
if !matches!(self.channels, 1 | 3 | 4) {
|
||||
return Err(format!("images have 1, 3 or 4 components, not {}", self.channels));
|
||||
}
|
||||
match &self.data {
|
||||
ImageData::Raw { bits, samples } => {
|
||||
if !matches!(bits, 8 | 16) {
|
||||
return Err(format!("images have 8 or 16 bits per component, not {bits}"));
|
||||
}
|
||||
let want = u64::from(self.width) * u64::from(self.height) * u64::from(self.channels) * u64::from(bits / 8);
|
||||
if samples.len() as u64 != want {
|
||||
return Err(format!("the image needs {want} bytes of samples, not {}", samples.len()));
|
||||
}
|
||||
}
|
||||
ImageData::Jpeg(data) => {
|
||||
if !data.starts_with(&[0xFF, 0xD8]) {
|
||||
return Err("the image isn't a JPEG stream".into());
|
||||
}
|
||||
}
|
||||
ImageData::Flate { bits, data } => {
|
||||
if !matches!(bits, 8 | 16) || data.is_empty() {
|
||||
return Err("a compressed image has 8 or 16 bits per component and some data".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Where an image goes on its page, in points from the page's bottom-left corner.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Placement {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
}
|
||||
|
||||
/// One page: its size in points and the images on it, drawn in order (later ones on top).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PdfPage {
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
pub images: Vec<(PdfImage, Placement)>,
|
||||
}
|
||||
|
||||
/// The printing condition the document is made for (PDF/X-style output intent).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct OutputIntent {
|
||||
/// The printer / paper profile.
|
||||
pub icc: Arc<Vec<u8>>,
|
||||
/// Its components: 1, 3 or 4.
|
||||
pub channels: u8,
|
||||
/// A name for the condition (the profile's description).
|
||||
pub condition: String,
|
||||
}
|
||||
|
||||
/// Document-level settings.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct PdfInfo {
|
||||
pub title: Option<String>,
|
||||
pub output_intent: Option<OutputIntent>,
|
||||
/// Application name, not a PDF/X conformance claim.
|
||||
pub producer: String,
|
||||
}
|
||||
|
||||
fn finite(v: f64) -> bool {
|
||||
v.is_finite()
|
||||
}
|
||||
|
||||
/// A number as PDF writes it: up to four decimals, no exponent, no trailing zeros.
|
||||
fn num(v: f64) -> String {
|
||||
let s = format!("{:.4}", if finite(v) { v } else { 0.0 });
|
||||
let s = s.trim_end_matches('0').trim_end_matches('.');
|
||||
if s == "-0" || s.is_empty() { "0".into() } else { s.into() }
|
||||
}
|
||||
|
||||
/// A text string: plain ASCII as a literal string, anything else as UTF-16BE with a byte order mark.
|
||||
fn text(s: &str) -> String {
|
||||
if s.bytes().all(|b| (0x20..0x7f).contains(&b)) {
|
||||
let mut out = String::with_capacity(s.len() + 2);
|
||||
out.push('(');
|
||||
for c in s.chars() {
|
||||
if matches!(c, '(' | ')' | '\\') {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out.push(')');
|
||||
out
|
||||
} else {
|
||||
super::pdf_unicode_string(s)
|
||||
}
|
||||
}
|
||||
|
||||
struct Writer {
|
||||
out: Vec<u8>,
|
||||
offsets: Vec<usize>,
|
||||
limit: usize,
|
||||
}
|
||||
impl Writer {
|
||||
fn put(&mut self, bytes: &[u8]) -> Result<(), String> {
|
||||
let end = self.out.len().checked_add(bytes.len()).ok_or("PDF output size overflow")?;
|
||||
if end > self.limit {
|
||||
return Err("PDF output byte limit exceeded".into());
|
||||
}
|
||||
self.out.try_reserve(bytes.len()).map_err(|_| "PDF output allocation failed")?;
|
||||
self.out.extend_from_slice(bytes);
|
||||
Ok(())
|
||||
}
|
||||
fn reserve(&mut self) -> Result<usize, String> {
|
||||
self.offsets.try_reserve(1).map_err(|_| "PDF object allocation failed")?;
|
||||
self.offsets.push(0);
|
||||
Ok(self.offsets.len())
|
||||
}
|
||||
fn object(&mut self, id: usize, dict: &str, stream: Option<&[u8]>) -> Result<(), String> {
|
||||
let pos = self.out.len();
|
||||
*self.offsets.get_mut(id.checked_sub(1).ok_or("invalid PDF object")?).ok_or("invalid PDF object")? = pos;
|
||||
self.put(format!("{id} 0 obj\n{dict}").as_bytes())?;
|
||||
if let Some(data) = stream {
|
||||
self.put(b"\nstream\n")?;
|
||||
self.put(data)?;
|
||||
self.put(b"\nendstream")?;
|
||||
}
|
||||
self.put(b"\nendobj\n")
|
||||
}
|
||||
}
|
||||
|
||||
fn deflate(data: &[u8], max: usize) -> Result<Vec<u8>, String> {
|
||||
struct Sink {
|
||||
bytes: Vec<u8>,
|
||||
max: usize,
|
||||
}
|
||||
impl std::io::Write for Sink {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
if bytes.len() > self.max.saturating_sub(self.bytes.len()) {
|
||||
return Err(std::io::Error::other("PDF compressed byte limit exceeded"));
|
||||
}
|
||||
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut encoder = flate2::write::ZlibEncoder::new(Sink { bytes: Vec::new(), max }, flate2::Compression::new(6));
|
||||
encoder.write_all(data).map_err(|e| e.to_string())?;
|
||||
Ok(encoder.finish().map_err(|e| e.to_string())?.bytes)
|
||||
}
|
||||
|
||||
/// Write a document. Errors name the first page or image that can't be written.
|
||||
pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result<Vec<u8>, String> {
|
||||
write_with_limits(pages, info, Limits::default())
|
||||
}
|
||||
|
||||
/// Write with explicit total input and serialized output budgets. ICC bytes are opaque:
|
||||
/// callers validate their profile semantics; resources are keyed by bytes AND channels.
|
||||
pub fn write_with_limits(pages: &[PdfPage], info: &PdfInfo, limits: Limits) -> Result<Vec<u8>, String> {
|
||||
let mut input = 0usize;
|
||||
let mut account = |n: usize| -> Result<(), String> {
|
||||
input = input.checked_add(n).ok_or("PDF input size overflow")?;
|
||||
if input > limits.input_bytes {
|
||||
return Err("PDF input byte limit exceeded".into());
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
account(info.producer.len())?;
|
||||
account(info.title.as_ref().map_or(0, String::len))?;
|
||||
if let Some(intent) = &info.output_intent {
|
||||
account(intent.icc.len())?;
|
||||
account(intent.condition.len())?;
|
||||
}
|
||||
|
||||
if pages.is_empty() {
|
||||
return Err("there are no pages to write".into());
|
||||
}
|
||||
if pages.len() > MAX_PAGES {
|
||||
return Err(format!("a document holds at most {MAX_PAGES} pages"));
|
||||
}
|
||||
for (n, page) in pages.iter().enumerate() {
|
||||
let n = n + 1;
|
||||
if !(finite(page.width) && finite(page.height))
|
||||
|| page.width <= 0.0
|
||||
|| page.height <= 0.0
|
||||
|| page.width > MAX_PAGE_POINTS
|
||||
|| page.height > MAX_PAGE_POINTS
|
||||
{
|
||||
return Err(format!("page {n} has no usable size"));
|
||||
}
|
||||
if page.images.len() > MAX_IMAGES_PER_PAGE {
|
||||
return Err(format!("page {n} has more than {MAX_IMAGES_PER_PAGE} images"));
|
||||
}
|
||||
for (image, at) in &page.images {
|
||||
account(match &image.data {
|
||||
ImageData::Raw { samples, .. } => samples.len(),
|
||||
ImageData::Jpeg(data) | ImageData::Flate { data, .. } => data.len(),
|
||||
})?;
|
||||
if let Some(icc) = &image.icc {
|
||||
account(icc.len())?;
|
||||
}
|
||||
image.check().map_err(|e| format!("page {n}: {e}"))?;
|
||||
if ![at.x, at.y, at.width, at.height].into_iter().all(finite) || at.width <= 0.0 || at.height <= 0.0 {
|
||||
return Err(format!("page {n}: an image has no usable placement"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(intent) = &info.output_intent
|
||||
&& !matches!(intent.channels, 1 | 3 | 4)
|
||||
{
|
||||
return Err("the output intent's profile has 1, 3 or 4 components".into());
|
||||
}
|
||||
|
||||
let mut w = Writer { out: Vec::new(), offsets: Vec::new(), limit: limits.output_bytes };
|
||||
w.put(b"%PDF-1.6\n%\xE2\xE3\xCF\xD3\n")?;
|
||||
let catalog = w.reserve()?;
|
||||
let pages_id = w.reserve()?;
|
||||
let info_id = w.reserve()?;
|
||||
|
||||
// ICC profiles, each written once (keyed by content) with its component count
|
||||
let mut profiles: HashMap<(Arc<Vec<u8>>, u8), usize> = HashMap::new();
|
||||
let mut profile = |w: &mut Writer, icc: &Arc<Vec<u8>>, n: u8| -> Result<usize, String> {
|
||||
if let Some(&id) = profiles.get(&(Arc::clone(icc), n)) {
|
||||
return Ok(id);
|
||||
}
|
||||
let id = w.reserve()?;
|
||||
let data = deflate(icc, limits.output_bytes)?;
|
||||
let alternate = match n {
|
||||
1 => "/DeviceGray",
|
||||
4 => "/DeviceCMYK",
|
||||
_ => "/DeviceRGB",
|
||||
};
|
||||
w.object(id, &format!("<< /N {n} /Alternate {alternate} /Filter /FlateDecode /Length {} >>", data.len()), Some(&data))?;
|
||||
profiles.try_reserve(1).map_err(|_| "PDF profile allocation failed")?;
|
||||
profiles.insert((Arc::clone(icc), n), id);
|
||||
Ok(id)
|
||||
};
|
||||
|
||||
let intent_id = info.output_intent.as_ref().map(|intent| profile(&mut w, &intent.icc, intent.channels)).transpose()?;
|
||||
|
||||
let mut kids = Vec::with_capacity(pages.len());
|
||||
for page in pages {
|
||||
let page_id = w.reserve()?;
|
||||
let mut xobjects = String::new();
|
||||
let mut content = String::new();
|
||||
for (k, (image, at)) in page.images.iter().enumerate() {
|
||||
let space = match &image.icc {
|
||||
Some(icc) => format!("[/ICCBased {} 0 R]", profile(&mut w, icc, image.channels)?),
|
||||
None => match image.channels {
|
||||
1 => "/DeviceGray".into(),
|
||||
4 => "/DeviceCMYK".into(),
|
||||
_ => "/DeviceRGB".into(),
|
||||
},
|
||||
};
|
||||
let id = w.reserve()?;
|
||||
let (bits, filter, data): (u8, &str, std::borrow::Cow<[u8]>) = match &image.data {
|
||||
ImageData::Raw { bits, samples } => (*bits, "/FlateDecode", deflate(samples, limits.output_bytes)?.into()),
|
||||
ImageData::Flate { bits, data } => (*bits, "/FlateDecode", data.as_slice().into()),
|
||||
ImageData::Jpeg(jpeg) => (8, "/DCTDecode", jpeg.as_slice().into()),
|
||||
};
|
||||
w.object(
|
||||
id,
|
||||
&format!(
|
||||
"<< /Type /XObject /Subtype /Image /Width {} /Height {} /ColorSpace {space} /BitsPerComponent {bits} /Filter {filter} /Length {} >>",
|
||||
image.width,
|
||||
image.height,
|
||||
data.len()
|
||||
),
|
||||
Some(&data),
|
||||
)?;
|
||||
xobjects.push_str(&format!("/Im{k} {id} 0 R "));
|
||||
content.push_str(&format!("q {} 0 0 {} {} {} cm /Im{k} Do Q\n", num(at.width), num(at.height), num(at.x), num(at.y)));
|
||||
}
|
||||
let content_id = w.reserve()?;
|
||||
let stream = deflate(content.as_bytes(), limits.output_bytes)?;
|
||||
w.object(content_id, &format!("<< /Filter /FlateDecode /Length {} >>", stream.len()), Some(&stream))?;
|
||||
w.object(
|
||||
page_id,
|
||||
&format!(
|
||||
"<< /Type /Page /Parent {pages_id} 0 R /MediaBox [0 0 {} {}] /Resources << /XObject << {xobjects}>> >> /Contents {content_id} 0 R >>",
|
||||
num(page.width),
|
||||
num(page.height)
|
||||
),
|
||||
None,
|
||||
)?;
|
||||
kids.push(format!("{page_id} 0 R"));
|
||||
}
|
||||
w.object(pages_id, &format!("<< /Type /Pages /Count {} /Kids [{}] >>", kids.len(), kids.join(" ")), None)?;
|
||||
|
||||
let intents = match (&info.output_intent, intent_id) {
|
||||
(Some(intent), Some(id)) => format!(
|
||||
" /OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX /OutputConditionIdentifier (Custom) /Info {} /DestOutputProfile {id} 0 R >>]",
|
||||
text(&intent.condition)
|
||||
),
|
||||
_ => String::new(),
|
||||
};
|
||||
w.object(catalog, &format!("<< /Type /Catalog /Pages {pages_id} 0 R{intents} >>"), None)?;
|
||||
let title = info.title.as_deref().map(|t| format!(" /Title {}", text(t))).unwrap_or_default();
|
||||
w.object(info_id, &format!("<< /Producer {}{title} >>", text(&info.producer)), None)?;
|
||||
|
||||
let xref = w.out.len();
|
||||
let mut table = format!("xref\n0 {}\n0000000000 65535 f \n", w.offsets.len() + 1);
|
||||
for offset in &w.offsets {
|
||||
table.push_str(&format!("{offset:010} 00000 n \n"));
|
||||
}
|
||||
table.push_str(&format!("trailer\n<< /Size {} /Root {catalog} 0 R /Info {info_id} 0 R >>\nstartxref\n{xref}\n%%EOF\n", w.offsets.len() + 1));
|
||||
w.put(table.as_bytes())?;
|
||||
Ok(w.out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The body of object `id` (dictionary and raw stream bytes), found through the xref table.
|
||||
fn object(pdf: &[u8], id: usize) -> (String, Vec<u8>) {
|
||||
// byte offsets: the streams are binary, so search the bytes, not a lossy string
|
||||
let start = pdf.windows(10).rposition(|w| w == b"startxref\n").unwrap() + 10;
|
||||
let tail = std::str::from_utf8(&pdf[start..]).unwrap();
|
||||
let xref: usize = tail.lines().next().unwrap().trim().parse().unwrap();
|
||||
let table = std::str::from_utf8(&pdf[xref..]).unwrap();
|
||||
let line = table.lines().nth(2 + id).unwrap();
|
||||
let offset: usize = line[..10].parse().unwrap();
|
||||
let head = format!("{id} 0 obj\n");
|
||||
assert_eq!(&pdf[offset..offset + head.len()], head.as_bytes(), "xref entry for {id}");
|
||||
let rest = &pdf[offset + head.len()..];
|
||||
let end = rest.windows(7).position(|w| w == b"\nendobj").unwrap();
|
||||
let body = &rest[..end];
|
||||
match body.windows(8).position(|w| w == b"\nstream\n") {
|
||||
Some(p) => {
|
||||
let data = &body[p + 8..body.len() - "\nendstream".len()];
|
||||
(String::from_utf8_lossy(&body[..p]).into_owned(), data.to_vec())
|
||||
}
|
||||
None => (String::from_utf8_lossy(body).into_owned(), vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
fn inflate(data: &[u8]) -> Vec<u8> {
|
||||
{
|
||||
use std::io::Read;
|
||||
let mut out = Vec::new();
|
||||
flate2::read::ZlibDecoder::new(data).read_to_end(&mut out).unwrap();
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Object ids of every image XObject, in file order.
|
||||
fn images(pdf: &[u8]) -> Vec<usize> {
|
||||
let s = String::from_utf8_lossy(pdf);
|
||||
s.match_indices(" 0 obj\n<< /Type /XObject /Subtype /Image").map(|(i, _)| s[..i].rsplit('\n').next().unwrap().parse().unwrap()).collect()
|
||||
}
|
||||
|
||||
fn reference(dict: &str, key: &str) -> usize {
|
||||
let at = dict.find(key).unwrap() + key.len();
|
||||
dict[at..].trim_start().split(' ').next().unwrap().parse().unwrap()
|
||||
}
|
||||
|
||||
fn srgb() -> Arc<Vec<u8>> {
|
||||
Arc::new(b"opaque RGB profile fixture".to_vec())
|
||||
}
|
||||
|
||||
fn page(images: Vec<(PdfImage, Placement)>) -> PdfPage {
|
||||
PdfPage { width: 612.0, height: 792.0, images }
|
||||
}
|
||||
|
||||
fn full(w: f64, h: f64) -> Placement {
|
||||
Placement { x: 0.0, y: 0.0, width: w, height: h }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_16_bit_icc_image_round_trips() {
|
||||
let samples: Vec<u16> = (0..2 * 3 * 3).map(|i| (i * 3851) as u16).collect();
|
||||
let img = PdfImage::from_u16(3, 2, 3, &samples, Some(srgb()));
|
||||
let pdf = write(&[page(vec![(img, full(612.0, 792.0))])], &PdfInfo::default()).unwrap();
|
||||
assert!(pdf.starts_with(b"%PDF-1.6\n"));
|
||||
assert!(pdf.ends_with(b"%%EOF\n"));
|
||||
let ids = images(&pdf);
|
||||
assert_eq!(ids.len(), 1);
|
||||
let (dict, data) = object(&pdf, ids[0]);
|
||||
assert!(dict.contains("/Width 3 /Height 2") && dict.contains("/BitsPerComponent 16") && dict.contains("/FlateDecode"), "{dict}");
|
||||
let back: Vec<u16> = inflate(&data).chunks(2).map(|b| u16::from_be_bytes([b[0], b[1]])).collect();
|
||||
assert_eq!(back, samples);
|
||||
let (profile, icc) = object(&pdf, reference(&dict, "/ICCBased"));
|
||||
assert!(profile.contains("/N 3") && profile.contains("/Alternate /DeviceRGB"), "{profile}");
|
||||
assert_eq!(inflate(&icc), *srgb());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pages_sizes_placements_and_a_shared_profile() {
|
||||
let px = |v: u16| PdfImage::from_u16(1, 1, 3, &[v, v, v], Some(srgb()));
|
||||
let at = Placement { x: 36.0, y: 72.5, width: 288.0, height: 216.0 };
|
||||
let pages = [page(vec![(px(1), at), (px(2), full(612.0, 792.0))]), PdfPage { width: 841.89, height: 595.28, images: vec![(px(3), at)] }];
|
||||
let pdf = write(&pages, &PdfInfo::default()).unwrap();
|
||||
let s = String::from_utf8_lossy(&pdf);
|
||||
assert!(s.contains("/Type /Pages /Count 2"), "two pages");
|
||||
assert!(s.contains("/MediaBox [0 0 612 792]") && s.contains("/MediaBox [0 0 841.89 595.28]"));
|
||||
// the sRGB profile is written once for all three images
|
||||
assert_eq!(s.matches("/N 3 /Alternate").count(), 1);
|
||||
let first: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
|
||||
assert!(first.windows(2).all(|w| w[0] == w[1]), "{first:?}");
|
||||
// placements reach the content streams
|
||||
let contents: Vec<String> = s
|
||||
.match_indices("/Contents ")
|
||||
.map(|(i, _)| s[i + 10..].split(' ').next().unwrap().parse::<usize>().unwrap())
|
||||
.map(|id| String::from_utf8(inflate(&object(&pdf, id).1)).unwrap())
|
||||
.collect();
|
||||
assert_eq!(contents[0], "q 288 0 0 216 36 72.5 cm /Im0 Do Q\nq 612 0 0 792 0 0 cm /Im1 Do Q\n");
|
||||
assert_eq!(write(&pages, &PdfInfo::default()).unwrap(), pdf, "deterministic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_intent_and_title() {
|
||||
let cmyk_like = srgb();
|
||||
let info = PdfInfo {
|
||||
title: Some("Garden – prints (2)".into()),
|
||||
producer: "Fixture".into(),
|
||||
output_intent: Some(OutputIntent { icc: cmyk_like.clone(), channels: 3, condition: "Glossy (Photo) \\ test".into() }),
|
||||
};
|
||||
let img = PdfImage::from_u16(1, 1, 3, &[0, 0, 0], Some(cmyk_like));
|
||||
let pdf = write(&[page(vec![(img, full(10.0, 10.0))])], &info).unwrap();
|
||||
let (catalog, _) = object(&pdf, 1);
|
||||
assert!(catalog.contains("/OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX"), "{catalog}");
|
||||
assert!(catalog.contains("/Info (Glossy \\(Photo\\) \\\\ test)"), "{catalog}");
|
||||
// the intent and the image share one profile object
|
||||
let dest = reference(&catalog, "/DestOutputProfile");
|
||||
assert_eq!(reference(&object(&pdf, images(&pdf)[0]).0, "/ICCBased"), dest);
|
||||
let (info_dict, _) = object(&pdf, 3);
|
||||
assert!(info_dict.contains("/Title <FEFF"), "non-ASCII titles are UTF-16: {info_dict}");
|
||||
assert!(info_dict.contains("2013"), "the en dash: {info_dict}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmyk_grey_jpeg_and_device_images() {
|
||||
let jpeg = vec![0xFF, 0xD8, 0xFF, 0xD9];
|
||||
let imgs = vec![
|
||||
(
|
||||
PdfImage { width: 1, height: 1, channels: 4, data: ImageData::Raw { bits: 8, samples: vec![0, 10, 20, 30] }, icc: None },
|
||||
full(1.0, 1.0),
|
||||
),
|
||||
(PdfImage { width: 2, height: 1, channels: 1, data: ImageData::Raw { bits: 8, samples: vec![0, 255] }, icc: None }, full(1.0, 1.0)),
|
||||
(PdfImage { width: 4, height: 4, channels: 3, data: ImageData::Jpeg(jpeg.clone()), icc: None }, full(1.0, 1.0)),
|
||||
];
|
||||
let pdf = write(&[page(imgs)], &PdfInfo::default()).unwrap();
|
||||
let ids = images(&pdf);
|
||||
let dicts: Vec<_> = ids.iter().map(|&id| object(&pdf, id)).collect();
|
||||
assert!(dicts[0].0.contains("/ColorSpace /DeviceCMYK") && dicts[0].0.contains("/BitsPerComponent 8"));
|
||||
assert!(dicts[1].0.contains("/ColorSpace /DeviceGray"));
|
||||
assert!(dicts[2].0.contains("/DCTDecode") && dicts[2].0.contains("/DeviceRGB"));
|
||||
assert_eq!(dicts[2].1, jpeg, "JPEG streams are copied as they are");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressing_early_writes_the_same_file() {
|
||||
let samples: Vec<u16> = (0..4 * 3 * 3).map(|i| (i * 977) as u16).collect();
|
||||
let raw = PdfImage::from_u16(4, 3, 3, &samples, Some(srgb()));
|
||||
let early = raw.clone().deflated().unwrap();
|
||||
assert!(matches!(early.data, ImageData::Flate { bits: 16, .. }));
|
||||
let at = full(100.0, 75.0);
|
||||
let a = write(&[page(vec![(raw, at)])], &PdfInfo::default()).unwrap();
|
||||
let b = write(&[page(vec![(early, at)])], &PdfInfo::default()).unwrap();
|
||||
assert_eq!(a, b);
|
||||
let mut short = PdfImage::from_u16(4, 3, 3, &samples, None);
|
||||
short.width = 5;
|
||||
assert!(short.deflated().is_err(), "checked before compressing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_documents_are_errors() {
|
||||
let ok = || PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
|
||||
let at = full(10.0, 10.0);
|
||||
let bad = |pages: Vec<PdfPage>| write(&pages, &PdfInfo::default()).is_err();
|
||||
assert!(bad(vec![]));
|
||||
assert!(bad(vec![PdfPage { width: f64::NAN, height: 10.0, images: vec![] }]));
|
||||
assert!(bad(vec![PdfPage { width: -1.0, height: 10.0, images: vec![] }]));
|
||||
assert!(bad(vec![PdfPage { width: 1e9, height: 10.0, images: vec![] }]));
|
||||
assert!(bad(vec![page(vec![(ok(), Placement { x: f64::INFINITY, ..at })])]));
|
||||
assert!(bad(vec![page(vec![(ok(), Placement { width: 0.0, ..at })])]));
|
||||
let mut short = ok();
|
||||
short.data = ImageData::Raw { bits: 16, samples: vec![0; 5] };
|
||||
assert!(bad(vec![page(vec![(short, at)])]));
|
||||
let mut empty = ok();
|
||||
empty.width = 0;
|
||||
assert!(bad(vec![page(vec![(empty, at)])]));
|
||||
let mut odd = ok();
|
||||
odd.channels = 2;
|
||||
assert!(bad(vec![page(vec![(odd, at)])]));
|
||||
let mut not_jpeg = ok();
|
||||
not_jpeg.data = ImageData::Jpeg(vec![1, 2, 3]);
|
||||
assert!(bad(vec![page(vec![(not_jpeg, at)])]));
|
||||
let too_many = (0..MAX_PAGES + 1).map(|_| page(vec![])).collect();
|
||||
assert!(bad(too_many));
|
||||
let info = PdfInfo {
|
||||
producer: String::new(),
|
||||
title: None,
|
||||
output_intent: Some(OutputIntent { icc: srgb(), channels: 2, condition: String::new() }),
|
||||
};
|
||||
assert!(write(&[page(vec![])], &info).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbers_and_strings() {
|
||||
assert_eq!(num(612.0), "612");
|
||||
assert_eq!(num(72.5), "72.5");
|
||||
assert_eq!(num(-0.00001), "0");
|
||||
assert_eq!(num(f64::NAN), "0");
|
||||
assert_eq!(num(1.0 / 3.0), "0.3333");
|
||||
assert_eq!(text("a(b)c\\"), "(a\\(b\\)c\\\\)");
|
||||
assert_eq!(text("é"), "<FEFF00E9>");
|
||||
}
|
||||
#[test]
|
||||
fn distinct_profiles_and_component_counts_are_not_conflated() {
|
||||
let common = Arc::new(b"opaque profile A".to_vec());
|
||||
let different = Arc::new(b"opaque profile B".to_vec());
|
||||
let image = |channels, profile| PdfImage {
|
||||
width: 1,
|
||||
height: 1,
|
||||
channels,
|
||||
data: ImageData::Raw { bits: 8, samples: vec![7; channels as usize] },
|
||||
icc: Some(profile),
|
||||
};
|
||||
let pages = [page(vec![
|
||||
(image(3, common.clone()), full(1.0, 1.0)),
|
||||
(image(4, common.clone()), full(1.0, 1.0)),
|
||||
(image(3, different.clone()), full(1.0, 1.0)),
|
||||
(image(3, common.clone()), full(1.0, 1.0)),
|
||||
])];
|
||||
let pdf = write(&pages, &PdfInfo::default()).unwrap();
|
||||
let refs: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
|
||||
assert_ne!(refs[0], refs[1]);
|
||||
assert_ne!(refs[0], refs[2]);
|
||||
assert_eq!(refs[0], refs[3]);
|
||||
for (i, channels, bytes) in [(0, 3, common.as_slice()), (1, 4, common.as_slice()), (2, 3, different.as_slice())] {
|
||||
let (dict, data) = object(&pdf, refs[i]);
|
||||
assert!(dict.contains(&format!("/N {channels}")));
|
||||
assert_eq!(inflate(&data), bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gray_rgb_cmyk_raw_depths_preserve_samples() {
|
||||
for channels in [1, 3, 4] {
|
||||
for bits in [8, 16] {
|
||||
let bytes: Vec<u8> = (0..channels * (bits / 8)).collect();
|
||||
let image = PdfImage { width: 1, height: 1, channels, data: ImageData::Raw { bits, samples: bytes.clone() }, icc: None };
|
||||
let pdf = write(&[page(vec![(image, full(1.0, 1.0))])], &PdfInfo::default()).unwrap();
|
||||
let (dict, data) = object(&pdf, images(&pdf)[0]);
|
||||
assert!(dict.contains(&format!("/BitsPerComponent {bits}")));
|
||||
assert_eq!(inflate(&data), bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_and_output_budgets_fail_without_partial_success() {
|
||||
let image = PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
|
||||
let pages = [page(vec![(image, full(1.0, 1.0))])];
|
||||
let info = PdfInfo::default();
|
||||
let expected = write(&pages, &info).unwrap();
|
||||
assert_eq!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() }).unwrap(), expected);
|
||||
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 5, output_bytes: expected.len() }).is_err());
|
||||
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() - 1 }).is_err());
|
||||
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: 0 }).is_err());
|
||||
}
|
||||
}
|
||||
@@ -465,6 +465,7 @@ mod tests {
|
||||
color: crate::ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Default::default(),
|
||||
|
||||
+126
-13
@@ -4,16 +4,19 @@
|
||||
|
||||
use crate::{RawError, Result};
|
||||
|
||||
/// A canonical-by-table Huffman decoder with a 12-bit lookup.
|
||||
/// A canonical-by-table Huffman decoder with a 12-bit lookup that also holds the difference when code and
|
||||
/// additional bits fit in 12 bits.
|
||||
pub(crate) struct Huffman {
|
||||
/// Indexed by the next 12 bits: (symbol, length); length 0 = invalid.
|
||||
lut: Vec<(u8, u8)>,
|
||||
/// Indexed by the next 12 bits. Per entry: bits 0-7 the symbol, bits 8-11 the code length (0 = invalid), bits
|
||||
/// 12-15 the length of code plus additional bits when both fit in 12 bits (0 = they do not), bits 16-31 the
|
||||
/// signed difference in that case.
|
||||
lut: Box<[u32; 4096]>,
|
||||
}
|
||||
|
||||
impl Huffman {
|
||||
/// From `(left-aligned 12-bit code, length)` per symbol.
|
||||
pub fn new(codes: &[(u16, u8)]) -> Result<Huffman> {
|
||||
let mut lut = vec![(0u8, 0u8); 4096];
|
||||
let mut lut = Box::new([0u32; 4096]);
|
||||
for (sym, &(code, len)) in codes.iter().enumerate() {
|
||||
if len == 0 {
|
||||
continue;
|
||||
@@ -21,10 +24,19 @@ impl Huffman {
|
||||
if len > 12 || sym > 16 {
|
||||
return Err(RawError::Unsupported(format!("PEF Huffman code of length {len}")));
|
||||
}
|
||||
let (len, s) = (len as u32, sym as u32);
|
||||
let first = (code >> (12 - len) << (12 - len)) as usize;
|
||||
for e in &mut lut[first..first + (1 << (12 - len))] {
|
||||
if e.1 == 0 {
|
||||
*e = (sym as u8, len);
|
||||
for (i, e) in lut[first..first + (1 << (12 - len))].iter_mut().enumerate() {
|
||||
if (*e >> 8) & 15 == 0 {
|
||||
let mut entry = (len << 8) | s;
|
||||
if s == 0 {
|
||||
entry |= len << 12;
|
||||
} else if len + s <= 12 {
|
||||
let v = (((first + i) >> (12 - len - s)) & ((1 << s) - 1)) as i32;
|
||||
let d = if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v };
|
||||
entry |= ((len + s) << 12) | ((d as i16 as u16 as u32) << 16);
|
||||
}
|
||||
*e = entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -85,21 +97,122 @@ impl<'a> Bits<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode one difference: Huffman category then T.81 additional bits.
|
||||
/// Decode one difference: Huffman category then T.81 additional bits, in one lookup when both fit in 12 bits.
|
||||
#[inline]
|
||||
pub(crate) fn diff(bits: &mut Bits, h: &Huffman) -> Option<i32> {
|
||||
let (sym, len) = h.lut[bits.peek(12) as usize];
|
||||
let e = h.lut[(bits.peek(12) & 0xfff) as usize];
|
||||
let len = (e >> 8) & 15;
|
||||
if len == 0 {
|
||||
return None;
|
||||
}
|
||||
bits.skip(len as u32);
|
||||
let s = sym as u32;
|
||||
if s == 0 {
|
||||
return Some(0);
|
||||
let total = (e >> 12) & 15;
|
||||
if total > 0 {
|
||||
bits.skip(total);
|
||||
return Some((e as i32) >> 16);
|
||||
}
|
||||
bits.skip(len);
|
||||
let s = e & 0xff;
|
||||
if s > 16 {
|
||||
return None;
|
||||
}
|
||||
let v = bits.get(s) as i32;
|
||||
Some(if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Canonical left-aligned 12-bit codes for symbols 0..lens.len() (shorter codes first, ties by symbol).
|
||||
fn canonical(lens: &[u8]) -> Vec<(u16, u8)> {
|
||||
let mut order: Vec<usize> = (0..lens.len()).collect();
|
||||
order.sort_by_key(|&s| (lens[s], s));
|
||||
let mut codes = vec![(0u16, 0u8); lens.len()];
|
||||
let (mut code, mut prev) = (0u32, 0u8);
|
||||
for s in order {
|
||||
code <<= lens[s] - prev;
|
||||
prev = lens[s];
|
||||
codes[s] = ((code << (12 - lens[s])) as u16, lens[s]);
|
||||
code += 1;
|
||||
}
|
||||
codes
|
||||
}
|
||||
|
||||
fn bit(src: &[u8], i: usize) -> u32 {
|
||||
src.get(i / 8).map_or(0, |b| ((b >> (7 - i % 8)) & 1) as u32)
|
||||
}
|
||||
|
||||
/// Reference: match the code bit by bit, then read the category's additional bits (T.81 F.1.2.1).
|
||||
fn reference(src: &[u8], pos: &mut usize, codes: &[(u16, u8)]) -> Option<i32> {
|
||||
let window: u32 = (0..12).fold(0, |a, k| (a << 1) | bit(src, *pos + k));
|
||||
let (sym, len) = codes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(s, &(c, l))| (l > 0 && (c as u32 >> (12 - l)) == window >> (12 - l)).then_some((s as u32, l as usize)))?;
|
||||
*pos += len;
|
||||
if sym == 0 {
|
||||
return Some(0);
|
||||
}
|
||||
let v = (0..sym as usize).fold(0u32, |a, k| (a << 1) | bit(src, *pos + k)) as i32;
|
||||
*pos += sym as usize;
|
||||
Some(if v < 1 << (sym - 1) { v - (1 << sym) + 1 } else { v })
|
||||
}
|
||||
|
||||
/// `diff` against the reference on random streams (invalid codes are stepped over) and on every (code, extra
|
||||
/// bits) pair of every symbol.
|
||||
fn check_table(lens: &[u8]) {
|
||||
let codes = canonical(lens);
|
||||
let h = Huffman::new(&codes).unwrap();
|
||||
for (sym, &(code, len)) in codes.iter().enumerate() {
|
||||
for v in 0..1u64 << sym {
|
||||
let (len, n) = (len as u32, sym as u32);
|
||||
let word = (((code >> (12 - len)) as u64) << n | v) << (64 - len - n);
|
||||
let src = word.to_be_bytes();
|
||||
let mut pos = 0;
|
||||
let want = reference(&src, &mut pos, &codes);
|
||||
let mut bits = Bits::new(&src);
|
||||
assert_eq!(diff(&mut bits, &h), want, "symbol {sym} bits {v:#b}");
|
||||
assert_eq!(bits.consumed_bits(), pos);
|
||||
}
|
||||
}
|
||||
let mut s = 0x1234_5678_9abc_def0u64;
|
||||
let (mut invalid, mut long) = (0, 0);
|
||||
for _ in 0..40 {
|
||||
let src: Vec<u8> = (0..400)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
(s >> 40) as u8
|
||||
})
|
||||
.collect();
|
||||
let mut bits = Bits::new(&src);
|
||||
let mut pos = 0;
|
||||
while pos < src.len() * 8 - 64 {
|
||||
let before = pos;
|
||||
let want = reference(&src, &mut pos, &codes);
|
||||
assert_eq!(diff(&mut bits, &h), want);
|
||||
if want.is_none() {
|
||||
// step over the invalid code
|
||||
invalid += 1;
|
||||
pos += 1;
|
||||
bits.skip(1);
|
||||
} else {
|
||||
long += (pos - before > 12) as u32;
|
||||
}
|
||||
assert_eq!(bits.consumed_bits(), pos);
|
||||
}
|
||||
}
|
||||
assert!(long > 0, "{invalid} invalid codes, no symbol longer than the lookup");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_matches_bitwise_reference_for_every_category() {
|
||||
// categories 0..=16, code lengths up to 12 so that short codes combine with their bits and long ones do not
|
||||
check_table(&[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_matches_reference_when_code_and_bits_fill_the_lookup_exactly() {
|
||||
// category 8 with a 4-bit code and category 7 with a 5-bit code: len + category == 12
|
||||
check_table(&[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12]);
|
||||
}
|
||||
}
|
||||
+129
-39
@@ -32,15 +32,17 @@ fn err(s: &str) -> RawError {
|
||||
/// Huffman decoding table (T.81 Annex C "BITS"/"HUFFVAL").
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Huffman {
|
||||
/// For peeks of `LOOKUP` bits: (code length, symbol); length 0 = slow path.
|
||||
fast: Vec<(u8, u8)>,
|
||||
/// Indexed by the next `LOOKUP` bits; length 0 = slow path. Per entry: bits 0-7 the symbol, bits 8-11 the code
|
||||
/// length, bits 12-15 the length of code plus extra bits when both fit in the lookup (0 = they do not), and
|
||||
/// bits 16-31 the signed difference in that case.
|
||||
fast: Vec<u32>,
|
||||
maxcode: [i32; 18],
|
||||
valptr: [i32; 17],
|
||||
mincode: [i32; 17],
|
||||
values: Vec<u8>,
|
||||
}
|
||||
|
||||
const LOOKUP: u32 = 10;
|
||||
const LOOKUP: u32 = 12;
|
||||
|
||||
impl Huffman {
|
||||
/// Build from the 16 code-length counts and the symbol list.
|
||||
@@ -52,7 +54,7 @@ impl Huffman {
|
||||
let mut maxcode = [-1i32; 18];
|
||||
let mut valptr = [0i32; 17];
|
||||
let mut mincode = [0i32; 17];
|
||||
let mut fast = vec![(0u8, 0u8); 1 << LOOKUP];
|
||||
let mut fast = vec![0u32; 1 << LOOKUP];
|
||||
let mut code = 0i32;
|
||||
let mut k = 0usize;
|
||||
for len in 1..=16usize {
|
||||
@@ -63,9 +65,10 @@ impl Huffman {
|
||||
if len as u32 <= LOOKUP {
|
||||
let shift = LOOKUP - len as u32;
|
||||
let base = (code as usize) << shift;
|
||||
let ssss = values[k] as u32;
|
||||
for i in 0..(1usize << shift) {
|
||||
if let Some(slot) = fast.get_mut(base + i) {
|
||||
*slot = (len as u8, values[k]);
|
||||
*slot = fast_entry(len as u32, ssss, base + i);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -84,12 +87,41 @@ impl Huffman {
|
||||
|
||||
#[inline]
|
||||
pub fn decode<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
|
||||
let peek = br.peek(LOOKUP);
|
||||
let (len, sym) = self.fast[peek as usize];
|
||||
let e = self.fast[br.peek(LOOKUP) as usize];
|
||||
let len = (e >> 8) & 15;
|
||||
if len > 0 {
|
||||
br.consume(len as u32);
|
||||
return Ok(sym);
|
||||
br.consume(len);
|
||||
return Ok(e as u8);
|
||||
}
|
||||
self.decode_slow(br)
|
||||
}
|
||||
|
||||
/// One symbol and its T.81 difference in a single lookup when code and extra bits fit in `LOOKUP` bits; the same
|
||||
/// as [`Huffman::decode`] followed by [`diff_value`]. `strict`: a category above 16 is an error (subsampled
|
||||
/// scans) instead of reading 16 bits.
|
||||
#[inline]
|
||||
pub fn decode_diff<B: Bits>(&self, br: &mut B, strict: bool) -> Result<i32, RawError> {
|
||||
let e = self.fast[br.peek(LOOKUP) as usize];
|
||||
let len = (e >> 8) & 15;
|
||||
let total = (e >> 12) & 15;
|
||||
if total > 0 {
|
||||
br.consume(total);
|
||||
return Ok((e as i32) >> 16);
|
||||
}
|
||||
let ssss = if len > 0 {
|
||||
br.consume(len);
|
||||
e as u8
|
||||
} else {
|
||||
self.decode_slow(br)?
|
||||
};
|
||||
if strict && ssss > 16 {
|
||||
return Err(err("invalid difference category"));
|
||||
}
|
||||
Ok(diff_value(br, ssss))
|
||||
}
|
||||
|
||||
#[cold]
|
||||
fn decode_slow<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
|
||||
let mut code = br.peek(LOOKUP + 1) as i32;
|
||||
let mut len = LOOKUP as usize + 1;
|
||||
while len <= 16 && code > self.maxcode[len] {
|
||||
@@ -105,6 +137,21 @@ impl Huffman {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lookup entry for a code of `len` bits (`len <= LOOKUP`) naming category `ssss`, at window position `index`
|
||||
/// (the code is the top `len` bits of the `LOOKUP`-bit window). See [`Huffman::fast`].
|
||||
fn fast_entry(len: u32, ssss: u32, index: usize) -> u32 {
|
||||
let base = (len << 8) | ssss;
|
||||
if ssss == 0 {
|
||||
return base | (len << 12);
|
||||
}
|
||||
if ssss > 11 || len + ssss > LOOKUP {
|
||||
return base;
|
||||
}
|
||||
let v = ((index >> (LOOKUP - len - ssss)) & ((1 << ssss) - 1)) as i32;
|
||||
let d = if v < (1 << (ssss - 1)) { v - (1 << ssss) + 1 } else { v };
|
||||
base | ((len + ssss) << 12) | ((d as i16 as u16 as u32) << 16)
|
||||
}
|
||||
|
||||
/// A source of entropy-coded bits, first bit most significant.
|
||||
pub trait Bits {
|
||||
/// The next `n` (1..=32) bits without consuming them; zeros past the end.
|
||||
@@ -614,11 +661,7 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
|
||||
Prediction::Sequential if mx == 0 && dx == 0 && dy == 0 => row_first[c],
|
||||
Prediction::Sequential => last[c],
|
||||
};
|
||||
let ssss = tables[c].decode(&mut br)?;
|
||||
if ssss > 16 {
|
||||
return Err(err("invalid difference category"));
|
||||
}
|
||||
let value = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16;
|
||||
let value = ((pred + tables[c].decode_diff(&mut br, true)?) & 0xffff) as u16;
|
||||
plane[i] = value;
|
||||
last[c] = i32::from(value);
|
||||
if mx == 0 && dx == 0 && dy == 0 {
|
||||
@@ -642,6 +685,44 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
|
||||
Ok(FrameSubsampled { width: h.width, height: h.height, vertical_subsampling: vertical, planes })
|
||||
}
|
||||
|
||||
/// T.81 Table H.1 predictor `pred` (1..=7; 0 = none) from the left, above and above-left neighbours.
|
||||
#[inline(always)]
|
||||
fn predict(pred: u8, ra: i32, rb: i32, rc: i32) -> i32 {
|
||||
match pred {
|
||||
0 => 0,
|
||||
1 => ra,
|
||||
2 => rb,
|
||||
3 => rc,
|
||||
4 => ra + rb - rc,
|
||||
5 => ra + ((rb - rc) >> 1),
|
||||
6 => rb + ((ra - rc) >> 1),
|
||||
_ => (ra + rb) >> 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode the pixels `x0..x1` (`x0 >= 1`) of one row with the fixed predictor `P`. `prev` is empty on the first
|
||||
/// line, which only predicts from the left (`P = 1`).
|
||||
#[inline(always)]
|
||||
fn decode_run<const P: u8>(
|
||||
tables: &[&Huffman],
|
||||
br: &mut BitReader<'_>,
|
||||
cur: &mut [u16],
|
||||
prev: &[u16],
|
||||
nc: usize,
|
||||
x0: usize,
|
||||
x1: usize,
|
||||
) -> Result<(), RawError> {
|
||||
for x in x0..x1 {
|
||||
for (c, table) in tables.iter().enumerate() {
|
||||
let i = x * nc + c;
|
||||
let ra = cur[i - nc] as i32;
|
||||
let p = if P <= 1 { predict(P, ra, 0, 0) } else { predict(P, ra, prev[i] as i32, prev[i - nc] as i32) };
|
||||
cur[i] = ((p + table.decode_diff(br, false)?) & 0xffff) as u16;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decode a lossless JPEG stream. `max_samples` bounds the allocation (hostile headers).
|
||||
pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
|
||||
let h = parse_header(d)?;
|
||||
@@ -680,18 +761,23 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
|
||||
let (prev_rows, cur_rows) = out.split_at_mut(y * row_len);
|
||||
let cur = &mut cur_rows[..row_len];
|
||||
let prev = if y > 0 { &prev_rows[(y - 1) * row_len..] } else { &[][..] };
|
||||
for x in 0..w {
|
||||
if h.restart > 0 {
|
||||
if restart_left == 0 {
|
||||
if !br.restart() {
|
||||
return Err(err("missing restart marker"));
|
||||
}
|
||||
restart_left = h.restart;
|
||||
first_line = true;
|
||||
reset_pending = true;
|
||||
let mut x = 0;
|
||||
while x < w {
|
||||
if h.restart > 0 && restart_left == 0 {
|
||||
if !br.restart() {
|
||||
return Err(err("missing restart marker"));
|
||||
}
|
||||
restart_left -= 1;
|
||||
restart_left = h.restart;
|
||||
first_line = true;
|
||||
reset_pending = true;
|
||||
}
|
||||
// pixels up to the next restart marker or the end of the row
|
||||
let mut end = w;
|
||||
if h.restart > 0 {
|
||||
end = w.min(x + restart_left);
|
||||
restart_left -= end - x;
|
||||
}
|
||||
// the first pixel of the run has the special predictions (start of the interval, first column)
|
||||
#[allow(clippy::needless_range_loop)] // `c` indexes the tables and the sample position alike
|
||||
for c in 0..nc {
|
||||
let i = x * nc + c;
|
||||
@@ -702,24 +788,28 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
|
||||
} else if x == 0 || pred == 0 {
|
||||
if pred == 0 { 0 } else { prev[i] as i32 }
|
||||
} else {
|
||||
let ra = cur[i - nc] as i32;
|
||||
let rb = prev[i] as i32;
|
||||
let rc = prev[i - nc] as i32;
|
||||
match pred {
|
||||
1 => ra,
|
||||
2 => rb,
|
||||
3 => rc,
|
||||
4 => ra + rb - rc,
|
||||
5 => ra + ((rb - rc) >> 1),
|
||||
6 => rb + ((ra - rc) >> 1),
|
||||
_ => (ra + rb) >> 1,
|
||||
}
|
||||
predict(pred, cur[i - nc] as i32, prev[i] as i32, prev[i - nc] as i32)
|
||||
};
|
||||
let ssss = tables[c].decode(&mut br)?;
|
||||
let diff = diff_value(&mut br, ssss);
|
||||
cur[i] = ((p + diff) & mask) as u16;
|
||||
cur[i] = ((p + tables[c].decode_diff(&mut br, false)?) & mask) as u16;
|
||||
}
|
||||
reset_pending = false;
|
||||
// the rest of the run: the predictor is fixed, so it is chosen once here, not per sample
|
||||
if first_line {
|
||||
// the first line of an interval predicts from the left neighbour
|
||||
decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?;
|
||||
} else {
|
||||
match pred {
|
||||
0 => decode_run::<0>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
1 => decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
2 => decode_run::<2>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
3 => decode_run::<3>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
4 => decode_run::<4>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
5 => decode_run::<5>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
6 => decode_run::<6>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
_ => decode_run::<7>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
}
|
||||
}
|
||||
x = end;
|
||||
}
|
||||
first_line = false;
|
||||
if br.overrun > 64 * 8 {
|
||||
|
||||
@@ -125,6 +125,121 @@ fn roundtrip_extremes_and_restarts() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-label an encoded stream as `precision` bits with point transform `pt`: the encoder codes the samples at
|
||||
/// `precision - pt` bits, which is what a transformed scan holds.
|
||||
fn with_point_transform(mut enc: Vec<u8>, precision: u8, pt: u8) -> Vec<u8> {
|
||||
enc[6] = precision; // SOF3 precision follows SOI, marker and length
|
||||
let sos = enc.windows(2).position(|w| w == [0xff, 0xda]).unwrap();
|
||||
let seg_len = u16::from_be_bytes([enc[sos + 2], enc[sos + 3]]) as usize;
|
||||
enc[sos + 2 + seg_len - 1] = pt;
|
||||
enc
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_point_transform_all_predictors() {
|
||||
for pred in 1..=7u8 {
|
||||
for comps in [1usize, 2, 3] {
|
||||
for (bits, pt) in [(12u8, 1u8), (14, 2), (16, 4), (8, 3)] {
|
||||
let (w, h) = (11, 6);
|
||||
let data = noise(w * h * comps, (bits - pt) as u32, pred as u64 * 17 + comps as u64);
|
||||
let enc = with_point_transform(encode(&data, w, h, comps, bits - pt, pred, 0), bits, pt);
|
||||
let f = decode(&enc, usize::MAX).unwrap();
|
||||
assert_eq!((f.precision, f.point_transform, f.predictor), (bits, pt, pred));
|
||||
let want: Vec<u16> = data.iter().map(|&v| v << pt).collect();
|
||||
assert_eq!(f.data, want, "pred {pred} comps {comps} bits {bits} pt {pt}");
|
||||
}
|
||||
}
|
||||
}
|
||||
// with restart intervals that end mid-row
|
||||
let data = noise(20 * 9 * 2, 10, 9);
|
||||
for rst in [3usize, 7, 20] {
|
||||
let enc = with_point_transform(encode(&data, 20, 9, 2, 10, 4, rst), 12, 2);
|
||||
let want: Vec<u16> = data.iter().map(|&v| v << 2).collect();
|
||||
assert_eq!(decode(&enc, usize::MAX).unwrap().data, want, "restart {rst}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_skewed_categories_with_long_codes() {
|
||||
// category k with probability 2^-(k+1): the table gets codes longer than the lookup, rare 16-bit differences
|
||||
let mut s = 0x9e37_79b9_7f4a_7c15u64;
|
||||
let mut last = 0u32;
|
||||
let data: Vec<u16> = (0..256 * 256)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
let r = (s >> 32) as u32;
|
||||
let cat = r.trailing_zeros().min(16);
|
||||
let mag = (((s >> 8) as u32) & ((1u32 << cat) - 1)) | (1 << cat) >> 1;
|
||||
let d = if s & 1 == 0 { mag } else { 0u32.wrapping_sub(mag) };
|
||||
last = last.wrapping_add(d) & 0xffff;
|
||||
last as u16
|
||||
})
|
||||
.collect();
|
||||
for pred in [1u8, 4, 7] {
|
||||
let enc = encode(&data, 256, 256, 1, 16, pred, 0);
|
||||
assert_eq!(decode(&enc, usize::MAX).unwrap().data, data, "pred {pred}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical decoding from the code lengths alone, bit by bit (T.81 Annex F, Figure F.16): the reference for the
|
||||
/// lookup tables.
|
||||
fn canonical_symbol(br: &mut BitReader, lens: &[u8]) -> Option<u8> {
|
||||
let window = br.peek(16);
|
||||
let (mut first, mut index) = (0u32, 0usize);
|
||||
for len in 1..=16u8 {
|
||||
let code = window >> (16 - len);
|
||||
let n = lens.iter().filter(|&&l| l == len).count() as u32;
|
||||
if code < first + n {
|
||||
br.consume(len as u32);
|
||||
return Some((index + (code - first) as usize) as u8);
|
||||
}
|
||||
index += n as usize;
|
||||
first = (first + n) << 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// `decode_diff` against canonical decoding and `diff_value`, for tables with codes inside and beyond the lookup
|
||||
/// and with `len + category` equal to the lookup width.
|
||||
#[test]
|
||||
fn decode_diff_matches_canonical_decoding() {
|
||||
let tables: [[u8; 17]; 3] = [
|
||||
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12],
|
||||
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 16],
|
||||
[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12],
|
||||
];
|
||||
for lens in tables {
|
||||
let mut counts = [0u8; 16];
|
||||
for &l in &lens {
|
||||
counts[l as usize - 1] += 1;
|
||||
}
|
||||
// symbols in code order: by length, ties by symbol
|
||||
let mut values: Vec<u8> = (0..17).collect();
|
||||
values.sort_by_key(|&v| (lens[v as usize], v));
|
||||
let sorted: Vec<u8> = values.iter().map(|&v| lens[v as usize]).collect();
|
||||
let t = Huffman::new(&counts, &values).unwrap();
|
||||
let mut s = 77u64;
|
||||
let src: Vec<u8> = (0..6000)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
((s >> 40) as u8).min(0xfe)
|
||||
})
|
||||
.collect();
|
||||
let (mut a, mut b) = (BitReader::new(&src), BitReader::new(&src));
|
||||
for n in 0..8000 {
|
||||
let want = canonical_symbol(&mut a, &sorted).map(|i| diff_value(&mut a, values[i as usize]));
|
||||
let got = t.decode_diff(&mut b, false);
|
||||
assert_eq!(got.ok(), want, "symbol {n}");
|
||||
assert_eq!(a.peek(32), b.peek(32), "symbol {n}");
|
||||
if want.is_none() {
|
||||
// step past the invalid code so the walk goes on
|
||||
a.consume(1);
|
||||
b.consume(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constant_image_single_symbol_table() {
|
||||
let data = vec![1234u16; 16 * 4];
|
||||
|
||||
@@ -46,7 +46,7 @@ assert_eq!((rows, sheet.map(|s| s.children.len())), (3, Ok(0)));
|
||||
| `max_input` | 256 MiB | `TooLarge` |
|
||||
| `max_depth` | 256 levels (the root is 1) | `TooDeep` |
|
||||
| `max_nodes` | 2,000,000 nodes kept in the tree: elements, text stretches, kept CDATA, instructions and comments | `TooManyNodes` |
|
||||
| `max_attrs` | 1024 on one start tag, namespace declarations and malformed ones included; with canonical names, also the declarations in scope at once | `TooManyAttrs` |
|
||||
| `max_attrs` | 1024 on one start tag, namespace declarations and malformed ones included; with canonical names or strict reads, also the declarations in scope at once | `TooManyAttrs` |
|
||||
| `max_attrs_total` | 4,000,000 attributes kept in the tree, over all its elements | `TooManyAttrsTotal` |
|
||||
|
||||
A limit exceeded is an error, never a panic and never a truncated tree. `parse_streaming_partial`
|
||||
@@ -73,9 +73,9 @@ A lenient read keeps what a damaged document had:
|
||||
- reading stops when the root element ends: nothing after it is looked at;
|
||||
- a malformed attribute is skipped; bytes that are not UTF-8 become U+FFFD.
|
||||
|
||||
A strict read makes each of those an error, and also a second root element and text outside the
|
||||
A strict read rejects duplicate raw or expanded attribute names (while allowing undeclared prefixes in fragments). It makes each of those an error, and also a second root element and text outside the
|
||||
root, in the pull reader as in the tree. Markup the tokenizer cannot read (`<a b='1`, an unterminated comment) is an error either
|
||||
way. A repeated attribute is kept each time in both.
|
||||
way. A lenient read keeps repeated attributes.
|
||||
|
||||
### Text
|
||||
|
||||
|
||||
+53
-2
@@ -1,6 +1,7 @@
|
||||
//! Reading: the pull reader over `quick-xml`, the tree builder on top of it, and streaming.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::BTreeSet;
|
||||
use std::ops::Range;
|
||||
|
||||
use quick_xml::Reader;
|
||||
@@ -12,6 +13,10 @@ use crate::write::is_xml_char;
|
||||
|
||||
/// The namespace the `xml` prefix is always bound to.
|
||||
pub const XML_NS: &str = "http://www.w3.org/XML/1998/namespace";
|
||||
/// [`Events::scope_ids`] id of the XML namespace, bound to `xml:` without a declaration.
|
||||
const XML_NS_ID: usize = usize::MAX;
|
||||
/// Expanded-name id of an attribute in no namespace (or with an undeclared prefix).
|
||||
const NO_NS_ID: usize = usize::MAX - 1;
|
||||
|
||||
/// What the reader refuses to exceed.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -469,6 +474,10 @@ pub struct Events<'o, 'i> {
|
||||
end_names: bool,
|
||||
open: Vec<Open>,
|
||||
scope: Vec<(String, String)>,
|
||||
/// For each binding in `scope`, an id shared by every binding of an equal URI (the index of
|
||||
/// the first such binding, or [`XML_NS_ID`]), with the URI's hash. Strict reads compare
|
||||
/// expanded attribute names by this id instead of copying the URI for each attribute.
|
||||
scope_ids: Vec<(u64, usize)>,
|
||||
/// The end of an empty element, due after its start.
|
||||
pending: bool,
|
||||
/// Whether a root element has started.
|
||||
@@ -500,7 +509,7 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
cfg.trim_text_start = false;
|
||||
cfg.trim_text_end = false;
|
||||
cfg.expand_empty_elements = false;
|
||||
Events { reader, opts, end_names, open: Vec::new(), scope: Vec::new(), pending: false, rooted: false, span: 0..0 }
|
||||
Events { reader, opts, end_names, open: Vec::new(), scope: Vec::new(), scope_ids: Vec::new(), pending: false, rooted: false, span: 0..0 }
|
||||
}
|
||||
|
||||
/// The bytes of the input the last event was read from. Both events of an empty element
|
||||
@@ -614,6 +623,7 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
fn end(&mut self) -> Event {
|
||||
let (scope, name) = self.open.pop().map_or((0, String::new()), |o| (o.scope, o.name));
|
||||
self.scope.truncate(scope);
|
||||
self.scope_ids.truncate(scope);
|
||||
Event::End { name }
|
||||
}
|
||||
|
||||
@@ -669,6 +679,7 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
if is_ns_decl(key) {
|
||||
let prefix = self.decode(key.get(6..).unwrap_or_default())?.into_owned();
|
||||
let uri = self.attr_value(&a.value)?;
|
||||
self.scope_ids.push(self.uri_id(&uri));
|
||||
self.scope.push((prefix, uri));
|
||||
}
|
||||
}
|
||||
@@ -680,6 +691,20 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The id an expanded name uses for `uri`: one hash and at most one full comparison per
|
||||
/// declaration, so a long URI costs its length once however many attributes use it.
|
||||
fn uri_id(&self, uri: &str) -> (u64, usize) {
|
||||
use std::hash::{Hash, Hasher};
|
||||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
||||
uri.hash(&mut h);
|
||||
let hash = h.finish();
|
||||
if uri == XML_NS {
|
||||
return (hash, XML_NS_ID);
|
||||
}
|
||||
let first = self.scope_ids.iter().zip(&self.scope).position(|((h, _), (_, u))| *h == hash && u == uri);
|
||||
(hash, first.map_or(self.scope.len(), |i| self.scope_ids.get(i).map_or(i, |(_, id)| *id)))
|
||||
}
|
||||
|
||||
/// In a strict read: is the reader before or after the root element?
|
||||
fn outside_root(&self) -> bool {
|
||||
self.opts.strict && self.open.is_empty()
|
||||
@@ -699,7 +724,7 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
_ => None,
|
||||
};
|
||||
let scope = self.scope.len();
|
||||
if canonical.is_some() {
|
||||
if canonical.is_some() || self.opts.strict {
|
||||
self.declare(e)?;
|
||||
}
|
||||
let raw = e.name();
|
||||
@@ -707,6 +732,8 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
let mut out = Vec::new();
|
||||
let mut attrs = e.attributes();
|
||||
attrs.with_checks(false);
|
||||
let mut raw_names = BTreeSet::new();
|
||||
let mut expanded_names = BTreeSet::new();
|
||||
for (seen, a) in attrs.enumerate() {
|
||||
if seen >= limits.max_attrs {
|
||||
return Err(Error::TooManyAttrs(limits.max_attrs));
|
||||
@@ -718,6 +745,30 @@ impl<'o, 'i> Events<'o, 'i> {
|
||||
};
|
||||
let key = a.key.as_ref();
|
||||
let decl = is_ns_decl(key);
|
||||
if self.opts.strict {
|
||||
let raw_key = self.decode(key)?;
|
||||
if !raw_names.insert(raw_key.to_string()) {
|
||||
return self.syntax(format!("duplicate attribute `{raw_key}`"));
|
||||
}
|
||||
if !decl {
|
||||
// Default namespaces do not apply to attributes. Keep undeclared prefixes
|
||||
// distinct so standalone fragments retain the reader's existing behaviour.
|
||||
let expanded = match raw_key.split_once(':') {
|
||||
Some(("xml", local)) => (XML_NS_ID, local),
|
||||
Some((prefix, local)) => self
|
||||
.scope
|
||||
.iter()
|
||||
.zip(&self.scope_ids)
|
||||
.rev()
|
||||
.find(|((p, _), _)| p == prefix)
|
||||
.map_or((NO_NS_ID, raw_key.as_ref()), |(_, (_, id))| (*id, local)),
|
||||
None => (NO_NS_ID, raw_key.as_ref()),
|
||||
};
|
||||
if !expanded_names.insert((expanded.0, expanded.1.to_string())) {
|
||||
return self.syntax(format!("duplicate expanded attribute `{raw_key}`"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if decl && !self.opts.ns_decls {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -117,7 +117,11 @@ fn an_attribute_flood_is_refused_at_the_limit() {
|
||||
let opts = opts.with_limits(limits);
|
||||
for (k, kind) in kinds.iter().enumerate() {
|
||||
let _ = opts.parse(tag(16, kind).as_bytes());
|
||||
assert_eq!(opts.parse(tag(17, kind).as_bytes()).err().map(|e| e.is_limit() || (opts.strict && k == 3)), Some(true), "kind {k}");
|
||||
assert_eq!(
|
||||
opts.parse(tag(17, kind).as_bytes()).err().map(|e| e.is_limit() || (opts.strict && (k == 1 || k == 3))),
|
||||
Some(true),
|
||||
"kind {k}"
|
||||
);
|
||||
assert!(opts.first_start(tag(17, kind).as_bytes(), "a").is_none());
|
||||
}
|
||||
assert_eq!(opts.parse(tag(17, kinds[0]).as_bytes()), Err(Error::TooManyAttrs(16)));
|
||||
@@ -524,3 +528,41 @@ fn a_strict_pull_reader_refuses_what_is_outside_the_root() {
|
||||
// white space, comments and instructions may stand outside the root
|
||||
assert!(read(strict, b"<?xml version='1.0'?>\r\n <!-- c -->\t<a>x</a>\n<?pi?> ").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_attributes_are_unique_before_name_rewriting() {
|
||||
use craft_xml::{Names, Options};
|
||||
for names in [Names::AsWritten, Names::Local] {
|
||||
let strict = Options { names, ..Options::default() }.strict();
|
||||
for xml in [
|
||||
"<r a='1' a='2'/>",
|
||||
"<r xmlns:p='urn:x' xmlns:q='urn:x' p:a='1' q:a='2'/>",
|
||||
"<r xml:a='1' xmlns:p='http://www.w3.org/XML/1998/namespace' p:a='2'/>",
|
||||
"<r xmlns:p='urn:x'><c xmlns:q='urn:x' p:a='1' q:a='2'/></r>",
|
||||
] {
|
||||
assert!(strict.parse(xml.as_bytes()).is_err(), "{xml}");
|
||||
let mut events = strict.events(xml.as_bytes());
|
||||
assert!(
|
||||
loop {
|
||||
match events.next() {
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break false,
|
||||
Err(_) => break true,
|
||||
}
|
||||
},
|
||||
"{xml}"
|
||||
);
|
||||
}
|
||||
for xml in [
|
||||
"<r xmlns='urn:x' xmlns:p='urn:x' a='1' p:a='2'/>",
|
||||
"<r p:a='1' q:a='2'/>",
|
||||
"<r xmlns:p='urn:x'><c xmlns:p='urn:y' xmlns:q='urn:x' p:a='1' q:a='2'/></r>",
|
||||
] {
|
||||
assert!(strict.parse(xml.as_bytes()).is_ok(), "{xml}");
|
||||
}
|
||||
assert!(Options { names, ..Options::default() }.parse(b"<r a='1' a='2'/>").is_ok());
|
||||
}
|
||||
let strict = Options { limits: craft_xml::Limits { max_attrs: 2, ..craft_xml::Limits::DEFAULT }, ..Options::default() }.strict();
|
||||
assert!(strict.parse(b"<r xmlns:a='a' xmlns:b='b'><c xmlns:c='c'/></r>").is_err());
|
||||
assert!(strict.parse(b"<r a='1' b='2' c='3'/>").is_err());
|
||||
}
|
||||
Reference in new issue
Block a user