diff --git a/Cargo.lock b/Cargo.lock index 95341b06d..eddc22221 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1903,6 +1903,10 @@ dependencies = [ "proptest", ] +[[package]] +name = "craft-http-url" +version = "0.1.0" + [[package]] name = "craft-i18n" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 9558946f5..e58ac33cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ rust-version = "1.95" repository = "https://github.com/storytold/craft-libs" [workspace.dependencies] +craft-http-url = { path = "crates/http-url" } craft-usd = { path = "crates/usd" } craft-smime = { path = "crates/smime" } craft-ber = { path = "crates/ber" } diff --git a/crates/cms/src/lutfile.rs b/crates/cms/src/lutfile.rs index b7cc99bf3..e2e0a5980 100644 --- a/crates/cms/src/lutfile.rs +++ b/crates/cms/src/lutfile.rs @@ -150,62 +150,142 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> { } } -/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table. -pub fn parse_cube(text: &str) -> Result { - let mut title = String::new(); - let (mut size3, mut size1) = (0usize, 0usize); - let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]); - let mut rows: Vec = Vec::new(); - for line in text.lines() { - // Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an - // unquoted title's comment must not turn that comment into part of the title. - if let Some(quoted) = line - .trim_start() - .strip_prefix("TITLE") - .filter(|rest| rest.starts_with(char::is_whitespace)) - .and_then(|rest| rest.trim_start().strip_prefix('"')) +/// A bounded, three-dimensional Cube profile with RGB rows in file order. +/// Unlike [`parse_cube`], this preserves the optional title and does not resample 1D curves. +#[derive(Clone, Debug, PartialEq)] +pub struct Cube3d { + pub title: Option, + pub size: usize, + pub data: Vec<[f32; 3]>, + pub domain_min: [f32; 3], + pub domain_max: [f32; 3], +} + +/// Read the simple 3D profile dialect used by camera-profile consumers. +/// +/// `max_size` must be in 2..=256; the row budget is checked before each allocation. +/// Whole-line comments and literal titles are preserved. Unknown alphabetic directives +/// are ignored; 1D tables are refused. Finite domains are returned unchanged so the +/// consumer retains its existing policy for equal or reversed endpoints. +/// Numeric fields must consist of exactly three finite numbers (never silently skip tokens). +pub fn parse_cube_3d(text: &str, max_size: usize) -> Result { + let p = read_cube(text, Some(max_size))?; + if p.size3 == 0 { + return err("not a 3D .cube LUT (no LUT_3D_SIZE)"); + } + let expected = p.size3.checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?; + if p.rows.len() != expected { + return err(format!("expected {expected} entries, found {}", p.rows.len())); + } + Ok(Cube3d { title: p.title, size: p.size3, data: p.rows, domain_min: p.min, domain_max: p.max }) +} + +struct CubeRows { + title: Option, + size3: usize, + size1: usize, + min: [f32; 3], + max: [f32; 3], + rows: Vec<[f32; 3]>, +} + +fn cube_triplet(text: &str) -> Result<[f32; 3], LutError> { + let mut it = text.split_whitespace(); + let mut row = [0.0; 3]; + for value in &mut row { + *value = it.next().and_then(|s| s.parse::().ok()).filter(|v| v.is_finite()).ok_or_else(|| LutError(format!("bad line `{text}`")))?; + } + if it.next().is_some() { + return err(format!("bad line `{text}`")); + } + Ok(row) +} + +fn read_cube(text: &str, profile_max: Option) -> Result { + if profile_max.is_some_and(|n| !(2..=256).contains(&n)) { + return err("3D profile size limit must be in 2..=256"); + } + let mut p = CubeRows { title: None, size3: 0, size1: 0, min: [0.0; 3], max: [1.0; 3], rows: Vec::new() }; + let row_limit = profile_max.unwrap_or(256).checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?; + for raw in text.lines() { + let profile = profile_max.is_some(); + let line = if profile { raw.trim() } else { raw.split('#').next().unwrap_or("").trim() }; + // The standard dialect protects '#' inside quoted titles; profile titles are literal. + if !profile + && let Some(quoted) = + raw.trim_start().strip_prefix("TITLE").filter(|r| r.starts_with(char::is_whitespace)).and_then(|r| r.trim_start().strip_prefix('"')) { - let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") }; - title = value.to_string(); + let Some((title, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") }; + p.title = Some(title.to_string()); continue; } - let line = line.split('#').next().unwrap_or("").trim(); - if line.is_empty() { + if line.is_empty() || line.starts_with('#') { continue; } - let mut it = line.split_whitespace(); - let key = it.next().unwrap_or(""); - let nums = |it: std::str::SplitWhitespace<'_>| -> Result<[f32; 3], LutError> { - let v: Vec = it.filter_map(|s| s.parse().ok()).collect(); - if v.len() == 3 { Ok([v[0], v[1], v[2]]) } else { err(format!("bad line `{line}`")) } + let (key, rest) = if profile { + ["TITLE", "LUT_3D_SIZE", "LUT_1D_SIZE", "DOMAIN_MIN", "DOMAIN_MAX"] + .into_iter() + .find_map(|key| line.strip_prefix(key).map(|rest| (key, rest.trim()))) + .unwrap_or_else(|| line.split_once(char::is_whitespace).unwrap_or((line, ""))) + } else { + line.split_once(char::is_whitespace).map_or((line, ""), |(key, rest)| (key, rest.trim())) }; match key { - "TITLE" => title = line[5..].trim().trim_matches('"').to_string(), - "LUT_3D_SIZE" => size3 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_3D_SIZE".into()))?, - "LUT_1D_SIZE" => size1 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?, - "DOMAIN_MIN" => dmin = nums(it)?, - "DOMAIN_MAX" => dmax = nums(it)?, - "LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" => { - let v: Vec = it.filter_map(|s| s.parse().ok()).collect(); - if v.len() == 2 { - dmin = [v[0]; 3]; - dmax = [v[1]; 3]; + "TITLE" => p.title = Some(rest.trim_matches('"').to_string()), + "LUT_3D_SIZE" => { + let value = if profile { rest } else { rest.split_whitespace().next().unwrap_or("") }; + p.size3 = value.parse().map_err(|_| LutError(format!("bad LUT_3D_SIZE `{value}`")))?; + let limit = profile_max.unwrap_or(MAX_SIZE); + if !(2..=limit).contains(&p.size3) { + return err(format!("LUT_3D_SIZE {} out of range", p.size3)); } } - k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => { - let mut v = vec![k.parse::().map_err(|_| LutError(format!("bad number in `{line}`")))?]; - for value in it { - v.push(value.parse::().map_err(|_| LutError(format!("bad number in `{line}`")))?); + "LUT_1D_SIZE" => { + if profile { + return err("1D LUTs are not supported (only 3D .cube files)"); } - if v.len() != 3 { - return err(format!("bad data line `{line}`")); + p.size1 = rest.split_whitespace().next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?; + if p.size1 > row_limit { + return err("1D LUT exceeds row budget"); + } + } + "DOMAIN_MIN" => p.min = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MIN".into()))?, + "DOMAIN_MAX" => p.max = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MAX".into()))?, + "LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" if !profile => { + let mut it = rest.split_whitespace(); + let min = it.next().and_then(|s| s.parse::().ok()); + let max = it.next().and_then(|s| s.parse::().ok()); + if let (Some(min), Some(max), None) = (min, max, it.next()) { + p.min = [min; 3]; + p.max = [max; 3]; + } + } + _ => { + let numeric = key.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.'); + let nonfinite = key.parse::().is_ok(); + if numeric || nonfinite || (profile && !key.starts_with(|c: char| c.is_ascii_alphabetic())) { + let row = cube_triplet(line)?; + if p.rows.len() >= row_limit { + return err("LUT exceeds row budget"); + } + // Explicit fallible growth prevents capacity growth past the configured budget. + if p.rows.len() == p.rows.capacity() { + let grow = (p.rows.capacity().max(1)).min(row_limit - p.rows.len()); + p.rows.try_reserve_exact(grow).map_err(|_| LutError("LUT allocation failed".into()))?; + } + p.rows.push(row); } - rows.extend(v); } - // Unknown keywords (e.g. Resolve's LUT_IN_VIDEO_RANGE) are ignored, as the spec allows. - _ => {} } } + Ok(p) +} + +/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table. +pub fn parse_cube(text: &str) -> Result { + let p = read_cube(text, None)?; + let (title, size3, size1, dmin, dmax) = (p.title.unwrap_or_default(), p.size3, p.size1, p.min, p.max); + let rows: Vec = p.rows.into_iter().flatten().collect(); let domain_ok = (0..3).all(|k| dmin[k] < dmax[k] && dmin[k].is_finite() && dmax[k].is_finite()); if (size3 > 0 || size1 >= 2) && !domain_ok { return err(format!("DOMAIN_MIN {:?} must lie below DOMAIN_MAX {:?}", dmin, dmax)); @@ -1386,3 +1466,52 @@ LUT_3D_SIZE 2 assert_eq!(back.data, [0.0, 0.25, 1.0].repeat(8)); } } + +#[cfg(test)] +mod profile_cube_tests { + use super::*; + + fn cube(prefix: &str) -> String { + format!("{prefix}\nLUT_3D_SIZE 2\n{}", "0.25 0.5 0.75\n".repeat(8)) + } + + #[test] + fn profile_preserves_optional_literal_title_and_domains() { + let no_title = parse_cube_3d(&cube(""), 256).unwrap(); + assert_eq!(no_title.title, None); + let value = parse_cube_3d(&cube("TITLE \"look # literal\"\nDOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1"), 256).unwrap(); + assert_eq!(value.title.as_deref(), Some("look # literal")); + assert_eq!(value.domain_min, [1.0, 0.0, 2.0]); + assert_eq!(value.domain_max, [0.0, 0.0, 1.0]); + assert_eq!(value.data, vec![[0.25, 0.5, 0.75]; 8]); + assert_eq!(parse_cube_3d(&cube("TITLE \"\""), 256).unwrap().title.as_deref(), Some("")); + assert!(parse_cube(&cube("DOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1")).is_err()); + } + + #[test] + fn profile_limits_are_checked_without_large_allocations() { + assert!(parse_cube_3d("LUT_3D_SIZE 256", 256).unwrap_err().0.contains("16777216")); + assert!(parse_cube_3d("LUT_3D_SIZE 257", 256).unwrap_err().0.contains("out of range")); + assert!(parse_cube_3d("LUT_3D_SIZE 256", 129).unwrap_err().0.contains("out of range")); + assert!(parse_cube_3d(&cube(""), usize::MAX).is_err()); + assert!(parse_cube_3d(&cube(""), 1).is_err()); + assert!(parse_cube_3d(&format!("{}0 0 0\n", cube("")), 2).unwrap_err().0.contains("budget")); + assert!(parse_cube_3d("LUT_1D_SIZE 2\n0 0 0\n1 1 1", 256).is_err()); + } + + #[test] + fn every_numeric_token_must_be_valid_and_finite() { + for row in ["0 ignored 0 0", "0 0 0 ignored", "NaN 0 0", "inf 0 0", "0 0 -inf", "1e39 0 0", "0 0", "0 0 0 0"] { + assert!(parse_cube_3d(&format!("LUT_3D_SIZE 2\n{row}\n{}", "0 0 0\n".repeat(7)), 256).is_err(), "{row}"); + assert!(parse_cube_3d(&cube(&format!("DOMAIN_MIN {row}")), 256).is_err(), "{row}"); + } + } + + #[test] + fn standard_reader_keeps_quoted_comments_and_1d_expansion() { + assert_eq!(parse_cube(&cube("TITLE \"look # preserved\" # comment")).unwrap().title, "look # preserved"); + let value = parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1").unwrap(); + assert_eq!(value.size, 33); + assert!(parse_cube_3d(&cube("DOMAIN_MIN 0 0 0 # not a whole-line comment"), 256).is_err()); + } +} diff --git a/crates/http-url/Cargo.toml b/crates/http-url/Cargo.toml new file mode 100644 index 000000000..f23d9bbc7 --- /dev/null +++ b/crates/http-url/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "craft-http-url" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license = "Apache-2.0" +repository.workspace = true +description = "Bounded HTTP URL parsing and redirect resolution without network access" + +[lints] +workspace = true diff --git a/crates/http-url/README.md b/crates/http-url/README.md new file mode 100644 index 000000000..cc3dc1bf5 --- /dev/null +++ b/crates/http-url/README.md @@ -0,0 +1,25 @@ +# craft-http-url + +HTTP/HTTPS URL parsing, dot-segment removal, relative redirects, HTTPS downgrade rejection, +and query-redacted display. No dependencies or network operations. Parsing rejects inputs +larger than 16 KiB and redirects are bounded before concatenation. + +This is not an SSRF filter, DNS resolver or HTTP client. Local/private hosts are allowed. +Applications keep destination permission, authentication, TLS, request/response limits and +redirect-count policy. Public fields preserve the original API; manually constructed values +must not be treated as validated URLs. + +## Attribution + +Extracted from LightCraft `crates/fetch/src/http.rs` at +`2dcb258ba40738230a07484a95eb27dc84ed66fe` (Apache-2.0). +Original parser: Daniel Voshart, `7fa7506db46d69d609e85e39b338f33786ed0293`. +HTTPS downgrade protection: MarekSpodBiedry, `4a91a1c218bf3d0ac49cc73ebb60621a21ae55e2`. +URL normalization, scheme, fragment and authority fixes: NetterSilver, including +`edc369565eef92161b56b8f1e39ef53629c05489`, +`78f0e6e650444886069f4d4ae4e6593f221b98d4`, +`cee93496587d31c514aaeced3e6b8027cfdb41a0`, +`dee5f7910ad8c24cf28bab528f8d007550af2b23`, +`c3a5d9b7f1f837dc14c01942e3d0293ecbf36345` and +`13849307a9343504339d68b086f1a32cdcbbb457`. +Source-author commits are retained separately in this extraction. diff --git a/crates/http-url/src/lib.rs b/crates/http-url/src/lib.rs new file mode 100644 index 000000000..1e3e5875f --- /dev/null +++ b/crates/http-url/src/lib.rs @@ -0,0 +1,277 @@ +//! Bounded HTTP/HTTPS URL parsing and redirect resolution, extracted from LightCraft. +//! +//! This is syntax handling, not a network permission or SSRF policy. It permits local/private +//! hosts and performs no DNS, socket or TLS operations. Callers own destination allowlists, +//! credential forwarding, redirect counts, connection deadlines and response limits. +//! Only parsed values have validated fields; constructing public fields bypasses parsing. + +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +/// A URL or redirect cannot be represented by this bounded HTTP URL syntax. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UrlError(pub String); + +impl std::fmt::Display for UrlError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} +impl std::error::Error for UrlError {} + +/// A parsed `http://` or `https://` URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Url { + pub tls: bool, + pub host: String, + pub port: u16, + /// Path and query, starting with `/`. + pub path: String, +} + +/// Remove `.` and `..` segments from the path part of `path` (RFC 3986 section 5.2.4); the query is kept as is. +fn remove_dot_segments(path: &str) -> String { + let (p, query) = match path.find('?') { + Some(i) => (path.get(..i).unwrap_or_default(), path.get(i..).unwrap_or_default()), + None => (path, ""), + }; + let mut out: Vec<&str> = Vec::new(); + let mut segments = p.split('/').skip(1).peekable(); + let mut trailing_slash = false; + while let Some(seg) = segments.next() { + let last = segments.peek().is_none(); + match seg { + "." => trailing_slash = last, + ".." => { + out.pop(); + trailing_slash = last; + } + _ => { + out.push(seg); + trailing_slash = false; + } + } + } + let mut result = format!("/{}", out.join("/")); + if trailing_slash && !result.ends_with('/') { + result.push('/'); + } + result.push_str(query); + result +} + +/// `s` without the leading `prefix`, matched ASCII case-insensitively (RFC 3986 §3.1). +fn strip_scheme<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { + let head = s.get(..prefix.len())?; + head.eq_ignore_ascii_case(prefix).then(|| s.get(prefix.len()..).unwrap_or_default()) +} + +impl Url { + pub fn parse(s: &str) -> Result { + let bad = |why: &str| UrlError(why.to_string()); + if s.len() > 16 * 1024 { + return Err(bad("URL exceeds 16 KiB")); + } + let s = s.trim(); + if s.chars().any(|c| c.is_control() || c == ' ') { + return Err(bad("contains spaces or control characters")); + } + let (tls, rest) = if let Some(r) = strip_scheme(s, "https://") { + (true, r) + } else if let Some(r) = strip_scheme(s, "http://") { + (false, r) + } else { + return Err(bad("only http:// and https:// are supported")); + }; + let rest = rest.split('#').next().unwrap_or_default(); + let (authority, path) = match rest.find(['/', '?']) { + Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()), + None => (rest, ""), + }; + let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") }; + let path = remove_dot_segments(&path); + if authority.contains('@') { + return Err(bad("user names and passwords in URLs are not supported")); + } + let (host, port) = if let Some(v6) = authority.strip_prefix('[') { + let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?; + let host = v6.get(..end).unwrap_or_default(); + let after = v6.get(end + 1..).unwrap_or_default(); + if !after.is_empty() && !after.starts_with(':') { + return Err(bad("unexpected text after IPv6 address")); + } + (host.to_string(), after.strip_prefix(':')) + } else { + match authority.rsplit_once(':') { + Some((h, p)) => (h.to_string(), Some(p)), + None => (authority.to_string(), None), + } + }; + let port = match port { + Some(p) if p.bytes().all(|b| b.is_ascii_digit()) => p.parse::().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?, + Some(_) => return Err(bad("bad port")), + None if tls => 443, + None => 80, + }; + if host.is_empty() || !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | ':' | '_')) { + return Err(bad("bad host name")); + } + Ok(Url { tls, host: host.to_ascii_lowercase(), port, path }) + } + + /// `Location` of a redirect, resolved against this URL. + pub fn join(&self, location: &str) -> Result { + // Reject before concatenating a hostile Location with the base URL. + if location.len() > 16 * 1024 || self.host.len().saturating_add(self.path.len()) > 16 * 1024 { + return Err(UrlError("URL exceeds 16 KiB".into())); + } + let l = location.trim(); + if strip_scheme(l, "http://").is_some() || strip_scheme(l, "https://").is_some() { + let url = Url::parse(l)?; + if self.tls && !url.tls { + return Err(UrlError("redirect from https to http refused".into())); + } + return Ok(url); + } + // another scheme (`ftp:`, `file:`, `data:`…) + if let Some((scheme, _)) = l.split_once(':') + && !scheme.is_empty() + && scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + { + return Err(UrlError(format!("redirect to an unsupported location `{scheme}:`"))); + } + let scheme = if self.tls { "https" } else { "http" }; + let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() }; + let base = format!("{scheme}://{host}:{}", self.port); + if let Some(rest) = l.strip_prefix("//") { + return Url::parse(&format!("{scheme}://{rest}")); + } + if l.starts_with('/') { + return Url::parse(&format!("{base}{l}")); + } + // empty or fragment-only: the same document + if l.is_empty() || l.starts_with('#') { + return Ok(self.clone()); + } + // relative to the current directory + let dir = self.path.split('?').next().unwrap_or("/"); + // a query-only reference keeps the whole path, file name included (RFC 3986 §5.2.2) + if l.starts_with('?') { + return Url::parse(&format!("{base}{dir}{l}")); + } + let dir = dir.rsplit_once('/').map_or("", |(d, _)| d); + Url::parse(&format!("{base}{dir}/{l}")) + } + + /// Authority formatted for an HTTP Host field. + pub fn host_header(&self) -> String { + let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() }; + let default = if self.tls { 443 } else { 80 }; + if self.port == default { host } else { format!("{host}:{}", self.port) } + } + + /// The URL without its query (for messages: a query may hold a signed token). + pub fn display(&self) -> String { + let scheme = if self.tls { "https" } else { "http" }; + format!("{scheme}://{}{}", self.host_header(), self.path.split('?').next().unwrap_or("/")) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::{Url, UrlError}; + + #[test] + fn fragment_only_or_empty_location_keeps_file_name_and_query() { + let base = Url::parse("https://a.example/x/y/file?q").unwrap(); + for l in ["#top", "", " "] { + let u = base.join(l).unwrap(); + assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/x/y/file?q"), "{l:?}"); + } + } + + #[test] + fn rejects_oversized_and_control_character_urls() { + let base = Url::parse("https://example.com/path").unwrap(); + assert!(base.join(&format!("#{}", "x".repeat(16 * 1024))).is_err()); + assert!(base.join(&"x".repeat(16 * 1024 + 1)).is_err()); + assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err()); + assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err()); + } + + #[test] + fn join_query_only_keeps_file_name() { + let u = Url::parse("https://a.example/x/y/file?q=1").unwrap(); + assert_eq!(u.join("?page=2").unwrap().path, "/x/y/file?page=2"); + } + + #[test] + fn join_removes_dot_segments() { + let base = Url::parse("https://a.example/x/y/file").unwrap(); + for (loc, want) in + [("../z", "/x/z"), ("./z", "/x/y/z"), ("../../../z", "/z"), ("/a/./b/../c", "/a/c"), ("../z?q=1", "/x/z?q=1"), ("..", "/x/")] + { + assert_eq!(base.join(loc).unwrap().path, want, "{loc}"); + } + } + + #[test] + fn url_scheme_is_matched_case_insensitively() { + let u = Url::parse("HTTPS://A.example/z").unwrap(); + assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/z")); + assert!(Url::parse("Http://a.example/").is_ok()); + let j = Url::parse("https://a.example/start").unwrap().join("HTTPS://cdn.example/file").unwrap(); + assert_eq!((j.tls, j.host.as_str(), j.path.as_str()), (true, "cdn.example", "/file")); + } + + #[test] + fn parses_urls() { + let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap(); + assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "cdn.example.com", 443, "/models/sam3/model.safetensors?sig=1")); + assert_eq!(u.display(), "https://cdn.example.com/models/sam3/model.safetensors"); + let u = Url::parse("http://127.0.0.1:8080").unwrap(); + assert_eq!((u.tls, u.port, u.path.as_str()), (false, 8080, "/")); + let u = Url::parse("http://[::1]:9/a").unwrap(); + assert_eq!((u.host.as_str(), u.port), ("::1", 9)); + for bad in ["ftp://x/y", "https://", "https://a b/", "https://u:p@h/", "http://h:0/", "http://h:99999/", "https://h/\r\nX: y", "file:///etc"] + { + assert!(Url::parse(bad).is_err(), "{bad}"); + } + } + + #[test] + fn rejects_signed_ports() { + for bad in ["http://example.com:+80/x", "http://example.com:-80/x", "http://[::1]:+80/x"] { + assert!(Url::parse(bad).is_err(), "{bad}"); + } + assert!(Url::parse("http://a.test/").unwrap().join("http://example.com:+80/x").is_err()); + assert_eq!(Url::parse("http://example.com:80/x").unwrap().port, 80); + } + + #[test] + fn rejects_text_after_a_bracketed_ipv6_host() { + for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] { + assert!(matches!(Url::parse(bad), Err(UrlError(_))), "{bad}"); + } + let u = Url::parse("http://[::1]:8080/x").unwrap(); + assert_eq!((u.host.as_str(), u.port), ("::1", 8080)); + let u = Url::parse("http://[::1]/x").unwrap(); + assert_eq!((u.host.as_str(), u.port), ("::1", 80)); + } + + #[test] + fn resolves_redirects() { + let u = Url::parse("https://a.example/x/y/file?q").unwrap(); + assert_eq!(u.join("https://b.example/z").unwrap().host, "b.example"); + assert_eq!(u.join("/root").unwrap().path, "/root"); + assert_eq!(u.join("other").unwrap().path, "/x/y/other"); + assert_eq!(u.join("//c.example/p").unwrap().host, "c.example"); + assert!(u.join("ftp://nope").is_err()); + // no downgrade from https to http, however the redirect is written + assert!(u.join("http://a.example/z").is_err()); + let plain = Url::parse("http://127.0.0.1:8080/a").unwrap(); + assert!(!plain.join("http://127.0.0.1:9/b").unwrap().tls); + assert!(plain.join("https://b.example/z").unwrap().tls); + } +} diff --git a/crates/image-codecs/src/codecs/avif_decode.rs b/crates/image-codecs/src/codecs/avif_decode.rs index 382f6a474..79b213326 100644 --- a/crates/image-codecs/src/codecs/avif_decode.rs +++ b/crates/image-codecs/src/codecs/avif_decode.rs @@ -842,6 +842,10 @@ fn read_metadata(c: &Container) -> (Option>, Option) { /// Decodes the primary image. `keep_orientation` leaves the container's `clap`, `irot` and /// `imir` unapplied (the stored pixels), as for HEIF. pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result { + decode_with_depth(bytes, limits, keep_orientation).map(|(image, _)| image) +} + +pub(crate) fn decode_with_depth(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<(Image, u8)> { let c = parse(bytes)?; let id = c.primary; let (w, h) = item_size(&c, id)?; @@ -890,9 +894,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> R // PQ or HLG: linear light in 32-bit float, as for HEIF. if cicp.is_hdr() { limits.check(w, h, layout, SampleType::F32)?; - return crate::cicp::linearize(img, cicp); + return crate::cicp::linearize(img, cicp).map(|image| (image, bits)); } - Ok(img) + Ok((img, bits)) } #[cfg(test)] @@ -1301,6 +1305,9 @@ mod tests { let src = Image::from_u8(w as u32, h as u32, ChannelLayout::Rgb, card(w, h, 3)).unwrap(); let file = encode_avif_with_speed(&src, 95, 10).unwrap(); let img = crate::decode(&file).unwrap(); + let (with_depth, depth) = crate::decode_avif_with_depth(&file, &crate::DecodeOptions::default()).unwrap(); + assert_eq!(depth, 10); + assert_eq!(with_depth, img); assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (48, 32, ChannelLayout::Rgb, SampleType::U16)); let diff = mean_abs_diff(&rgb8(&img), &card(w, h, 3)); assert!(diff < 2.5, "mean difference {diff}"); diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index e918fafda..b3016c8f1 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -121,6 +121,14 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res finish_format(format, bytes, img, opts) } +/// Decode AVIF pixels and report their original AV1 bit depth (8, 10 or 12). +/// The returned image may use a wider sample type, particularly for linear HDR. +#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))] +pub fn decode_avif_with_depth(bytes: &[u8], opts: &DecodeOptions) -> Result<(Image, u8), CodecError> { + let (image, depth) = codecs::avif_decode::decode_with_depth(bytes, &opts.limits, opts.keep_orientation)?; + Ok((finish_format(Format::Avif, bytes, image, opts)?, depth)) +} + /// What [`decode_as_with`] does with a PNG decoded in steps by [`PngDecodeRows`] (`img` is its /// output): the resolution from the metadata, the upright orientation and the final limit check, /// so the result equals [`decode_as_with`]'s on the same file. diff --git a/crates/pdf/src/write/raster.rs b/crates/pdf/src/write/raster.rs index 1aa64ec3c..bcc32c23f 100644 --- a/crates/pdf/src/write/raster.rs +++ b/crates/pdf/src/write/raster.rs @@ -3,6 +3,10 @@ //! Callers map their page and color models to these neutral records. This module owns PDF object //! layout, image streams, ICC/Separation resources, page labels, marks, xref and trailer syntax. +/// Multi-image 8/16-bit print pages with per-image ICC resources and explicit byte budgets. +#[path = "raster_print.rs"] +pub mod print; + use std::collections::BTreeSet; use std::fmt::Write as _; use std::io::Write as _; diff --git a/crates/pdf/src/write/raster_print.rs b/crates/pdf/src/write/raster_print.rs new file mode 100644 index 000000000..9866d73e1 --- /dev/null +++ b/crates/pdf/src/write/raster_print.rs @@ -0,0 +1,658 @@ +//! A small PDF writer for print output: pages of a given size in points, each with raster images +//! placed on it, tagged with their ICC profile (`ICCBased`) so the numbers mean the same colour in +//! every reader and RIP, Flate-compressed (lossless) at 8 or 16 bits per component, or carried as +//! JPEG (`DCTDecode`) where small files matter more than exact samples. Optionally an output intent: +//! the printing condition (the printer / paper profile) the file was made for, as PDF/X readers +//! expect it in the catalog. Profiles shared by several images are written once. +//! +//! The file is PDF 1.6 (16-bit images need 1.5): classic cross-reference table, no object streams, +//! deterministic (no dates or IDs), so the same pages always give the same bytes. +//! +//! Encoded JPEG and Flate payloads are copied, not decoded or verified against declared +//! dimensions. Callers own their encoding validity and ICC/profile-channel compatibility. +//! An output intent describes a printing condition; this API does not certify PDF/X. +//! Byte limits bound accounted input and emitted output, not peak process memory or +//! caller-owned image construction. This is not the streaming slideshow writer. +//! +//! Sources: ISO 32000-1:2008 (§7.3 objects, §7.5 file structure, §8.6.5.5 ICCBased colour spaces, +//! §8.9 images, §14.11.5 output intents). Own design. + +use std::collections::HashMap; +use std::io::Write as _; +use std::sync::Arc; + +/// Pages per document: a print job beyond this is a mistake, not a book. +pub const MAX_PAGES: usize = 1000; +/// Images per page. +pub const MAX_IMAGES_PER_PAGE: usize = 256; +/// Largest page side in points (200 inches): more than any printer's roll. +pub const MAX_PAGE_POINTS: f64 = 14_400.0; +/// Largest image, in pixels per side. +pub const MAX_IMAGE_SIDE: u32 = 65_535; + +/// Serialization budgets; input records already belong to the caller. These are not a +/// process-wide peak-memory guarantee. Repeated references count toward the input budget. +#[derive(Clone, Copy, Debug)] +pub struct Limits { + pub input_bytes: usize, + pub output_bytes: usize, +} +impl Default for Limits { + fn default() -> Self { + Self { input_bytes: 512 * 1024 * 1024, output_bytes: 512 * 1024 * 1024 } + } +} + +/// How an image's samples are stored. +#[derive(Clone, Debug, PartialEq)] +pub enum ImageData { + /// Interleaved samples, `bits` (8 or 16) per component, 16-bit values big-endian: compressed + /// losslessly in the file. + Raw { bits: u8, samples: Vec }, + /// A baseline JPEG stream, copied into the file as it is. + Jpeg(Vec), + /// Samples already compressed with zlib (see [`PdfImage::deflated`]), `bits` per component. + Flate { bits: u8, data: Vec }, +} + +/// A raster image and the colour space its samples are in. +#[derive(Clone, Debug, PartialEq)] +pub struct PdfImage { + pub width: u32, + pub height: u32, + /// Components per pixel: 1 (grey), 3 (RGB) or 4 (CMYK). + pub channels: u8, + pub data: ImageData, + /// The ICC profile the samples are in; without one the image is device colour. + pub icc: Option>>, +} + +impl PdfImage { + /// An image from interleaved 16-bit samples. + pub fn from_u16(width: u32, height: u32, channels: u8, samples: &[u16], icc: Option>>) -> PdfImage { + let bytes = samples.iter().flat_map(|v| v.to_be_bytes()).collect(); + PdfImage { width, height, channels, data: ImageData::Raw { bits: 16, samples: bytes }, icc } + } + + /// The same image with its samples compressed now (checked first), so a long print doesn't + /// hold every page's uncompressed samples until the file is written. + pub fn deflated(self) -> Result { + self.check()?; + if matches!(&self.data, ImageData::Raw { samples, .. } if samples.len() > Limits::default().input_bytes) { + return Err("PDF input byte limit exceeded".into()); + } + let data = match self.data { + ImageData::Raw { bits, samples } => ImageData::Flate { bits, data: deflate(&samples, Limits::default().output_bytes)? }, + other => other, + }; + Ok(PdfImage { data, ..self }) + } + + fn check(&self) -> Result<(), String> { + if self.width == 0 || self.height == 0 || self.width > MAX_IMAGE_SIDE || self.height > MAX_IMAGE_SIDE { + return Err(format!("an image of {} × {} pixels can't be written", self.width, self.height)); + } + if !matches!(self.channels, 1 | 3 | 4) { + return Err(format!("images have 1, 3 or 4 components, not {}", self.channels)); + } + match &self.data { + ImageData::Raw { bits, samples } => { + if !matches!(bits, 8 | 16) { + return Err(format!("images have 8 or 16 bits per component, not {bits}")); + } + let want = u64::from(self.width) * u64::from(self.height) * u64::from(self.channels) * u64::from(bits / 8); + if samples.len() as u64 != want { + return Err(format!("the image needs {want} bytes of samples, not {}", samples.len())); + } + } + ImageData::Jpeg(data) => { + if !data.starts_with(&[0xFF, 0xD8]) { + return Err("the image isn't a JPEG stream".into()); + } + } + ImageData::Flate { bits, data } => { + if !matches!(bits, 8 | 16) || data.is_empty() { + return Err("a compressed image has 8 or 16 bits per component and some data".into()); + } + } + } + Ok(()) + } +} + +/// Where an image goes on its page, in points from the page's bottom-left corner. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Placement { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +/// One page: its size in points and the images on it, drawn in order (later ones on top). +#[derive(Clone, Debug, PartialEq)] +pub struct PdfPage { + pub width: f64, + pub height: f64, + pub images: Vec<(PdfImage, Placement)>, +} + +/// The printing condition the document is made for (PDF/X-style output intent). +#[derive(Clone, Debug, PartialEq)] +pub struct OutputIntent { + /// The printer / paper profile. + pub icc: Arc>, + /// Its components: 1, 3 or 4. + pub channels: u8, + /// A name for the condition (the profile's description). + pub condition: String, +} + +/// Document-level settings. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PdfInfo { + pub title: Option, + pub output_intent: Option, + /// Application name, not a PDF/X conformance claim. + pub producer: String, +} + +fn finite(v: f64) -> bool { + v.is_finite() +} + +/// A number as PDF writes it: up to four decimals, no exponent, no trailing zeros. +fn num(v: f64) -> String { + let s = format!("{:.4}", if finite(v) { v } else { 0.0 }); + let s = s.trim_end_matches('0').trim_end_matches('.'); + if s == "-0" || s.is_empty() { "0".into() } else { s.into() } +} + +/// A text string: plain ASCII as a literal string, anything else as UTF-16BE with a byte order mark. +fn text(s: &str) -> String { + if s.bytes().all(|b| (0x20..0x7f).contains(&b)) { + let mut out = String::with_capacity(s.len() + 2); + out.push('('); + for c in s.chars() { + if matches!(c, '(' | ')' | '\\') { + out.push('\\'); + } + out.push(c); + } + out.push(')'); + out + } else { + super::pdf_unicode_string(s) + } +} + +struct Writer { + out: Vec, + offsets: Vec, + limit: usize, +} +impl Writer { + fn put(&mut self, bytes: &[u8]) -> Result<(), String> { + let end = self.out.len().checked_add(bytes.len()).ok_or("PDF output size overflow")?; + if end > self.limit { + return Err("PDF output byte limit exceeded".into()); + } + self.out.try_reserve(bytes.len()).map_err(|_| "PDF output allocation failed")?; + self.out.extend_from_slice(bytes); + Ok(()) + } + fn reserve(&mut self) -> Result { + self.offsets.try_reserve(1).map_err(|_| "PDF object allocation failed")?; + self.offsets.push(0); + Ok(self.offsets.len()) + } + fn object(&mut self, id: usize, dict: &str, stream: Option<&[u8]>) -> Result<(), String> { + let pos = self.out.len(); + *self.offsets.get_mut(id.checked_sub(1).ok_or("invalid PDF object")?).ok_or("invalid PDF object")? = pos; + self.put(format!("{id} 0 obj\n{dict}").as_bytes())?; + if let Some(data) = stream { + self.put(b"\nstream\n")?; + self.put(data)?; + self.put(b"\nendstream")?; + } + self.put(b"\nendobj\n") + } +} + +fn deflate(data: &[u8], max: usize) -> Result, String> { + struct Sink { + bytes: Vec, + max: usize, + } + impl std::io::Write for Sink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.len() > self.max.saturating_sub(self.bytes.len()) { + return Err(std::io::Error::other("PDF compressed byte limit exceeded")); + } + self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?; + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut encoder = flate2::write::ZlibEncoder::new(Sink { bytes: Vec::new(), max }, flate2::Compression::new(6)); + encoder.write_all(data).map_err(|e| e.to_string())?; + Ok(encoder.finish().map_err(|e| e.to_string())?.bytes) +} + +/// Write a document. Errors name the first page or image that can't be written. +pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result, String> { + write_with_limits(pages, info, Limits::default()) +} + +/// Write with explicit total input and serialized output budgets. ICC bytes are opaque: +/// callers validate their profile semantics; resources are keyed by bytes AND channels. +pub fn write_with_limits(pages: &[PdfPage], info: &PdfInfo, limits: Limits) -> Result, String> { + let mut input = 0usize; + let mut account = |n: usize| -> Result<(), String> { + input = input.checked_add(n).ok_or("PDF input size overflow")?; + if input > limits.input_bytes { + return Err("PDF input byte limit exceeded".into()); + } + Ok(()) + }; + account(info.producer.len())?; + account(info.title.as_ref().map_or(0, String::len))?; + if let Some(intent) = &info.output_intent { + account(intent.icc.len())?; + account(intent.condition.len())?; + } + + if pages.is_empty() { + return Err("there are no pages to write".into()); + } + if pages.len() > MAX_PAGES { + return Err(format!("a document holds at most {MAX_PAGES} pages")); + } + for (n, page) in pages.iter().enumerate() { + let n = n + 1; + if !(finite(page.width) && finite(page.height)) + || page.width <= 0.0 + || page.height <= 0.0 + || page.width > MAX_PAGE_POINTS + || page.height > MAX_PAGE_POINTS + { + return Err(format!("page {n} has no usable size")); + } + if page.images.len() > MAX_IMAGES_PER_PAGE { + return Err(format!("page {n} has more than {MAX_IMAGES_PER_PAGE} images")); + } + for (image, at) in &page.images { + account(match &image.data { + ImageData::Raw { samples, .. } => samples.len(), + ImageData::Jpeg(data) | ImageData::Flate { data, .. } => data.len(), + })?; + if let Some(icc) = &image.icc { + account(icc.len())?; + } + image.check().map_err(|e| format!("page {n}: {e}"))?; + if ![at.x, at.y, at.width, at.height].into_iter().all(finite) || at.width <= 0.0 || at.height <= 0.0 { + return Err(format!("page {n}: an image has no usable placement")); + } + } + } + if let Some(intent) = &info.output_intent + && !matches!(intent.channels, 1 | 3 | 4) + { + return Err("the output intent's profile has 1, 3 or 4 components".into()); + } + + let mut w = Writer { out: Vec::new(), offsets: Vec::new(), limit: limits.output_bytes }; + w.put(b"%PDF-1.6\n%\xE2\xE3\xCF\xD3\n")?; + let catalog = w.reserve()?; + let pages_id = w.reserve()?; + let info_id = w.reserve()?; + + // ICC profiles, each written once (keyed by content) with its component count + let mut profiles: HashMap<(Arc>, u8), usize> = HashMap::new(); + let mut profile = |w: &mut Writer, icc: &Arc>, n: u8| -> Result { + if let Some(&id) = profiles.get(&(Arc::clone(icc), n)) { + return Ok(id); + } + let id = w.reserve()?; + let data = deflate(icc, limits.output_bytes)?; + let alternate = match n { + 1 => "/DeviceGray", + 4 => "/DeviceCMYK", + _ => "/DeviceRGB", + }; + w.object(id, &format!("<< /N {n} /Alternate {alternate} /Filter /FlateDecode /Length {} >>", data.len()), Some(&data))?; + profiles.try_reserve(1).map_err(|_| "PDF profile allocation failed")?; + profiles.insert((Arc::clone(icc), n), id); + Ok(id) + }; + + let intent_id = info.output_intent.as_ref().map(|intent| profile(&mut w, &intent.icc, intent.channels)).transpose()?; + + let mut kids = Vec::with_capacity(pages.len()); + for page in pages { + let page_id = w.reserve()?; + let mut xobjects = String::new(); + let mut content = String::new(); + for (k, (image, at)) in page.images.iter().enumerate() { + let space = match &image.icc { + Some(icc) => format!("[/ICCBased {} 0 R]", profile(&mut w, icc, image.channels)?), + None => match image.channels { + 1 => "/DeviceGray".into(), + 4 => "/DeviceCMYK".into(), + _ => "/DeviceRGB".into(), + }, + }; + let id = w.reserve()?; + let (bits, filter, data): (u8, &str, std::borrow::Cow<[u8]>) = match &image.data { + ImageData::Raw { bits, samples } => (*bits, "/FlateDecode", deflate(samples, limits.output_bytes)?.into()), + ImageData::Flate { bits, data } => (*bits, "/FlateDecode", data.as_slice().into()), + ImageData::Jpeg(jpeg) => (8, "/DCTDecode", jpeg.as_slice().into()), + }; + w.object( + id, + &format!( + "<< /Type /XObject /Subtype /Image /Width {} /Height {} /ColorSpace {space} /BitsPerComponent {bits} /Filter {filter} /Length {} >>", + image.width, + image.height, + data.len() + ), + Some(&data), + )?; + xobjects.push_str(&format!("/Im{k} {id} 0 R ")); + content.push_str(&format!("q {} 0 0 {} {} {} cm /Im{k} Do Q\n", num(at.width), num(at.height), num(at.x), num(at.y))); + } + let content_id = w.reserve()?; + let stream = deflate(content.as_bytes(), limits.output_bytes)?; + w.object(content_id, &format!("<< /Filter /FlateDecode /Length {} >>", stream.len()), Some(&stream))?; + w.object( + page_id, + &format!( + "<< /Type /Page /Parent {pages_id} 0 R /MediaBox [0 0 {} {}] /Resources << /XObject << {xobjects}>> >> /Contents {content_id} 0 R >>", + num(page.width), + num(page.height) + ), + None, + )?; + kids.push(format!("{page_id} 0 R")); + } + w.object(pages_id, &format!("<< /Type /Pages /Count {} /Kids [{}] >>", kids.len(), kids.join(" ")), None)?; + + let intents = match (&info.output_intent, intent_id) { + (Some(intent), Some(id)) => format!( + " /OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX /OutputConditionIdentifier (Custom) /Info {} /DestOutputProfile {id} 0 R >>]", + text(&intent.condition) + ), + _ => String::new(), + }; + w.object(catalog, &format!("<< /Type /Catalog /Pages {pages_id} 0 R{intents} >>"), None)?; + let title = info.title.as_deref().map(|t| format!(" /Title {}", text(t))).unwrap_or_default(); + w.object(info_id, &format!("<< /Producer {}{title} >>", text(&info.producer)), None)?; + + let xref = w.out.len(); + let mut table = format!("xref\n0 {}\n0000000000 65535 f \n", w.offsets.len() + 1); + for offset in &w.offsets { + table.push_str(&format!("{offset:010} 00000 n \n")); + } + table.push_str(&format!("trailer\n<< /Size {} /Root {catalog} 0 R /Info {info_id} 0 R >>\nstartxref\n{xref}\n%%EOF\n", w.offsets.len() + 1)); + w.put(table.as_bytes())?; + Ok(w.out) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + /// The body of object `id` (dictionary and raw stream bytes), found through the xref table. + fn object(pdf: &[u8], id: usize) -> (String, Vec) { + // byte offsets: the streams are binary, so search the bytes, not a lossy string + let start = pdf.windows(10).rposition(|w| w == b"startxref\n").unwrap() + 10; + let tail = std::str::from_utf8(&pdf[start..]).unwrap(); + let xref: usize = tail.lines().next().unwrap().trim().parse().unwrap(); + let table = std::str::from_utf8(&pdf[xref..]).unwrap(); + let line = table.lines().nth(2 + id).unwrap(); + let offset: usize = line[..10].parse().unwrap(); + let head = format!("{id} 0 obj\n"); + assert_eq!(&pdf[offset..offset + head.len()], head.as_bytes(), "xref entry for {id}"); + let rest = &pdf[offset + head.len()..]; + let end = rest.windows(7).position(|w| w == b"\nendobj").unwrap(); + let body = &rest[..end]; + match body.windows(8).position(|w| w == b"\nstream\n") { + Some(p) => { + let data = &body[p + 8..body.len() - "\nendstream".len()]; + (String::from_utf8_lossy(&body[..p]).into_owned(), data.to_vec()) + } + None => (String::from_utf8_lossy(body).into_owned(), vec![]), + } + } + + fn inflate(data: &[u8]) -> Vec { + { + use std::io::Read; + let mut out = Vec::new(); + flate2::read::ZlibDecoder::new(data).read_to_end(&mut out).unwrap(); + out + } + } + + /// Object ids of every image XObject, in file order. + fn images(pdf: &[u8]) -> Vec { + let s = String::from_utf8_lossy(pdf); + s.match_indices(" 0 obj\n<< /Type /XObject /Subtype /Image").map(|(i, _)| s[..i].rsplit('\n').next().unwrap().parse().unwrap()).collect() + } + + fn reference(dict: &str, key: &str) -> usize { + let at = dict.find(key).unwrap() + key.len(); + dict[at..].trim_start().split(' ').next().unwrap().parse().unwrap() + } + + fn srgb() -> Arc> { + Arc::new(b"opaque RGB profile fixture".to_vec()) + } + + fn page(images: Vec<(PdfImage, Placement)>) -> PdfPage { + PdfPage { width: 612.0, height: 792.0, images } + } + + fn full(w: f64, h: f64) -> Placement { + Placement { x: 0.0, y: 0.0, width: w, height: h } + } + + #[test] + fn a_16_bit_icc_image_round_trips() { + let samples: Vec = (0..2 * 3 * 3).map(|i| (i * 3851) as u16).collect(); + let img = PdfImage::from_u16(3, 2, 3, &samples, Some(srgb())); + let pdf = write(&[page(vec![(img, full(612.0, 792.0))])], &PdfInfo::default()).unwrap(); + assert!(pdf.starts_with(b"%PDF-1.6\n")); + assert!(pdf.ends_with(b"%%EOF\n")); + let ids = images(&pdf); + assert_eq!(ids.len(), 1); + let (dict, data) = object(&pdf, ids[0]); + assert!(dict.contains("/Width 3 /Height 2") && dict.contains("/BitsPerComponent 16") && dict.contains("/FlateDecode"), "{dict}"); + let back: Vec = inflate(&data).chunks(2).map(|b| u16::from_be_bytes([b[0], b[1]])).collect(); + assert_eq!(back, samples); + let (profile, icc) = object(&pdf, reference(&dict, "/ICCBased")); + assert!(profile.contains("/N 3") && profile.contains("/Alternate /DeviceRGB"), "{profile}"); + assert_eq!(inflate(&icc), *srgb()); + } + + #[test] + fn pages_sizes_placements_and_a_shared_profile() { + let px = |v: u16| PdfImage::from_u16(1, 1, 3, &[v, v, v], Some(srgb())); + let at = Placement { x: 36.0, y: 72.5, width: 288.0, height: 216.0 }; + let pages = [page(vec![(px(1), at), (px(2), full(612.0, 792.0))]), PdfPage { width: 841.89, height: 595.28, images: vec![(px(3), at)] }]; + let pdf = write(&pages, &PdfInfo::default()).unwrap(); + let s = String::from_utf8_lossy(&pdf); + assert!(s.contains("/Type /Pages /Count 2"), "two pages"); + assert!(s.contains("/MediaBox [0 0 612 792]") && s.contains("/MediaBox [0 0 841.89 595.28]")); + // the sRGB profile is written once for all three images + assert_eq!(s.matches("/N 3 /Alternate").count(), 1); + let first: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect(); + assert!(first.windows(2).all(|w| w[0] == w[1]), "{first:?}"); + // placements reach the content streams + let contents: Vec = s + .match_indices("/Contents ") + .map(|(i, _)| s[i + 10..].split(' ').next().unwrap().parse::().unwrap()) + .map(|id| String::from_utf8(inflate(&object(&pdf, id).1)).unwrap()) + .collect(); + assert_eq!(contents[0], "q 288 0 0 216 36 72.5 cm /Im0 Do Q\nq 612 0 0 792 0 0 cm /Im1 Do Q\n"); + assert_eq!(write(&pages, &PdfInfo::default()).unwrap(), pdf, "deterministic"); + } + + #[test] + fn output_intent_and_title() { + let cmyk_like = srgb(); + let info = PdfInfo { + title: Some("Garden – prints (2)".into()), + producer: "Fixture".into(), + output_intent: Some(OutputIntent { icc: cmyk_like.clone(), channels: 3, condition: "Glossy (Photo) \\ test".into() }), + }; + let img = PdfImage::from_u16(1, 1, 3, &[0, 0, 0], Some(cmyk_like)); + let pdf = write(&[page(vec![(img, full(10.0, 10.0))])], &info).unwrap(); + let (catalog, _) = object(&pdf, 1); + assert!(catalog.contains("/OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX"), "{catalog}"); + assert!(catalog.contains("/Info (Glossy \\(Photo\\) \\\\ test)"), "{catalog}"); + // the intent and the image share one profile object + let dest = reference(&catalog, "/DestOutputProfile"); + assert_eq!(reference(&object(&pdf, images(&pdf)[0]).0, "/ICCBased"), dest); + let (info_dict, _) = object(&pdf, 3); + assert!(info_dict.contains("/Title = ids.iter().map(|&id| object(&pdf, id)).collect(); + assert!(dicts[0].0.contains("/ColorSpace /DeviceCMYK") && dicts[0].0.contains("/BitsPerComponent 8")); + assert!(dicts[1].0.contains("/ColorSpace /DeviceGray")); + assert!(dicts[2].0.contains("/DCTDecode") && dicts[2].0.contains("/DeviceRGB")); + assert_eq!(dicts[2].1, jpeg, "JPEG streams are copied as they are"); + } + + #[test] + fn compressing_early_writes_the_same_file() { + let samples: Vec = (0..4 * 3 * 3).map(|i| (i * 977) as u16).collect(); + let raw = PdfImage::from_u16(4, 3, 3, &samples, Some(srgb())); + let early = raw.clone().deflated().unwrap(); + assert!(matches!(early.data, ImageData::Flate { bits: 16, .. })); + let at = full(100.0, 75.0); + let a = write(&[page(vec![(raw, at)])], &PdfInfo::default()).unwrap(); + let b = write(&[page(vec![(early, at)])], &PdfInfo::default()).unwrap(); + assert_eq!(a, b); + let mut short = PdfImage::from_u16(4, 3, 3, &samples, None); + short.width = 5; + assert!(short.deflated().is_err(), "checked before compressing"); + } + + #[test] + fn hostile_documents_are_errors() { + let ok = || PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None); + let at = full(10.0, 10.0); + let bad = |pages: Vec| write(&pages, &PdfInfo::default()).is_err(); + assert!(bad(vec![])); + assert!(bad(vec![PdfPage { width: f64::NAN, height: 10.0, images: vec![] }])); + assert!(bad(vec![PdfPage { width: -1.0, height: 10.0, images: vec![] }])); + assert!(bad(vec![PdfPage { width: 1e9, height: 10.0, images: vec![] }])); + assert!(bad(vec![page(vec![(ok(), Placement { x: f64::INFINITY, ..at })])])); + assert!(bad(vec![page(vec![(ok(), Placement { width: 0.0, ..at })])])); + let mut short = ok(); + short.data = ImageData::Raw { bits: 16, samples: vec![0; 5] }; + assert!(bad(vec![page(vec![(short, at)])])); + let mut empty = ok(); + empty.width = 0; + assert!(bad(vec![page(vec![(empty, at)])])); + let mut odd = ok(); + odd.channels = 2; + assert!(bad(vec![page(vec![(odd, at)])])); + let mut not_jpeg = ok(); + not_jpeg.data = ImageData::Jpeg(vec![1, 2, 3]); + assert!(bad(vec![page(vec![(not_jpeg, at)])])); + let too_many = (0..MAX_PAGES + 1).map(|_| page(vec![])).collect(); + assert!(bad(too_many)); + let info = PdfInfo { + producer: String::new(), + title: None, + output_intent: Some(OutputIntent { icc: srgb(), channels: 2, condition: String::new() }), + }; + assert!(write(&[page(vec![])], &info).is_err()); + } + + #[test] + fn numbers_and_strings() { + assert_eq!(num(612.0), "612"); + assert_eq!(num(72.5), "72.5"); + assert_eq!(num(-0.00001), "0"); + assert_eq!(num(f64::NAN), "0"); + assert_eq!(num(1.0 / 3.0), "0.3333"); + assert_eq!(text("a(b)c\\"), "(a\\(b\\)c\\\\)"); + assert_eq!(text("é"), ""); + } + #[test] + fn distinct_profiles_and_component_counts_are_not_conflated() { + let common = Arc::new(b"opaque profile A".to_vec()); + let different = Arc::new(b"opaque profile B".to_vec()); + let image = |channels, profile| PdfImage { + width: 1, + height: 1, + channels, + data: ImageData::Raw { bits: 8, samples: vec![7; channels as usize] }, + icc: Some(profile), + }; + let pages = [page(vec![ + (image(3, common.clone()), full(1.0, 1.0)), + (image(4, common.clone()), full(1.0, 1.0)), + (image(3, different.clone()), full(1.0, 1.0)), + (image(3, common.clone()), full(1.0, 1.0)), + ])]; + let pdf = write(&pages, &PdfInfo::default()).unwrap(); + let refs: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect(); + assert_ne!(refs[0], refs[1]); + assert_ne!(refs[0], refs[2]); + assert_eq!(refs[0], refs[3]); + for (i, channels, bytes) in [(0, 3, common.as_slice()), (1, 4, common.as_slice()), (2, 3, different.as_slice())] { + let (dict, data) = object(&pdf, refs[i]); + assert!(dict.contains(&format!("/N {channels}"))); + assert_eq!(inflate(&data), bytes); + } + } + + #[test] + fn gray_rgb_cmyk_raw_depths_preserve_samples() { + for channels in [1, 3, 4] { + for bits in [8, 16] { + let bytes: Vec = (0..channels * (bits / 8)).collect(); + let image = PdfImage { width: 1, height: 1, channels, data: ImageData::Raw { bits, samples: bytes.clone() }, icc: None }; + let pdf = write(&[page(vec![(image, full(1.0, 1.0))])], &PdfInfo::default()).unwrap(); + let (dict, data) = object(&pdf, images(&pdf)[0]); + assert!(dict.contains(&format!("/BitsPerComponent {bits}"))); + assert_eq!(inflate(&data), bytes); + } + } + } + + #[test] + fn input_and_output_budgets_fail_without_partial_success() { + let image = PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None); + let pages = [page(vec![(image, full(1.0, 1.0))])]; + let info = PdfInfo::default(); + let expected = write(&pages, &info).unwrap(); + assert_eq!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() }).unwrap(), expected); + assert!(write_with_limits(&pages, &info, Limits { input_bytes: 5, output_bytes: expected.len() }).is_err()); + assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() - 1 }).is_err()); + assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: 0 }).is_err()); + } +} diff --git a/crates/raw/README.md b/crates/raw/README.md index e15e355c1..f0faf0670 100644 --- a/crates/raw/README.md +++ b/crates/raw/README.md @@ -33,7 +33,7 @@ The format layer ends where arithmetic on the picture begins. |---|---| | Recognising containers (`probe`), walking TIFF/IFD structure on `craft-tiff` | Black subtraction, white scaling, cropping to the active area | | Decompressing the samples: unpacking, lossless JPEG, lossy JPEG, Deflate, JPEG XL | Demosaicing, binned previews, hot-pixel and highlight handling | -| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values) | Raising samples stuck below the stated white level (`lift_clipped_samples`) | +| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values; left unapplied when `OpcodeList1` is present, so the develop pipeline can run that list on the stored codes first) | Raising samples stuck below the stated white level (`lift_clipped_samples`) | | Black and white levels, CFA layout, active area, default crop, orientation, pixel aspect | The DNG colour model's arithmetic (illuminant interpolation, camera to working space) | | Colour tags as data: matrices, calibration illuminants, as-shot neutral / white | White balance, Kelvin scales, spectral or fitted camera colour | | Opcode lists parsed and written (`opcodes`), profile tables and gain table maps as data | Applying opcodes (gain maps, warps, bad pixels), profile look tables, tone curves | diff --git a/crates/raw/src/arw.rs b/crates/raw/src/arw.rs index 1b25aacda..389c464a5 100644 --- a/crates/raw/src/arw.rs +++ b/crates/raw/src/arw.rs @@ -852,6 +852,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Result Result = table.iter().map(|&x| x.min(65535) as u16).collect(); - let last = table.len() - 1; - v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]); - linearized = true; - table_max = table.iter().copied().max().filter(|&m| m > 0); - } + let table_max = linearization_table.as_deref().and_then(|t| t.iter().copied().max().filter(|&m| m > 0)); // a reduced copy's own levels, where it states them let levels = |tag| if pixels.contains(tag) { pixels } else { raw }; @@ -442,10 +436,6 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f None }; - let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default(); - let opcodes = - crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) }; - let color = color_data(ifd0, raw, tiff.order); let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2)) && color.profile.gain_table_map.is_none(); @@ -465,6 +455,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f color, wb_multipliers: None, linearized, + linearization_table, pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()), opcodes, camera: Camera::from_ifd0(ifd0), @@ -477,6 +468,31 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f Ok(img) } +/// Parse opcode lists and, for integer data, the LinearizationTable. +/// +/// OpcodeList1 runs on the stored codes, then the table maps them to sensor values (DNG 1.7 +/// chapter 7). Applying opcodes is the develop pipeline's work, so a non-empty list 1 leaves the +/// table unapplied (`linearized` is false) and available on the image. +fn linearization_and_opcodes(pixels: &Ifd, raw: &Ifd, data: &mut RawData) -> (bool, Option>, crate::OpcodeLists) { + let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default(); + let opcodes = + crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) }; + let linearization_table = if matches!(data, RawData::U16(_)) { + pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty()).map(|table| table.iter().map(|&x| x.min(65535) as u16).collect::>()) + } else { + None + }; + let mut linearized = false; + if opcodes.list1.is_empty() + && let (RawData::U16(v), Some(table)) = (data, linearization_table.as_deref()) + { + let last = table.len() - 1; + v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]); + linearized = true; + } + (linearized, linearization_table, opcodes) +} + /// CinemaDNG `TimeCodes` (CinemaDNG Image Data Format Specification 1.0, tag 51043). const TIME_CODES: u16 = 51043; /// CinemaDNG `FrameRate` (tag 51044). diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index ace0c07f6..384152e66 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -165,6 +165,14 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { } else { ifd.set(t::WHITE_LEVEL, Value::Long(white)); } + // A table still to be applied (OpcodeList1 has to run on the stored codes first) is written + // with the samples. Once the samples have been mapped, writing the tag would apply it twice + // on the next read of a file without list 1. + if !raw.linearized + && let Some(table) = raw.linearization_table.as_deref().filter(|t| !t.is_empty()) + { + ifd.set(t::LINEARIZATION_TABLE, Value::Short(table.to_vec())); + } let a = raw.active_area; ifd.set(t::ACTIVE_AREA, Value::Long(vec![a.y as u32, a.x as u32, (a.y + a.height) as u32, (a.x + a.width) as u32])); ifd.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![raw.crop.x as u32, raw.crop.y as u32])); @@ -457,6 +465,7 @@ mod tests { color: crate::ColorData::default(), wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera: Default::default(), @@ -514,6 +523,7 @@ mod tests { color: crate::ColorData::default(), wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera: Default::default(), diff --git a/crates/raw/src/erf.rs b/crates/raw/src/erf.rs index fcdfa43cd..b7dd9d8be 100644 --- a/crates/raw/src/erf.rs +++ b/crates/raw/src/erf.rs @@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result, + /// Indexed by the next 12 bits. Per entry: bits 0-7 the symbol, bits 8-11 the code length (0 = invalid), bits + /// 12-15 the length of code plus additional bits when both fit in 12 bits (0 = they do not), bits 16-31 the + /// signed difference in that case. + lut: Box<[u32; 4096]>, } impl Huffman { /// From `(left-aligned 12-bit code, length)` per symbol. pub fn new(codes: &[(u16, u8)]) -> Result { - let mut lut = vec![(0u8, 0u8); 4096]; + let mut lut = Box::new([0u32; 4096]); for (sym, &(code, len)) in codes.iter().enumerate() { if len == 0 { continue; @@ -21,10 +24,19 @@ impl Huffman { if len > 12 || sym > 16 { return Err(RawError::Unsupported(format!("PEF Huffman code of length {len}"))); } + let (len, s) = (len as u32, sym as u32); let first = (code >> (12 - len) << (12 - len)) as usize; - for e in &mut lut[first..first + (1 << (12 - len))] { - if e.1 == 0 { - *e = (sym as u8, len); + for (i, e) in lut[first..first + (1 << (12 - len))].iter_mut().enumerate() { + if (*e >> 8) & 15 == 0 { + let mut entry = (len << 8) | s; + if s == 0 { + entry |= len << 12; + } else if len + s <= 12 { + let v = (((first + i) >> (12 - len - s)) & ((1 << s) - 1)) as i32; + let d = if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v }; + entry |= ((len + s) << 12) | ((d as i16 as u16 as u32) << 16); + } + *e = entry; } } } @@ -85,21 +97,122 @@ impl<'a> Bits<'a> { } } -/// Decode one difference: Huffman category then T.81 additional bits. +/// Decode one difference: Huffman category then T.81 additional bits, in one lookup when both fit in 12 bits. #[inline] pub(crate) fn diff(bits: &mut Bits, h: &Huffman) -> Option { - let (sym, len) = h.lut[bits.peek(12) as usize]; + let e = h.lut[(bits.peek(12) & 0xfff) as usize]; + let len = (e >> 8) & 15; if len == 0 { return None; } - bits.skip(len as u32); - let s = sym as u32; - if s == 0 { - return Some(0); + let total = (e >> 12) & 15; + if total > 0 { + bits.skip(total); + return Some((e as i32) >> 16); } + bits.skip(len); + let s = e & 0xff; if s > 16 { return None; } let v = bits.get(s) as i32; Some(if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Canonical left-aligned 12-bit codes for symbols 0..lens.len() (shorter codes first, ties by symbol). + fn canonical(lens: &[u8]) -> Vec<(u16, u8)> { + let mut order: Vec = (0..lens.len()).collect(); + order.sort_by_key(|&s| (lens[s], s)); + let mut codes = vec![(0u16, 0u8); lens.len()]; + let (mut code, mut prev) = (0u32, 0u8); + for s in order { + code <<= lens[s] - prev; + prev = lens[s]; + codes[s] = ((code << (12 - lens[s])) as u16, lens[s]); + code += 1; + } + codes + } + + fn bit(src: &[u8], i: usize) -> u32 { + src.get(i / 8).map_or(0, |b| ((b >> (7 - i % 8)) & 1) as u32) + } + + /// Reference: match the code bit by bit, then read the category's additional bits (T.81 F.1.2.1). + fn reference(src: &[u8], pos: &mut usize, codes: &[(u16, u8)]) -> Option { + let window: u32 = (0..12).fold(0, |a, k| (a << 1) | bit(src, *pos + k)); + let (sym, len) = codes + .iter() + .enumerate() + .find_map(|(s, &(c, l))| (l > 0 && (c as u32 >> (12 - l)) == window >> (12 - l)).then_some((s as u32, l as usize)))?; + *pos += len; + if sym == 0 { + return Some(0); + } + let v = (0..sym as usize).fold(0u32, |a, k| (a << 1) | bit(src, *pos + k)) as i32; + *pos += sym as usize; + Some(if v < 1 << (sym - 1) { v - (1 << sym) + 1 } else { v }) + } + + /// `diff` against the reference on random streams (invalid codes are stepped over) and on every (code, extra + /// bits) pair of every symbol. + fn check_table(lens: &[u8]) { + let codes = canonical(lens); + let h = Huffman::new(&codes).unwrap(); + for (sym, &(code, len)) in codes.iter().enumerate() { + for v in 0..1u64 << sym { + let (len, n) = (len as u32, sym as u32); + let word = (((code >> (12 - len)) as u64) << n | v) << (64 - len - n); + let src = word.to_be_bytes(); + let mut pos = 0; + let want = reference(&src, &mut pos, &codes); + let mut bits = Bits::new(&src); + assert_eq!(diff(&mut bits, &h), want, "symbol {sym} bits {v:#b}"); + assert_eq!(bits.consumed_bits(), pos); + } + } + let mut s = 0x1234_5678_9abc_def0u64; + let (mut invalid, mut long) = (0, 0); + for _ in 0..40 { + let src: Vec = (0..400) + .map(|_| { + s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (s >> 40) as u8 + }) + .collect(); + let mut bits = Bits::new(&src); + let mut pos = 0; + while pos < src.len() * 8 - 64 { + let before = pos; + let want = reference(&src, &mut pos, &codes); + assert_eq!(diff(&mut bits, &h), want); + if want.is_none() { + // step over the invalid code + invalid += 1; + pos += 1; + bits.skip(1); + } else { + long += (pos - before > 12) as u32; + } + assert_eq!(bits.consumed_bits(), pos); + } + } + assert!(long > 0, "{invalid} invalid codes, no symbol longer than the lookup"); + } + + #[test] + fn diff_matches_bitwise_reference_for_every_category() { + // categories 0..=16, code lengths up to 12 so that short codes combine with their bits and long ones do not + check_table(&[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12]); + } + + #[test] + fn diff_matches_reference_when_code_and_bits_fill_the_lookup_exactly() { + // category 8 with a 4-bit code and category 7 with a 5-bit code: len + category == 12 + check_table(&[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12]); + } +} diff --git a/crates/raw/src/kodak.rs b/crates/raw/src/kodak.rs index 0af4404f1..12e68b4a4 100644 --- a/crates/raw/src/kodak.rs +++ b/crates/raw/src/kodak.rs @@ -115,6 +115,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Option { } // A damaged/unsupported pixel IFD must not hide usable orientation metadata from preview // fallback. CR3 keeps IFD0 in its CMT1 box; TIFF-based raws keep it at the file start. + // The `cr3` module exists only with the `cr3` feature: this gate is required so + // `cargo check -p craft-raw --no-default-features --features dng` builds (E0433 otherwise). #[cfg(feature = "cr3")] if probe(bytes) == Some(RawFormat::Cr3) { return cr3::orientation(bytes); @@ -1041,6 +1045,12 @@ pub struct RawImage { pub wb_multipliers: Option<[f32; 3]>, /// Whether a `LinearizationTable` was applied to `data`. pub linearized: bool, + /// The file's `LinearizationTable`, when present. `data` has already been mapped through it when + /// [`Self::linearized`] is true. When `linearized` is false and this is `Some`, the table was left + /// unapplied so `OpcodeList1` can run on the stored samples first (DNG chapter 7); call + /// [`Self::apply_linearization`] after that list. + #[serde(default)] + pub linearization_table: Option>, /// Height of a sensor pixel over its width (1.0 for square pixels). The D1X and D1H write 2.0: their frame is /// 4028 x 1324 and the picture is 3:2 only once the rows are doubled. A developer stretches the rows by /// this factor. @@ -1119,6 +1129,23 @@ impl RawImage { self.check(true) } + /// Map integer `data` through [`Self::linearization_table`] when the table has not yet been applied. + /// No-op when [`Self::linearized`] is already true, the table is missing, or the samples are not + /// integers. Call this after `OpcodeList1`, which DNG applies to the stored codes (chapter 7). + pub fn apply_linearization(&mut self) { + if self.linearized { + return; + } + let Some(table) = self.linearization_table.as_deref().filter(|t| !t.is_empty()) else { + return; + }; + if let RawData::U16(v) = &mut self.data { + let last = table.len() - 1; + v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]); + self.linearized = true; + } + } + fn check(&self, samples: bool) -> Result<()> { if self.width == 0 || self.height == 0 || !(1..=4).contains(&self.cpp) { return Err(RawError::Corrupt("bad dimensions".into())); diff --git a/crates/raw/src/ljpeg.rs b/crates/raw/src/ljpeg.rs index 56b07a533..498a8302f 100644 --- a/crates/raw/src/ljpeg.rs +++ b/crates/raw/src/ljpeg.rs @@ -32,15 +32,17 @@ fn err(s: &str) -> RawError { /// Huffman decoding table (T.81 Annex C "BITS"/"HUFFVAL"). #[derive(Clone, Debug)] pub struct Huffman { - /// For peeks of `LOOKUP` bits: (code length, symbol); length 0 = slow path. - fast: Vec<(u8, u8)>, + /// Indexed by the next `LOOKUP` bits; length 0 = slow path. Per entry: bits 0-7 the symbol, bits 8-11 the code + /// length, bits 12-15 the length of code plus extra bits when both fit in the lookup (0 = they do not), and + /// bits 16-31 the signed difference in that case. + fast: Vec, maxcode: [i32; 18], valptr: [i32; 17], mincode: [i32; 17], values: Vec, } -const LOOKUP: u32 = 10; +const LOOKUP: u32 = 12; impl Huffman { /// Build from the 16 code-length counts and the symbol list. @@ -52,7 +54,7 @@ impl Huffman { let mut maxcode = [-1i32; 18]; let mut valptr = [0i32; 17]; let mut mincode = [0i32; 17]; - let mut fast = vec![(0u8, 0u8); 1 << LOOKUP]; + let mut fast = vec![0u32; 1 << LOOKUP]; let mut code = 0i32; let mut k = 0usize; for len in 1..=16usize { @@ -63,9 +65,10 @@ impl Huffman { if len as u32 <= LOOKUP { let shift = LOOKUP - len as u32; let base = (code as usize) << shift; + let ssss = values[k] as u32; for i in 0..(1usize << shift) { if let Some(slot) = fast.get_mut(base + i) { - *slot = (len as u8, values[k]); + *slot = fast_entry(len as u32, ssss, base + i); } } } @@ -84,12 +87,41 @@ impl Huffman { #[inline] pub fn decode(&self, br: &mut B) -> Result { - let peek = br.peek(LOOKUP); - let (len, sym) = self.fast[peek as usize]; + let e = self.fast[br.peek(LOOKUP) as usize]; + let len = (e >> 8) & 15; if len > 0 { - br.consume(len as u32); - return Ok(sym); + br.consume(len); + return Ok(e as u8); } + self.decode_slow(br) + } + + /// One symbol and its T.81 difference in a single lookup when code and extra bits fit in `LOOKUP` bits; the same + /// as [`Huffman::decode`] followed by [`diff_value`]. `strict`: a category above 16 is an error (subsampled + /// scans) instead of reading 16 bits. + #[inline] + pub fn decode_diff(&self, br: &mut B, strict: bool) -> Result { + let e = self.fast[br.peek(LOOKUP) as usize]; + let len = (e >> 8) & 15; + let total = (e >> 12) & 15; + if total > 0 { + br.consume(total); + return Ok((e as i32) >> 16); + } + let ssss = if len > 0 { + br.consume(len); + e as u8 + } else { + self.decode_slow(br)? + }; + if strict && ssss > 16 { + return Err(err("invalid difference category")); + } + Ok(diff_value(br, ssss)) + } + + #[cold] + fn decode_slow(&self, br: &mut B) -> Result { let mut code = br.peek(LOOKUP + 1) as i32; let mut len = LOOKUP as usize + 1; while len <= 16 && code > self.maxcode[len] { @@ -105,6 +137,21 @@ impl Huffman { } } +/// Lookup entry for a code of `len` bits (`len <= LOOKUP`) naming category `ssss`, at window position `index` +/// (the code is the top `len` bits of the `LOOKUP`-bit window). See [`Huffman::fast`]. +fn fast_entry(len: u32, ssss: u32, index: usize) -> u32 { + let base = (len << 8) | ssss; + if ssss == 0 { + return base | (len << 12); + } + if ssss > 11 || len + ssss > LOOKUP { + return base; + } + let v = ((index >> (LOOKUP - len - ssss)) & ((1 << ssss) - 1)) as i32; + let d = if v < (1 << (ssss - 1)) { v - (1 << ssss) + 1 } else { v }; + base | ((len + ssss) << 12) | ((d as i16 as u16 as u32) << 16) +} + /// A source of entropy-coded bits, first bit most significant. pub trait Bits { /// The next `n` (1..=32) bits without consuming them; zeros past the end. @@ -614,11 +661,7 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) - Prediction::Sequential if mx == 0 && dx == 0 && dy == 0 => row_first[c], Prediction::Sequential => last[c], }; - let ssss = tables[c].decode(&mut br)?; - if ssss > 16 { - return Err(err("invalid difference category")); - } - let value = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16; + let value = ((pred + tables[c].decode_diff(&mut br, true)?) & 0xffff) as u16; plane[i] = value; last[c] = i32::from(value); if mx == 0 && dx == 0 && dy == 0 { @@ -642,6 +685,44 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) - Ok(FrameSubsampled { width: h.width, height: h.height, vertical_subsampling: vertical, planes }) } +/// T.81 Table H.1 predictor `pred` (1..=7; 0 = none) from the left, above and above-left neighbours. +#[inline(always)] +fn predict(pred: u8, ra: i32, rb: i32, rc: i32) -> i32 { + match pred { + 0 => 0, + 1 => ra, + 2 => rb, + 3 => rc, + 4 => ra + rb - rc, + 5 => ra + ((rb - rc) >> 1), + 6 => rb + ((ra - rc) >> 1), + _ => (ra + rb) >> 1, + } +} + +/// Decode the pixels `x0..x1` (`x0 >= 1`) of one row with the fixed predictor `P`. `prev` is empty on the first +/// line, which only predicts from the left (`P = 1`). +#[inline(always)] +fn decode_run( + tables: &[&Huffman], + br: &mut BitReader<'_>, + cur: &mut [u16], + prev: &[u16], + nc: usize, + x0: usize, + x1: usize, +) -> Result<(), RawError> { + for x in x0..x1 { + for (c, table) in tables.iter().enumerate() { + let i = x * nc + c; + let ra = cur[i - nc] as i32; + let p = if P <= 1 { predict(P, ra, 0, 0) } else { predict(P, ra, prev[i] as i32, prev[i - nc] as i32) }; + cur[i] = ((p + table.decode_diff(br, false)?) & 0xffff) as u16; + } + } + Ok(()) +} + /// Decode a lossless JPEG stream. `max_samples` bounds the allocation (hostile headers). pub fn decode(d: &[u8], max_samples: usize) -> Result { let h = parse_header(d)?; @@ -680,18 +761,23 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result { let (prev_rows, cur_rows) = out.split_at_mut(y * row_len); let cur = &mut cur_rows[..row_len]; let prev = if y > 0 { &prev_rows[(y - 1) * row_len..] } else { &[][..] }; - for x in 0..w { - if h.restart > 0 { - if restart_left == 0 { - if !br.restart() { - return Err(err("missing restart marker")); - } - restart_left = h.restart; - first_line = true; - reset_pending = true; + let mut x = 0; + while x < w { + if h.restart > 0 && restart_left == 0 { + if !br.restart() { + return Err(err("missing restart marker")); } - restart_left -= 1; + restart_left = h.restart; + first_line = true; + reset_pending = true; } + // pixels up to the next restart marker or the end of the row + let mut end = w; + if h.restart > 0 { + end = w.min(x + restart_left); + restart_left -= end - x; + } + // the first pixel of the run has the special predictions (start of the interval, first column) #[allow(clippy::needless_range_loop)] // `c` indexes the tables and the sample position alike for c in 0..nc { let i = x * nc + c; @@ -702,24 +788,28 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result { } else if x == 0 || pred == 0 { if pred == 0 { 0 } else { prev[i] as i32 } } else { - let ra = cur[i - nc] as i32; - let rb = prev[i] as i32; - let rc = prev[i - nc] as i32; - match pred { - 1 => ra, - 2 => rb, - 3 => rc, - 4 => ra + rb - rc, - 5 => ra + ((rb - rc) >> 1), - 6 => rb + ((ra - rc) >> 1), - _ => (ra + rb) >> 1, - } + predict(pred, cur[i - nc] as i32, prev[i] as i32, prev[i - nc] as i32) }; - let ssss = tables[c].decode(&mut br)?; - let diff = diff_value(&mut br, ssss); - cur[i] = ((p + diff) & mask) as u16; + cur[i] = ((p + tables[c].decode_diff(&mut br, false)?) & mask) as u16; } reset_pending = false; + // the rest of the run: the predictor is fixed, so it is chosen once here, not per sample + if first_line { + // the first line of an interval predicts from the left neighbour + decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?; + } else { + match pred { + 0 => decode_run::<0>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 1 => decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 2 => decode_run::<2>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 3 => decode_run::<3>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 4 => decode_run::<4>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 5 => decode_run::<5>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + 6 => decode_run::<6>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + _ => decode_run::<7>(&tables, &mut br, cur, prev, nc, x + 1, end)?, + } + } + x = end; } first_line = false; if br.overrun > 64 * 8 { diff --git a/crates/raw/src/ljpeg/tests.rs b/crates/raw/src/ljpeg/tests.rs index c5c6ccf25..a77425d40 100644 --- a/crates/raw/src/ljpeg/tests.rs +++ b/crates/raw/src/ljpeg/tests.rs @@ -125,6 +125,121 @@ fn roundtrip_extremes_and_restarts() { } } +/// Re-label an encoded stream as `precision` bits with point transform `pt`: the encoder codes the samples at +/// `precision - pt` bits, which is what a transformed scan holds. +fn with_point_transform(mut enc: Vec, precision: u8, pt: u8) -> Vec { + enc[6] = precision; // SOF3 precision follows SOI, marker and length + let sos = enc.windows(2).position(|w| w == [0xff, 0xda]).unwrap(); + let seg_len = u16::from_be_bytes([enc[sos + 2], enc[sos + 3]]) as usize; + enc[sos + 2 + seg_len - 1] = pt; + enc +} + +#[test] +fn roundtrip_point_transform_all_predictors() { + for pred in 1..=7u8 { + for comps in [1usize, 2, 3] { + for (bits, pt) in [(12u8, 1u8), (14, 2), (16, 4), (8, 3)] { + let (w, h) = (11, 6); + let data = noise(w * h * comps, (bits - pt) as u32, pred as u64 * 17 + comps as u64); + let enc = with_point_transform(encode(&data, w, h, comps, bits - pt, pred, 0), bits, pt); + let f = decode(&enc, usize::MAX).unwrap(); + assert_eq!((f.precision, f.point_transform, f.predictor), (bits, pt, pred)); + let want: Vec = data.iter().map(|&v| v << pt).collect(); + assert_eq!(f.data, want, "pred {pred} comps {comps} bits {bits} pt {pt}"); + } + } + } + // with restart intervals that end mid-row + let data = noise(20 * 9 * 2, 10, 9); + for rst in [3usize, 7, 20] { + let enc = with_point_transform(encode(&data, 20, 9, 2, 10, 4, rst), 12, 2); + let want: Vec = data.iter().map(|&v| v << 2).collect(); + assert_eq!(decode(&enc, usize::MAX).unwrap().data, want, "restart {rst}"); + } +} + +#[test] +fn roundtrip_skewed_categories_with_long_codes() { + // category k with probability 2^-(k+1): the table gets codes longer than the lookup, rare 16-bit differences + let mut s = 0x9e37_79b9_7f4a_7c15u64; + let mut last = 0u32; + let data: Vec = (0..256 * 256) + .map(|_| { + s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + let r = (s >> 32) as u32; + let cat = r.trailing_zeros().min(16); + let mag = (((s >> 8) as u32) & ((1u32 << cat) - 1)) | (1 << cat) >> 1; + let d = if s & 1 == 0 { mag } else { 0u32.wrapping_sub(mag) }; + last = last.wrapping_add(d) & 0xffff; + last as u16 + }) + .collect(); + for pred in [1u8, 4, 7] { + let enc = encode(&data, 256, 256, 1, 16, pred, 0); + assert_eq!(decode(&enc, usize::MAX).unwrap().data, data, "pred {pred}"); + } +} + +/// Canonical decoding from the code lengths alone, bit by bit (T.81 Annex F, Figure F.16): the reference for the +/// lookup tables. +fn canonical_symbol(br: &mut BitReader, lens: &[u8]) -> Option { + let window = br.peek(16); + let (mut first, mut index) = (0u32, 0usize); + for len in 1..=16u8 { + let code = window >> (16 - len); + let n = lens.iter().filter(|&&l| l == len).count() as u32; + if code < first + n { + br.consume(len as u32); + return Some((index + (code - first) as usize) as u8); + } + index += n as usize; + first = (first + n) << 1; + } + None +} + +/// `decode_diff` against canonical decoding and `diff_value`, for tables with codes inside and beyond the lookup +/// and with `len + category` equal to the lookup width. +#[test] +fn decode_diff_matches_canonical_decoding() { + let tables: [[u8; 17]; 3] = [ + [2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12], + [2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 16], + [2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12], + ]; + for lens in tables { + let mut counts = [0u8; 16]; + for &l in &lens { + counts[l as usize - 1] += 1; + } + // symbols in code order: by length, ties by symbol + let mut values: Vec = (0..17).collect(); + values.sort_by_key(|&v| (lens[v as usize], v)); + let sorted: Vec = values.iter().map(|&v| lens[v as usize]).collect(); + let t = Huffman::new(&counts, &values).unwrap(); + let mut s = 77u64; + let src: Vec = (0..6000) + .map(|_| { + s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + ((s >> 40) as u8).min(0xfe) + }) + .collect(); + let (mut a, mut b) = (BitReader::new(&src), BitReader::new(&src)); + for n in 0..8000 { + let want = canonical_symbol(&mut a, &sorted).map(|i| diff_value(&mut a, values[i as usize])); + let got = t.decode_diff(&mut b, false); + assert_eq!(got.ok(), want, "symbol {n}"); + assert_eq!(a.peek(32), b.peek(32), "symbol {n}"); + if want.is_none() { + // step past the invalid code so the walk goes on + a.consume(1); + b.consume(1); + } + } + } +} + #[test] fn constant_image_single_symbol_table() { let data = vec![1234u16; 16 * 4]; diff --git a/crates/raw/src/mef.rs b/crates/raw/src/mef.rs index 36ba18e0f..1180fe063 100644 --- a/crates/raw/src/mef.rs +++ b/crates/raw/src/mef.rs @@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Result Result Result Result Result Result, cpp: usize) -> RawImage { color, wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera, @@ -1343,12 +1344,77 @@ fn white_level_defaults_to_the_linearization_tables_maximum() { }; let img = build(None); assert!(img.linearized); + assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice())); assert_eq!(img.data, RawData::U16(codes.iter().map(|&c| table[c as usize]).collect())); assert_eq!(img.white, vec![51000.0], "255 × 200, the table's top"); // a stated level still wins assert_eq!(build(Some(40000)).white, vec![40000.0]); } +/// DNG chapter 7: OpcodeList1 runs on the stored codes, then the LinearizationTable maps them to +/// sensor values. When list 1 is present, decode leaves the table unapplied so the develop pipeline +/// can keep that order. Applying the table first would give a different picture. +#[test] +fn opcode_list1_runs_on_stored_samples_before_linearization() { + let (w, h) = (4, 2); + let stored = vec![2u16; w * h]; + // table: 2 → 20, 7 → 30, 20 → 99 (so the two orders disagree) + let mut table = vec![0u16; 32]; + table[2] = 20; + table[7] = 30; + table[20] = 99; + // MapTable: 2 → 7, 20 → 8 + let mut map: Vec = (0..32).collect(); + map[2] = 7; + map[20] = 8; + let area = Area { top: 0, left: 0, bottom: h as u32, right: w as u32, plane: 0, planes: 1, row_pitch: 1, col_pitch: 1 }; + let list1 = opcodes::write_list(&[Opcode::MapTable { area, table: map.clone() }]); + + let mut raw = base_ifd(w, h, 16, 1, true); + raw.set(t::COMPRESSION, Value::Short(vec![compression::NONE])); + raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![u16_bytes(&stored, ByteOrder::Little)] }); + raw.set(t::LINEARIZATION_TABLE, Value::Short(table.clone())); + raw.set(t::OPCODE_LIST_1, Value::Undefined(list1)); + raw.set(t::WHITE_LEVEL, Value::Long(vec![100])); + let img = decode(&dng(raw, ByteOrder::Little)).unwrap(); + assert!(!img.linearized, "list 1 is present, so the table must wait"); + assert_eq!(img.data, RawData::U16(stored.clone())); + assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice())); + assert_eq!(img.opcodes.list1.len(), 1); + + let apply_map = |samples: &mut [u16]| { + for s in samples { + let i = (*s as usize).min(map.len() - 1); + *s = map[i]; + } + }; + + let mut correct = img.clone(); + if let RawData::U16(v) = &mut correct.data { + apply_map(v); + } + correct.apply_linearization(); + assert!(correct.linearized); + // 2 → 7 (MapTable) → 30 (linearization) + assert_eq!(correct.data, RawData::U16(vec![30; w * h])); + + let mut wrong = img.clone(); + wrong.apply_linearization(); + if let RawData::U16(v) = &mut wrong.data { + apply_map(v); + } + // 2 → 20 (linearization) → 8 (MapTable) + assert_eq!(wrong.data, RawData::U16(vec![8; w * h])); + assert_ne!(correct.data, wrong.data); + + // the unapplied table and list 1 survive a write / decode round trip + let back = decode(&write_dng(&img, &DngWriteOptions { compression: DngCompression::Uncompressed, ..Default::default() }).unwrap()).unwrap(); + assert!(!back.linearized); + assert_eq!(back.data, RawData::U16(stored)); + assert_eq!(back.linearization_table.as_deref(), Some(table.as_slice())); + assert_eq!(back.opcodes.list1.len(), 1); +} + /// PhotoCraft: `AsShotWhiteXY` is a chromaticity, so both coordinates lie inside (0, 1). #[test] fn an_as_shot_white_outside_the_chromaticity_range_is_ignored() { diff --git a/crates/raw/tests/it/robust.rs b/crates/raw/tests/it/robust.rs index 5ad1d1b32..28640a803 100644 --- a/crates/raw/tests/it/robust.rs +++ b/crates/raw/tests/it/robust.rs @@ -22,6 +22,7 @@ fn sample(comp: DngCompression) -> Vec { color: ColorData::default(), wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera: Default::default(), diff --git a/crates/usd/README.md b/crates/usd/README.md index 9e9635185..fcf585760 100644 --- a/crates/usd/README.md +++ b/crates/usd/README.md @@ -2,6 +2,6 @@ Bounded wire primitives for USD Crate, independent of application scene and material models. -Provides bounded integer-delta array decoding and Crate-framed LZ4 decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and export remain outside this extraction. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation. +Provides bounded integer-delta array decoding and Crate-framed LZ4 compression/decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and full scene export remain outside this extraction. Compression uses a caller cap on worst-case reserved output and fallible allocation. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation. Adapted from TextureCraft `crates/formats/src/mesh_import/usd/codec.rs`, introduced by bflatastic in `bdbb52c9b66b221a55b370105b42f485a76e6fc7`. The LZ4 backend is MIT-licensed `lz4_flex`. diff --git a/crates/usd/src/compress.rs b/crates/usd/src/compress.rs new file mode 100644 index 000000000..b92fa3be8 --- /dev/null +++ b/crates/usd/src/compress.rs @@ -0,0 +1,76 @@ +//! Bounded single-block USD Crate LZ4 writing. + +/// Compression admission, allocation or backend failure. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Input or worst-case output exceeds the supported/caller budget. + #[error("USD Crate LZ4 compression exceeds budget")] + Limit, + /// The bounded output reservation failed. + #[error("cannot allocate USD Crate LZ4 output")] + Allocation, + /// The backend could not encode into the preflighted output buffer. + #[error("USD Crate LZ4 compression: {0}")] + Encode(#[from] lz4_flex::block::CompressError), +} + +fn capacity(input: usize, max_output_bytes: usize) -> Result { + // Keep inputs below the LZ4 single-block limit and preflight the backend's + // 20 + input*110/100 sizing arithmetic before calling it, including the count byte. + if input > 0x7e00_0000 { + return Err(Error::Limit); + } + input + .checked_mul(110) + .map(|n| n / 100) + .and_then(|n| n.checked_add(21)) + .filter(|n| *n <= max_output_bytes && *n <= isize::MAX as usize) + .ok_or(Error::Limit) +} + +/// Write one Crate-framed LZ4 block, beginning with a zero chunk-count byte. +/// +/// `max_output_bytes` bounds the reserved output including framing. Admission uses +/// the backend's worst-case size, so even highly compressible input must fit that bound. +/// Allocation is fallible; backend scratch memory is not included in the output cap. +/// Uses TextureCraft's original LZ4 backend/configuration and produces the same block bytes. +pub fn compress(input: &[u8], max_output_bytes: usize) -> Result, Error> { + let capacity = capacity(input.len(), max_output_bytes)?; + let mut output = Vec::new(); + output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?; + output.resize(capacity, 0); + let target = output.get_mut(1..).ok_or(Error::Limit)?; + let written = lz4_flex::block::compress_into(input, target)?; + output.truncate(written.checked_add(1).ok_or(Error::Limit)?); + Ok(output) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn original_wire_bytes_and_roundtrip() { + for size in [0, 1, 15, 16, 255, 4096, 65536] { + for repeated in [false, true] { + let input: Vec = (0..size).map(|i| if repeated { 17 } else { (i % 251) as u8 }).collect(); + let mut old = vec![0]; + old.extend(lz4_flex::block::compress(&input)); + let encoded = compress(&input, 1 << 20).unwrap(); + assert_eq!(encoded, old); + assert_eq!(crate::decompress(&encoded, size).unwrap(), input); + } + } + } + + #[test] + fn checks_worst_case_before_allocation() { + let input = [0; 100]; + assert!(matches!(compress(&input, 130), Err(Error::Limit))); + assert!(compress(&input, 131).is_ok()); + assert!(capacity(usize::MAX, usize::MAX).is_err()); + assert!(capacity(0x7e00_0001, usize::MAX).is_err()); + assert_eq!(capacity(0, 21).unwrap(), 21); + } +} diff --git a/crates/usd/src/integers.rs b/crates/usd/src/integers.rs index c50dd0e0f..0d8e67df4 100644 --- a/crates/usd/src/integers.rs +++ b/crates/usd/src/integers.rs @@ -80,6 +80,72 @@ pub fn decode(data: &[u8], count: usize, width: Width, max_output_bytes: usize) Ok(output) } +/// Encode integers with their original signed wrapping and largest-delta tie-break policy. +/// +/// `max_bytes` bounds the combined worst-case output reservation and sorted delta scratch. +/// Both reservations are fallible. Sorting uses no extra heap storage; input storage and +/// allocator bookkeeping are outside this budget. Empty input emits no bytes. +pub fn encode(values: &[i64], width: Width, max_bytes: usize) -> Result, Error> { + if values.is_empty() { + return Ok(Vec::new()); + } + let size = width.bytes(); + let codes_len = values.len().div_ceil(4); + let capacity = values.len().checked_mul(size).and_then(|n| n.checked_add(size)).and_then(|n| n.checked_add(codes_len)).ok_or(Error::Limit)?; + let scratch = values.len().checked_mul(8).ok_or(Error::Limit)?; + if capacity > isize::MAX as usize || scratch > isize::MAX as usize || capacity.checked_add(scratch).is_none_or(|n| n > max_bytes) { + return Err(Error::Limit); + } + let wrap = |v: i64| match width { + Width::I32 => i64::from(v as i32), + Width::I64 => v, + }; + let mut deltas = Vec::new(); + deltas.try_reserve_exact(values.len()).map_err(|_| Error::Allocation)?; + let mut previous = 0i64; + for &value in values { + deltas.push(wrap(value.wrapping_sub(previous))); + previous = value; + } + deltas.sort_unstable(); + let mut best = (0usize, 0i64); + let mut at = 0; + while at < deltas.len() { + let value = deltas[at]; + let start = at; + while at < deltas.len() && deltas[at] == value { + at += 1; + } + best = best.max((at - start, value)); + } + let common = best.1; + let mut output = Vec::new(); + output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?; + output.extend_from_slice(&common.to_le_bytes()[..size]); + output.resize(size + codes_len, 0); + previous = 0; + for (i, &value) in values.iter().enumerate() { + let delta = wrap(value.wrapping_sub(previous)); + previous = value; + let small = |bits: u32| delta >= -(1i64 << (bits - 1)) && delta < (1i64 << (bits - 1)); + let code: u8 = if delta == common { + 0 + } else if small(size as u32 * 2) { + 1 + } else if small(size as u32 * 4) { + 2 + } else { + 3 + }; + if code != 0 { + let bytes = (size >> (3 - usize::from(code))).max(1); + output.extend_from_slice(&delta.to_le_bytes()[..bytes]); + } + output[size + i / 4] |= code << (2 * (i % 4)); + } + Ok(output) +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { @@ -125,3 +191,81 @@ mod tests { } } } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod encode_tests { + use super::*; + fn oracle(values: &[i64], width: usize) -> Vec { + if values.is_empty() { + return Vec::new(); + } + let wrap = |v: i64| if width == 4 { i64::from(v as i32) } else { v }; + let mut prev = 0i64; + let deltas: Vec = values + .iter() + .map(|&v| { + let d = wrap(v.wrapping_sub(prev)); + prev = v; + d + }) + .collect(); + let mut counts = std::collections::HashMap::new(); + for d in &deltas { + *counts.entry(*d).or_insert(0usize) += 1; + } + let common = counts.into_iter().max_by_key(|&(v, c)| (c, v)).map_or(0, |(v, _)| v); + let mut out = Vec::with_capacity(width + values.len().div_ceil(4) + values.len() * width); + out.extend_from_slice(&common.to_le_bytes()[..width]); + let codes_at = out.len(); + out.resize(codes_at + values.len().div_ceil(4), 0); + for (i, &d) in deltas.iter().enumerate() { + let small = |bits: u32| d >= -(1i64 << (bits - 1)) && d < (1i64 << (bits - 1)); + let code: u8 = if d == common { + 0 + } else if small(width as u32 * 2) { + 1 + } else if small(width as u32 * 4) { + 2 + } else { + 3 + }; + if code > 0 { + let w = (width >> (3 - code as usize)).max(1); + out.extend_from_slice(&d.to_le_bytes()[..w]); + } + if let Some(b) = out.get_mut(codes_at + i / 4) { + *b |= code << (2 * (i % 4)); + } + } + out + } + + #[test] + fn matches_original_bytes_and_wrapping() { + for width in [Width::I32, Width::I64] { + for n in [0, 1, 2, 3, 4, 5, 127, 1024] { + let mut seed = 17u64; + let values: Vec = (0..n) + .map(|_| { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + seed as i64 + }) + .collect(); + let bytes = encode(&values, width, 32768).unwrap(); + assert_eq!(bytes, oracle(&values, width.bytes())); + let expected: Vec = values.iter().map(|&v| if width == Width::I32 { i64::from(v as i32) } else { v }).collect(); + assert_eq!(decode(&bytes, n, width, 32768).unwrap(), expected); + } + for values in [vec![1, 3, 4, 6], vec![i64::MAX, i64::MIN, 0], vec![7; 32]] { + assert_eq!(encode(&values, width, 4096).unwrap(), oracle(&values, width.bytes())); + } + } + } + #[test] + fn combined_reservation_budget_is_enforced() { + assert_eq!(encode(&[], Width::I64, 0).unwrap(), Vec::::new()); + assert!(matches!(encode(&[1], Width::I32, 16), Err(Error::Limit))); + assert!(encode(&[1], Width::I32, 17).is_ok()); + } +} diff --git a/crates/usd/src/lib.rs b/crates/usd/src/lib.rs index edaeb89a2..33c7f3484 100644 --- a/crates/usd/src/lib.rs +++ b/crates/usd/src/lib.rs @@ -5,6 +5,9 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +/// Bounded single-block Crate LZ4 compression. +pub mod compress; + /// Bounded Crate integer delta arrays. pub mod integers; diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index b8669288e..9d8e1b785 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -83,17 +83,22 @@ for new pictures (`vm = i + 1`) and the `XLDAPR` dynamic-array metadata (`cm = 1 The parts this crate writes use the transitional namespaces; a strict file's kept parts stay as they were. A binary workbook is not written. -A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not -read (each cell keeps its format index). Its cell formulas are decoded from their tokens into the -text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four -ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in -the formula, numbers as Excel spells them), with -`FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell -metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the -file keeps the tokens once. A formula whose decoding needs something the reader does not read -keeps only its saved result, with a warning saying why: user defined names, table references, -other workbooks, Python code stored outside the cell, data tables. The XLSB function table -(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. +A binary workbook gives the same `Workbook`, strings, rows and merged cells, and the same `Styles`: +number formats, fonts, fills, borders, cell and cell style formats, cell styles, differential +formats and the indexed colours, as Excel writes them in `styles.xml` (default fill colours and +empty settings left out, a cell style format's `apply*="0"`, a number format with a newer locale tag +in its Excel 2007 spelling, as `formatCode` has it). Custom table styles and recent colours are not +read; a damaged styles part is a warning and no styles, while one past a cap (65,536 entries in a +table, 1,024 gradient stops) or past `binary_text` is too large. Its cell formulas are decoded from +their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from +the four ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks +typed in the formula, numbers as Excel spells them), with `FormulaKind::Shared` or +`FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a +shared formula gets its own text, decoded at the cell, since the file keeps the tokens once. A +formula whose decoding needs something the reader does not read keeps only its saved result, with a +warning saying why: user defined names, table references, other workbooks, Python code stored +outside the cell, data tables. The XLSB function table (`xlsb/ftab.rs`) and the expression printer +are this crate's own, apart from `craft-xls`'s. ## Hostile input diff --git a/crates/xlsx/src/document.rs b/crates/xlsx/src/document.rs index 26e6a84c6..1be23029e 100644 --- a/crates/xlsx/src/document.rs +++ b/crates/xlsx/src/document.rs @@ -209,8 +209,8 @@ pub struct Document { } impl Document { - /// Reads every part. A binary workbook is refused: its formulas and styles are not read, so - /// it cannot be written back as SpreadsheetML without loss. + /// Reads every part. A binary workbook is refused: only some of its parts are read, so it + /// cannot be written back as SpreadsheetML without loss. pub fn read(bytes: &[u8], limits: &Limits) -> Result { let mut p = Package::open(bytes, limits)?; if p.is_binary() { diff --git a/crates/xlsx/src/package.rs b/crates/xlsx/src/package.rs index d43a15a8c..62076b3fc 100644 --- a/crates/xlsx/src/package.rs +++ b/crates/xlsx/src/package.rs @@ -220,16 +220,28 @@ impl<'a> Package<'a> { Ok(items) } - /// The styles part, `None` when the workbook has none (or is binary: its styles are not read). + /// The styles part, `None` when the workbook has none. A damaged one is a warning and `None`; + /// only a limit is an error. pub fn styles(&mut self) -> Result, Error> { - if self.binary.is_some() { - return Ok(None); - } let rels = self.workbook_rels()?; let Some(rel) = rels.first("styles").filter(|r| !r.external) else { self.warn("no styles part"); return Ok(None); }; + if let Some(types) = &self.binary { + let read = xlsb::check_type(types, &rel.target, xlsb::STYLES).and_then(|()| { + let bytes = self.opc.read(&rel.target)?.ok_or_else(|| xlsb::invalid(format!("missing part {}", rel.target)))?; + xlsb::styles(&bytes, &self.limits, &self.budget, &mut self.warnings) + }); + return match read { + Ok(styles) => Ok(Some(styles)), + Err(Error::TooLarge(m)) => Err(Error::TooLarge(m)), + Err(e) => { + self.warn(format!("skipped {}: {e}", rel.target)); + Ok(None) + } + }; + } Ok(self.optional_xml(&rel.target.clone())?.map(Styles::read)) } diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index b3f94b5fa..c8fdba69f 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -1,11 +1,13 @@ //! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheet cells, from //! their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an //! `.xlsx` file holds; one that can't be keeps only its saved result, with a warning saying why. -//! Styles are not read. A record that does not parse fails the read. +//! The styles part is read into the same [`crate::styles::Styles`] ([`styles`]). A record that +//! does not parse fails the read. mod expr; mod formula; mod ftab; +mod styles; use std::collections::HashMap; use std::sync::Arc; @@ -19,11 +21,13 @@ use crate::workbook::{SheetEntry, SheetKind, Workbook}; use crate::{Error, Limits, Warnings}; pub(crate) use formula::Links; use formula::{Book, Mode, Name, Xti}; +pub(crate) use styles::styles; pub(crate) const MAIN: &str = "application/vnd.ms-excel.main"; const BINARY_MAIN: &str = "application/vnd.ms-excel.sheet.binary.macroEnabled.main"; pub(crate) const WORKSHEET: &str = "application/vnd.ms-excel.worksheet"; pub(crate) const STRINGS: &str = "application/vnd.ms-excel.sharedStrings"; +pub(crate) const STYLES: &str = "application/vnd.ms-excel.styles"; /// Most sheets in the workbook. const MAX_SHEETS: usize = 1024; /// Supporting links (this workbook, add-ins, other workbooks) read at most; past it, formulas @@ -174,12 +178,16 @@ struct Records<'a, 'l> { bytes: Bytes<'a>, limits: &'l Limits, budget: &'l Budget, - blocks: Vec, + /// Each open block's end record, and whether its records are read. + blocks: Vec<(u16, bool)>, + /// Read the alternate content meant for Excel 2007 (version 12), as an `.xlsx` reader sees + /// the main attribute where Excel keeps a newer spelling in an extension. + excel_2007: bool, } impl<'a, 'l> Records<'a, 'l> { fn new(bytes: &'a [u8], limits: &'l Limits, budget: &'l Budget) -> Self { - Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![] } + Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![], excel_2007: false } } fn next(&mut self, warnings: &mut Warnings) -> Result, Error> { @@ -208,6 +216,7 @@ impl<'a, 'l> Records<'a, 'l> { // BrtFRTBegin, BrtACBegin. 35 | 37 => { let mut b = Bytes::new(payload); + let mut read = false; if id == 35 { b.take(4)?; } else { @@ -215,22 +224,32 @@ impl<'a, 'l> Records<'a, 'l> { if count == 0 { return Err(invalid("empty alternate-content version list")); } - b.take(usize::from(count) * 4)?; + // ACProductVersion: fileVersion, then fileProduct (15 bits) and + // fileExtension (the version and later). + for _ in 0..count { + let (version, product) = (b.u16()?, b.u16()?); + let later = product & 0x8000 != 0; + read |= self.excel_2007 && product & 0x7FFF == 0 && (version == 12 || later && version <= 12); + } } b.finish()?; if self.blocks.len() >= 64 { return Err(invalid("extension blocks nested too deeply")); } - self.blocks.push(id + 1); - warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped"); + // Inside a skipped block, everything is skipped. + let read = read && self.blocks.iter().all(|(_, r)| *r); + self.blocks.push((id + 1, read)); + if !read { + warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped"); + } } // BrtFRTEnd, BrtACEnd. 36 | 38 => { - if self.blocks.pop() != Some(id) || !payload.is_empty() { + if self.blocks.pop().map(|(end, _)| end) != Some(id) || !payload.is_empty() { return Err(invalid("mismatched extension block end")); } } - _ if !self.blocks.is_empty() => {} + _ if self.blocks.iter().any(|(_, read)| !read) => {} _ => return Ok(Some((id, payload))), } } diff --git a/crates/xlsx/src/xlsb/styles.rs b/crates/xlsx/src/xlsb/styles.rs new file mode 100644 index 000000000..88c6bee51 --- /dev/null +++ b/crates/xlsx/src/xlsb/styles.rs @@ -0,0 +1,484 @@ +//! The styles part of a binary workbook (`styles.bin`, [MS-XLSB] 2.1.7.50), read into the same +//! [`Styles`] the XML part gives: what Excel writes in `styles.xml` for the same workbook. Table +//! styles, recent colours and future records are not read. + +use craft_sheet::{BorderStyle, HAlign, Pattern, Script, Underline, VAlign}; +use craft_xml::El; + +use super::{Budget, Bytes, Records, invalid}; +use crate::styles::{ + Alignment, Border, BorderEdge, CellStyle, Color, ColorSource, Dxf, Fill, Font, FontScheme, GradientFill, NumFmt, PatternFill, Protection, Styles, + Xf, +}; +use crate::{Error, Limits, Warnings}; + +/// Characters of a font name or number format kept, as the XML side keeps of a font name. +const MAX_NAME_CHARS: usize = 255; +/// Longest string read from the part (Excel's own limits are 31 and 255 characters). +const MAX_STRING_UNITS: u32 = 32_767; +/// Most stops in one gradient fill (Excel writes a handful). +const MAX_STOPS: u32 = 1024; +/// Most entries in one table of the part (Excel allows about 64,000 cell formats). Each entry is +/// an allocation, and the record allowance alone would let a small part hold millions. +const MAX_ENTRIES: usize = 65_536; + +/// `BrtColor` and `XFPropColor`: the same eight bytes. `None` for "not set" (`XFPropColor` type 4). +fn color(b: &mut Bytes<'_>) -> Result, Error> { + let kind = b.u8()? >> 1; + let index = b.u8()?; + let tint = i16::from_le_bytes(<[u8; 2]>::try_from(b.take(2)?).map_err(|_| invalid("truncated colour"))?); + let [r, g, bl, a] = <[u8; 4]>::try_from(b.take(4)?).map_err(|_| invalid("truncated colour"))?; + let source = match kind { + 0 => ColorSource::Auto, + 1 => ColorSource::Indexed(u32::from(index)), + 2 => ColorSource::Rgb(u32::from_be_bytes([a, r, g, bl])), + 3 => ColorSource::Theme(u32::from(index)), + _ => return Ok(None), + }; + Ok(Some(Color { source, tint: (f64::from(tint) / 32767.0).clamp(-1.0, 1.0) })) +} + +fn pattern(fls: u32) -> Result { + const PATTERNS: [Pattern; 19] = [ + Pattern::None, + Pattern::Solid, + Pattern::Gray50, + Pattern::Gray75, + Pattern::Gray25, + Pattern::DarkHorizontal, + Pattern::DarkVertical, + Pattern::DarkDown, + Pattern::DarkUp, + Pattern::DarkGrid, + Pattern::DarkTrellis, + Pattern::LightHorizontal, + Pattern::LightVertical, + Pattern::LightDown, + Pattern::LightUp, + Pattern::LightGrid, + Pattern::LightTrellis, + Pattern::Gray125, + Pattern::Gray0625, + ]; + PATTERNS.get(fls as usize).copied().ok_or_else(|| invalid("unknown fill pattern")) +} + +fn border_style(dg: u16) -> Result { + const STYLES: [BorderStyle; 14] = [ + BorderStyle::None, + BorderStyle::Thin, + BorderStyle::Medium, + BorderStyle::Dashed, + BorderStyle::Dotted, + BorderStyle::Thick, + BorderStyle::Double, + BorderStyle::Hair, + BorderStyle::MediumDashed, + BorderStyle::DashDot, + BorderStyle::MediumDashDot, + BorderStyle::DashDotDot, + BorderStyle::MediumDashDotDot, + BorderStyle::SlantDashDot, + ]; + STYLES.get(usize::from(dg)).copied().ok_or_else(|| invalid("unknown border style")) +} + +fn halign(alc: u32) -> Option { + [HAlign::General, HAlign::Left, HAlign::Center, HAlign::Right, HAlign::Fill, HAlign::Justify, HAlign::CenterAcross, HAlign::Distributed] + .get(alc as usize) + .copied() +} + +fn valign(alcv: u32) -> Option { + [VAlign::Top, VAlign::Center, VAlign::Bottom, VAlign::Justify, VAlign::Distributed].get(alcv as usize).copied() +} + +fn underline(uls: u32) -> Underline { + match uls { + 0 => Underline::None, + 2 => Underline::Double, + 0x21 => Underline::SingleAccounting, + 0x22 => Underline::DoubleAccounting, + _ => Underline::Single, + } +} + +fn script(sss: u32) -> Script { + match sss { + 1 => Script::Superscript, + 2 => Script::Subscript, + _ => Script::Baseline, + } +} + +fn scheme(s: u32) -> Option { + match s { + 0 => Some(FontScheme::None), + 1 => Some(FontScheme::Major), + 2 => Some(FontScheme::Minor), + _ => None, + } +} + +fn name(text: String) -> String { + text.chars().take(MAX_NAME_CHARS).collect() +} + +/// `BrtFont`: only what Excel's `.xlsx` states for the same font (no ``, no scheme +/// `none`, no family or character set 0). +fn font(b: &mut Bytes<'_>) -> Result { + let height = b.u16()?; + let flags = b.u16()?; + let weight = b.u16()?; + let sss = b.u16()?; + let uls = b.u8()?; + let family = b.u8()?; + let charset = b.u8()?; + b.u8()?; + let color = color(b)?; + let font_scheme = b.u8()?; + let font_name = name(b.text(MAX_STRING_UNITS)?); + let on = |bit: u16| (flags & bit != 0).then_some(true); + Ok(Font { + name: Some(font_name), + size: Some(f64::from(height) / 20.0), + bold: (weight >= 0x2BC).then_some(true), + italic: on(0x02), + strike: on(0x08), + outline: on(0x10), + shadow: on(0x20), + condense: on(0x40), + extend: on(0x80), + underline: (uls != 0).then(|| underline(u32::from(uls))), + vert_align: (sss != 0).then(|| script(u32::from(sss))), + family: (family != 0).then_some(u32::from(family)), + charset: (charset != 0).then_some(u32::from(charset)), + scheme: scheme(u32::from(font_scheme)).filter(|s| *s != FontScheme::None), + // An automatic colour is a font with no `` in `.xlsx`. + color: color.filter(|c| c.source != ColorSource::Auto), + }) +} + +/// A gradient's angle and inner rectangle, as `BrtFill` and `XFPropGradient` hold them. +fn gradient(b: &mut Bytes<'_>, path: bool) -> Result { + Ok(GradientFill { path, degree: b.f64()?, left: b.f64()?, right: b.f64()?, top: b.f64()?, bottom: b.f64()?, stops: vec![] }) +} + +/// `BrtFill`. Excel's `.xlsx` leaves out the default colours: system foreground (indexed 64) as +/// the pattern colour and system background (indexed 65) as the background, unless the pattern +/// colour has a tint (as the built-in "20% - Accent1" styles do), when it states the background. +fn fill(b: &mut Bytes<'_>) -> Result { + let fls = b.u32()?; + let fg = color(b)?; + let bg = color(b)?; + let kind = b.u32()?; + let mut g = gradient(b, kind == 1)?; + let count = b.u32()?; + if fls == 0x28 { + if count > MAX_STOPS { + return Err(Error::TooLarge(format!("more than {MAX_STOPS} gradient stops"))); + } + for _ in 0..count { + let c = color(b)?.unwrap_or_default(); + g.stops.push((b.f64()?, c)); + } + return Ok(Fill::Gradient(g)); + } + let system = |c: Color, i: u32| c.source == ColorSource::Indexed(i) && c.tint == 0.0; + let tinted = fg.is_some_and(|c| c.tint != 0.0); + Ok(Fill::Pattern(PatternFill { pattern: Some(pattern(fls)?), fg: fg.filter(|c| !system(*c, 64)), bg: bg.filter(|c| tinted || !system(*c, 65)) })) +} + +/// `BrtBorder`: the five edges a cell border has, each stated (`` when it has no line). +fn border(b: &mut Bytes<'_>) -> Result { + let flags = b.u8()?; + let mut edge = || -> Result { + let style = border_style(u16::from(b.u8()?))?; + b.u8()?; + let c = color(b)?; + Ok(BorderEdge { style, color: c.filter(|_| style != BorderStyle::None) }) + }; + let (top, bottom, left, right, diagonal) = (edge()?, edge()?, edge()?, edge()?, edge()?); + Ok(Border { + left: Some(left), + right: Some(right), + top: Some(top), + bottom: Some(bottom), + diagonal: Some(diagonal), + vertical: None, + horizontal: None, + diagonal_up: flags & 2 != 0, + diagonal_down: flags & 1 != 0, + outline: true, + }) +} + +/// `BrtXF`. A cell style format (`style`) marks what it leaves out (`apply*="0"`); a cell format +/// marks what it sets apart from its style (`apply*="1"`). +fn xf(b: &mut Bytes<'_>, style: bool) -> Result { + const APPLY: [(u8, &str); 6] = [ + (0x01, "applyNumberFormat"), + (0x02, "applyFont"), + (0x10, "applyFill"), + (0x08, "applyBorder"), + (0x04, "applyAlignment"), + (0x20, "applyProtection"), + ]; + let parent = b.u16()?; + let num_fmt_id = u32::from(b.u16()?); + let font_id = u32::from(b.u16()?); + let fill_id = u32::from(b.u16()?); + let border_id = u32::from(b.u16()?); + let rotation = u32::from(b.u8()?); + let indent = u32::from(b.u8()?); + let flags = u32::from(b.u16()?); + let apply = b.u8()?; + b.u8()?; + let alignment = Alignment { + horizontal: halign(flags & 7).filter(|h| *h != HAlign::General), + vertical: valign((flags >> 3) & 7).filter(|v| *v != VAlign::Bottom), + text_rotation: (rotation != 0).then_some(rotation), + wrap_text: flags & 0x40 != 0, + justify_last_line: flags & 0x80 != 0, + shrink_to_fit: flags & 0x100 != 0, + indent: (indent != 0).then_some(indent), + reading_order: Some((flags >> 10) & 3).filter(|r| *r != 0), + }; + let protection = Protection { locked: (flags & 0x1000 == 0).then_some(false), hidden: (flags & 0x2000 != 0).then_some(true) }; + let mut el = El::new("xf"); + for (bit, attr) in APPLY { + if apply & bit != 0 { + el = el.with_attr(attr, if style { "0" } else { "1" }); + } + } + Ok(Xf { + num_fmt_id, + font_id, + fill_id, + border_id, + xf_id: (!style).then_some(u32::from(parent)), + alignment: (alignment != Alignment::default()).then_some(alignment), + protection: (protection != Protection::default()).then_some(protection), + quote_prefix: flags & 0x8000 != 0, + el, + }) +} + +/// `BrtStyle`. +fn cell_style(b: &mut Bytes<'_>) -> Result { + let xf_id = b.u32()?; + let flags = b.u16()?; + let builtin = b.u8()?; + let level = b.u8()?; + let style_name = b.string(MAX_STRING_UNITS, true)?.map(name).unwrap_or_default(); + let is_builtin = flags & 1 != 0; + Ok(CellStyle { + name: style_name, + xf_id, + builtin_id: is_builtin.then_some(u32::from(builtin)), + level: (is_builtin && matches!(builtin, 1 | 2)).then_some(u32::from(level)), + hidden: flags & 2 != 0, + custom_builtin: flags & 4 != 0, + }) +} + +/// An unsigned integer of the size a property has (1, 2 or 4 bytes). +fn uint(data: &[u8]) -> Result { + match *data { + [a] => Ok(u32::from(a)), + [a, b] => Ok(u32::from(u16::from_le_bytes([a, b]))), + [a, b, c, d] => Ok(u32::from_le_bytes([a, b, c, d])), + _ => Err(invalid("unexpected formatting property size")), + } +} + +/// A string with a 2-byte length (`LPWideString`, and the number format of `XFProp` 0x26). +fn short_string(b: &mut Bytes<'_>) -> Result { + let count = usize::from(b.u16()?); + let mut text = String::new(); + for ch in char::decode_utf16(b.take(count * 2)?.as_chunks::<2>().0.iter().copied().map(u16::from_le_bytes)) { + text.push(ch.map_err(|_| invalid("invalid UTF-16 string"))?); + } + Ok(name(text)) +} + +fn dxf_font(d: &mut Dxf) -> &mut Font { + d.font.get_or_insert_with(Font::default) +} + +/// `BrtDXF`: its `XFProps`, each stating one property. +fn dxf(b: &mut Bytes<'_>) -> Result { + b.u16()?; + b.u16()?; + let count = b.u16()?; + let mut d = Dxf::default(); + let mut gradient_fill: Option = None; + let mut pattern_fill: Option = None; + for _ in 0..count { + let kind = b.u16()?; + let size = usize::from(b.u16()?).checked_sub(4).ok_or_else(|| invalid("formatting property too short"))?; + let data = b.take(size)?; + let mut p = Bytes::new(data); + match kind { + 0x00 => pattern_fill.get_or_insert_with(PatternFill::default).pattern = Some(pattern(uint(data)?)?), + 0x01 => pattern_fill.get_or_insert_with(PatternFill::default).fg = color(&mut p)?, + 0x02 => pattern_fill.get_or_insert_with(PatternFill::default).bg = color(&mut p)?, + 0x03 => { + let path = p.u32()? == 1; + gradient_fill = Some(gradient(&mut p, path)?); + } + 0x04 => { + let g = gradient_fill.as_mut().ok_or_else(|| invalid("gradient stop without a gradient"))?; + if g.stops.len() >= MAX_STOPS as usize { + return Err(Error::TooLarge(format!("more than {MAX_STOPS} gradient stops"))); + } + p.u16()?; + let position = p.f64()?; + g.stops.push((position, color(&mut p)?.unwrap_or_default())); + } + 0x05 => dxf_font(&mut d).color = color(&mut p)?, + 0x06..=0x0C => { + let c = color(&mut p)?; + let edge = Some(BorderEdge { style: border_style(p.u16()?)?, color: c }); + let border = d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }); + match kind { + 0x06 => border.top = edge, + 0x07 => border.bottom = edge, + 0x08 => border.left = edge, + 0x09 => border.right = edge, + 0x0A => border.diagonal = edge, + 0x0B => border.vertical = edge, + _ => border.horizontal = edge, + } + } + 0x0D => d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }).diagonal_up = uint(data)? != 0, + 0x0E => d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }).diagonal_down = uint(data)? != 0, + 0x0F..=0x16 => { + let v = uint(data)?; + let a = d.alignment.get_or_insert_with(Alignment::default); + match kind { + 0x0F => a.horizontal = halign(v), + 0x10 => a.vertical = valign(v), + 0x11 => a.text_rotation = Some(v), + 0x12 => a.indent = Some(v), + 0x13 => a.reading_order = Some(v), + 0x14 => a.wrap_text = v != 0, + 0x15 => a.justify_last_line = v != 0, + _ => a.shrink_to_fit = v != 0, + } + } + 0x18 => dxf_font(&mut d).name = Some(short_string(&mut p)?), + 0x19 => dxf_font(&mut d).bold = Some(uint(data)? >= 0x2BC), + 0x1A => dxf_font(&mut d).underline = Some(underline(uint(data)?)), + 0x1B => dxf_font(&mut d).vert_align = Some(script(uint(data)?)), + 0x1C => dxf_font(&mut d).italic = Some(uint(data)? != 0), + 0x1D => dxf_font(&mut d).strike = Some(uint(data)? != 0), + 0x1E => dxf_font(&mut d).outline = Some(uint(data)? != 0), + 0x1F => dxf_font(&mut d).shadow = Some(uint(data)? != 0), + 0x20 => dxf_font(&mut d).condense = Some(uint(data)? != 0), + 0x21 => dxf_font(&mut d).extend = Some(uint(data)? != 0), + 0x22 => dxf_font(&mut d).charset = Some(uint(data)?), + 0x23 => dxf_font(&mut d).family = Some(uint(data)?), + 0x24 => dxf_font(&mut d).size = Some(f64::from(uint(data)?) / 20.0), + 0x25 => dxf_font(&mut d).scheme = scheme(uint(data)?), + 0x26 => d.num_fmt.get_or_insert_with(NumFmt::default).code = Some(short_string(&mut p)?), + 0x29 => d.num_fmt.get_or_insert_with(NumFmt::default).id = Some(uint(data)?), + 0x2B => d.protection.get_or_insert_with(Protection::default).locked = Some(uint(data)? != 0), + 0x2C => d.protection.get_or_insert_with(Protection::default).hidden = Some(uint(data)? != 0), + // Merged cells (0x17) and relative indents (0x2A) have no place in ``. + _ => {} + } + } + d.fill = gradient_fill.map(Fill::Gradient).or(pattern_fill.map(Fill::Pattern)); + Ok(d) +} + +/// The styles part: `BrtBeginStyleSheet` to `BrtEndStyleSheet`. +pub(crate) fn styles(bytes: &[u8], limits: &Limits, budget: &Budget, warnings: &mut Warnings) -> Result { + let mut records = Records::new(bytes, limits, budget); + // Excel keeps a number format with a newer locale tag in two alternate-content blocks: the + // newer spelling for itself, and for Excel 2007 the one `.xlsx` has as `formatCode`. + records.excel_2007 = true; + records.first(278, warnings)?; + let mut s = Styles::default(); + // Inside BrtBeginCellStyleXFs (true) or BrtBeginCellXFs (false). + let mut xfs: Option = None; + let mut palette = false; + while let Some((id, payload)) = records.next(warnings)? { + let mut b = Bytes::new(payload); + let entries = match id { + 44 => s.num_fmts.len(), + 43 => s.fonts.len(), + 45 => s.fills.len(), + 46 => s.borders.len(), + 47 => s.cell_style_xfs.len().max(s.cell_xfs.len()), + 48 => s.cell_styles.len(), + 507 => s.dxfs.len(), + 475 => s.indexed_colors.len(), + _ => 0, + }; + if entries >= MAX_ENTRIES { + return Err(Error::TooLarge(format!("more than {MAX_ENTRIES} entries in a styles table"))); + } + match id { + 279 => { + records.end(payload, warnings)?; + return Ok(s); + } + 44 => { + let id = u32::from(b.u16()?); + let code = name(b.text(MAX_STRING_UNITS)?); + budget.text(code.len(), limits)?; + s.num_fmts.push((id, code)); + } + 43 => { + let f = font(&mut b)?; + budget.text(f.name.as_ref().map_or(0, String::len), limits)?; + s.fonts.push(f); + } + 45 => s.fills.push(fill(&mut b)?), + 46 => s.borders.push(border(&mut b)?), + 626 => xfs = Some(true), + 617 => xfs = Some(false), + 627 | 618 => xfs = None, + 47 => match xfs { + Some(true) => s.cell_style_xfs.push(xf(&mut b, true)?), + Some(false) => s.cell_xfs.push(xf(&mut b, false)?), + None => return Err(invalid("cell format outside its list")), + }, + 48 => { + let c = cell_style(&mut b)?; + budget.text(c.name.len(), limits)?; + s.cell_styles.push(c); + } + 507 => { + let d = dxf(&mut b)?; + // What it keeps of its strings, after a later property has replaced an earlier. + let font = d.font.as_ref().and_then(|f| f.name.as_ref()).map_or(0, String::len); + let format = d.num_fmt.as_ref().and_then(|n| n.code.as_ref()).map_or(0, String::len); + budget.text(font + format, limits)?; + s.dxfs.push(d); + } + // BrtBeginIndexedColors, BrtEndIndexedColors. + 565 => palette = true, + 566 => palette = false, + 475 if palette => { + let [r, g, bl, _] = <[u8; 4]>::try_from(b.take(4)?).map_err(|_| invalid("truncated colour"))?; + s.indexed_colors.push(u32::from_be_bytes([0, r, g, bl])); + } + // BrtBeginTableStyles: the count of custom table styles, which are not read. + 508 => { + if b.u32()? > 0 { + warnings.warn("custom table styles in a binary workbook were not read"); + } + continue; + } + _ => continue, + } + // The records read above are read whole; the rest are skipped. + if matches!(id, 44 | 43 | 45 | 46 | 47 | 48 | 507 | 475) { + b.finish()?; + } + } + Err(invalid("missing styles end record")) +} diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 748d1b86d..18a833388 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -5,7 +5,10 @@ mod common; use common::{read, read_with}; use craft_xlsx::testing::zip; -use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Range, SheetKind, Value, Visibility, is_xlsb}; +use craft_xlsx::{ + CellError, CellRef, Color, ColorSource, Error, Fill, FormulaKind, Limits, Package, Pattern, PatternFill, Protection, Range, SheetKind, Styles, + Value, Visibility, is_xlsb, +}; const ROOT: &str = r#""#; const RELS: &str = r#""#; @@ -931,6 +934,280 @@ fn a_dynamic_array_kept_as_its_value_has_no_cell_metadata() { assert!(warned(&wb, "missing from the file"), "{:?}", wb.warnings); } +// Styles ([MS-XLSB] 2.1.7.50): a styles part assembled from records, and the `styles.xml` Excel +// writes for the same formats when it saves the workbook as `.xlsx`. + +const STYLES_RELS: &str = r#""#; +const STYLES_TYPES: &str = r#""#; + +fn styled_file(styles: &[u8], types: &str) -> Vec { + let sheet = worksheet(&[row(0), cell(5, 0, &1.0f64.to_le_bytes())].concat(), &[]); + zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", ROOT.as_bytes()), + ("records/book.bin", &book(false)), + ("records/_rels/book.bin.rels", STYLES_RELS.as_bytes()), + ("data/one.bin", &sheet), + ("data/two.bin", &worksheet(&[], &[])), + ("look/styles.bin", styles), + ]) +} + +/// The styles part of a package, and the warnings reading it gave. +fn styles_of(bytes: &[u8]) -> (Option, Vec) { + let mut p = Package::open(bytes, &Limits::default()).unwrap(); + let styles = p.styles().unwrap(); + (styles, p.warnings().to_vec()) +} + +/// `BrtColor`: auto 0, indexed 1, RGB 2, theme 3. +fn color(kind: u8, index: u8, tint: i16, rgba: [u8; 4]) -> Vec { + [&[kind << 1 | 1, index][..], &tint.to_le_bytes(), &rgba].concat() +} +fn indexed(i: u8) -> Vec { + color(1, i, 0, [0, 0, 0, 255]) +} +fn theme(i: u8, tint: i16) -> Vec { + color(3, i, tint, [0, 0, 0, 255]) +} +fn rgb(r: u8, g: u8, b: u8) -> Vec { + color(2, 255, 0, [r, g, b, 255]) +} +#[allow(clippy::too_many_arguments)] +fn font(twips: u16, flags: u16, weight: u16, script: u16, underline: u8, family: u8, charset: u8, c: &[u8], scheme: u8, name: &str) -> Vec { + let head = + [&twips.to_le_bytes()[..], &flags.to_le_bytes(), &weight.to_le_bytes(), &script.to_le_bytes(), &[underline, family, charset, 0]].concat(); + record(43, &[&head[..], c, &[scheme], &wide(name)].concat()) +} +fn pattern_fill(fls: u32, fg: &[u8], bg: &[u8]) -> Vec { + record(45, &[&fls.to_le_bytes()[..], fg, bg, &[0; 48]].concat()) +} +fn edge(dg: u8, c: &[u8]) -> Vec { + [&[dg, 0][..], c].concat() +} +#[allow(clippy::too_many_arguments)] +fn brt_xf(parent: u16, fmt: u16, font: u16, fill: u16, border: u16, rotation: u8, indent: u8, flags: u16, apply: u8) -> Vec { + let ids = [parent, fmt, font, fill, border].map(u16::to_le_bytes).concat(); + record(47, &[&ids[..], &[rotation, indent], &flags.to_le_bytes(), &[apply, 0]].concat()) +} +fn style(xf: u32, flags: u16, builtin: u8, level: u8, name: &str) -> Vec { + record(48, &[&xf.to_le_bytes()[..], &flags.to_le_bytes(), &[builtin, level], &wide(name)].concat()) +} +fn prop(kind: u16, data: &[u8]) -> Vec { + [&kind.to_le_bytes()[..], &(data.len() as u16 + 4).to_le_bytes(), data].concat() +} +fn short(text: &str) -> Vec { + let units: Vec = text.encode_utf16().collect(); + [(units.len() as u16).to_le_bytes().to_vec(), units.iter().flat_map(|u| u.to_le_bytes()).collect()].concat() +} +fn brt_dxf(props: &[Vec]) -> Vec { + record(507, &[&[0, 0x80, 0, 0][..], &(props.len() as u16).to_le_bytes(), &props.concat()].concat()) +} +fn begin(id: u16, count: u32) -> Vec { + record(id, &count.to_le_bytes()) +} + +/// Every kind of record the styles part has, as Excel writes them. +fn styles_bin() -> Vec { + let normal = font(220, 0, 400, 0, 0, 2, 0, &theme(1, 0), 2, "Aptos Narrow"); + // Bold, italic, struck out, superscript, double accounting underline; family and + // character set 0 and scheme "none" are not written in `.xlsx`. + let fancy = font(290, 0x0A, 700, 1, 0x22, 0, 2, &rgb(0, 128, 255), 0, "Wingdings"); + let heading = font(220, 0, 400, 0, 0, 2, 0, &theme(4, 13106), 1, "Aptos Display"); + // An automatic colour: no `` in `.xlsx`. + let auto = font(220, 0x02, 400, 0, 0, 2, 0, &color(0, 64, 0, [0, 0, 0, 255]), 2, "Aptos Narrow"); + let gradient = [ + &0x28u32.to_le_bytes()[..], + &indexed(64), + &indexed(65), + &1u32.to_le_bytes(), + &[0.0f64, 0.5, 0.5, 0.5, 0.5].map(f64::to_le_bytes).concat(), + &2u32.to_le_bytes(), + &theme(0, 0), + &0.0f64.to_le_bytes(), + &theme(4, 0), + &1.0f64.to_le_bytes(), + ] + .concat(); + let none = edge(0, &color(0, 0, 0, [0, 0, 0, 0])); + let parts: Vec> = vec![ + record(278, &[]), + begin(615, 2), + record(44, &[&164u16.to_le_bytes()[..], &wide(r"yyyy\-mm\-dd")].concat()), + // A format in two alternate-content blocks: the newer spelling for version 13 and later + // (skipped), and the one for versions 12 and 13, which `.xlsx` has as `formatCode`. + record(37, &[1, 0, 13, 0, 0, 0x80]), + record(44, &[&165u16.to_le_bytes()[..], &wide("[$-ja-JP-x-gannen]ge")].concat()), + record(38, &[]), + record(37, &[2, 0, 12, 0, 0, 0, 13, 0, 0, 0]), + record(44, &[&165u16.to_le_bytes()[..], &wide("[$]ge")].concat()), + record(38, &[]), + record(616, &[]), + begin(611, 4), + normal, + fancy, + heading, + auto, + record(612, &[]), + begin(603, 6), + pattern_fill(0, &indexed(64), &indexed(65)), + pattern_fill(17, &indexed(64), &indexed(65)), + pattern_fill(1, &rgb(255, 255, 0), &indexed(64)), + pattern_fill(9, &theme(5, -8191), &indexed(65)), + record(45, &gradient), + // A tinted pattern colour: `.xlsx` states the system background, as for darkGrid above. + pattern_fill(1, &theme(4, 26213), &indexed(65)), + record(604, &[]), + begin(613, 2), + record(46, &[&[0][..], &none, &none, &none, &none, &none].concat()), + record( + 46, + &[&[3][..], &edge(2, &rgb(200, 0, 0)), &none, &edge(7, &indexed(64)), &edge(13, &color(0, 0, 0, [0, 0, 0, 0])), &edge(1, &theme(4, 0))] + .concat(), + ), + record(614, &[]), + begin(626, 3), + brt_xf(0xFFFF, 0, 0, 0, 0, 0, 0, 0x1010, 0), + brt_xf(0xFFFF, 9, 0, 0, 0, 0, 0, 0x1010, 0x3E), + brt_xf(0xFFFF, 0, 2, 0, 0, 0, 0, 0x1010, 0), + record(627, &[]), + begin(617, 4), + brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0), + brt_xf(0, 164, 0, 0, 0, 0, 0, 0x1010, 0x01), + // Left, top, wrapped, shrunk, right to left, unlocked, hidden, quote prefix. + brt_xf(0, 0, 0, 0, 0, 180, 2, 0x01 | 0x40 | 0x100 | 0x800 | 0x2000 | 0x8000, 0x24), + brt_xf(1, 0, 1, 4, 1, 0, 0, 0x1010, 0x1A), + record(618, &[]), + begin(619, 3), + style(0, 1, 0, 0, "Normal"), + // Excel wraps each style in alternate content, which is skipped. + record(37, &[1, 0, 2, 0x0e, 0, 0x80]), + record(3072, &[0; 16]), + record(38, &[]), + style(1, 1 | 4, 1, 1, "RowLevel_2"), + style(2, 2, 0, 0, "Probe Style"), + record(620, &[]), + begin(505, 2), + brt_dxf(&[ + prop(0x05, &rgb(0x9C, 0, 6)), + prop(0x19, &700u16.to_le_bytes()), + prop(0x1C, &[0]), + prop(0x1A, &0x21u16.to_le_bytes()), + prop(0x24, &240u32.to_le_bytes()), + prop(0x18, &short("Arial")), + prop(0x29, &170u16.to_le_bytes()), + prop(0x26, &short("0.0")), + prop(0x02, &rgb(0xFF, 0xC7, 0xCE)), + prop(0x0F, &[2]), + prop(0x14, &[1]), + prop(0x2B, &[0]), + prop(0x07, &[&color(0, 0, 0, [0, 0, 0, 0])[..], &1u16.to_le_bytes()].concat()), + ]), + brt_dxf(&[ + prop(0x03, &[&0u32.to_le_bytes()[..], &[90.0f64, 0.0, 0.0, 0.0, 0.0].map(f64::to_le_bytes).concat()].concat()), + prop(0x04, &[&[0, 0][..], &0.0f64.to_le_bytes(), &rgb(255, 255, 255)].concat()), + prop(0x04, &[&[0, 0][..], &1.0f64.to_le_bytes(), &theme(4, 0)].concat()), + ]), + record(506, &[]), + record(508, &[&0u32.to_le_bytes()[..], &wide("TableStyleMedium2"), &wide("PivotStyleLight16")].concat()), + record(509, &[]), + record(473, &[]), + record(565, &[]), + record(475, &[0x11, 0x22, 0x33, 0]), + record(475, &[0xFF, 0xFF, 0xFF, 0]), + record(566, &[]), + record(474, &[]), + record(279, &[]), + ]; + parts.concat() +} + +const STYLES_XML: &str = r#""#; + +/// The same styles from a SpreadsheetML package. +fn xml_styles(xml: &str) -> Styles { + let types = r#""#; + let root = r#""#; + let rels = r#""#; + let bytes = zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", root.as_bytes()), + ("xl/workbook.xml", br#""#), + ("xl/_rels/workbook.xml.rels", rels.as_bytes()), + ("xl/styles.xml", xml.as_bytes()), + ]); + styles_of(&bytes).0.unwrap() +} + +/// What both readers lift out: an ``'s element keeps only its `apply*` flags, and the XML +/// part's own element is dropped. +fn lifted(mut s: Styles) -> Styles { + for xf in s.cell_style_xfs.iter_mut().chain(&mut s.cell_xfs) { + let attrs = xf.el.attrs.iter().filter(|(k, _)| k.starts_with("apply")).cloned().collect(); + xf.el = craft_xml::El::new("xf"); + xf.el.attrs = attrs; + } + s.root = craft_xml::El::default(); + s +} + +#[test] +fn styles_are_read_as_their_xlsx_spelling() { + let (binary, warnings) = styles_of(&styled_file(&styles_bin(), STYLES_TYPES)); + let binary = binary.expect("styles are read"); + assert_eq!(lifted(binary.clone()), lifted(xml_styles(STYLES_XML))); + // A few of the conversions, spelled out. + assert_eq!(binary.fonts[1].size, Some(14.5)); + assert_eq!(binary.fonts[1].family, None); + let solid = Fill::Pattern(PatternFill { + pattern: Some(Pattern::Solid), + fg: Some(Color { source: ColorSource::Rgb(0xFFFF_FF00), tint: 0.0 }), + bg: Some(Color { source: ColorSource::Indexed(64), tint: 0.0 }), + }); + assert_eq!(binary.fills[2], solid); + assert_eq!(binary.cell_xfs[2].protection, Some(Protection { locked: Some(false), hidden: Some(true) })); + assert_eq!(binary.palette()[0], 0x11_2233); + // Written back as SpreadsheetML, a cell style format says what it leaves out, as Excel's does. + let written = String::from_utf8(binary.to_xml()).unwrap(); + assert!(written.contains(r#"applyFont="0" applyFill="0" applyBorder="0" applyAlignment="0" applyProtection="0""#), "{written}"); + assert!(warnings.iter().any(|w| w.contains("alternate-content")), "{warnings:?}"); + assert!(!warnings.iter().any(|w| w.contains("table styles")), "{warnings:?}"); +} + +#[test] +fn damaged_styles_are_a_warning_and_the_workbook_opens() { + let good = styles_bin(); + let end = record(279, &[]); + let body = &good[..good.len() - end.len()]; + let with = |extra: &[u8]| [body, extra, &end].concat(); + let cases: Vec<(&str, Vec, &str)> = vec![ + ("truncated", good[..good.len() - 1].to_vec(), STYLES_TYPES), + ("no end record", body.to_vec(), STYLES_TYPES), + ("unknown fill pattern", with(&pattern_fill(99, &indexed(64), &indexed(65))), STYLES_TYPES), + ("unknown border style", with(&record(46, &[&[0][..], &edge(14, &indexed(64)), &[0; 40]].concat())), STYLES_TYPES), + ("cell format outside its list", with(&brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0)), STYLES_TYPES), + ("short property", with(&brt_dxf(&[vec![0x19, 0, 2, 0]])), STYLES_TYPES), + ("wrong content type", good.clone(), TYPES), + ]; + for (why, part, types) in cases { + let bytes = styled_file(&part, types); + let (styles, warnings) = styles_of(&bytes); + assert!(styles.is_none(), "{why}"); + assert!(warnings.iter().any(|w| w.contains("look/styles.bin")), "{why}: {warnings:?}"); + assert_eq!(read(&bytes).unwrap().sheets[0].value(CellRef::new(0, 0)), Value::Number(1.0), "{why}"); + } + // A table past 65,536 entries is past a fixed cap: refused before it is held, as too large. + let xfs = brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0).repeat(65_537); + let bytes = styled_file(&with(&[&begin(617, 65_537)[..], &xfs, &record(618, &[])].concat()), STYLES_TYPES); + let mut p = Package::open(&bytes, &Limits::default()).unwrap(); + assert!(matches!(p.styles(), Err(Error::TooLarge(m)) if m.contains("65536 entries"))); + // Custom table styles are not read, and say so. + let table_styles = [body, &record(508, &[&1u32.to_le_bytes()[..], &wide("Probe Table"), &wide("")].concat()), &end].concat(); + let (styles, warnings) = styles_of(&styled_file(&table_styles, STYLES_TYPES)); + assert!(styles.is_some()); + assert!(warnings.iter().any(|w| w.contains("table styles")), "{warnings:?}"); +} + /// `PtgAttrSpace` ([MS-XLSB] 2.5.98.31): its type (0 spaces before the next token, 1 line breaks /// before it, 2 spaces before a `(`, 4 spaces before a `)`) and count. fn space(kind: u8, n: u8) -> Vec { @@ -1063,6 +1340,16 @@ fn a_spent_formula_allowance_skips_decoding() { assert!(!warned(&wb, "unknown formula token"), "{:?}", wb.warnings); } +#[test] +fn differential_format_strings_are_charged_to_the_text_allowance() { + // A DXF's font name and number format are text the styles part keeps, charged like a font's. + let dxf = |name: &str| [record(278, &[]), begin(505, 1), brt_dxf(&[prop(0x18, &short(name))]), record(506, &[]), record(279, &[])].concat(); + let limits = Limits { binary_text: 100, ..Limits::default() }; + let open = |part: &[u8]| Package::open(&styled_file(part, STYLES_TYPES), &limits).unwrap().styles(); + assert!(open(&dxf("Arial")).is_ok_and(|s| s.is_some_and(|s| s.dxfs.len() == 1))); + assert!(matches!(open(&dxf(&"x".repeat(200))), Err(Error::TooLarge(_)))); +} + #[test] fn module_sheets_and_supporting_links_are_bounded() { // 1,025 very hidden module sheets (no relationship) before the worksheets: each keeps a place diff --git a/crates/xml/README.md b/crates/xml/README.md index adb178159..a61a46832 100644 --- a/crates/xml/README.md +++ b/crates/xml/README.md @@ -46,7 +46,7 @@ assert_eq!((rows, sheet.map(|s| s.children.len())), (3, Ok(0))); | `max_input` | 256 MiB | `TooLarge` | | `max_depth` | 256 levels (the root is 1) | `TooDeep` | | `max_nodes` | 2,000,000 nodes kept in the tree: elements, text stretches, kept CDATA, instructions and comments | `TooManyNodes` | -| `max_attrs` | 1024 on one start tag, namespace declarations and malformed ones included; with canonical names, also the declarations in scope at once | `TooManyAttrs` | +| `max_attrs` | 1024 on one start tag, namespace declarations and malformed ones included; with canonical names or strict reads, also the declarations in scope at once | `TooManyAttrs` | | `max_attrs_total` | 4,000,000 attributes kept in the tree, over all its elements | `TooManyAttrsTotal` | A limit exceeded is an error, never a panic and never a truncated tree. `parse_streaming_partial` @@ -73,9 +73,9 @@ A lenient read keeps what a damaged document had: - reading stops when the root element ends: nothing after it is looked at; - a malformed attribute is skipped; bytes that are not UTF-8 become U+FFFD. -A strict read makes each of those an error, and also a second root element and text outside the +A strict read rejects duplicate raw or expanded attribute names (while allowing undeclared prefixes in fragments). It makes each of those an error, and also a second root element and text outside the root, in the pull reader as in the tree. Markup the tokenizer cannot read (` { end_names: bool, open: Vec, scope: Vec<(String, String)>, + /// For each binding in `scope`, an id shared by every binding of an equal URI (the index of + /// the first such binding, or [`XML_NS_ID`]), with the URI's hash. Strict reads compare + /// expanded attribute names by this id instead of copying the URI for each attribute. + scope_ids: Vec<(u64, usize)>, /// The end of an empty element, due after its start. pending: bool, /// Whether a root element has started. @@ -500,7 +509,7 @@ impl<'o, 'i> Events<'o, 'i> { cfg.trim_text_start = false; cfg.trim_text_end = false; cfg.expand_empty_elements = false; - Events { reader, opts, end_names, open: Vec::new(), scope: Vec::new(), pending: false, rooted: false, span: 0..0 } + Events { reader, opts, end_names, open: Vec::new(), scope: Vec::new(), scope_ids: Vec::new(), pending: false, rooted: false, span: 0..0 } } /// The bytes of the input the last event was read from. Both events of an empty element @@ -614,6 +623,7 @@ impl<'o, 'i> Events<'o, 'i> { fn end(&mut self) -> Event { let (scope, name) = self.open.pop().map_or((0, String::new()), |o| (o.scope, o.name)); self.scope.truncate(scope); + self.scope_ids.truncate(scope); Event::End { name } } @@ -669,6 +679,7 @@ impl<'o, 'i> Events<'o, 'i> { if is_ns_decl(key) { let prefix = self.decode(key.get(6..).unwrap_or_default())?.into_owned(); let uri = self.attr_value(&a.value)?; + self.scope_ids.push(self.uri_id(&uri)); self.scope.push((prefix, uri)); } } @@ -680,6 +691,20 @@ impl<'o, 'i> Events<'o, 'i> { Ok(()) } + /// The id an expanded name uses for `uri`: one hash and at most one full comparison per + /// declaration, so a long URI costs its length once however many attributes use it. + fn uri_id(&self, uri: &str) -> (u64, usize) { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + uri.hash(&mut h); + let hash = h.finish(); + if uri == XML_NS { + return (hash, XML_NS_ID); + } + let first = self.scope_ids.iter().zip(&self.scope).position(|((h, _), (_, u))| *h == hash && u == uri); + (hash, first.map_or(self.scope.len(), |i| self.scope_ids.get(i).map_or(i, |(_, id)| *id))) + } + /// In a strict read: is the reader before or after the root element? fn outside_root(&self) -> bool { self.opts.strict && self.open.is_empty() @@ -699,7 +724,7 @@ impl<'o, 'i> Events<'o, 'i> { _ => None, }; let scope = self.scope.len(); - if canonical.is_some() { + if canonical.is_some() || self.opts.strict { self.declare(e)?; } let raw = e.name(); @@ -707,6 +732,8 @@ impl<'o, 'i> Events<'o, 'i> { let mut out = Vec::new(); let mut attrs = e.attributes(); attrs.with_checks(false); + let mut raw_names = BTreeSet::new(); + let mut expanded_names = BTreeSet::new(); for (seen, a) in attrs.enumerate() { if seen >= limits.max_attrs { return Err(Error::TooManyAttrs(limits.max_attrs)); @@ -718,6 +745,30 @@ impl<'o, 'i> Events<'o, 'i> { }; let key = a.key.as_ref(); let decl = is_ns_decl(key); + if self.opts.strict { + let raw_key = self.decode(key)?; + if !raw_names.insert(raw_key.to_string()) { + return self.syntax(format!("duplicate attribute `{raw_key}`")); + } + if !decl { + // Default namespaces do not apply to attributes. Keep undeclared prefixes + // distinct so standalone fragments retain the reader's existing behaviour. + let expanded = match raw_key.split_once(':') { + Some(("xml", local)) => (XML_NS_ID, local), + Some((prefix, local)) => self + .scope + .iter() + .zip(&self.scope_ids) + .rev() + .find(|((p, _), _)| p == prefix) + .map_or((NO_NS_ID, raw_key.as_ref()), |(_, (_, id))| (*id, local)), + None => (NO_NS_ID, raw_key.as_ref()), + }; + if !expanded_names.insert((expanded.0, expanded.1.to_string())) { + return self.syntax(format!("duplicate expanded attribute `{raw_key}`")); + } + } + } if decl && !self.opts.ns_decls { continue; } diff --git a/crates/xml/tests/hostile.rs b/crates/xml/tests/hostile.rs index 2d387519b..4ef151914 100644 --- a/crates/xml/tests/hostile.rs +++ b/crates/xml/tests/hostile.rs @@ -117,7 +117,11 @@ fn an_attribute_flood_is_refused_at_the_limit() { let opts = opts.with_limits(limits); for (k, kind) in kinds.iter().enumerate() { let _ = opts.parse(tag(16, kind).as_bytes()); - assert_eq!(opts.parse(tag(17, kind).as_bytes()).err().map(|e| e.is_limit() || (opts.strict && k == 3)), Some(true), "kind {k}"); + assert_eq!( + opts.parse(tag(17, kind).as_bytes()).err().map(|e| e.is_limit() || (opts.strict && (k == 1 || k == 3))), + Some(true), + "kind {k}" + ); assert!(opts.first_start(tag(17, kind).as_bytes(), "a").is_none()); } assert_eq!(opts.parse(tag(17, kinds[0]).as_bytes()), Err(Error::TooManyAttrs(16))); @@ -524,3 +528,41 @@ fn a_strict_pull_reader_refuses_what_is_outside_the_root() { // white space, comments and instructions may stand outside the root assert!(read(strict, b"\r\n \tx\n ").is_ok()); } + +#[test] +fn strict_attributes_are_unique_before_name_rewriting() { + use craft_xml::{Names, Options}; + for names in [Names::AsWritten, Names::Local] { + let strict = Options { names, ..Options::default() }.strict(); + for xml in [ + "", + "", + "", + "", + ] { + assert!(strict.parse(xml.as_bytes()).is_err(), "{xml}"); + let mut events = strict.events(xml.as_bytes()); + assert!( + loop { + match events.next() { + Ok(Some(_)) => {} + Ok(None) => break false, + Err(_) => break true, + } + }, + "{xml}" + ); + } + for xml in [ + "", + "", + "", + ] { + assert!(strict.parse(xml.as_bytes()).is_ok(), "{xml}"); + } + assert!(Options { names, ..Options::default() }.parse(b"").is_ok()); + } + let strict = Options { limits: craft_xml::Limits { max_attrs: 2, ..craft_xml::Limits::DEFAULT }, ..Options::default() }.strict(); + assert!(strict.parse(b"").is_err()); + assert!(strict.parse(b"").is_err()); +}