Merge remote-tracking branch 'origin/main' into w-r-shx

# Conflicts:
#	Cargo.toml
This commit is contained in:
FredApps committed 2026-11-16 01:15:16 +01:00
commit 38b5ba8b7d
92 files changed
+8053 -917

No files matched your search

Generated
+4
View File
@@ -1903,6 +1903,10 @@ dependencies = [
"proptest",
]
[[package]]
name = "craft-http-url"
version = "0.1.0"
[[package]]
name = "craft-i18n"
version = "0.1.0"
+1
View File
@@ -12,6 +12,7 @@ repository = "https://github.com/storytold/craft-libs"
[workspace.dependencies]
craft-shx = { path = "crates/shx" }
craft-http-url = { path = "crates/http-url" }
craft-usd = { path = "crates/usd" }
craft-smime = { path = "crates/smime" }
craft-ber = { path = "crates/ber" }
+171 -42
View File
@@ -150,62 +150,142 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> {
}
}
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
let mut title = String::new();
let (mut size3, mut size1) = (0usize, 0usize);
let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]);
let mut rows: Vec<f32> = Vec::new();
for line in text.lines() {
// Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an
// unquoted title's comment must not turn that comment into part of the title.
if let Some(quoted) = line
.trim_start()
.strip_prefix("TITLE")
.filter(|rest| rest.starts_with(char::is_whitespace))
.and_then(|rest| rest.trim_start().strip_prefix('"'))
/// A bounded, three-dimensional Cube profile with RGB rows in file order.
/// Unlike [`parse_cube`], this preserves the optional title and does not resample 1D curves.
#[derive(Clone, Debug, PartialEq)]
pub struct Cube3d {
pub title: Option<String>,
pub size: usize,
pub data: Vec<[f32; 3]>,
pub domain_min: [f32; 3],
pub domain_max: [f32; 3],
}
/// Read the simple 3D profile dialect used by camera-profile consumers.
///
/// `max_size` must be in 2..=256; the row budget is checked before each allocation.
/// Whole-line comments and literal titles are preserved. Unknown alphabetic directives
/// are ignored; 1D tables are refused. Finite domains are returned unchanged so the
/// consumer retains its existing policy for equal or reversed endpoints.
/// Numeric fields must consist of exactly three finite numbers (never silently skip tokens).
pub fn parse_cube_3d(text: &str, max_size: usize) -> Result<Cube3d, LutError> {
let p = read_cube(text, Some(max_size))?;
if p.size3 == 0 {
return err("not a 3D .cube LUT (no LUT_3D_SIZE)");
}
let expected = p.size3.checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
if p.rows.len() != expected {
return err(format!("expected {expected} entries, found {}", p.rows.len()));
}
Ok(Cube3d { title: p.title, size: p.size3, data: p.rows, domain_min: p.min, domain_max: p.max })
}
struct CubeRows {
title: Option<String>,
size3: usize,
size1: usize,
min: [f32; 3],
max: [f32; 3],
rows: Vec<[f32; 3]>,
}
fn cube_triplet(text: &str) -> Result<[f32; 3], LutError> {
let mut it = text.split_whitespace();
let mut row = [0.0; 3];
for value in &mut row {
*value = it.next().and_then(|s| s.parse::<f32>().ok()).filter(|v| v.is_finite()).ok_or_else(|| LutError(format!("bad line `{text}`")))?;
}
if it.next().is_some() {
return err(format!("bad line `{text}`"));
}
Ok(row)
}
fn read_cube(text: &str, profile_max: Option<usize>) -> Result<CubeRows, LutError> {
if profile_max.is_some_and(|n| !(2..=256).contains(&n)) {
return err("3D profile size limit must be in 2..=256");
}
let mut p = CubeRows { title: None, size3: 0, size1: 0, min: [0.0; 3], max: [1.0; 3], rows: Vec::new() };
let row_limit = profile_max.unwrap_or(256).checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
for raw in text.lines() {
let profile = profile_max.is_some();
let line = if profile { raw.trim() } else { raw.split('#').next().unwrap_or("").trim() };
// The standard dialect protects '#' inside quoted titles; profile titles are literal.
if !profile
&& let Some(quoted) =
raw.trim_start().strip_prefix("TITLE").filter(|r| r.starts_with(char::is_whitespace)).and_then(|r| r.trim_start().strip_prefix('"'))
{
let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
title = value.to_string();
let Some((title, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
p.title = Some(title.to_string());
continue;
}
let line = line.split('#').next().unwrap_or("").trim();
if line.is_empty() {
if line.is_empty() || line.starts_with('#') {
continue;
}
let mut it = line.split_whitespace();
let key = it.next().unwrap_or("");
let nums = |it: std::str::SplitWhitespace<'_>| -> Result<[f32; 3], LutError> {
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
if v.len() == 3 { Ok([v[0], v[1], v[2]]) } else { err(format!("bad line `{line}`")) }
let (key, rest) = if profile {
["TITLE", "LUT_3D_SIZE", "LUT_1D_SIZE", "DOMAIN_MIN", "DOMAIN_MAX"]
.into_iter()
.find_map(|key| line.strip_prefix(key).map(|rest| (key, rest.trim())))
.unwrap_or_else(|| line.split_once(char::is_whitespace).unwrap_or((line, "")))
} else {
line.split_once(char::is_whitespace).map_or((line, ""), |(key, rest)| (key, rest.trim()))
};
match key {
"TITLE" => title = line[5..].trim().trim_matches('"').to_string(),
"LUT_3D_SIZE" => size3 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_3D_SIZE".into()))?,
"LUT_1D_SIZE" => size1 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?,
"DOMAIN_MIN" => dmin = nums(it)?,
"DOMAIN_MAX" => dmax = nums(it)?,
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" => {
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
if v.len() == 2 {
dmin = [v[0]; 3];
dmax = [v[1]; 3];
"TITLE" => p.title = Some(rest.trim_matches('"').to_string()),
"LUT_3D_SIZE" => {
let value = if profile { rest } else { rest.split_whitespace().next().unwrap_or("") };
p.size3 = value.parse().map_err(|_| LutError(format!("bad LUT_3D_SIZE `{value}`")))?;
let limit = profile_max.unwrap_or(MAX_SIZE);
if !(2..=limit).contains(&p.size3) {
return err(format!("LUT_3D_SIZE {} out of range", p.size3));
}
}
k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => {
let mut v = vec![k.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?];
for value in it {
v.push(value.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?);
"LUT_1D_SIZE" => {
if profile {
return err("1D LUTs are not supported (only 3D .cube files)");
}
if v.len() != 3 {
return err(format!("bad data line `{line}`"));
p.size1 = rest.split_whitespace().next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?;
if p.size1 > row_limit {
return err("1D LUT exceeds row budget");
}
}
"DOMAIN_MIN" => p.min = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MIN".into()))?,
"DOMAIN_MAX" => p.max = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MAX".into()))?,
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" if !profile => {
let mut it = rest.split_whitespace();
let min = it.next().and_then(|s| s.parse::<f32>().ok());
let max = it.next().and_then(|s| s.parse::<f32>().ok());
if let (Some(min), Some(max), None) = (min, max, it.next()) {
p.min = [min; 3];
p.max = [max; 3];
}
}
_ => {
let numeric = key.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.');
let nonfinite = key.parse::<f32>().is_ok();
if numeric || nonfinite || (profile && !key.starts_with(|c: char| c.is_ascii_alphabetic())) {
let row = cube_triplet(line)?;
if p.rows.len() >= row_limit {
return err("LUT exceeds row budget");
}
// Explicit fallible growth prevents capacity growth past the configured budget.
if p.rows.len() == p.rows.capacity() {
let grow = (p.rows.capacity().max(1)).min(row_limit - p.rows.len());
p.rows.try_reserve_exact(grow).map_err(|_| LutError("LUT allocation failed".into()))?;
}
p.rows.push(row);
}
rows.extend(v);
}
// Unknown keywords (e.g. Resolve's LUT_IN_VIDEO_RANGE) are ignored, as the spec allows.
_ => {}
}
}
Ok(p)
}
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
let p = read_cube(text, None)?;
let (title, size3, size1, dmin, dmax) = (p.title.unwrap_or_default(), p.size3, p.size1, p.min, p.max);
let rows: Vec<f32> = p.rows.into_iter().flatten().collect();
let domain_ok = (0..3).all(|k| dmin[k] < dmax[k] && dmin[k].is_finite() && dmax[k].is_finite());
if (size3 > 0 || size1 >= 2) && !domain_ok {
return err(format!("DOMAIN_MIN {:?} must lie below DOMAIN_MAX {:?}", dmin, dmax));
@@ -1386,3 +1466,52 @@ LUT_3D_SIZE 2
assert_eq!(back.data, [0.0, 0.25, 1.0].repeat(8));
}
}
#[cfg(test)]
mod profile_cube_tests {
use super::*;
fn cube(prefix: &str) -> String {
format!("{prefix}\nLUT_3D_SIZE 2\n{}", "0.25 0.5 0.75\n".repeat(8))
}
#[test]
fn profile_preserves_optional_literal_title_and_domains() {
let no_title = parse_cube_3d(&cube(""), 256).unwrap();
assert_eq!(no_title.title, None);
let value = parse_cube_3d(&cube("TITLE \"look # literal\"\nDOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1"), 256).unwrap();
assert_eq!(value.title.as_deref(), Some("look # literal"));
assert_eq!(value.domain_min, [1.0, 0.0, 2.0]);
assert_eq!(value.domain_max, [0.0, 0.0, 1.0]);
assert_eq!(value.data, vec![[0.25, 0.5, 0.75]; 8]);
assert_eq!(parse_cube_3d(&cube("TITLE \"\""), 256).unwrap().title.as_deref(), Some(""));
assert!(parse_cube(&cube("DOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1")).is_err());
}
#[test]
fn profile_limits_are_checked_without_large_allocations() {
assert!(parse_cube_3d("LUT_3D_SIZE 256", 256).unwrap_err().0.contains("16777216"));
assert!(parse_cube_3d("LUT_3D_SIZE 257", 256).unwrap_err().0.contains("out of range"));
assert!(parse_cube_3d("LUT_3D_SIZE 256", 129).unwrap_err().0.contains("out of range"));
assert!(parse_cube_3d(&cube(""), usize::MAX).is_err());
assert!(parse_cube_3d(&cube(""), 1).is_err());
assert!(parse_cube_3d(&format!("{}0 0 0\n", cube("")), 2).unwrap_err().0.contains("budget"));
assert!(parse_cube_3d("LUT_1D_SIZE 2\n0 0 0\n1 1 1", 256).is_err());
}
#[test]
fn every_numeric_token_must_be_valid_and_finite() {
for row in ["0 ignored 0 0", "0 0 0 ignored", "NaN 0 0", "inf 0 0", "0 0 -inf", "1e39 0 0", "0 0", "0 0 0 0"] {
assert!(parse_cube_3d(&format!("LUT_3D_SIZE 2\n{row}\n{}", "0 0 0\n".repeat(7)), 256).is_err(), "{row}");
assert!(parse_cube_3d(&cube(&format!("DOMAIN_MIN {row}")), 256).is_err(), "{row}");
}
}
#[test]
fn standard_reader_keeps_quoted_comments_and_1d_expansion() {
assert_eq!(parse_cube(&cube("TITLE \"look # preserved\" # comment")).unwrap().title, "look # preserved");
let value = parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1").unwrap();
assert_eq!(value.size, 33);
assert!(parse_cube_3d(&cube("DOMAIN_MIN 0 0 0 # not a whole-line comment"), 256).is_err());
}
}
+362 -65
View File
@@ -6,7 +6,7 @@
//! [`spec_of`] turns a chart part back into the model only when writing that model again gives
//! the same markup, so a chart with anything the model leaves out stays as it was read.
use craft_wordmodel::chart::{ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES};
use craft_wordmodel::chart::{ChartAxis, ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES};
use craft_wordmodel::graphic::GraphicItem;
use craft_wordmodel::props::Rgb;
@@ -102,11 +102,24 @@ fn bounded(spec: &ChartSpec) -> ChartSpec {
.series
.iter()
.take(MAX_SERIES)
.map(|s| ChartSeries { name: s.name.clone(), values: s.values.iter().take(MAX_POINTS).copied().collect() })
.map(|s| ChartSeries {
name: s.name.clone(),
values: s.values.iter().take(MAX_POINTS).copied().collect(),
fill: s.fill,
no_fill: s.no_fill,
line_w: s.line_w,
no_line: s.no_line,
markers: s.markers,
smooth: s.smooth,
labels: s.labels,
fmt: s.fmt.clone(),
})
.collect(),
title: spec.title.clone(),
legend: spec.legend,
data_labels: spec.data_labels,
cat_axis: spec.cat_axis.clone(),
val_axis: spec.val_axis.clone(),
}
.sanitized()
}
@@ -185,17 +198,19 @@ fn plot(w: &mut W, spec: &ChartSpec) {
match k {
ChartType::Pie | ChartType::Doughnut => {}
ChartType::Scatter => {
value_axis(w, CAT_AX, VAL_AX, "b", false, "midCat");
value_axis(w, VAL_AX, CAT_AX, "l", true, "midCat");
value_axis(w, CAT_AX, VAL_AX, "b", "midCat", &spec.cat_axis);
value_axis(w, VAL_AX, CAT_AX, "l", "midCat", &spec.val_axis);
}
_ => {
category_axis(w, if horizontal { "l" } else { "b" });
value_axis(w, VAL_AX, CAT_AX, if horizontal { "b" } else { "l" }, true, "between");
category_axis(w, if horizontal { "l" } else { "b" }, &spec.cat_axis);
value_axis(w, VAL_AX, CAT_AX, if horizontal { "b" } else { "l" }, "between", &spec.val_axis);
}
}
}
/// One `c:ser` (ECMA-376 element order: idx, order, tx, spPr, marker, cat/xVal, val/yVal, smooth).
/// One `c:ser` (ECMA-376 element order: idx, order, tx, spPr, marker, invertIfNegative, dLbls,
/// cat/xVal, val/yVal, smooth): the series' formatting (fill, line, markers, its own labels) sits
/// in `c:spPr`, `c:marker` and `c:dLbls`; the values carry their number format.
fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) {
let k = spec.kind;
let i = i.to_string();
@@ -207,25 +222,26 @@ fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) {
w.leaf("c:v", &[], &s.name);
w.close("c:tx");
}
match k {
// Markers only: the series has no line.
ChartType::Scatter => {
w.open("c:spPr", &[]);
w.open("a:ln", &[("w", "19050")]);
w.empty("a:noFill", &[]);
w.close("a:ln");
w.close("c:spPr");
}
ChartType::Column | ChartType::ColumnStacked | ChartType::Bar | ChartType::BarStacked => {
w.empty("c:invertIfNegative", &[("val", "0")]);
}
ChartType::Line => {
w.open("c:marker", &[]);
w.empty("c:symbol", &[("val", "none")]);
w.close("c:marker");
}
_ => {}
series_props(w, s);
series_marker(w, k, s);
if matches!(k, ChartType::Column | ChartType::ColumnStacked | ChartType::Bar | ChartType::BarStacked) {
w.empty("c:invertIfNegative", &[("val", "0")]);
}
if let Some(on) = s.labels {
w.open("c:dLbls", &[]);
for (name, show) in [
("c:showLegendKey", false),
("c:showVal", on),
("c:showCatName", false),
("c:showSerName", false),
("c:showPercent", false),
("c:showBubbleSize", false),
] {
w.empty(name, &[("val", if show { "1" } else { "0" })]);
}
w.close("c:dLbls");
}
let code = s.fmt.as_deref().unwrap_or("General");
if k == ChartType::Scatter {
let xs: Option<Vec<Option<f64>>> = spec
.categories
@@ -234,27 +250,85 @@ fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) {
.collect();
w.open("c:xVal", &[]);
match xs {
Some(xs) => num_lit(w, &xs),
Some(xs) => num_lit(w, &xs, "General"),
None => str_lit(w, &spec.categories),
}
w.close("c:xVal");
w.open("c:yVal", &[]);
num_lit(w, &s.values);
num_lit(w, &s.values, code);
w.close("c:yVal");
} else {
w.open("c:cat", &[]);
str_lit(w, &spec.categories);
w.close("c:cat");
w.open("c:val", &[]);
num_lit(w, &s.values);
num_lit(w, &s.values, code);
w.close("c:val");
}
if matches!(k, ChartType::Line | ChartType::Scatter) {
w.empty("c:smooth", &[("val", "0")]);
w.empty("c:smooth", &[("val", if s.smooth { "1" } else { "0" })]);
}
w.close("c:ser");
}
/// `c:spPr`: the series' fill (`a:solidFill` / `a:noFill`) and line (`a:ln`), written when the
/// model sets them. `a:ln` without `w` is the default width, `w` a fixed one, and `a:noFill`
/// inside it "no line" (a scatter series keeps its markers with no line).
fn series_props(w: &mut W, s: &ChartSeries) {
let fill_set = s.no_fill || s.fill.is_some();
let ln_set = s.no_line || s.line_w.is_some();
if !(fill_set || ln_set) {
return;
}
w.open("c:spPr", &[]);
if fill_set {
// "No fill" wins over a stored colour, so the pair reads back as it was written.
match &s.fill {
Some(c) if !s.no_fill => {
w.open("a:solidFill", &[]);
w.empty("a:srgbClr", &[("val", &c.hex())]);
w.close("a:solidFill");
}
_ => w.empty("a:noFill", &[]),
}
}
if ln_set {
let w_attr = s.line_w.map(emu).unwrap_or_default();
let attrs = [("w", w_attr.as_str())];
w.open("a:ln", if w_attr.is_empty() { &[] } else { &attrs });
if s.no_line {
w.empty("a:noFill", &[]);
}
w.close("a:ln");
}
w.close("c:spPr");
}
/// `c:marker`: line and scatter series say which symbol each point gets; other types have none.
fn series_marker(w: &mut W, k: ChartType, s: &ChartSeries) {
match k {
ChartType::Line => {
w.open("c:marker", &[]);
w.empty("c:symbol", &[("val", if s.markers == Some(true) { "circle" } else { "none" })]);
w.close("c:marker");
}
// Unset: no `c:marker`, so Word picks its automatic symbol for each series.
ChartType::Scatter => {
if let Some(on) = s.markers {
w.open("c:marker", &[]);
w.empty("c:symbol", &[("val", if on { "circle" } else { "none" })]);
w.close("c:marker");
}
}
_ => {}
}
}
/// A line width in points as the `w` attribute DrawingML wants (EMU).
fn emu(pt: f32) -> String {
((pt * 12700.0).round().max(0.0) as i64).to_string()
}
fn str_lit(w: &mut W, items: &[String]) {
w.open("c:strLit", &[]);
w.empty("c:ptCount", &[("val", &items.len().to_string())]);
@@ -266,9 +340,9 @@ fn str_lit(w: &mut W, items: &[String]) {
w.close("c:strLit");
}
fn num_lit(w: &mut W, values: &[Option<f64>]) {
fn num_lit(w: &mut W, values: &[Option<f64>], code: &str) {
w.open("c:numLit", &[]);
w.leaf("c:formatCode", &[], "General");
w.leaf("c:formatCode", &[], code);
w.empty("c:ptCount", &[("val", &values.len().to_string())]);
for (i, v) in values.iter().enumerate() {
if let Some(v) = v.filter(|v| v.is_finite()) {
@@ -285,17 +359,21 @@ fn num(v: f64) -> String {
if v == 0.0 { "0".to_string() } else { format!("{v}") }
}
fn category_axis(w: &mut W, pos: &str) {
fn category_axis(w: &mut W, pos: &str, ax: &ChartAxis) {
w.open("c:catAx", &[]);
w.empty("c:axId", &[("val", CAT_AX)]);
w.open("c:scaling", &[]);
w.empty("c:orientation", &[("val", "minMax")]);
w.close("c:scaling");
scaling(w, ax);
w.empty("c:delete", &[("val", "0")]);
w.empty("c:axPos", &[("val", pos)]);
if ax.gridlines {
w.empty("c:majorGridlines", &[]);
}
axis_title(w, ax);
num_fmt(w, ax);
w.empty("c:majorTickMark", &[("val", "none")]);
w.empty("c:minorTickMark", &[("val", "none")]);
w.empty("c:tickLblPos", &[("val", "nextTo")]);
w.empty("c:tickLblPos", &[("val", if ax.labels { "nextTo" } else { "none" })]);
axis_line(w, ax);
w.empty("c:crossAx", &[("val", VAL_AX)]);
w.empty("c:crosses", &[("val", "autoZero")]);
w.empty("c:auto", &[("val", "1")]);
@@ -305,34 +383,81 @@ fn category_axis(w: &mut W, pos: &str) {
w.close("c:catAx");
}
/// A value axis: with gridlines and no axis line (`grid`), else with its line and no gridlines.
fn value_axis(w: &mut W, id: &str, cross: &str, pos: &str, grid: bool, between: &str) {
/// A value axis (scatter x axes included): its scale, gridlines, title, labels and line, as the
/// model sets them.
fn value_axis(w: &mut W, id: &str, cross: &str, pos: &str, between: &str, ax: &ChartAxis) {
w.open("c:valAx", &[]);
w.empty("c:axId", &[("val", id)]);
w.open("c:scaling", &[]);
w.empty("c:orientation", &[("val", "minMax")]);
w.close("c:scaling");
scaling(w, ax);
w.empty("c:delete", &[("val", "0")]);
w.empty("c:axPos", &[("val", pos)]);
if grid {
if ax.gridlines {
w.empty("c:majorGridlines", &[]);
}
axis_title(w, ax);
num_fmt(w, ax);
w.empty("c:majorTickMark", &[("val", "none")]);
w.empty("c:minorTickMark", &[("val", "none")]);
w.empty("c:tickLblPos", &[("val", "nextTo")]);
if grid {
w.open("c:spPr", &[]);
w.open("a:ln", &[]);
w.empty("a:noFill", &[]);
w.close("a:ln");
w.close("c:spPr");
}
w.empty("c:tickLblPos", &[("val", if ax.labels { "nextTo" } else { "none" })]);
axis_line(w, ax);
w.empty("c:crossAx", &[("val", cross)]);
w.empty("c:crosses", &[("val", "autoZero")]);
w.empty("c:crossBetween", &[("val", between)]);
w.close("c:valAx");
}
/// `c:scaling`: the orientation, then the fixed bounds when the model sets them.
fn scaling(w: &mut W, ax: &ChartAxis) {
w.open("c:scaling", &[]);
w.empty("c:orientation", &[("val", "minMax")]);
for (name, v) in [("c:max", ax.max), ("c:min", ax.min)] {
if let Some(v) = v.filter(|v| v.is_finite()) {
w.empty(name, &[("val", &num(v))]);
}
}
w.close("c:scaling");
}
/// The axis title, in the same rich-text shape the chart title uses.
fn axis_title(w: &mut W, ax: &ChartAxis) {
let Some(t) = &ax.title else { return };
w.open("c:title", &[]);
w.open("c:tx", &[]);
w.open("c:rich", &[]);
w.empty("a:bodyPr", &[]);
w.empty("a:lstStyle", &[]);
w.open("a:p", &[]);
w.open("a:r", &[]);
w.leaf("a:t", &[], t);
w.close("a:r");
w.close("a:p");
w.close("c:rich");
w.close("c:tx");
w.empty("c:overlay", &[("val", "0")]);
w.close("c:title");
}
/// Tick-label number format: the model's code, unlinked from the source, else the default.
fn num_fmt(w: &mut W, ax: &ChartAxis) {
match &ax.fmt {
Some(code) => w.empty("c:numFmt", &[("formatCode", code), ("sourceLinked", "0")]),
None => w.empty("c:numFmt", &[("formatCode", "General"), ("sourceLinked", "1")]),
}
}
/// `c:spPr`: an axis line only when the model keeps it (the default writes nothing, which reads
/// as a line); `a:noFill` when the axis line is off.
fn axis_line(w: &mut W, ax: &ChartAxis) {
if ax.line {
return;
}
w.open("c:spPr", &[]);
w.open("a:ln", &[]);
w.empty("a:noFill", &[]);
w.close("a:ln");
w.close("c:spPr");
}
/// The chart a chart part holds, when the part is exactly what [`chart_xml`] writes for it (so
/// editing and saving it loses nothing). `None` for any other chart.
pub(crate) fn spec_of(space: &El) -> Option<ChartSpec> {
@@ -365,15 +490,40 @@ pub(crate) fn spec_of(space: &El) -> Option<ChartSpec> {
let categories = sers.first().and_then(|s| s.child(if scatter { "c:xVal" } else { "c:cat" })).map(literal).unwrap_or_default();
let series = sers
.iter()
.map(|s| ChartSeries {
name: s.child("c:tx").and_then(|t| t.child("c:v")).map(|v| v.text()).unwrap_or_default(),
values: s
.child(if scatter { "c:yVal" } else { "c:val" })
.map(literal)
.unwrap_or_default()
.iter()
.map(|v| v.parse::<f64>().ok().filter(|v| v.is_finite()))
.collect(),
.map(|s| {
let ln = s.child("c:spPr").and_then(|p| p.child("a:ln"));
let symbol = s.child("c:marker").and_then(|m| m.child("c:symbol")).and_then(|m| m.attr("val"));
ChartSeries {
name: s.child("c:tx").and_then(|t| t.child("c:v")).map(|v| v.text()).unwrap_or_default(),
values: s
.child(if scatter { "c:yVal" } else { "c:val" })
.map(literal)
.unwrap_or_default()
.iter()
.map(|v| v.parse::<f64>().ok().filter(|v| v.is_finite()))
.collect(),
fill: s
.child("c:spPr")
.and_then(|p| p.child("a:solidFill"))
.and_then(|f| f.child("a:srgbClr"))
.and_then(|c| c.attr("val"))
.and_then(Rgb::parse),
no_fill: s.child("c:spPr").and_then(|p| p.child("a:noFill")).is_some(),
line_w: ln.and_then(|l| l.attr("w")).and_then(|w| w.parse::<f64>().ok()).filter(|w| w.is_finite()).map(|w| (w / 12700.0) as f32),
no_line: ln.and_then(|l| l.child("a:noFill")).is_some(),
// Line series show no markers by default: only a setting that differs from that
// belongs in the model. A scatter series keeps what the file says, so a missing
// `c:marker` (Word's automatic symbols) stays unset and is not written back.
markers: symbol.map(|v| v != "none").filter(|m| scatter || *m),
smooth: s.child("c:smooth").and_then(|v| v.attr("val")).is_some_and(|v| v == "1"),
labels: s.child("c:dLbls").and_then(|d| d.child("c:showVal")).and_then(|v| v.attr("val")).map(|v| v == "1"),
fmt: s
.child(if scatter { "c:yVal" } else { "c:val" })
.and_then(|v| v.els().find(|e| e.name == "c:numLit"))
.and_then(|l| l.child("c:formatCode"))
.map(|f| f.text())
.filter(|f| *f != "General"),
}
})
.collect();
let title = chart.child("c:title").and_then(|t| t.find("a:t")).map(|t| t.text());
@@ -384,12 +534,48 @@ pub(crate) fn spec_of(space: &El) -> Option<ChartSpec> {
_ => LegendPos::Right,
});
let data_labels = ty.child("c:dLbls").and_then(|d| d.child("c:showVal")).and_then(|v| v.attr("val")) == Some("1");
let spec = ChartSpec { kind, categories, series, title, legend, data_labels }.sanitized();
// Editable only when nothing would be lost: writing the model again gives the same part.
let again = xml::parse(chart_xml(&spec).as_bytes()).ok()?;
let (cat_a, val_a) = crate::read::chart::chart_axes(plot, scatter);
let axis = |a: crate::read::chart::Axis| ChartAxis {
title: a.title,
min: a.min,
max: a.max,
fmt: a.fmt,
gridlines: a.grid,
labels: a.labels,
line: a.line,
};
let spec = ChartSpec { kind, categories, series, title, legend, data_labels, cat_axis: axis(cat_a), val_axis: axis(val_a) }.sanitized();
// Editable only when nothing would be lost: writing the model again gives the same part, as
// this writer or the earlier one wrote it.
let mut again = xml::parse(chart_xml(&spec).as_bytes()).ok()?;
if same(space, &again, 0) {
return Some(spec);
}
as_previous_writer(&mut again);
same(space, &again, 0).then_some(spec)
}
/// `part` (what [`chart_xml`] writes) as the earlier WordCraft writer wrote the same model: it
/// had no axis `c:numFmt`. Only that element goes, and only with the value its absence reads as
/// (General, linked to the source), so a part that matches still holds nothing the model leaves out.
fn as_previous_writer(part: &mut El) {
fn els_mut(e: &mut El) -> impl Iterator<Item = &mut El> {
e.kids.iter_mut().filter_map(|n| match n {
Node::El(e) => Some(e),
Node::Text(_) => None,
})
}
fn drop_els(e: &mut El, drop: impl Fn(&El) -> bool) {
e.kids.retain(|n| !matches!(n, Node::El(c) if drop(c)));
}
let Some(plot) = els_mut(part).find(|e| e.name == "c:chart").and_then(|c| els_mut(c).find(|e| e.name == "c:plotArea")) else { return };
for e in els_mut(plot) {
if matches!(e.name.as_str(), "c:catAx" | "c:valAx") {
drop_els(e, |c| c.name == "c:numFmt" && c.attr("sourceLinked") == Some("1"));
}
}
}
/// The points of a `c:strLit` / `c:numLit` (or a cache) as texts by index, `ptCount` long.
fn literal(e: &El) -> Vec<String> {
let Some(lit) = e.els().find(|c| matches!(c.name.as_str(), "c:strLit" | "c:numLit")) else { return Vec::new() };
@@ -461,10 +647,11 @@ mod tests {
let spec = ChartSpec {
kind: ChartType::Column,
categories: (0..4500).map(|i| i.to_string()).collect(),
series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500] }).collect(),
series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500], ..Default::default() }).collect(),
title: None,
legend: None,
data_labels: false,
..Default::default()
};
let x = chart_xml(&spec);
let series = x.matches("<c:ser>").count();
@@ -491,6 +678,33 @@ mod tests {
assert_eq!(spec_of(&parse(&plain)), Some(plain));
}
#[test]
fn no_fill_wins_over_a_stored_colour() {
let mut spec = ChartSpec::sample(ChartType::Column);
spec.series[0].fill = Some(Rgb(255, 0, 0));
spec.series[0].no_fill = true;
let x = chart_xml(&spec);
assert!(!x.contains("FF0000"), "colour written next to no_fill");
let back = spec_of(&parse(&spec)).expect("stays editable");
assert!(back.series[0].no_fill);
assert_eq!(back.series[0].fill, None);
}
#[test]
fn an_axis_format_is_limited_in_characters_not_bytes() {
use craft_wordmodel::chart::MAX_FMT;
let cjk = format!("0\"{}\"", "\u{5143}".repeat(90));
let ascii = format!("0\"{}\"", "x".repeat(90));
for fmt in [cjk, ascii] {
assert!(fmt.chars().count() <= MAX_FMT);
let mut spec = ChartSpec::sample(ChartType::Column);
spec.val_axis.fmt = Some(fmt.clone());
let spec = spec.sanitized();
assert_eq!(spec.val_axis.fmt.as_deref(), Some(fmt.as_str()));
assert_eq!(spec_of(&parse(&spec)).as_ref(), Some(&spec), "{} chars", fmt.chars().count());
}
}
#[test]
fn a_chart_with_more_than_the_model_stays_read_only() {
let spec = ChartSpec::sample(ChartType::Column);
@@ -500,6 +714,61 @@ mod tests {
assert_eq!(spec_of(&xml::parse(x.as_bytes()).unwrap()), None);
}
/// Chart parts as the previous WordCraft chart writer saved them (its output for
/// `ChartSpec::sample`): no axis `c:numFmt`, and scatter series with no `c:marker`.
const PREVIOUS_COLUMN: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?><c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><c:roundedCorners val="0"/><c:chart><c:autoTitleDeleted val="1"/><c:plotArea><c:layout/><c:barChart><c:barDir val="col"/><c:grouping val="clustered"/><c:varyColors val="0"/><c:ser><c:idx val="0"/><c:order val="0"/><c:tx><c:v>Series 1</c:v></c:tx><c:invertIfNegative val="0"/><c:cat><c:strLit><c:ptCount val="4"/><c:pt idx="0"><c:v>Q1</c:v></c:pt><c:pt idx="1"><c:v>Q2</c:v></c:pt><c:pt idx="2"><c:v>Q3</c:v></c:pt><c:pt idx="3"><c:v>Q4</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>3.2</c:v></c:pt><c:pt idx="1"><c:v>4.1</c:v></c:pt><c:pt idx="2"><c:v>3.6</c:v></c:pt><c:pt idx="3"><c:v>5</c:v></c:pt></c:numLit></c:val></c:ser><c:ser><c:idx val="1"/><c:order val="1"/><c:tx><c:v>Series 2</c:v></c:tx><c:invertIfNegative val="0"/><c:cat><c:strLit><c:ptCount val="4"/><c:pt idx="0"><c:v>Q1</c:v></c:pt><c:pt idx="1"><c:v>Q2</c:v></c:pt><c:pt idx="2"><c:v>Q3</c:v></c:pt><c:pt idx="3"><c:v>Q4</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>2</c:v></c:pt><c:pt idx="1"><c:v>2.7</c:v></c:pt><c:pt idx="2"><c:v>3.9</c:v></c:pt><c:pt idx="3"><c:v>3.3</c:v></c:pt></c:numLit></c:val></c:ser><c:ser><c:idx val="2"/><c:order val="2"/><c:tx><c:v>Series 3</c:v></c:tx><c:invertIfNegative val="0"/><c:cat><c:strLit><c:ptCount val="4"/><c:pt idx="0"><c:v>Q1</c:v></c:pt><c:pt idx="1"><c:v>Q2</c:v></c:pt><c:pt idx="2"><c:v>Q3</c:v></c:pt><c:pt idx="3"><c:v>Q4</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1.5</c:v></c:pt><c:pt idx="1"><c:v>2.2</c:v></c:pt><c:pt idx="2"><c:v>2.4</c:v></c:pt><c:pt idx="3"><c:v>3.8</c:v></c:pt></c:numLit></c:val></c:ser><c:gapWidth val="150"/><c:axId val="500000001"/><c:axId val="500000002"/></c:barChart><c:catAx><c:axId val="500000001"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="b"/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:crossAx val="500000002"/><c:crosses val="autoZero"/><c:auto val="1"/><c:lblAlgn val="ctr"/><c:lblOffset val="100"/><c:noMultiLvlLbl val="0"/></c:catAx><c:valAx><c:axId val="500000002"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="l"/><c:majorGridlines/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:spPr><a:ln><a:noFill/></a:ln></c:spPr><c:crossAx val="500000001"/><c:crosses val="autoZero"/><c:crossBetween val="between"/></c:valAx></c:plotArea><c:legend><c:legendPos val="b"/><c:overlay val="0"/></c:legend><c:plotVisOnly val="1"/><c:dispBlanksAs val="gap"/></c:chart><c:spPr><a:solidFill><a:schemeClr val="bg1"/></a:solidFill><a:ln><a:noFill/></a:ln></c:spPr></c:chartSpace>"#;
const PREVIOUS_SCATTER: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?><c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><c:roundedCorners val="0"/><c:chart><c:autoTitleDeleted val="1"/><c:plotArea><c:layout/><c:scatterChart><c:scatterStyle val="lineMarker"/><c:varyColors val="0"/><c:ser><c:idx val="0"/><c:order val="0"/><c:tx><c:v>Series 1</c:v></c:tx><c:spPr><a:ln w="19050"><a:noFill/></a:ln></c:spPr><c:xVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1</c:v></c:pt><c:pt idx="1"><c:v>2</c:v></c:pt><c:pt idx="2"><c:v>3</c:v></c:pt><c:pt idx="3"><c:v>4</c:v></c:pt></c:numLit></c:xVal><c:yVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>2.1</c:v></c:pt><c:pt idx="1"><c:v>3.4</c:v></c:pt><c:pt idx="2"><c:v>3</c:v></c:pt><c:pt idx="3"><c:v>4.6</c:v></c:pt></c:numLit></c:yVal><c:smooth val="0"/></c:ser><c:ser><c:idx val="1"/><c:order val="1"/><c:tx><c:v>Series 2</c:v></c:tx><c:spPr><a:ln w="19050"><a:noFill/></a:ln></c:spPr><c:xVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1</c:v></c:pt><c:pt idx="1"><c:v>2</c:v></c:pt><c:pt idx="2"><c:v>3</c:v></c:pt><c:pt idx="3"><c:v>4</c:v></c:pt></c:numLit></c:xVal><c:yVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1.2</c:v></c:pt><c:pt idx="1"><c:v>1.9</c:v></c:pt><c:pt idx="2"><c:v>2.8</c:v></c:pt><c:pt idx="3"><c:v>3.1</c:v></c:pt></c:numLit></c:yVal><c:smooth val="0"/></c:ser><c:axId val="500000001"/><c:axId val="500000002"/></c:scatterChart><c:valAx><c:axId val="500000001"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="b"/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:crossAx val="500000002"/><c:crosses val="autoZero"/><c:crossBetween val="midCat"/></c:valAx><c:valAx><c:axId val="500000002"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="l"/><c:majorGridlines/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:spPr><a:ln><a:noFill/></a:ln></c:spPr><c:crossAx val="500000001"/><c:crosses val="autoZero"/><c:crossBetween val="midCat"/></c:valAx></c:plotArea><c:legend><c:legendPos val="b"/><c:overlay val="0"/></c:legend><c:plotVisOnly val="1"/><c:dispBlanksAs val="gap"/></c:chart><c:spPr><a:solidFill><a:schemeClr val="bg1"/></a:solidFill><a:ln><a:noFill/></a:ln></c:spPr></c:chartSpace>"#;
#[test]
fn charts_saved_by_the_previous_writer_stay_editable() {
for (x, kind) in [(PREVIOUS_COLUMN, ChartType::Column), (PREVIOUS_SCATTER, ChartType::Scatter)] {
let spec = spec_of(&xml::parse(x.as_bytes()).unwrap()).unwrap_or_else(|| panic!("{kind:?}: the earlier part is read-only"));
assert_eq!((spec.kind, spec.series.len()), (kind, if kind == ChartType::Scatter { 2 } else { 3 }));
assert_eq!((spec.cat_axis.fmt.as_deref(), spec.val_axis.fmt.as_deref()), (None, None), "{kind:?}");
assert!(spec.series.iter().all(|s| s.markers.is_none()), "{kind:?}: markers unset");
// Saved again, it is in the current shape and still reads back as the same model.
assert_eq!(spec_of(&parse(&spec)).as_ref(), Some(&spec), "{kind:?}");
}
// What neither writer writes still keeps such a part read-only: an axis format linked to
// the source that isn't General, a scatter marker other than the default, a marker size.
let edited = |x: &str, from: &str, to: &str| {
assert!(x.contains(from), "{from}");
spec_of(&xml::parse(x.replacen(from, to, 1).as_bytes()).unwrap())
};
let ax = r#"<c:axPos val="l"/><c:majorGridlines/>"#;
assert_eq!(edited(PREVIOUS_COLUMN, ax, &format!(r#"{ax}<c:numFmt formatCode="0.0" sourceLinked="1"/>"#)), None);
let ln = r#"<a:ln w="19050"><a:noFill/></a:ln></c:spPr>"#;
assert_eq!(edited(PREVIOUS_SCATTER, ln, &format!(r#"{ln}<c:marker><c:symbol val="square"/></c:marker>"#)), None);
assert_eq!(edited(PREVIOUS_SCATTER, ln, &format!(r#"{ln}<c:marker><c:symbol val="circle"/><c:size val="9"/></c:marker>"#)), None);
assert_eq!(edited(PREVIOUS_COLUMN, "<c:roundedCorners val=\"0\"/>", "<c:roundedCorners val=\"0\"/><c:style val=\"2\"/>"), None);
}
/// A scatter series with no `c:marker` (Word's automatic symbols) stays without one after an
/// unrelated edit and a save; an explicit circle or "none" stays explicit.
#[test]
fn scatter_series_without_a_marker_stay_without_after_a_save() {
let marker_of = |x: &El| -> Vec<Option<String>> {
let sc = x.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:scatterChart").unwrap();
sc.children("c:ser").map(|s| s.child("c:marker").map(|m| m.child("c:symbol").unwrap().attr("val").unwrap().to_string())).collect()
};
let mut spec = spec_of(&xml::parse(PREVIOUS_SCATTER.as_bytes()).unwrap()).unwrap();
assert_eq!(marker_of(&xml::parse(PREVIOUS_SCATTER.as_bytes()).unwrap()), [None, None]);
spec.title = Some("Edited".into());
spec.series[0].name = "Renamed".into();
let saved = parse(&spec);
assert_eq!(marker_of(&saved), [None, None]);
let again = spec_of(&saved).unwrap();
assert_eq!(again, spec);
assert!(again.series.iter().all(|s| s.markers.is_none()));
// Explicit choices are kept as written.
spec.series[0].markers = Some(true);
spec.series[1].markers = Some(false);
let saved = parse(&spec);
assert_eq!(marker_of(&saved), [Some("circle".to_string()), Some("none".to_string())]);
let again = spec_of(&saved).unwrap();
assert_eq!((again.series[0].markers, again.series[1].markers), (Some(true), Some(false)));
}
#[test]
fn elements_come_in_schema_order() {
let mut spec = ChartSpec::sample(ChartType::ColumnStacked);
@@ -517,6 +786,11 @@ mod tests {
["c:barDir", "c:grouping", "c:varyColors", "c:ser", "c:ser", "c:ser", "c:dLbls", "c:gapWidth", "c:overlap", "c:axId", "c:axId"]
);
assert_eq!(names(bar.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:invertIfNegative", "c:cat", "c:val"]);
// A series with its own labels keeps `c:invertIfNegative` ahead of `c:dLbls` (CT_BarSer).
spec.series[0].labels = Some(true);
let labelled = parse(&spec);
let ser = labelled.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:barChart").unwrap().child("c:ser").unwrap();
assert_eq!(names(ser), ["c:idx", "c:order", "c:tx", "c:invertIfNegative", "c:dLbls", "c:cat", "c:val"]);
let val_ax = plot.child("c:valAx").unwrap();
assert_eq!(
names(val_ax),
@@ -526,6 +800,7 @@ mod tests {
"c:delete",
"c:axPos",
"c:majorGridlines",
"c:numFmt",
"c:majorTickMark",
"c:minorTickMark",
"c:tickLblPos",
@@ -535,13 +810,35 @@ mod tests {
"c:crossBetween"
]
);
let cat_ax = plot.child("c:catAx").unwrap();
assert_eq!(
names(cat_ax),
[
"c:axId",
"c:scaling",
"c:delete",
"c:axPos",
"c:numFmt",
"c:majorTickMark",
"c:minorTickMark",
"c:tickLblPos",
"c:crossAx",
"c:crosses",
"c:auto",
"c:lblAlgn",
"c:lblOffset",
"c:noMultiLvlLbl"
]
);
let line = parse(&ChartSpec { data_labels: true, ..ChartSpec::sample(ChartType::Line) });
let lc = line.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:lineChart").unwrap();
assert_eq!(names(lc), ["c:grouping", "c:varyColors", "c:ser", "c:ser", "c:ser", "c:dLbls", "c:marker", "c:axId", "c:axId"]);
assert_eq!(names(lc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:marker", "c:cat", "c:val", "c:smooth"]);
let sc = parse(&ChartSpec::sample(ChartType::Scatter));
let mut scatter = ChartSpec::sample(ChartType::Scatter);
scatter.series.iter_mut().for_each(|s| s.markers = Some(true));
let sc = parse(&scatter);
let sc = sc.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:scatterChart").unwrap();
assert_eq!(names(sc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:spPr", "c:xVal", "c:yVal", "c:smooth"]);
assert_eq!(names(sc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:spPr", "c:marker", "c:xVal", "c:yVal", "c:smooth"]);
let dn = parse(&ChartSpec::sample(ChartType::Doughnut));
let dn = dn.child("c:chart").unwrap().child("c:plotArea").unwrap();
assert_eq!(names(dn), ["c:layout", "c:doughnutChart"]);
+3 -1
View File
@@ -29,7 +29,9 @@ pub(super) use canvas::text_w;
use canvas::{Rect, clip_text, legend, text};
#[cfg(test)]
use format::{civil, excel_stamp, format_value};
use model::{Axis, Grouping, Kind, Pos, Series};
pub(crate) use model::Axis;
use model::{Grouping, Kind, Pos, Series};
pub(crate) use parse::chart_axes;
#[cfg(test)]
use parse::{axis, category_text, labels, numbers};
use parse::{read_series, text_size, title_of};
+6 -1
View File
@@ -41,8 +41,13 @@ pub(super) fn widest(items: &[String], size: f32) -> f32 {
items.iter().map(|s| text_w(s, size)).fold(0.0, f32::max)
}
/// Text in `rect`; `angle` degrees clockwise turns it about the rect's centre (axis titles).
pub(super) fn text(out: &mut Vec<GraphicItem>, rect: Rect, s: &str, size: f32, align: TextAlign, bold: bool) {
out.push(GraphicItem::Text { rect: rect.arr(), text: s.to_string(), size, color: TEXT, bold, align, font: None });
text_angle(out, rect, s, size, align, bold, 0.0);
}
pub(super) fn text_angle(out: &mut Vec<GraphicItem>, rect: Rect, s: &str, size: f32, align: TextAlign, bold: bool, angle: f32) {
out.push(GraphicItem::Text { rect: rect.arr(), text: s.to_string(), size, color: TEXT, bold, align, font: None, angle });
}
/// Text centred on `cx` with its top at `y`.
+94 -8
View File
@@ -7,6 +7,33 @@ pub(super) struct DataLabel {
pub(super) y: f32,
pub(super) text: String,
pub(super) inside: bool,
/// Where a label that is not `inside` sits relative to (`x`, `y`).
pub(super) pos: LabelPos,
/// Distance from (`x`, `y`) to the label for `pos` other than `Center`, points (half the marker).
pub(super) off: f32,
}
/// Where a data label sits beside its point (`c:dLblPos`: `t`, `b`, `l`, `r`, `ctr`).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum LabelPos {
Above,
Below,
Left,
Right,
Center,
}
impl LabelPos {
pub(super) fn parse(v: &str) -> Option<LabelPos> {
Some(match v.trim() {
"t" => LabelPos::Above,
"b" => LabelPos::Below,
"l" => LabelPos::Left,
"r" => LabelPos::Right,
"ctr" => LabelPos::Center,
_ => return None,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -52,16 +79,56 @@ pub(super) enum Pos {
/// One value axis (or category axis) as the file states it.
#[derive(Default)]
pub(super) struct Axis {
pub(super) deleted: bool,
pub(super) grid: bool,
pub(super) line: bool,
pub(crate) struct Axis {
pub(crate) deleted: bool,
pub(crate) grid: bool,
pub(crate) line: bool,
/// `c:crossBetween val="midCat"`: categories sit on the tick marks, not between them.
pub(super) mid: bool,
pub(super) min: Option<f64>,
pub(super) max: Option<f64>,
pub(crate) min: Option<f64>,
pub(crate) max: Option<f64>,
/// Number format of the tick labels, when not linked to the source data.
pub(super) fmt: Option<String>,
pub(crate) fmt: Option<String>,
/// The axis title, when the file gives it one.
pub(crate) title: Option<String>,
/// The title's `a:bodyPr/@rot` in degrees clockwise (`-5400000` is 90° counter-clockwise),
/// when the file gives one.
pub(crate) title_rot: Option<f32>,
/// Tick labels, as `c:tickLblPos` says (`none` hides them; a file that says nothing shows
/// them, as Word does).
pub(crate) labels: bool,
}
/// The shape of a series' point markers (`ST_MarkerStyle`, ECMA-376 §21.2.3.28).
#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
pub(super) enum Symbol {
#[default]
Circle,
Square,
Diamond,
Triangle,
X,
Plus,
Star,
Dash,
Dot,
}
impl Symbol {
/// The symbol a `c:symbol` value names; anything unknown (`auto`, `picture`) is a circle.
pub(super) fn parse(name: &str) -> Symbol {
match name {
"square" => Symbol::Square,
"diamond" => Symbol::Diamond,
"triangle" => Symbol::Triangle,
"x" => Symbol::X,
"plus" => Symbol::Plus,
"star" => Symbol::Star,
"dash" => Symbol::Dash,
"dot" => Symbol::Dot,
_ => Symbol::Circle,
}
}
}
pub(super) struct Series {
@@ -70,9 +137,21 @@ pub(super) struct Series {
/// when missing or not a valid number.
pub(super) idx: Option<usize>,
pub(super) color: Option<Rgb>,
pub(super) line_w: f32,
/// Fixed line width in points, `None`: the chart's default.
pub(super) line_w: Option<f32>,
/// Whether the series is drawn as a line (scatter series without one get markers).
pub(super) line: bool,
/// The series' own fill: `Some` colour, `None` colour with `no_fill` set, else the chart's
/// accent for the series.
pub(super) no_fill: bool,
/// Markers on the series' points, when the file says.
pub(super) markers: Option<bool>,
/// The marker shape (`c:marker/c:symbol`; a circle when missing or unknown) and its size in
/// points (`c:marker/c:size`), when the file gives one.
pub(super) symbol: Symbol,
pub(super) marker_size: Option<f32>,
/// Draw the series' line as a smooth curve (`c:smooth`).
pub(super) smooth: bool,
pub(super) values: Vec<Option<f64>>,
/// Scatter x values (index + 1 when the file has none).
pub(super) xs: Vec<Option<f64>>,
@@ -83,6 +162,13 @@ pub(super) struct Series {
pub(super) x_fmt: Option<String>,
/// Data labels on or off for this series (`c:dLbls` of its own), when it says.
pub(super) labels: Option<bool>,
/// The labels' number format (`c:dLbls/c:numFmt`): `None` when not given (the chart type's
/// applies), `Some(None)` when linked to the source, else `Some(Some(code))`.
pub(super) label_fmt: Option<Option<String>>,
/// The data labels' `c:dLblPos`, when the file gives a known one.
pub(super) label_pos: Option<LabelPos>,
/// The marker's own fill (`c:marker/c:spPr`); without it markers take the automatic colour.
pub(super) marker_color: Option<Rgb>,
/// The values as the file has them, for data labels (`values` become shares in a 100%
/// stacked chart); empty when the series shows no labels.
pub(super) raw: Vec<Option<f64>>,
+76 -7
View File
@@ -1,7 +1,7 @@
use super::format::{date_text, excel_stamp, format_value, is_date_code};
use super::model::{Axis, Grouping, Kind, Pos, Series};
use super::model::{Axis, Grouping, Kind, LabelPos, Pos, Series, Symbol};
use super::plot::label;
use super::{Chart, El, FONT, LINE_W, MAX_CELLS, MAX_POINTS, MAX_RADAR_SERIES, MAX_SERIES, MAX_SPOKES, Rgb, fill_colour, int, measure, slot};
use super::{Chart, El, FONT, MAX_CELLS, MAX_POINTS, MAX_RADAR_SERIES, MAX_SERIES, MAX_SPOKES, Rgb, fill_colour, int, measure, slot};
impl Chart {
pub(super) fn parse(space: &El, theme: &[Rgb]) -> Option<Chart> {
@@ -39,10 +39,15 @@ impl Chart {
cats.resize(n, String::new());
let radar_style = ty.child("c:radarStyle").and_then(|r| r.attr("val"));
let type_labels = show_values(ty.child("c:dLbls"));
let type_fmt = label_format(ty.child("c:dLbls"));
for s in &mut series {
s.values.resize(n, None);
s.xs.resize(n, None);
s.labels = Some(s.labels.or(type_labels).unwrap_or(false));
// Absent (`None`) inherits the chart type's; explicitly source-linked (`Some(None)`) does not.
if s.label_fmt.is_none() {
s.label_fmt.clone_from(&type_fmt);
}
if s.labels == Some(true) {
s.raw = s.values.clone();
}
@@ -98,6 +103,13 @@ impl Chart {
s.and_then(|s| s.color).unwrap_or_else(|| self.accent(s.and_then(|s| s.idx).unwrap_or(j)))
}
/// Colour of series `j`'s markers: the marker's own fill, else the automatic colour of the
/// series index (not the series' explicit line colour, as Word draws them).
pub(super) fn marker_colour(&self, j: usize) -> Rgb {
let s = self.series.get(j);
s.and_then(|s| s.marker_color).unwrap_or_else(|| self.accent(s.and_then(|s| s.idx).unwrap_or(j)))
}
/// Colour of point `i` of series `j`: its `c:dPt` override, else the varied accent (pie) or
/// the series colour.
pub(super) fn point_colour(&self, j: usize, i: usize) -> Rgb {
@@ -149,7 +161,7 @@ fn to_percent(series: &mut [Series], n: usize) {
}
/// The category axis (scatter: the horizontal value axis) and the value axis of a plot area.
fn chart_axes(plot: &El, scatter: bool) -> (Axis, Axis) {
pub(crate) fn chart_axes(plot: &El, scatter: bool) -> (Axis, Axis) {
let vals_ax: Vec<&El> = plot.children("c:valAx").collect();
let horiz_pos = |a: &&El| matches!(a.child("c:axPos").and_then(|p| p.attr("val")), Some("b" | "t"));
let (val_el, x_el) = if scatter {
@@ -180,6 +192,17 @@ fn show_values(d: Option<&El>) -> Option<bool> {
d.child("c:showVal").map(|v| v.attr("val").is_none_or(truthy))
}
/// The number format of data labels (`c:dLbls/c:numFmt`): `None` when the labels say nothing,
/// `Some(None)` when they are linked to the source (so a chart-level format does not apply),
/// `Some(Some(code))` for their own format code.
fn label_format(d: Option<&El>) -> Option<Option<String>> {
let f = d?.child("c:numFmt")?;
if f.attr("sourceLinked").is_some_and(truthy) {
return Some(None);
}
f.attr("formatCode").filter(|f| f.chars().count() <= craft_wordmodel::chart::MAX_FMT).map(|f| Some(f.to_string()))
}
fn legend_pos(chart: &El) -> Option<Pos> {
chart.child("c:legend").map(|l| match l.child("c:legendPos").and_then(|p| p.attr("val")).unwrap_or("r") {
"l" => Pos::Left,
@@ -206,9 +229,24 @@ impl Series {
let sp = ser.child("c:spPr");
let ln = sp.and_then(|p| p.child("a:ln"));
let marker = ser.child("c:marker").and_then(|m| m.child("c:spPr")).and_then(|p| fill_colour(p, theme));
let marker_color = marker;
let color = sp.and_then(|p| fill_colour(p, theme)).or_else(|| ln.and_then(|l| fill_colour(l, theme))).or(marker);
let line_w = ln.and_then(|l| l.attr("w")).and_then(|v| measure(v, 12_700.0)).map_or(LINE_W, |w| w.clamp(0.0, 20.0));
let no_fill = sp.and_then(|p| p.child("a:noFill")).is_some();
let line_w = ln.and_then(|l| l.attr("w")).and_then(|v| measure(v, 12_700.0)).map(|w| w.clamp(0.0, 20.0));
let line = ln.is_none_or(|l| l.child("a:noFill").is_none());
// `c:marker/c:symbol`: which symbol each point gets (`none` for no markers). Scatter
// series show markers unless the file says not; line series only when it says so.
let symbol = ser.child("c:marker").and_then(|m| m.child("c:symbol")).and_then(|s| s.attr("val")).map(str::to_string);
let markers = symbol.as_deref().map(|s| s != "none").or_else(|| scatter.then_some(true));
let symbol = symbol.as_deref().map_or(Symbol::Circle, Symbol::parse);
let marker_size = ser
.child("c:marker")
.and_then(|m| m.child("c:size"))
.and_then(|s| s.attr("val"))
.and_then(|v| v.trim().parse::<f32>().ok())
.filter(|v| v.is_finite())
.map(|v| v.clamp(2.0, 72.0));
let smooth = ser.child("c:smooth").and_then(|s| s.attr("val")).is_some_and(truthy);
let (values, fmt) = numbers(ser.child(if scatter { "c:yVal" } else { "c:val" }), cap);
let (mut xs, x_fmt) = if scatter { numbers(ser.child("c:xVal"), cap) } else { (Vec::new(), None) };
if scatter && xs.is_empty() {
@@ -223,7 +261,30 @@ impl Series {
}
}
let labels = show_values(ser.child("c:dLbls"));
Series { name, idx, color, line_w, line, values, xs, points, fmt, x_fmt, labels, raw: Vec::new() }
let label_fmt = label_format(ser.child("c:dLbls"));
let label_pos = ser.child("c:dLbls").and_then(|d| d.child("c:dLblPos")).and_then(|p| p.attr("val")).and_then(LabelPos::parse);
Series {
name,
idx,
color,
line_w,
line,
no_fill,
markers,
symbol,
marker_size,
smooth,
values,
xs,
points,
fmt,
x_fmt,
labels,
label_fmt,
label_pos,
marker_color,
raw: Vec::new(),
}
}
}
@@ -290,7 +351,7 @@ fn pts(cache: &El, cap: usize) -> Vec<Option<String>> {
}
pub(super) fn axis(a: Option<&El>) -> Axis {
let Some(a) = a else { return Axis { deleted: true, ..Default::default() } };
let Some(a) = a else { return Axis { deleted: true, labels: true, line: true, ..Default::default() } };
let scaling = a.child("c:scaling");
let num =
|n: &str| scaling.and_then(|s| s.child(n)).and_then(|e| e.attr("val")).and_then(|v| v.trim().parse::<f64>().ok()).filter(|v| v.is_finite());
@@ -305,8 +366,16 @@ pub(super) fn axis(a: Option<&El>) -> Axis {
.child("c:numFmt")
.filter(|f| f.attr("sourceLinked") != Some("1"))
.and_then(|f| f.attr("formatCode"))
.filter(|f| f.len() <= MAX_FORMAT_CODE)
.filter(|f| f.chars().count() <= craft_wordmodel::chart::MAX_FMT)
.map(str::to_string),
title: a.child("c:title").and_then(title_text),
title_rot: a
.child("c:title")
.and_then(|t| t.find("a:bodyPr"))
.and_then(|b| b.attr("rot"))
.and_then(|v| v.trim().parse::<i64>().ok())
.map(|v| (v as f64 / 60_000.0).clamp(-360.0, 360.0) as f32),
labels: a.child("c:tickLblPos").and_then(|p| p.attr("val")) != Some("none"),
}
}
+246 -32
View File
@@ -1,9 +1,12 @@
use super::canvas::{Rect, fill, rect_segs, runs, seg, stroke, text, text_at, text_w, widest, wrap};
use super::canvas::{Rect, clip_text, fill, rect_segs, runs, seg, stroke, text, text_angle, text_at, text_w, widest, wrap};
use super::format::format_value;
use super::model::{DataLabel, Grouping, Kind, Series};
use super::model::{DataLabel, Grouping, Kind, LabelPos, Series, Symbol};
use super::parse::category_text;
use super::polar::{draw_data_labels, label_of};
use super::{AXIS, Chart, El, GAP, GRID, GraphicItem, LINE, MAX_LABEL, MAX_TICKS, PathSeg, Rgb, ShapeKind, TextAlign, fill_colour, measure};
use super::{AXIS, Chart, El, GAP, GRID, GraphicItem, LINE, LINE_W, MAX_LABEL, MAX_TICKS, PathSeg, Rgb, ShapeKind, TextAlign, fill_colour, measure};
/// Line width of the stroked marker symbols (x, plus, star), points.
const MARKER_STROKE: f32 = 1.5;
pub(super) fn label(s: &str) -> String {
s.chars().take(MAX_LABEL).collect()
@@ -142,6 +145,8 @@ impl Axes {
/// The room around the plot for axis labels, and the category labels as they'll be drawn.
struct Margins {
left: f32,
/// Part of `left` taken by the rotated category-axis title (bars): the labels start after it.
title: f32,
top: f32,
bottom: f32,
/// Category labels in lines (none for scatter charts or a deleted category axis).
@@ -151,22 +156,25 @@ struct Margins {
}
/// The margins the axis labels need around the plot in `area`. Category labels under a column
/// chart wrap to the band; labels that still don't fit show every k-th category.
/// chart wrap to the band; labels that still don't fit show every k-th category. Hidden tick
/// labels take no room; an axis title adds one line on its side.
fn axis_margins(c: &Chart, ax: &Axes, area: Rect) -> Margins {
let fs = c.text_size;
let lh = fs * LINE;
let scatter = c.kind == Kind::Scatter;
let (cat_on, val_on) = (!c.cat.deleted, !c.val.deleted);
let (cat_labels, val_labels) = (cat_on && c.cat.labels, val_on && c.val.labels);
let title = title_left(c, cat_on, val_on, area.w);
let left = if c.horizontal {
if cat_on { widest(&c.cats, fs).min(area.w * 0.4) + GAP } else { 0.0 }
} else if val_on {
if cat_labels { widest(&c.cats, fs).min(area.w * 0.4) + GAP } else { 0.0 }
} else if val_labels {
ax.vticks.iter().map(|(_, s)| text_w(s, fs)).fold(0.0, f32::max) + GAP
} else {
0.0
};
} + title;
let band_w = (area.w - left).max(1.0) / c.n as f32;
let wrapped: Vec<Vec<String>> = if cat_on && !scatter {
let width = if c.horizontal { left - GAP } else { band_w };
let wrapped: Vec<Vec<String>> = if cat_labels && !scatter {
let width = if c.horizontal { left - title - GAP } else { band_w };
c.cats.iter().map(|s| wrap(s, width, fs)).collect()
} else {
Vec::new()
@@ -174,15 +182,48 @@ fn axis_margins(c: &Chart, ax: &Axes, area: Rect) -> Margins {
let widest_line = wrapped.iter().flatten().map(|s| text_w(s, fs)).fold(0.0, f32::max);
let skip = ((widest_line / band_w).ceil() as usize).max(1);
let label_lines = wrapped.iter().map(Vec::len).max().unwrap_or(1) as f32;
let bottom = if cat_on && !c.horizontal {
let bottom = (if cat_labels && !c.horizontal {
if scatter { lh + GAP } else { label_lines * lh + GAP }
} else if c.horizontal && val_on {
} else if c.horizontal && val_labels {
lh + GAP
} else {
0.0
};
let top = if !c.horizontal && val_on { lh / 2.0 } else { 0.0 };
Margins { left, top, bottom, wrapped, skip }
}) + title_bottom(c, cat_on, val_on, lh);
let top = if !c.horizontal && val_labels { lh / 2.0 } else { 0.0 };
Margins { left, title, top, bottom, wrapped, skip }
}
/// Room for the vertical axis title on the plot's left: one line when the axis that sits there
/// (the value axis for columns, the category axis for bars) has a title.
/// A horizontal title (`rot="0"`) gets a strip as wide as its wrapped lines, at most a quarter of
/// `area_w`, so the plot keeps its room.
fn title_left(c: &Chart, cat_on: bool, val_on: bool, area_w: f32) -> f32 {
let side = if c.horizontal { cat_on && c.cat.title.is_some() } else { val_on && c.val.title.is_some() };
if !side {
return 0.0;
}
match flat_title_lines(c, area_w) {
Some(lines) => widest(&lines, c.text_size).min((area_w * 0.25).max(0.0)) + GAP,
None => c.text_size * LINE,
}
}
/// The wrapped lines of the left axis's title when the file keeps it horizontal (`rot="0"`).
fn flat_title_lines(c: &Chart, area_w: f32) -> Option<Vec<String>> {
let left = if c.horizontal { &c.cat } else { &c.val };
let r = left.title_rot?.rem_euclid(360.0);
if !(r < 1.0 || r > 359.0) {
return None;
}
let width = if area_w.is_finite() { (area_w * 0.18).max(1.0) } else { 1.0 };
left.title.as_ref().map(|t| wrap(t, width, c.text_size))
}
/// Room for the horizontal axis title under the plot: one line when the axis that sits there
/// (the category axis for columns, the value axis for bars) has a title.
fn title_bottom(c: &Chart, cat_on: bool, val_on: bool, lh: f32) -> f32 {
let side = if c.horizontal { val_on && c.val.title.is_some() } else { cat_on && c.cat.title.is_some() };
if side { lh } else { 0.0 }
}
/// A bar, column, line, area or scatter chart in `area`: gridlines, series, axes, then labels.
@@ -200,6 +241,9 @@ pub(super) fn cartesian(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
if c.val.grid {
draw_grid(out, &g, &ax.vticks);
}
if c.cat.grid {
draw_cat_grid(out, &g, &ax.xticks, c.kind);
}
let mut labels = Vec::new();
match c.kind {
Kind::Bar => bars(out, c, &g, &mut labels),
@@ -210,12 +254,49 @@ pub(super) fn cartesian(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
}
draw_axes(out, c, &g);
draw_data_labels(out, &labels, c.text_size, g.horiz && c.kind == Kind::Bar);
if !c.val.deleted {
draw_value_labels(out, c, &g, &ax.vticks, Rect { w: m.left, ..area });
if !c.val.deleted && c.val.labels {
let room = title_left(c, !c.cat.deleted, true, area.w);
draw_value_labels(out, c, &g, &ax.vticks, Rect { x: area.x + room, w: m.left - room, ..area });
}
if !c.cat.deleted {
if !c.cat.deleted && c.cat.labels {
draw_category_labels(out, c, &g, &ax.xticks, &m, area);
}
draw_axis_titles(out, c, &g, area);
}
/// The axis titles, one line: the title of the vertical axis (the value axis for columns, the
/// category axis for bars) is turned 90° on the plot's left; the horizontal axis's title sits
/// under the plot.
fn draw_axis_titles(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, area: Rect) {
let lh = c.text_size * LINE;
// The axis drawn along the plot's left (the value axis for columns, the category axis for
// bars) gets its title rotated there; the other axis's title sits under the plot.
let (left, bottom) = if c.horizontal { (&c.cat, &c.val) } else { (&c.val, &c.cat) };
// Word turns the vertical axis's title counter-clockwise (reading upwards) unless the file
// turns it the other way or keeps it horizontal (`rot="0"`, wrapped in its own strip); any
// other rotation would not fit the strip and is read as counter-clockwise.
let up = if left.title_rot.is_some_and(|r| (r.rem_euclid(360.0) - 90.0).abs() < 1.0) { 90.0 } else { -90.0 };
if !left.deleted
&& let Some(lines) = flat_title_lines(c, area.w)
{
let strip = (g.plot.x - area.x - GAP).max(1.0);
let top = g.plot.y + g.plot.h / 2.0 - lines.len() as f32 * lh / 2.0;
for (k, l) in lines.iter().enumerate() {
let l = clip_text(l, strip, c.text_size);
text_angle(out, Rect { x: area.x, y: top + k as f32 * lh, w: strip, h: lh }, &l, c.text_size, TextAlign::Center, false, 0.0);
}
} else if !left.deleted
&& let Some(t) = &left.title
{
let t = clip_text(t, g.plot.h, c.text_size);
text_angle(out, Rect { x: area.x, y: g.plot.y, w: lh, h: g.plot.h }, &t, c.text_size, TextAlign::Center, false, up);
}
if !bottom.deleted
&& let Some(t) = &bottom.title
{
let t = clip_text(t, g.plot.w, c.text_size);
text_angle(out, Rect { x: g.plot.x, y: area.bottom() - lh, w: g.plot.w, h: lh }, &t, c.text_size, TextAlign::Center, false, 0.0);
}
}
/// Major gridlines at the value ticks.
@@ -228,6 +309,31 @@ fn draw_grid(out: &mut Vec<GraphicItem>, g: &Frame, ticks: &[(f64, String)]) {
}
}
/// Gridlines along the category axis: across the plot at each category's centre (where line and
/// area points sit, on the tick marks when the axis crosses "midCat"), at the x ticks of a
/// scatter chart.
fn draw_cat_grid(out: &mut Vec<GraphicItem>, g: &Frame, xticks: &[(f64, String)], kind: Kind) {
let plot = g.plot;
if kind == Kind::Scatter {
for (t, _) in xticks {
let p = g.scat_x(*t);
stroke(out, seg(p, plot.y, p, plot.bottom()), GRID, 0.75);
}
return;
}
for i in 0..g.n {
let segs = if g.horiz {
let p = g.band_start(i) + g.band() / 2.0;
seg(plot.x, p, plot.right(), p)
} else {
// Bars stay in their bands; only line and area points move onto the ticks.
let p = if matches!(kind, Kind::Line | Kind::Area) { g.cat_x(i) } else { g.band_start(i) + g.band() / 2.0 };
seg(p, plot.y, p, plot.bottom())
};
stroke(out, segs, GRID, 0.75);
}
}
/// The category axis line, the plot area's outline and the value axis line, as the file shows them.
fn draw_axes(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame) {
let plot = g.plot;
@@ -271,7 +377,7 @@ fn draw_category_labels(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, xticks
for (i, lines) in m.wrapped.iter().enumerate() {
let top = g.band_start(i) + g.band() / 2.0 - lines.len() as f32 * lh / 2.0;
for (k, s) in lines.iter().enumerate() {
text(out, Rect { x: area.x, y: top + k as f32 * lh, w: m.left - GAP, h: lh }, s, fs, TextAlign::Right, false);
text(out, Rect { x: area.x + m.title, y: top + k as f32 * lh, w: m.left - m.title - GAP, h: lh }, s, fs, TextAlign::Right, false);
}
}
} else {
@@ -361,7 +467,13 @@ fn bars(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataL
let (p0, p1) = (g.val(v0), g.val(v1));
let (p0, p1) = (p0.min(p1), p0.max(p1));
let segs = if g.horiz { rect_segs(p0, a, p1, a + thick) } else { rect_segs(a, p0, a + thick, p1) };
fill(out, segs, c.point_colour(j, i));
if !s.no_fill {
fill(out, segs.clone(), c.point_colour(j, i));
}
// An explicit outline shows without a fill too (the series' colour is the line's here).
if let (true, true, Some(w)) = (s.line, s.no_fill, s.line_w) {
stroke(out, segs, c.point_colour(j, i), w);
}
if let Some(text) = label_of(s, i) {
let mid = a + thick / 2.0;
let (x, y) = match (g.horiz, stacked) {
@@ -371,7 +483,7 @@ fn bars(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataL
(true, false) => (if v >= 0.0 { p1 } else { p0 }, mid),
(false, false) => (mid, if v >= 0.0 { p0 } else { p1 }),
};
labels.push(DataLabel { x, y, text, inside: stacked });
labels.push(DataLabel { x, y, text, inside: stacked, pos: LabelPos::Above, off: 0.0 });
}
}
}
@@ -401,13 +513,109 @@ fn lines(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<Data
.collect();
for (i, p) in pts.iter().enumerate() {
if let (Some((x, y)), Some(text)) = (p, label_of(s, i)) {
labels.push(DataLabel { x: *x, y: *y, text, inside: false });
labels.push(DataLabel {
x: *x,
y: *y,
text,
inside: false,
pos: s.label_pos.unwrap_or(LabelPos::Right),
off: marker_half(s, s.markers == Some(true)) + 2.0,
});
}
}
stroke(out, runs(pts.into_iter()), c.series_colour(j), s.line_w);
if s.line {
stroke(out, curve(pts.iter().copied(), s.smooth), c.series_colour(j), s.line_w.unwrap_or(LINE_W));
}
if s.markers == Some(true) {
markers(out, pts.iter().flatten().copied(), c.marker_colour(j), s);
}
}
}
/// A series' point markers in the series' symbol and size (`colour` fills or strokes them), for
/// lines and scatter charts.
/// Half a series' marker size, points; zero when it draws no markers.
fn marker_half(s: &Series, drawn: bool) -> f32 {
if drawn { s.marker_size.filter(|v| v.is_finite()).unwrap_or(6.0).clamp(2.0, 72.0) / 2.0 } else { 0.0 }
}
fn markers(out: &mut Vec<GraphicItem>, pts: impl Iterator<Item = (f32, f32)>, colour: Rgb, s: &Series) {
const MARKER: f32 = 6.0;
let size = s.marker_size.filter(|v| v.is_finite()).unwrap_or(MARKER).clamp(2.0, 72.0);
let h = size / 2.0;
let shape = |out: &mut Vec<GraphicItem>, rect: [f32; 4], kind: ShapeKind| {
out.push(GraphicItem::Shape { rect, kind, fill: Some(colour), stroke: None, stroke_width: 0.0 });
};
let mut strokes = Vec::new();
for (x, y) in pts {
match s.symbol {
Symbol::Circle => shape(out, [x - h, y - h, size, size], ShapeKind::Ellipse),
Symbol::Square => shape(out, [x - h, y - h, size, size], ShapeKind::Rectangle),
Symbol::Dot => shape(out, [x - h / 2.0, y - h / 2.0, h, h], ShapeKind::Ellipse),
Symbol::Dash => shape(out, [x - h, y - size / 6.0, size, size / 3.0], ShapeKind::Rectangle),
Symbol::Diamond => fill(
out,
vec![PathSeg::Move(x, y - h), PathSeg::Line(x + h, y), PathSeg::Line(x, y + h), PathSeg::Line(x - h, y), PathSeg::Close],
colour,
),
Symbol::Triangle => {
fill(out, vec![PathSeg::Move(x, y - h), PathSeg::Line(x + h, y + h), PathSeg::Line(x - h, y + h), PathSeg::Close], colour)
}
Symbol::X | Symbol::Plus | Symbol::Star => {
if s.symbol != Symbol::Plus {
strokes.extend(seg(x - h, y - h, x + h, y + h));
strokes.extend(seg(x - h, y + h, x + h, y - h));
}
if s.symbol != Symbol::X {
strokes.extend(seg(x - h, y, x + h, y));
strokes.extend(seg(x, y - h, x, y + h));
}
}
}
}
stroke(out, strokes, colour, MARKER_STROKE);
}
/// A polyline through `pts`, a smooth curve (Catmull-Rom pieces) when `smooth` and there are at
/// least three points in a row; gaps (`None`) end a run.
fn curve(pts: impl Iterator<Item = Option<(f32, f32)>>, smooth: bool) -> Vec<PathSeg> {
if !smooth {
return runs(pts);
}
let pts: Vec<Option<(f32, f32)>> = pts.collect();
let mut segs = Vec::new();
let mut i = 0;
while i < pts.len() {
let Some(first) = pts[i] else {
i += 1;
continue;
};
// Collect one run of points.
let mut run = vec![first];
while let Some(p) = pts.get(i + 1).copied().flatten() {
run.push(p);
i += 1;
}
i += 1;
if run.len() < 3 {
segs.extend(runs(run.into_iter().map(Some)));
continue;
}
segs.push(PathSeg::Move(run[0].0, run[0].1));
for k in 0..run.len() - 1 {
// Catmull-Rom: tangent at p1 from p0 to p2 (ends mirror their own tangent).
let p0 = run[k.max(1) - 1];
let p1 = run[k];
let p2 = run[k + 1];
let p3 = run.get(k + 2).copied().unwrap_or(p2);
let c1 = (p1.0 + (p2.0 - p0.0) / 6.0, p1.1 + (p2.1 - p0.1) / 6.0);
let c2 = (p2.0 - (p3.0 - p1.0) / 6.0, p2.1 - (p3.1 - p1.1) / 6.0);
segs.push(PathSeg::Cubic(c1.0, c1.1, c2.0, c2.1, p2.0, p2.1));
}
}
segs
}
fn areas(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataLabel>) {
let mut base = vec![0.0f64; g.n];
for (j, s) in c.series.iter().enumerate().filter(|(_, s)| has_values(s)) {
@@ -422,10 +630,12 @@ fn areas(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<Data
segs.push(PathSeg::Line(g.cat_x(i), g.val(*b)));
}
segs.push(PathSeg::Close);
fill(out, segs, c.series_colour(j));
if !s.no_fill {
fill(out, segs, c.series_colour(j));
}
for (i, (t, b)) in top.iter().zip(&base).enumerate() {
if let Some(text) = label_of(s, i) {
labels.push(DataLabel { x: g.cat_x(i), y: g.val((t + b) / 2.0), text, inside: true });
labels.push(DataLabel { x: g.cat_x(i), y: g.val((t + b) / 2.0), text, inside: true, pos: LabelPos::Above, off: 0.0 });
}
}
if c.stacked() {
@@ -434,9 +644,8 @@ fn areas(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<Data
}
}
/// Scatter series as lines, or as small markers when the series has no line.
/// Scatter series as lines, and as small markers when the series asks for them.
fn draw_scatter(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataLabel>) {
const MARKER: f32 = 6.0;
for (j, s) in c.series.iter().enumerate() {
let pts: Vec<Option<(f32, f32)>> =
s.xs.iter()
@@ -448,16 +657,21 @@ fn draw_scatter(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut V
.collect();
for (i, p) in pts.iter().enumerate() {
if let (Some((x, y)), Some(text)) = (p, label_of(s, i)) {
labels.push(DataLabel { x: *x, y: *y - MARKER / 2.0, text, inside: false });
labels.push(DataLabel {
x: *x,
y: *y,
text,
inside: false,
pos: s.label_pos.unwrap_or(LabelPos::Right),
off: marker_half(s, s.markers != Some(false)) + 2.0,
});
}
}
if s.line {
stroke(out, runs(pts.iter().copied()), c.series_colour(j), s.line_w);
continue;
stroke(out, curve(pts.iter().copied(), s.smooth), c.series_colour(j), s.line_w.unwrap_or(LINE_W));
}
for (x, y) in pts.iter().flatten() {
let rect = [x - MARKER / 2.0, y - MARKER / 2.0, MARKER, MARKER];
out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(c.series_colour(j)), stroke: None, stroke_width: 0.0 });
if s.markers != Some(false) {
markers(out, pts.iter().flatten().copied(), c.marker_colour(j), s);
}
}
}
+17 -8
View File
@@ -1,8 +1,8 @@
use super::canvas::{Rect, clip_text, fill, stroke, text, text_w, widest};
use super::format::format_value;
use super::model::{DataLabel, Kind, Series};
use super::model::{DataLabel, Kind, LabelPos, Series};
use super::plot::{Axes, has_values};
use super::{Chart, FRAC_PI_2, GAP, GRID, GraphicItem, LINE, PathSeg, Rgb, ShapeKind, TAU, TEXT, TextAlign};
use super::{Chart, FRAC_PI_2, GAP, GRID, GraphicItem, LINE, LINE_W, PathSeg, Rgb, ShapeKind, TAU, TEXT, TextAlign};
pub(super) fn pie(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
let Some(s) = c.series.first() else { return };
@@ -38,7 +38,7 @@ pub(super) fn pie(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
// Halfway across the ring (pie: two thirds out from the centre).
let at = if inner > 0.0 { (r + inner) / 2.0 } else { r * 0.65 };
let (x, y) = polar(cx, cy, at, (a + b) / 2.0);
labels.push(DataLabel { x: x as f32, y: y as f32, text, inside: true });
labels.push(DataLabel { x: x as f32, y: y as f32, text, inside: true, pos: LabelPos::Above, off: 0.0 });
}
a = b;
}
@@ -92,14 +92,14 @@ pub(super) fn radar(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
if c.radar_fill {
fill(out, segs.clone(), col);
}
stroke(out, segs, col, s.line_w);
stroke(out, segs, col, s.line_w.unwrap_or(LINE_W));
for (i, p) in ring.iter().enumerate() {
if c.radar_marks && s.values.get(i).copied().flatten().is_some() {
let rect = [p.0 as f32 - MARKER / 2.0, p.1 as f32 - MARKER / 2.0, MARKER, MARKER];
out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(col), stroke: None, stroke_width: 0.0 });
}
if let Some(text) = label_of(s, i) {
labels.push(DataLabel { x: p.0 as f32, y: p.1 as f32, text, inside: false });
labels.push(DataLabel { x: p.0 as f32, y: p.1 as f32, text, inside: false, pos: LabelPos::Above, off: 0.0 });
}
}
}
@@ -134,13 +134,13 @@ pub(super) fn radar(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
}
/// The data label of point `i` of series `s`, when it shows one: the value as the file has it,
/// in the series' number format.
/// in the labels' own number format, else the series' one.
pub(super) fn label_of(s: &Series, i: usize) -> Option<String> {
if s.labels != Some(true) {
return None;
}
let v = s.raw.get(i).copied().flatten()?;
let fmt = s.fmt.as_deref().filter(|f| !f.eq_ignore_ascii_case("General"));
let fmt = s.label_fmt.as_ref().map_or(s.fmt.as_deref(), |f| f.as_deref().or(s.fmt.as_deref())).filter(|f| !f.eq_ignore_ascii_case("General"));
Some(match fmt {
Some(f) => format_value(v, f, 1.0),
None => general_value(v),
@@ -165,13 +165,22 @@ pub(super) fn draw_data_labels(out: &mut Vec<GraphicItem>, labels: &[DataLabel],
let w = text_w(&l.text, fs) + 4.0;
let (rect, align) = if l.inside {
(Rect { x: l.x - w / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Center)
} else if l.off > 0.0 || l.pos != LabelPos::Above {
let o = l.off;
match l.pos {
LabelPos::Right => (Rect { x: l.x + o, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Left),
LabelPos::Left => (Rect { x: l.x - o - w, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Right),
LabelPos::Below => (Rect { x: l.x - w / 2.0, y: l.y + o, w, h: lh }, TextAlign::Center),
LabelPos::Center => (Rect { x: l.x - w / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Center),
LabelPos::Above => (Rect { x: l.x - w / 2.0, y: l.y - o - lh, w, h: lh }, TextAlign::Center),
}
} else if beside_right {
(Rect { x: l.x + GAP / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Left)
} else {
(Rect { x: l.x - w / 2.0, y: l.y - lh, w, h: lh }, TextAlign::Center)
};
let color = if l.inside { Rgb::WHITE } else { TEXT };
out.push(GraphicItem::Text { rect: rect.arr(), text: l.text.clone(), size: fs, color, bold: false, align, font: None });
out.push(GraphicItem::Text { rect: rect.arr(), text: l.text.clone(), size: fs, color, bold: false, align, font: None, angle: 0.0 });
}
}
+311 -2
View File
@@ -386,9 +386,13 @@ fn axis_format_code_length_is_capped() {
let x = format!(r#"<c:valAx {NS}><c:numFmt formatCode="{code}" sourceLinked="0"/></c:valAx>"#);
axis(Some(&crate::xml::parse(x.as_bytes()).unwrap())).fmt
};
let ok = format!("{}0", "x".repeat(254));
use craft_wordmodel::chart::MAX_FMT;
// The limit is the model's, counted in characters (a CJK character is one).
let ok = format!("{}0", "x".repeat(MAX_FMT - 1));
assert_eq!(fmt(&ok), Some(ok.clone()));
assert_eq!(fmt(&format!("{}0", "x".repeat(300))), None);
let cjk = format!("{}0", "\u{5143}".repeat(MAX_FMT - 1));
assert_eq!(fmt(&cjk), Some(cjk.clone()));
assert_eq!(fmt(&format!("{}0", "x".repeat(MAX_FMT))), None);
}
#[test]
@@ -590,3 +594,308 @@ fn series_without_a_usable_c_idx_take_their_place() {
let big = [renumbered(&a, &format!(r#"<c:idx val="{}"/>"#, u32::MAX))];
assert_eq!(column_colours(&big).0.len(), 1);
}
#[test]
fn bar_category_labels_start_after_the_axis_title() {
let long = "A category label that is much too long for its column";
let title = r#"<c:title><c:tx><c:rich><a:p><a:r><a:t>Region</a:t></a:r></a:p></c:rich></c:tx></c:title>"#;
let plot = format!(
r#"<c:barChart><c:barDir val="bar"/><c:grouping val="clustered"/>{}<c:axId val="1"/><c:axId val="2"/></c:barChart><c:catAx><c:axId val="1"/>{title}</c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#,
ser(0, "S", Some("FF0000"), &[long, "Short"], &["4", "8"]),
);
let items = chart_items(&space(&plot), &theme(), 300.0, 200.0);
let rect_of = |f: &dyn Fn(&str, &TextAlign) -> bool| -> Vec<[f32; 4]> {
items
.iter()
.filter_map(|it| if let GraphicItem::Text { rect, text, align, .. } = it { f(text, align).then_some(*rect) } else { None })
.collect()
};
let title_rect = rect_of(&|t, _| t == "Region");
assert_eq!(title_rect.len(), 1);
let strip_end = title_rect[0][0] + title_rect[0][2];
let labels = rect_of(&|t, a| matches!(a, TextAlign::Right) && t != "Region" && (long.contains(t) || t == "Short"));
assert!(!labels.is_empty());
assert!(labels.iter().all(|r| r[0] >= strip_end - 1e-3), "labels {labels:?} overlap the title strip ending at {strip_end}");
assert!(labels.iter().all(|r| r[2] <= 0.4 * 300.0), "{labels:?}");
}
#[test]
fn category_gridlines_follow_the_points_when_the_axis_crosses_mid() {
let line = |cross: &str| {
let plot = format!(
r#"<c:lineChart><c:grouping val="standard"/>{}<c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/><c:majorGridlines/></c:catAx><c:valAx><c:axId val="2"/><c:crossBetween val="{cross}"/></c:valAx>"#,
ser(0, "L", Some("FF0000"), &["Q1", "Q2", "Q3"], &["1", "5", "3"]),
);
let items = chart_items(&space(&plot), &theme(), 300.0, 200.0);
let grid: Vec<f32> = items
.iter()
.filter_map(|it| match it {
GraphicItem::Path { segs, stroke: Some(c), .. } if *c == GRID => match segs.first() {
Some(PathSeg::Move(x, _)) => Some(*x),
_ => None,
},
_ => None,
})
.collect();
let pts: Vec<f32> = items
.iter()
.filter_map(|it| match it {
GraphicItem::Path { segs, stroke: Some(c), .. } if *c == Rgb(255, 0, 0) => Some(
segs.iter().filter_map(|s| if let PathSeg::Move(x, _) | PathSeg::Line(x, _) = s { Some(*x) } else { None }).collect::<Vec<f32>>(),
),
_ => None,
})
.next()
.unwrap_or_default();
(grid, pts)
};
for cross in ["midCat", "between"] {
let (grid, pts) = line(cross);
assert_eq!(grid.len(), 3, "{cross}");
assert_eq!(pts.len(), 3, "{cross}");
assert!(grid.iter().zip(&pts).all(|(g, p)| (g - p).abs() < 1e-3), "{cross}: grid {grid:?} points {pts:?}");
}
}
/// A line chart with one series; `extra` goes into the series (`c:marker`, `c:dLbls`), `axes`
/// into the value axis.
fn line_with(extra: &str, val_axis: &str) -> Vec<GraphicItem> {
let s = ser(0, "S", Some("FF0000"), &["Q1", "Q2", "Q3"], &["3.2", "4.1", "5"]).replace("<c:cat>", &format!("{extra}<c:cat>"));
let plot = format!(
r#"<c:lineChart><c:grouping val="standard"/>{s}<c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/>{val_axis}</c:valAx>"#
);
chart_items(&space(&plot), &theme(), 300.0, 200.0)
}
fn marker_xml(symbol: &str, size: u32) -> String {
format!(r#"<c:marker><c:symbol val="{symbol}"/><c:size val="{size}"/></c:marker>"#)
}
#[test]
fn marker_symbols_are_drawn_as_their_shape() {
let shapes = |items: &[GraphicItem], kind: ShapeKind| {
items
.iter()
.filter(|it| matches!(it, GraphicItem::Shape { kind: k, rect, .. } if *k == kind && rect[2] > 3.0 && rect[2] < 20.0 && rect[3] > 1.0))
.count()
};
let paths = |items: &[GraphicItem], filled: bool| {
items
.iter()
.filter(|it| matches!(it, GraphicItem::Path { fill, segs, .. } if fill.is_some() == filled && segs.len() <= 12 && segs.len() >= 3))
.count()
};
// One marker per point, in the series' size.
let circle = line_with(&marker_xml("circle", 9), "");
assert_eq!(shapes(&circle, ShapeKind::Ellipse), 3);
assert!(circle.iter().any(|it| matches!(it, GraphicItem::Shape { kind: ShapeKind::Ellipse, rect, .. } if (rect[2] - 9.0).abs() < 1e-3)));
let square = line_with(&marker_xml("square", 7), "");
assert_eq!(shapes(&square, ShapeKind::Rectangle) - shapes(&circle, ShapeKind::Rectangle), 3, "a square per point");
assert_eq!(shapes(&square, ShapeKind::Ellipse), 0, "no circles");
// Diamonds and triangles are filled polygons; x, plus and star are strokes.
let base = paths(&line_with("", ""), true);
assert_eq!(paths(&line_with(&marker_xml("diamond", 7), ""), true) - base, 3);
assert_eq!(paths(&line_with(&marker_xml("triangle", 7), ""), true) - base, 3);
for symbol in ["x", "plus", "star"] {
let items = line_with(&marker_xml(symbol, 7), "");
assert!(items.iter().any(|it| matches!(it, GraphicItem::Path { fill: None, stroke: Some(_), segs, .. } if segs.len() >= 6)), "{symbol}");
assert_eq!(shapes(&items, ShapeKind::Ellipse), 0, "{symbol} is not a circle");
}
// Dash and dot are small shapes; `none` and unknown names never crash (unknown: a circle).
assert_eq!(shapes(&line_with(&marker_xml("dash", 7), ""), ShapeKind::Rectangle) - shapes(&circle, ShapeKind::Rectangle), 3);
assert_eq!(shapes(&line_with(&marker_xml("dot", 7), ""), ShapeKind::Ellipse), 3);
assert_eq!(shapes(&line_with(&marker_xml("none", 7), ""), ShapeKind::Ellipse), 0);
assert_eq!(shapes(&line_with(&marker_xml("wibble", 7), ""), ShapeKind::Ellipse), 3);
// Hostile sizes are clamped.
for size in ["0", "99999999999", "-4", "x"] {
let items = line_with(&format!(r#"<c:marker><c:symbol val="square"/><c:size val="{size}"/></c:marker>"#), "");
assert!(items.iter().all(|it| match it {
GraphicItem::Shape { rect, .. } => rect.iter().all(|v| v.is_finite()) && rect[2] <= 300.0,
_ => true,
}));
}
}
#[test]
fn data_labels_use_their_own_format_code() {
let dl = |num: &str| format!(r#"<c:dLbls>{num}<c:showVal val="1"/></c:dLbls>"#);
let shown = |items: &[GraphicItem]| -> Vec<String> {
items.iter().filter_map(|it| if let GraphicItem::Text { text, .. } = it { Some(text.clone()) } else { None }).collect()
};
let own = shown(&line_with(&dl(r#"<c:numFmt formatCode="0.00" sourceLinked="0"/>"#), ""));
assert!(own.contains(&"3.20".to_string()) && own.contains(&"5.00".to_string()), "{own:?}");
// Linked to the source: the series' format (General here).
let linked = shown(&line_with(&dl(r#"<c:numFmt formatCode="0.00" sourceLinked="1"/>"#), ""));
assert!(linked.contains(&"3.2".to_string()) && linked.contains(&"5".to_string()), "{linked:?}");
// A format code beyond the cap is ignored, not applied.
let long = "0".repeat(craft_wordmodel::chart::MAX_FMT + 1);
let capped = shown(&line_with(&dl(&format!(r#"<c:numFmt formatCode="{long}" sourceLinked="0"/>"#)), ""));
assert!(capped.contains(&"3.2".to_string()), "{capped:?}");
}
#[test]
fn value_axis_title_turns_counter_clockwise_like_word() {
let title = |body: &str| format!(r#"<c:title><c:tx><c:rich>{body}<a:p><a:r><a:t>Revenue</a:t></a:r></a:p></c:rich></c:tx></c:title>"#);
let angle = |items: &[GraphicItem]| {
items.iter().find_map(|it| if let GraphicItem::Text { text, angle, .. } = it { (text == "Revenue").then_some(*angle) } else { None })
};
// The file's rotation (`-5400000` is 90° counter-clockwise), or Word's default when it says none.
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="-5400000" vert="horz"/>"#))), Some(-90.0));
assert_eq!(angle(&line_with("", &title("<a:bodyPr/>"))), Some(-90.0));
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="5400000" vert="horz"/>"#))), Some(90.0));
// A horizontal title (`rot="0"`) stays horizontal: the plot makes room for it (see below).
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="0" vert="horz"/>"#))), Some(0.0));
// Hostile values stay finite.
let odd = angle(&line_with("", &title(r#"<a:bodyPr rot="99999999999999999999"/>"#)));
assert!(odd.is_some_and(f32::is_finite));
}
fn first_ellipse(items: &[GraphicItem]) -> Option<([f32; 4], Rgb)> {
items.iter().find_map(|it| match it {
GraphicItem::Shape { kind: ShapeKind::Ellipse, rect, fill: Some(c), .. } if rect[2] > 3.0 && rect[2] < 20.0 => Some((*rect, *c)),
_ => None,
})
}
#[test]
fn markers_use_the_automatic_series_colour_not_the_line_colour() {
// The series has an explicit red line/fill but no marker fill: Word draws the markers in the
// palette colour of the series index (accent 1 here).
let items = line_with(&marker_xml("circle", 9), "");
assert_eq!(first_ellipse(&items).map(|e| e.1), Some(theme()[4]));
// A marker's own fill wins.
let own =
r#"<c:marker><c:symbol val="circle"/><c:size val="9"/><c:spPr><a:solidFill><a:srgbClr val="00AA00"/></a:solidFill></c:spPr></c:marker>"#;
assert_eq!(first_ellipse(&line_with(own, "")).map(|e| e.1), Some(Rgb(0, 0xAA, 0)));
}
#[test]
fn line_data_labels_default_to_the_right_and_honour_dlblpos() {
let dl = |pos: &str| format!(r#"{}<c:dLbls>{pos}<c:showVal val="1"/></c:dLbls>"#, marker_xml("circle", 9));
let text_rect = |items: &[GraphicItem]| {
items.iter().find_map(|it| if let GraphicItem::Text { text, rect, .. } = it { (text == "3.2").then_some(*rect) } else { None })
};
let at = |pos: &str| {
let items = line_with(&dl(pos), "");
let (m, _) = first_ellipse(&items).unwrap();
let t = text_rect(&items).unwrap();
// Centre of the marker and of the label box.
((t[0] + t[2] / 2.0) - (m[0] + m[2] / 2.0), (t[1] + t[3] / 2.0) - (m[1] + m[3] / 2.0), t, m)
};
let (dx, dy, t, m) = at("");
assert!(t[0] >= m[0] + m[2] && dy.abs() < 1.0, "default is right of the marker: {dx} {dy}");
let (dx, dy, t, m) = at(r#"<c:dLblPos val="r"/>"#);
assert!(t[0] >= m[0] + m[2] && dy.abs() < 1.0, "{dx} {dy}");
let (dx, dy, _, _) = at(r#"<c:dLblPos val="t"/>"#);
assert!(dx.abs() < 1.0 && dy < -4.0, "{dx} {dy}");
let (dx, dy, _, _) = at(r#"<c:dLblPos val="b"/>"#);
assert!(dx.abs() < 1.0 && dy > 4.0, "{dx} {dy}");
let (dx, dy, t, m) = at(r#"<c:dLblPos val="l"/>"#);
assert!(t[0] + t[2] <= m[0] && dy.abs() < 1.0, "{dx} {dy}");
let (dx, dy, _, _) = at(r#"<c:dLblPos val="ctr"/>"#);
assert!(dx.abs() < 1.0 && dy.abs() < 1.0, "{dx} {dy}");
// Unknown positions fall back to the default.
let (_, _, t, m) = at(r#"<c:dLblPos val="wibble"/>"#);
assert!(t[0] >= m[0] + m[2]);
}
#[test]
fn horizontal_value_axis_title_gets_its_own_strip() {
let title = |rot: &str, name: &str| {
format!(r#"<c:title><c:tx><c:rich><a:bodyPr{rot}/><a:p><a:r><a:t>{name}</a:t></a:r></a:p></c:rich></c:tx></c:title>"#)
};
let plot_left = |items: &[GraphicItem]| {
// The first gridline's start is the plot's left edge.
items.iter().find_map(|it| match it {
GraphicItem::Path { segs, stroke: Some(_), fill: None, .. } if segs.len() == 2 => match segs.first() {
Some(PathSeg::Move(x, _)) => Some(*x),
_ => None,
},
_ => None,
})
};
let rect_of = |items: &[GraphicItem], name: &str| {
items.iter().find_map(|it| if let GraphicItem::Text { text, rect, .. } = it { text.starts_with(name).then_some(*rect) } else { None })
};
let turned = line_with("", &title(r#" rot="-5400000" vert="horz""#, "Revenue"));
let flat = line_with("", &title(r#" rot="0" vert="horz""#, "Revenue"));
let (a, b) = (plot_left(&turned).unwrap(), plot_left(&flat).unwrap());
assert!(b > a + 10.0, "a horizontal title takes more room than a turned one: {a} {b}");
// The title box lies left of the plot and inside the chart.
let r = rect_of(&flat, "Revenue").unwrap();
assert!(r[0] >= 0.0 && r[0] + r[2] <= b + 0.5, "{r:?} {b}");
// A hostile title never takes more than a quarter of the width and stays finite.
let long = "Wibble ".repeat(500);
let items = line_with("", &title(r#" rot="0""#, &long));
let p = plot_left(&items).unwrap();
assert!(p.is_finite() && p <= 300.0 * 0.3, "{p}");
}
/// A one-series chart whose series carries `sp` as its `c:spPr`.
fn chart_with_sp(kind: &str, sp: &str) -> Vec<GraphicItem> {
let s = ser(0, "S", None, &["Q1", "Q2"], &["4", "8"]).replace("<c:cat>", &format!("{sp}<c:cat>"));
let plot = match kind {
"area" => format!(
r#"<c:areaChart><c:grouping val="standard"/>{s}<c:axId val="1"/><c:axId val="2"/></c:areaChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#
),
dir => format!(
r#"<c:barChart><c:barDir val="{dir}"/><c:grouping val="clustered"/>{s}<c:axId val="1"/><c:axId val="2"/></c:barChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#
),
};
chart_items(&space(&plot), &theme(), 300.0, 200.0)
}
/// The series' closed shapes: (filled, stroked with the given width).
fn series_shapes(items: &[GraphicItem], width: f32) -> (usize, usize) {
let closed = |segs: &[PathSeg]| segs.iter().any(|s| matches!(s, PathSeg::Close));
let fills = items.iter().filter(|it| matches!(it, GraphicItem::Path { segs, fill: Some(_), .. } if closed(segs))).count();
let strokes = items
.iter()
.filter(|it| matches!(it, GraphicItem::Path { segs, fill: None, stroke: Some(_), stroke_width, .. } if closed(segs) && (*stroke_width - width).abs() < 0.01))
.count();
(fills, strokes)
}
#[test]
fn unfilled_bars_keep_their_explicit_outline() {
let outlined = r#"<c:spPr><a:noFill/><a:ln w="25400"><a:solidFill><a:srgbClr val="FF0000"/></a:solidFill></a:ln></c:spPr>"#;
let no_line = r#"<c:spPr><a:noFill/><a:ln><a:noFill/></a:ln></c:spPr>"#;
for dir in ["col", "bar"] {
// Two points, two outlines, no fill.
assert_eq!(series_shapes(&chart_with_sp(dir, outlined), 2.0), (0, 2), "{dir}");
// Control: no fill and no line draws nothing.
assert_eq!(series_shapes(&chart_with_sp(dir, no_line), 2.0), (0, 0), "{dir}");
}
}
#[test]
fn no_fill_area_series_is_not_painted() {
let filled = r#"<c:spPr><a:solidFill><a:srgbClr val="FF0000"/></a:solidFill></c:spPr>"#;
let none = r#"<c:spPr><a:noFill/><a:ln w="25400"><a:solidFill><a:srgbClr val="FF0000"/></a:solidFill></a:ln></c:spPr>"#;
assert_eq!(series_shapes(&chart_with_sp("area", filled), 2.0).0, 1);
assert_eq!(series_shapes(&chart_with_sp("area", none), 2.0).0, 0);
}
#[test]
fn source_linked_series_labels_do_not_inherit_the_chart_format() {
let dl = |fmt: &str| format!(r#"<c:dLbls><c:numFmt formatCode="{fmt}" sourceLinked="{{}}"/><c:showVal val="1"/></c:dLbls>"#);
let shown = |items: &[GraphicItem]| -> Vec<String> {
items.iter().filter_map(|it| if let GraphicItem::Text { text, .. } = it { Some(text.clone()) } else { None }).collect()
};
// Chart-level 0%; the series' own labels are `linked` (or have no labels element at all).
let run = |series_dl: &str| {
let s = ser(0, "S", Some("FF0000"), &["Q1", "Q2", "Q3"], &["3.2", "4.1", "5"]).replace("<c:cat>", &format!("{series_dl}<c:cat>"));
let chart_dl = dl("0%").replace("{}", "0");
let plot = format!(
r#"<c:lineChart><c:grouping val="standard"/>{s}{chart_dl}<c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#
);
shown(&chart_items(&space(&plot), &theme(), 300.0, 200.0))
};
for linked in ["1", "true"] {
let own = run(&dl("General").replace("{}", linked));
assert!(own.contains(&"3.2".to_string()) && !own.contains(&"320%".to_string()), "sourceLinked={linked}: {own:?}");
}
// Control: a series without a label format still takes the chart's.
let inherited = run(r#"<c:dLbls><c:showVal val="1"/></c:dLbls>"#);
assert!(inherited.contains(&"320%".to_string()), "{inherited:?}");
}
+10 -1
View File
@@ -463,7 +463,16 @@ fn text(sp: &El, xf: Xf, b: [f64; 4], theme: &[Rgb], filled: bool, fit: f32, out
let mut y = y0 + (th - total) / 2.0;
for l in lines {
let lh = l.size * LINE;
out.push(GraphicItem::Text { rect: [x0, y, tw, lh], text: l.text, size: l.size, color: l.color, bold: l.bold, align: l.align, font: l.font });
out.push(GraphicItem::Text {
rect: [x0, y, tw, lh],
text: l.text,
size: l.size,
color: l.color,
bold: l.bold,
align: l.align,
font: l.font,
angle: 0.0,
});
y += lh;
}
}
+2 -1
View File
@@ -884,10 +884,11 @@ mod tests {
let spec = ChartSpec {
kind: ChartType::Column,
categories: vec!["a".into(), "b".into()],
series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2] }).collect(),
series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2], ..Default::default() }).collect(),
title: None,
legend: None,
data_labels: false,
..Default::default()
};
let graphic = Graphic { kind: GraphicKind::Chart, w: 100.0, h: 100.0, chart: Some(Arc::new(spec)), ..Default::default() };
InlineObject::Graphic { w: 100.0, h: 100.0, alt: String::new(), float: Default::default(), graphic: Arc::new(graphic) }
+8 -2
View File
@@ -493,6 +493,8 @@ fn flow_paras_in(doc: &Document, p: &Paragraph, opts: FlowOpts) -> Vec<Para> {
kind,
list,
align: p.props.align,
// Direction can come from the paragraph's style (`w:bidi` in the style's pPr).
rtl: doc.styles.resolve_para(&p.props).bidi,
page_break: p.props.page_break_before.unwrap_or(false),
keep_next: p.props.keep_next.unwrap_or(false),
inlines: Vec::new(),
@@ -536,6 +538,7 @@ fn flow_paras_in(doc: &Document, p: &Paragraph, opts: FlowOpts) -> Vec<Para> {
kind,
list: None,
align: p.props.align,
rtl: out.rtl,
page_break: true,
keep_next: p.props.keep_next.unwrap_or(false),
inlines: Vec::new(),
@@ -658,7 +661,10 @@ fn flow_paras_in(doc: &Document, p: &Paragraph, opts: FlowOpts) -> Vec<Para> {
{
(li.list, li.label) = (0, None);
}
let old = Para { kind, list, align: c.old.align, page_break: c.old.page_break_before.unwrap_or(false), ..Default::default() };
// Direction resolved like the current paragraph's: the old properties' own `w:bidi`, else
// the style's.
let rtl = doc.styles.resolve_para(&c.old).bidi;
let old = Para { kind, list, align: c.old.align, rtl, page_break: c.old.page_break_before.unwrap_or(false), ..Default::default() };
Box::new(FmtChange { mark: rev_mark(doc, c.rev), old })
});
}
@@ -749,7 +755,7 @@ fn graphic_group(
let inline = Inline::Shape { kind: *kind, w: rect[2] * sx, h: rect[3] * sy, float: None, blocks: Vec::new(), look, path: None };
Some(Member { x: rect[0] * sx, y: rect[1] * sy, inline })
}
GraphicItem::Text { rect, text, size, color, bold, align, font } if finite(rect) && !text.trim().is_empty() => {
GraphicItem::Text { rect, text, size, color, bold, align, font, .. } if finite(rect) && !text.trim().is_empty() => {
let size = if size.is_finite() { size.clamp(1.0, 400.0) } else { 11.0 };
// One line, centred vertically in its place as the graphic draws it.
let line = size * 1.2;
+3
View File
@@ -159,6 +159,9 @@ fn para_props(p: &Para, list_num: Option<u32>) -> ParaProps {
out.numbering = Some(NumRef { num, level: li.level.min(8) });
}
out.align = p.align;
if p.rtl {
out.bidi = Some(true);
}
let sp = p.spacing;
out.space_before = sp.before;
out.space_after = sp.after;
+2
View File
@@ -676,6 +676,8 @@ pub struct Para {
pub kind: Kind,
pub list: Option<ListInfo>,
pub align: Option<Align>,
/// The paragraph reads right to left (OOXML `w:bidi`, RTF `\rtlpar`).
pub rtl: bool,
pub page_break: bool,
/// Keep the paragraph on the same page as the next one (ODT `fo:keep-with-next`).
pub keep_next: bool,
+11
View File
@@ -0,0 +1,11 @@
[package]
name = "craft-http-url"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license = "Apache-2.0"
repository.workspace = true
description = "Bounded HTTP URL parsing and redirect resolution without network access"
[lints]
workspace = true
+25
View File
@@ -0,0 +1,25 @@
# craft-http-url
HTTP/HTTPS URL parsing, dot-segment removal, relative redirects, HTTPS downgrade rejection,
and query-redacted display. No dependencies or network operations. Parsing rejects inputs
larger than 16 KiB and redirects are bounded before concatenation.
This is not an SSRF filter, DNS resolver or HTTP client. Local/private hosts are allowed.
Applications keep destination permission, authentication, TLS, request/response limits and
redirect-count policy. Public fields preserve the original API; manually constructed values
must not be treated as validated URLs.
## Attribution
Extracted from LightCraft `crates/fetch/src/http.rs` at
`2dcb258ba40738230a07484a95eb27dc84ed66fe` (Apache-2.0).
Original parser: Daniel Voshart, `7fa7506db46d69d609e85e39b338f33786ed0293`.
HTTPS downgrade protection: MarekSpodBiedry, `4a91a1c218bf3d0ac49cc73ebb60621a21ae55e2`.
URL normalization, scheme, fragment and authority fixes: NetterSilver, including
`edc369565eef92161b56b8f1e39ef53629c05489`,
`78f0e6e650444886069f4d4ae4e6593f221b98d4`,
`cee93496587d31c514aaeced3e6b8027cfdb41a0`,
`dee5f7910ad8c24cf28bab528f8d007550af2b23`,
`c3a5d9b7f1f837dc14c01942e3d0293ecbf36345` and
`13849307a9343504339d68b086f1a32cdcbbb457`.
Source-author commits are retained separately in this extraction.
+277
View File
@@ -0,0 +1,277 @@
//! Bounded HTTP/HTTPS URL parsing and redirect resolution, extracted from LightCraft.
//!
//! This is syntax handling, not a network permission or SSRF policy. It permits local/private
//! hosts and performs no DNS, socket or TLS operations. Callers own destination allowlists,
//! credential forwarding, redirect counts, connection deadlines and response limits.
//! Only parsed values have validated fields; constructing public fields bypasses parsing.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
/// A URL or redirect cannot be represented by this bounded HTTP URL syntax.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UrlError(pub String);
impl std::fmt::Display for UrlError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for UrlError {}
/// A parsed `http://` or `https://` URL.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Url {
pub tls: bool,
pub host: String,
pub port: u16,
/// Path and query, starting with `/`.
pub path: String,
}
/// Remove `.` and `..` segments from the path part of `path` (RFC 3986 section 5.2.4); the query is kept as is.
fn remove_dot_segments(path: &str) -> String {
let (p, query) = match path.find('?') {
Some(i) => (path.get(..i).unwrap_or_default(), path.get(i..).unwrap_or_default()),
None => (path, ""),
};
let mut out: Vec<&str> = Vec::new();
let mut segments = p.split('/').skip(1).peekable();
let mut trailing_slash = false;
while let Some(seg) = segments.next() {
let last = segments.peek().is_none();
match seg {
"." => trailing_slash = last,
".." => {
out.pop();
trailing_slash = last;
}
_ => {
out.push(seg);
trailing_slash = false;
}
}
}
let mut result = format!("/{}", out.join("/"));
if trailing_slash && !result.ends_with('/') {
result.push('/');
}
result.push_str(query);
result
}
/// `s` without the leading `prefix`, matched ASCII case-insensitively (RFC 3986 §3.1).
fn strip_scheme<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
let head = s.get(..prefix.len())?;
head.eq_ignore_ascii_case(prefix).then(|| s.get(prefix.len()..).unwrap_or_default())
}
impl Url {
pub fn parse(s: &str) -> Result<Url, UrlError> {
let bad = |why: &str| UrlError(why.to_string());
if s.len() > 16 * 1024 {
return Err(bad("URL exceeds 16 KiB"));
}
let s = s.trim();
if s.chars().any(|c| c.is_control() || c == ' ') {
return Err(bad("contains spaces or control characters"));
}
let (tls, rest) = if let Some(r) = strip_scheme(s, "https://") {
(true, r)
} else if let Some(r) = strip_scheme(s, "http://") {
(false, r)
} else {
return Err(bad("only http:// and https:// are supported"));
};
let rest = rest.split('#').next().unwrap_or_default();
let (authority, path) = match rest.find(['/', '?']) {
Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()),
None => (rest, ""),
};
let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") };
let path = remove_dot_segments(&path);
if authority.contains('@') {
return Err(bad("user names and passwords in URLs are not supported"));
}
let (host, port) = if let Some(v6) = authority.strip_prefix('[') {
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
let host = v6.get(..end).unwrap_or_default();
let after = v6.get(end + 1..).unwrap_or_default();
if !after.is_empty() && !after.starts_with(':') {
return Err(bad("unexpected text after IPv6 address"));
}
(host.to_string(), after.strip_prefix(':'))
} else {
match authority.rsplit_once(':') {
Some((h, p)) => (h.to_string(), Some(p)),
None => (authority.to_string(), None),
}
};
let port = match port {
Some(p) if p.bytes().all(|b| b.is_ascii_digit()) => p.parse::<u16>().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?,
Some(_) => return Err(bad("bad port")),
None if tls => 443,
None => 80,
};
if host.is_empty() || !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | ':' | '_')) {
return Err(bad("bad host name"));
}
Ok(Url { tls, host: host.to_ascii_lowercase(), port, path })
}
/// `Location` of a redirect, resolved against this URL.
pub fn join(&self, location: &str) -> Result<Url, UrlError> {
// Reject before concatenating a hostile Location with the base URL.
if location.len() > 16 * 1024 || self.host.len().saturating_add(self.path.len()) > 16 * 1024 {
return Err(UrlError("URL exceeds 16 KiB".into()));
}
let l = location.trim();
if strip_scheme(l, "http://").is_some() || strip_scheme(l, "https://").is_some() {
let url = Url::parse(l)?;
if self.tls && !url.tls {
return Err(UrlError("redirect from https to http refused".into()));
}
return Ok(url);
}
// another scheme (`ftp:`, `file:`, `data:`…)
if let Some((scheme, _)) = l.split_once(':')
&& !scheme.is_empty()
&& scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
{
return Err(UrlError(format!("redirect to an unsupported location `{scheme}:`")));
}
let scheme = if self.tls { "https" } else { "http" };
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
let base = format!("{scheme}://{host}:{}", self.port);
if let Some(rest) = l.strip_prefix("//") {
return Url::parse(&format!("{scheme}://{rest}"));
}
if l.starts_with('/') {
return Url::parse(&format!("{base}{l}"));
}
// empty or fragment-only: the same document
if l.is_empty() || l.starts_with('#') {
return Ok(self.clone());
}
// relative to the current directory
let dir = self.path.split('?').next().unwrap_or("/");
// a query-only reference keeps the whole path, file name included (RFC 3986 §5.2.2)
if l.starts_with('?') {
return Url::parse(&format!("{base}{dir}{l}"));
}
let dir = dir.rsplit_once('/').map_or("", |(d, _)| d);
Url::parse(&format!("{base}{dir}/{l}"))
}
/// Authority formatted for an HTTP Host field.
pub fn host_header(&self) -> String {
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
let default = if self.tls { 443 } else { 80 };
if self.port == default { host } else { format!("{host}:{}", self.port) }
}
/// The URL without its query (for messages: a query may hold a signed token).
pub fn display(&self) -> String {
let scheme = if self.tls { "https" } else { "http" };
format!("{scheme}://{}{}", self.host_header(), self.path.split('?').next().unwrap_or("/"))
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::{Url, UrlError};
#[test]
fn fragment_only_or_empty_location_keeps_file_name_and_query() {
let base = Url::parse("https://a.example/x/y/file?q").unwrap();
for l in ["#top", "", " "] {
let u = base.join(l).unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/x/y/file?q"), "{l:?}");
}
}
#[test]
fn rejects_oversized_and_control_character_urls() {
let base = Url::parse("https://example.com/path").unwrap();
assert!(base.join(&format!("#{}", "x".repeat(16 * 1024))).is_err());
assert!(base.join(&"x".repeat(16 * 1024 + 1)).is_err());
assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err());
assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err());
}
#[test]
fn join_query_only_keeps_file_name() {
let u = Url::parse("https://a.example/x/y/file?q=1").unwrap();
assert_eq!(u.join("?page=2").unwrap().path, "/x/y/file?page=2");
}
#[test]
fn join_removes_dot_segments() {
let base = Url::parse("https://a.example/x/y/file").unwrap();
for (loc, want) in
[("../z", "/x/z"), ("./z", "/x/y/z"), ("../../../z", "/z"), ("/a/./b/../c", "/a/c"), ("../z?q=1", "/x/z?q=1"), ("..", "/x/")]
{
assert_eq!(base.join(loc).unwrap().path, want, "{loc}");
}
}
#[test]
fn url_scheme_is_matched_case_insensitively() {
let u = Url::parse("HTTPS://A.example/z").unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/z"));
assert!(Url::parse("Http://a.example/").is_ok());
let j = Url::parse("https://a.example/start").unwrap().join("HTTPS://cdn.example/file").unwrap();
assert_eq!((j.tls, j.host.as_str(), j.path.as_str()), (true, "cdn.example", "/file"));
}
#[test]
fn parses_urls() {
let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "cdn.example.com", 443, "/models/sam3/model.safetensors?sig=1"));
assert_eq!(u.display(), "https://cdn.example.com/models/sam3/model.safetensors");
let u = Url::parse("http://127.0.0.1:8080").unwrap();
assert_eq!((u.tls, u.port, u.path.as_str()), (false, 8080, "/"));
let u = Url::parse("http://[::1]:9/a").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 9));
for bad in ["ftp://x/y", "https://", "https://a b/", "https://u:p@h/", "http://h:0/", "http://h:99999/", "https://h/\r\nX: y", "file:///etc"]
{
assert!(Url::parse(bad).is_err(), "{bad}");
}
}
#[test]
fn rejects_signed_ports() {
for bad in ["http://example.com:+80/x", "http://example.com:-80/x", "http://[::1]:+80/x"] {
assert!(Url::parse(bad).is_err(), "{bad}");
}
assert!(Url::parse("http://a.test/").unwrap().join("http://example.com:+80/x").is_err());
assert_eq!(Url::parse("http://example.com:80/x").unwrap().port, 80);
}
#[test]
fn rejects_text_after_a_bracketed_ipv6_host() {
for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] {
assert!(matches!(Url::parse(bad), Err(UrlError(_))), "{bad}");
}
let u = Url::parse("http://[::1]:8080/x").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 8080));
let u = Url::parse("http://[::1]/x").unwrap();
assert_eq!((u.host.as_str(), u.port), ("::1", 80));
}
#[test]
fn resolves_redirects() {
let u = Url::parse("https://a.example/x/y/file?q").unwrap();
assert_eq!(u.join("https://b.example/z").unwrap().host, "b.example");
assert_eq!(u.join("/root").unwrap().path, "/root");
assert_eq!(u.join("other").unwrap().path, "/x/y/other");
assert_eq!(u.join("//c.example/p").unwrap().host, "c.example");
assert!(u.join("ftp://nope").is_err());
// no downgrade from https to http, however the redirect is written
assert!(u.join("http://a.example/z").is_err());
let plain = Url::parse("http://127.0.0.1:8080/a").unwrap();
assert!(!plain.join("http://127.0.0.1:9/b").unwrap().tls);
assert!(plain.join("https://b.example/z").unwrap().tls);
}
}
@@ -842,6 +842,10 @@ fn read_metadata(c: &Container) -> (Option<Vec<u8>>, Option<String>) {
/// Decodes the primary image. `keep_orientation` leaves the container's `clap`, `irot` and
/// `imir` unapplied (the stored pixels), as for HEIF.
pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<Image> {
decode_with_depth(bytes, limits, keep_orientation).map(|(image, _)| image)
}
pub(crate) fn decode_with_depth(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<(Image, u8)> {
let c = parse(bytes)?;
let id = c.primary;
let (w, h) = item_size(&c, id)?;
@@ -890,9 +894,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> R
// PQ or HLG: linear light in 32-bit float, as for HEIF.
if cicp.is_hdr() {
limits.check(w, h, layout, SampleType::F32)?;
return crate::cicp::linearize(img, cicp);
return crate::cicp::linearize(img, cicp).map(|image| (image, bits));
}
Ok(img)
Ok((img, bits))
}
#[cfg(test)]
@@ -1301,6 +1305,9 @@ mod tests {
let src = Image::from_u8(w as u32, h as u32, ChannelLayout::Rgb, card(w, h, 3)).unwrap();
let file = encode_avif_with_speed(&src, 95, 10).unwrap();
let img = crate::decode(&file).unwrap();
let (with_depth, depth) = crate::decode_avif_with_depth(&file, &crate::DecodeOptions::default()).unwrap();
assert_eq!(depth, 10);
assert_eq!(with_depth, img);
assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (48, 32, ChannelLayout::Rgb, SampleType::U16));
let diff = mean_abs_diff(&rgb8(&img), &card(w, h, 3));
assert!(diff < 2.5, "mean difference {diff}");
+8
View File
@@ -120,6 +120,14 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
finish_format(format, bytes, img, opts)
}
/// Decode AVIF pixels and report their original AV1 bit depth (8, 10 or 12).
/// The returned image may use a wider sample type, particularly for linear HDR.
#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))]
pub fn decode_avif_with_depth(bytes: &[u8], opts: &DecodeOptions) -> Result<(Image, u8), CodecError> {
let (image, depth) = codecs::avif_decode::decode_with_depth(bytes, &opts.limits, opts.keep_orientation)?;
Ok((finish_format(Format::Avif, bytes, image, opts)?, depth))
}
/// What [`decode_as_with`] does with a PNG decoded in steps by [`PngDecodeRows`] (`img` is its
/// output): the resolution from the metadata, the upright orientation and the final limit check,
/// so the result equals [`decode_as_with`]'s on the same file.
+5 -2
View File
@@ -229,7 +229,7 @@ impl Reader {
};
let name = s.label.as_deref().and_then(|a| cells_of(a).first().copied()).map(|at| label(self.cell(at))).unwrap_or_default();
let name = if name.is_empty() { format!("Series {}", i + 1) } else { name };
ChartSeries { name, values }
ChartSeries { name, values, ..Default::default() }
})
.collect();
// A scatter chart's x values: its first series' domain.
@@ -248,7 +248,10 @@ impl Reader {
_ => LegendPos::Right,
});
let title = self.title.as_deref().map(str::trim).filter(|t| !t.is_empty()).map(str::to_string);
Some(ChartSpec { kind, categories, series, title, legend, data_labels }.sanitized())
// ODF axis formatting is not read; the value axis keeps the look every chart had before
// axes were formattable (gridlines, no axis line), a round chart has none.
let val_axis = if kind.is_round() { Default::default() } else { craft_wordmodel::chart::value_axis() };
Some(ChartSpec { kind, categories, series, title, legend, data_labels, val_axis, ..Default::default() }.sanitized())
}
}
+14 -4
View File
@@ -57,8 +57,8 @@ fn spec(chart: &str) -> Option<ChartSpec> {
fn north_south() -> Vec<ChartSeries> {
vec![
ChartSeries { name: "North".into(), values: vec![Some(10.0), Some(14.0), Some(8.0), Some(16.0)] },
ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)] },
ChartSeries { name: "North".into(), values: vec![Some(10.0), Some(14.0), Some(8.0), Some(16.0)], ..Default::default() },
ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)], ..Default::default() },
]
}
@@ -78,6 +78,8 @@ fn a_column_chart_reads_its_data_title_and_legend() {
title: Some("Sales by quarter".into()),
legend: Some(LegendPos::Right),
data_labels: false,
val_axis: craft_wordmodel::chart::value_axis(),
..Default::default()
}
);
// The frame's size, in points.
@@ -135,7 +137,7 @@ fn line_area_pie_doughnut_and_scatter_classes() {
let s = spec(&chart_xml("chart:scatter", "", "", "", scatter)).unwrap();
assert_eq!(s.kind, ChartType::Scatter);
assert_eq!(s.categories, vec!["10", "14", "8", "16"]);
assert_eq!(s.series, vec![ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)] }]);
assert_eq!(s.series, vec![ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)], ..Default::default() }]);
}
#[test]
@@ -249,7 +251,15 @@ fn every_chart_type_is_saved_as_a_chart_object_and_reads_back() {
let bytes = crate::text::export(&doc_with(spec.clone(), false)).unwrap();
let back = reread(&bytes);
assert_eq!(back.len(), 1, "{kind:?}");
assert_eq!(*back[0].spec, spec.clone().sanitized(), "{kind:?}");
// ODF keeps no series or axis formatting here: only the data, the title and the legend
// come back; the value axis has the gridlines every chart had before axes were formattable.
let mut expected = spec.clone().sanitized();
for s in &mut expected.series {
*s = ChartSeries { name: std::mem::take(&mut s.name), values: std::mem::take(&mut s.values), ..Default::default() };
}
expected.cat_axis = Default::default();
expected.val_axis = if kind.is_round() { Default::default() } else { craft_wordmodel::chart::value_axis() };
assert_eq!(*back[0].spec, expected, "{kind:?}");
assert_eq!((back[0].w, back[0].h, back[0].alt.as_str()), (300.0, 180.0, "Sales chart"), "{kind:?}");
// The package lists the chart object.
let z = crate::Package::open(&bytes, &crate::Limits::DEFAULT).unwrap();
+1
View File
@@ -130,6 +130,7 @@ fn a_diagram_is_saved_as_a_group_of_its_drawing() {
bold: false,
align: TextAlign::Center,
font: None,
angle: 0.0,
},
GraphicItem::Shape { rect: [90.0, 9.0, 18.0, 18.0], kind: ShapeKind::Arrow, fill: Some(blue), stroke: None, stroke_width: 0.0 },
],
+4
View File
@@ -3,6 +3,10 @@
//! Callers map their page and color models to these neutral records. This module owns PDF object
//! layout, image streams, ICC/Separation resources, page labels, marks, xref and trailer syntax.
/// Multi-image 8/16-bit print pages with per-image ICC resources and explicit byte budgets.
#[path = "raster_print.rs"]
pub mod print;
use std::collections::BTreeSet;
use std::fmt::Write as _;
use std::io::Write as _;
+658
View File
@@ -0,0 +1,658 @@
//! A small PDF writer for print output: pages of a given size in points, each with raster images
//! placed on it, tagged with their ICC profile (`ICCBased`) so the numbers mean the same colour in
//! every reader and RIP, Flate-compressed (lossless) at 8 or 16 bits per component, or carried as
//! JPEG (`DCTDecode`) where small files matter more than exact samples. Optionally an output intent:
//! the printing condition (the printer / paper profile) the file was made for, as PDF/X readers
//! expect it in the catalog. Profiles shared by several images are written once.
//!
//! The file is PDF 1.6 (16-bit images need 1.5): classic cross-reference table, no object streams,
//! deterministic (no dates or IDs), so the same pages always give the same bytes.
//!
//! Encoded JPEG and Flate payloads are copied, not decoded or verified against declared
//! dimensions. Callers own their encoding validity and ICC/profile-channel compatibility.
//! An output intent describes a printing condition; this API does not certify PDF/X.
//! Byte limits bound accounted input and emitted output, not peak process memory or
//! caller-owned image construction. This is not the streaming slideshow writer.
//!
//! Sources: ISO 32000-1:2008 (§7.3 objects, §7.5 file structure, §8.6.5.5 ICCBased colour spaces,
//! §8.9 images, §14.11.5 output intents). Own design.
use std::collections::HashMap;
use std::io::Write as _;
use std::sync::Arc;
/// Pages per document: a print job beyond this is a mistake, not a book.
pub const MAX_PAGES: usize = 1000;
/// Images per page.
pub const MAX_IMAGES_PER_PAGE: usize = 256;
/// Largest page side in points (200 inches): more than any printer's roll.
pub const MAX_PAGE_POINTS: f64 = 14_400.0;
/// Largest image, in pixels per side.
pub const MAX_IMAGE_SIDE: u32 = 65_535;
/// Serialization budgets; input records already belong to the caller. These are not a
/// process-wide peak-memory guarantee. Repeated references count toward the input budget.
#[derive(Clone, Copy, Debug)]
pub struct Limits {
pub input_bytes: usize,
pub output_bytes: usize,
}
impl Default for Limits {
fn default() -> Self {
Self { input_bytes: 512 * 1024 * 1024, output_bytes: 512 * 1024 * 1024 }
}
}
/// How an image's samples are stored.
#[derive(Clone, Debug, PartialEq)]
pub enum ImageData {
/// Interleaved samples, `bits` (8 or 16) per component, 16-bit values big-endian: compressed
/// losslessly in the file.
Raw { bits: u8, samples: Vec<u8> },
/// A baseline JPEG stream, copied into the file as it is.
Jpeg(Vec<u8>),
/// Samples already compressed with zlib (see [`PdfImage::deflated`]), `bits` per component.
Flate { bits: u8, data: Vec<u8> },
}
/// A raster image and the colour space its samples are in.
#[derive(Clone, Debug, PartialEq)]
pub struct PdfImage {
pub width: u32,
pub height: u32,
/// Components per pixel: 1 (grey), 3 (RGB) or 4 (CMYK).
pub channels: u8,
pub data: ImageData,
/// The ICC profile the samples are in; without one the image is device colour.
pub icc: Option<Arc<Vec<u8>>>,
}
impl PdfImage {
/// An image from interleaved 16-bit samples.
pub fn from_u16(width: u32, height: u32, channels: u8, samples: &[u16], icc: Option<Arc<Vec<u8>>>) -> PdfImage {
let bytes = samples.iter().flat_map(|v| v.to_be_bytes()).collect();
PdfImage { width, height, channels, data: ImageData::Raw { bits: 16, samples: bytes }, icc }
}
/// The same image with its samples compressed now (checked first), so a long print doesn't
/// hold every page's uncompressed samples until the file is written.
pub fn deflated(self) -> Result<PdfImage, String> {
self.check()?;
if matches!(&self.data, ImageData::Raw { samples, .. } if samples.len() > Limits::default().input_bytes) {
return Err("PDF input byte limit exceeded".into());
}
let data = match self.data {
ImageData::Raw { bits, samples } => ImageData::Flate { bits, data: deflate(&samples, Limits::default().output_bytes)? },
other => other,
};
Ok(PdfImage { data, ..self })
}
fn check(&self) -> Result<(), String> {
if self.width == 0 || self.height == 0 || self.width > MAX_IMAGE_SIDE || self.height > MAX_IMAGE_SIDE {
return Err(format!("an image of {} × {} pixels can't be written", self.width, self.height));
}
if !matches!(self.channels, 1 | 3 | 4) {
return Err(format!("images have 1, 3 or 4 components, not {}", self.channels));
}
match &self.data {
ImageData::Raw { bits, samples } => {
if !matches!(bits, 8 | 16) {
return Err(format!("images have 8 or 16 bits per component, not {bits}"));
}
let want = u64::from(self.width) * u64::from(self.height) * u64::from(self.channels) * u64::from(bits / 8);
if samples.len() as u64 != want {
return Err(format!("the image needs {want} bytes of samples, not {}", samples.len()));
}
}
ImageData::Jpeg(data) => {
if !data.starts_with(&[0xFF, 0xD8]) {
return Err("the image isn't a JPEG stream".into());
}
}
ImageData::Flate { bits, data } => {
if !matches!(bits, 8 | 16) || data.is_empty() {
return Err("a compressed image has 8 or 16 bits per component and some data".into());
}
}
}
Ok(())
}
}
/// Where an image goes on its page, in points from the page's bottom-left corner.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Placement {
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
}
/// One page: its size in points and the images on it, drawn in order (later ones on top).
#[derive(Clone, Debug, PartialEq)]
pub struct PdfPage {
pub width: f64,
pub height: f64,
pub images: Vec<(PdfImage, Placement)>,
}
/// The printing condition the document is made for (PDF/X-style output intent).
#[derive(Clone, Debug, PartialEq)]
pub struct OutputIntent {
/// The printer / paper profile.
pub icc: Arc<Vec<u8>>,
/// Its components: 1, 3 or 4.
pub channels: u8,
/// A name for the condition (the profile's description).
pub condition: String,
}
/// Document-level settings.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct PdfInfo {
pub title: Option<String>,
pub output_intent: Option<OutputIntent>,
/// Application name, not a PDF/X conformance claim.
pub producer: String,
}
fn finite(v: f64) -> bool {
v.is_finite()
}
/// A number as PDF writes it: up to four decimals, no exponent, no trailing zeros.
fn num(v: f64) -> String {
let s = format!("{:.4}", if finite(v) { v } else { 0.0 });
let s = s.trim_end_matches('0').trim_end_matches('.');
if s == "-0" || s.is_empty() { "0".into() } else { s.into() }
}
/// A text string: plain ASCII as a literal string, anything else as UTF-16BE with a byte order mark.
fn text(s: &str) -> String {
if s.bytes().all(|b| (0x20..0x7f).contains(&b)) {
let mut out = String::with_capacity(s.len() + 2);
out.push('(');
for c in s.chars() {
if matches!(c, '(' | ')' | '\\') {
out.push('\\');
}
out.push(c);
}
out.push(')');
out
} else {
super::pdf_unicode_string(s)
}
}
struct Writer {
out: Vec<u8>,
offsets: Vec<usize>,
limit: usize,
}
impl Writer {
fn put(&mut self, bytes: &[u8]) -> Result<(), String> {
let end = self.out.len().checked_add(bytes.len()).ok_or("PDF output size overflow")?;
if end > self.limit {
return Err("PDF output byte limit exceeded".into());
}
self.out.try_reserve(bytes.len()).map_err(|_| "PDF output allocation failed")?;
self.out.extend_from_slice(bytes);
Ok(())
}
fn reserve(&mut self) -> Result<usize, String> {
self.offsets.try_reserve(1).map_err(|_| "PDF object allocation failed")?;
self.offsets.push(0);
Ok(self.offsets.len())
}
fn object(&mut self, id: usize, dict: &str, stream: Option<&[u8]>) -> Result<(), String> {
let pos = self.out.len();
*self.offsets.get_mut(id.checked_sub(1).ok_or("invalid PDF object")?).ok_or("invalid PDF object")? = pos;
self.put(format!("{id} 0 obj\n{dict}").as_bytes())?;
if let Some(data) = stream {
self.put(b"\nstream\n")?;
self.put(data)?;
self.put(b"\nendstream")?;
}
self.put(b"\nendobj\n")
}
}
fn deflate(data: &[u8], max: usize) -> Result<Vec<u8>, String> {
struct Sink {
bytes: Vec<u8>,
max: usize,
}
impl std::io::Write for Sink {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
if bytes.len() > self.max.saturating_sub(self.bytes.len()) {
return Err(std::io::Error::other("PDF compressed byte limit exceeded"));
}
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
self.bytes.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let mut encoder = flate2::write::ZlibEncoder::new(Sink { bytes: Vec::new(), max }, flate2::Compression::new(6));
encoder.write_all(data).map_err(|e| e.to_string())?;
Ok(encoder.finish().map_err(|e| e.to_string())?.bytes)
}
/// Write a document. Errors name the first page or image that can't be written.
pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result<Vec<u8>, String> {
write_with_limits(pages, info, Limits::default())
}
/// Write with explicit total input and serialized output budgets. ICC bytes are opaque:
/// callers validate their profile semantics; resources are keyed by bytes AND channels.
pub fn write_with_limits(pages: &[PdfPage], info: &PdfInfo, limits: Limits) -> Result<Vec<u8>, String> {
let mut input = 0usize;
let mut account = |n: usize| -> Result<(), String> {
input = input.checked_add(n).ok_or("PDF input size overflow")?;
if input > limits.input_bytes {
return Err("PDF input byte limit exceeded".into());
}
Ok(())
};
account(info.producer.len())?;
account(info.title.as_ref().map_or(0, String::len))?;
if let Some(intent) = &info.output_intent {
account(intent.icc.len())?;
account(intent.condition.len())?;
}
if pages.is_empty() {
return Err("there are no pages to write".into());
}
if pages.len() > MAX_PAGES {
return Err(format!("a document holds at most {MAX_PAGES} pages"));
}
for (n, page) in pages.iter().enumerate() {
let n = n + 1;
if !(finite(page.width) && finite(page.height))
|| page.width <= 0.0
|| page.height <= 0.0
|| page.width > MAX_PAGE_POINTS
|| page.height > MAX_PAGE_POINTS
{
return Err(format!("page {n} has no usable size"));
}
if page.images.len() > MAX_IMAGES_PER_PAGE {
return Err(format!("page {n} has more than {MAX_IMAGES_PER_PAGE} images"));
}
for (image, at) in &page.images {
account(match &image.data {
ImageData::Raw { samples, .. } => samples.len(),
ImageData::Jpeg(data) | ImageData::Flate { data, .. } => data.len(),
})?;
if let Some(icc) = &image.icc {
account(icc.len())?;
}
image.check().map_err(|e| format!("page {n}: {e}"))?;
if ![at.x, at.y, at.width, at.height].into_iter().all(finite) || at.width <= 0.0 || at.height <= 0.0 {
return Err(format!("page {n}: an image has no usable placement"));
}
}
}
if let Some(intent) = &info.output_intent
&& !matches!(intent.channels, 1 | 3 | 4)
{
return Err("the output intent's profile has 1, 3 or 4 components".into());
}
let mut w = Writer { out: Vec::new(), offsets: Vec::new(), limit: limits.output_bytes };
w.put(b"%PDF-1.6\n%\xE2\xE3\xCF\xD3\n")?;
let catalog = w.reserve()?;
let pages_id = w.reserve()?;
let info_id = w.reserve()?;
// ICC profiles, each written once (keyed by content) with its component count
let mut profiles: HashMap<(Arc<Vec<u8>>, u8), usize> = HashMap::new();
let mut profile = |w: &mut Writer, icc: &Arc<Vec<u8>>, n: u8| -> Result<usize, String> {
if let Some(&id) = profiles.get(&(Arc::clone(icc), n)) {
return Ok(id);
}
let id = w.reserve()?;
let data = deflate(icc, limits.output_bytes)?;
let alternate = match n {
1 => "/DeviceGray",
4 => "/DeviceCMYK",
_ => "/DeviceRGB",
};
w.object(id, &format!("<< /N {n} /Alternate {alternate} /Filter /FlateDecode /Length {} >>", data.len()), Some(&data))?;
profiles.try_reserve(1).map_err(|_| "PDF profile allocation failed")?;
profiles.insert((Arc::clone(icc), n), id);
Ok(id)
};
let intent_id = info.output_intent.as_ref().map(|intent| profile(&mut w, &intent.icc, intent.channels)).transpose()?;
let mut kids = Vec::with_capacity(pages.len());
for page in pages {
let page_id = w.reserve()?;
let mut xobjects = String::new();
let mut content = String::new();
for (k, (image, at)) in page.images.iter().enumerate() {
let space = match &image.icc {
Some(icc) => format!("[/ICCBased {} 0 R]", profile(&mut w, icc, image.channels)?),
None => match image.channels {
1 => "/DeviceGray".into(),
4 => "/DeviceCMYK".into(),
_ => "/DeviceRGB".into(),
},
};
let id = w.reserve()?;
let (bits, filter, data): (u8, &str, std::borrow::Cow<[u8]>) = match &image.data {
ImageData::Raw { bits, samples } => (*bits, "/FlateDecode", deflate(samples, limits.output_bytes)?.into()),
ImageData::Flate { bits, data } => (*bits, "/FlateDecode", data.as_slice().into()),
ImageData::Jpeg(jpeg) => (8, "/DCTDecode", jpeg.as_slice().into()),
};
w.object(
id,
&format!(
"<< /Type /XObject /Subtype /Image /Width {} /Height {} /ColorSpace {space} /BitsPerComponent {bits} /Filter {filter} /Length {} >>",
image.width,
image.height,
data.len()
),
Some(&data),
)?;
xobjects.push_str(&format!("/Im{k} {id} 0 R "));
content.push_str(&format!("q {} 0 0 {} {} {} cm /Im{k} Do Q\n", num(at.width), num(at.height), num(at.x), num(at.y)));
}
let content_id = w.reserve()?;
let stream = deflate(content.as_bytes(), limits.output_bytes)?;
w.object(content_id, &format!("<< /Filter /FlateDecode /Length {} >>", stream.len()), Some(&stream))?;
w.object(
page_id,
&format!(
"<< /Type /Page /Parent {pages_id} 0 R /MediaBox [0 0 {} {}] /Resources << /XObject << {xobjects}>> >> /Contents {content_id} 0 R >>",
num(page.width),
num(page.height)
),
None,
)?;
kids.push(format!("{page_id} 0 R"));
}
w.object(pages_id, &format!("<< /Type /Pages /Count {} /Kids [{}] >>", kids.len(), kids.join(" ")), None)?;
let intents = match (&info.output_intent, intent_id) {
(Some(intent), Some(id)) => format!(
" /OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX /OutputConditionIdentifier (Custom) /Info {} /DestOutputProfile {id} 0 R >>]",
text(&intent.condition)
),
_ => String::new(),
};
w.object(catalog, &format!("<< /Type /Catalog /Pages {pages_id} 0 R{intents} >>"), None)?;
let title = info.title.as_deref().map(|t| format!(" /Title {}", text(t))).unwrap_or_default();
w.object(info_id, &format!("<< /Producer {}{title} >>", text(&info.producer)), None)?;
let xref = w.out.len();
let mut table = format!("xref\n0 {}\n0000000000 65535 f \n", w.offsets.len() + 1);
for offset in &w.offsets {
table.push_str(&format!("{offset:010} 00000 n \n"));
}
table.push_str(&format!("trailer\n<< /Size {} /Root {catalog} 0 R /Info {info_id} 0 R >>\nstartxref\n{xref}\n%%EOF\n", w.offsets.len() + 1));
w.put(table.as_bytes())?;
Ok(w.out)
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use super::*;
/// The body of object `id` (dictionary and raw stream bytes), found through the xref table.
fn object(pdf: &[u8], id: usize) -> (String, Vec<u8>) {
// byte offsets: the streams are binary, so search the bytes, not a lossy string
let start = pdf.windows(10).rposition(|w| w == b"startxref\n").unwrap() + 10;
let tail = std::str::from_utf8(&pdf[start..]).unwrap();
let xref: usize = tail.lines().next().unwrap().trim().parse().unwrap();
let table = std::str::from_utf8(&pdf[xref..]).unwrap();
let line = table.lines().nth(2 + id).unwrap();
let offset: usize = line[..10].parse().unwrap();
let head = format!("{id} 0 obj\n");
assert_eq!(&pdf[offset..offset + head.len()], head.as_bytes(), "xref entry for {id}");
let rest = &pdf[offset + head.len()..];
let end = rest.windows(7).position(|w| w == b"\nendobj").unwrap();
let body = &rest[..end];
match body.windows(8).position(|w| w == b"\nstream\n") {
Some(p) => {
let data = &body[p + 8..body.len() - "\nendstream".len()];
(String::from_utf8_lossy(&body[..p]).into_owned(), data.to_vec())
}
None => (String::from_utf8_lossy(body).into_owned(), vec![]),
}
}
fn inflate(data: &[u8]) -> Vec<u8> {
{
use std::io::Read;
let mut out = Vec::new();
flate2::read::ZlibDecoder::new(data).read_to_end(&mut out).unwrap();
out
}
}
/// Object ids of every image XObject, in file order.
fn images(pdf: &[u8]) -> Vec<usize> {
let s = String::from_utf8_lossy(pdf);
s.match_indices(" 0 obj\n<< /Type /XObject /Subtype /Image").map(|(i, _)| s[..i].rsplit('\n').next().unwrap().parse().unwrap()).collect()
}
fn reference(dict: &str, key: &str) -> usize {
let at = dict.find(key).unwrap() + key.len();
dict[at..].trim_start().split(' ').next().unwrap().parse().unwrap()
}
fn srgb() -> Arc<Vec<u8>> {
Arc::new(b"opaque RGB profile fixture".to_vec())
}
fn page(images: Vec<(PdfImage, Placement)>) -> PdfPage {
PdfPage { width: 612.0, height: 792.0, images }
}
fn full(w: f64, h: f64) -> Placement {
Placement { x: 0.0, y: 0.0, width: w, height: h }
}
#[test]
fn a_16_bit_icc_image_round_trips() {
let samples: Vec<u16> = (0..2 * 3 * 3).map(|i| (i * 3851) as u16).collect();
let img = PdfImage::from_u16(3, 2, 3, &samples, Some(srgb()));
let pdf = write(&[page(vec![(img, full(612.0, 792.0))])], &PdfInfo::default()).unwrap();
assert!(pdf.starts_with(b"%PDF-1.6\n"));
assert!(pdf.ends_with(b"%%EOF\n"));
let ids = images(&pdf);
assert_eq!(ids.len(), 1);
let (dict, data) = object(&pdf, ids[0]);
assert!(dict.contains("/Width 3 /Height 2") && dict.contains("/BitsPerComponent 16") && dict.contains("/FlateDecode"), "{dict}");
let back: Vec<u16> = inflate(&data).chunks(2).map(|b| u16::from_be_bytes([b[0], b[1]])).collect();
assert_eq!(back, samples);
let (profile, icc) = object(&pdf, reference(&dict, "/ICCBased"));
assert!(profile.contains("/N 3") && profile.contains("/Alternate /DeviceRGB"), "{profile}");
assert_eq!(inflate(&icc), *srgb());
}
#[test]
fn pages_sizes_placements_and_a_shared_profile() {
let px = |v: u16| PdfImage::from_u16(1, 1, 3, &[v, v, v], Some(srgb()));
let at = Placement { x: 36.0, y: 72.5, width: 288.0, height: 216.0 };
let pages = [page(vec![(px(1), at), (px(2), full(612.0, 792.0))]), PdfPage { width: 841.89, height: 595.28, images: vec![(px(3), at)] }];
let pdf = write(&pages, &PdfInfo::default()).unwrap();
let s = String::from_utf8_lossy(&pdf);
assert!(s.contains("/Type /Pages /Count 2"), "two pages");
assert!(s.contains("/MediaBox [0 0 612 792]") && s.contains("/MediaBox [0 0 841.89 595.28]"));
// the sRGB profile is written once for all three images
assert_eq!(s.matches("/N 3 /Alternate").count(), 1);
let first: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
assert!(first.windows(2).all(|w| w[0] == w[1]), "{first:?}");
// placements reach the content streams
let contents: Vec<String> = s
.match_indices("/Contents ")
.map(|(i, _)| s[i + 10..].split(' ').next().unwrap().parse::<usize>().unwrap())
.map(|id| String::from_utf8(inflate(&object(&pdf, id).1)).unwrap())
.collect();
assert_eq!(contents[0], "q 288 0 0 216 36 72.5 cm /Im0 Do Q\nq 612 0 0 792 0 0 cm /Im1 Do Q\n");
assert_eq!(write(&pages, &PdfInfo::default()).unwrap(), pdf, "deterministic");
}
#[test]
fn output_intent_and_title() {
let cmyk_like = srgb();
let info = PdfInfo {
title: Some("Garden – prints (2)".into()),
producer: "Fixture".into(),
output_intent: Some(OutputIntent { icc: cmyk_like.clone(), channels: 3, condition: "Glossy (Photo) \\ test".into() }),
};
let img = PdfImage::from_u16(1, 1, 3, &[0, 0, 0], Some(cmyk_like));
let pdf = write(&[page(vec![(img, full(10.0, 10.0))])], &info).unwrap();
let (catalog, _) = object(&pdf, 1);
assert!(catalog.contains("/OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX"), "{catalog}");
assert!(catalog.contains("/Info (Glossy \\(Photo\\) \\\\ test)"), "{catalog}");
// the intent and the image share one profile object
let dest = reference(&catalog, "/DestOutputProfile");
assert_eq!(reference(&object(&pdf, images(&pdf)[0]).0, "/ICCBased"), dest);
let (info_dict, _) = object(&pdf, 3);
assert!(info_dict.contains("/Title <FEFF"), "non-ASCII titles are UTF-16: {info_dict}");
assert!(info_dict.contains("2013"), "the en dash: {info_dict}");
}
#[test]
fn cmyk_grey_jpeg_and_device_images() {
let jpeg = vec![0xFF, 0xD8, 0xFF, 0xD9];
let imgs = vec![
(
PdfImage { width: 1, height: 1, channels: 4, data: ImageData::Raw { bits: 8, samples: vec![0, 10, 20, 30] }, icc: None },
full(1.0, 1.0),
),
(PdfImage { width: 2, height: 1, channels: 1, data: ImageData::Raw { bits: 8, samples: vec![0, 255] }, icc: None }, full(1.0, 1.0)),
(PdfImage { width: 4, height: 4, channels: 3, data: ImageData::Jpeg(jpeg.clone()), icc: None }, full(1.0, 1.0)),
];
let pdf = write(&[page(imgs)], &PdfInfo::default()).unwrap();
let ids = images(&pdf);
let dicts: Vec<_> = ids.iter().map(|&id| object(&pdf, id)).collect();
assert!(dicts[0].0.contains("/ColorSpace /DeviceCMYK") && dicts[0].0.contains("/BitsPerComponent 8"));
assert!(dicts[1].0.contains("/ColorSpace /DeviceGray"));
assert!(dicts[2].0.contains("/DCTDecode") && dicts[2].0.contains("/DeviceRGB"));
assert_eq!(dicts[2].1, jpeg, "JPEG streams are copied as they are");
}
#[test]
fn compressing_early_writes_the_same_file() {
let samples: Vec<u16> = (0..4 * 3 * 3).map(|i| (i * 977) as u16).collect();
let raw = PdfImage::from_u16(4, 3, 3, &samples, Some(srgb()));
let early = raw.clone().deflated().unwrap();
assert!(matches!(early.data, ImageData::Flate { bits: 16, .. }));
let at = full(100.0, 75.0);
let a = write(&[page(vec![(raw, at)])], &PdfInfo::default()).unwrap();
let b = write(&[page(vec![(early, at)])], &PdfInfo::default()).unwrap();
assert_eq!(a, b);
let mut short = PdfImage::from_u16(4, 3, 3, &samples, None);
short.width = 5;
assert!(short.deflated().is_err(), "checked before compressing");
}
#[test]
fn hostile_documents_are_errors() {
let ok = || PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
let at = full(10.0, 10.0);
let bad = |pages: Vec<PdfPage>| write(&pages, &PdfInfo::default()).is_err();
assert!(bad(vec![]));
assert!(bad(vec![PdfPage { width: f64::NAN, height: 10.0, images: vec![] }]));
assert!(bad(vec![PdfPage { width: -1.0, height: 10.0, images: vec![] }]));
assert!(bad(vec![PdfPage { width: 1e9, height: 10.0, images: vec![] }]));
assert!(bad(vec![page(vec![(ok(), Placement { x: f64::INFINITY, ..at })])]));
assert!(bad(vec![page(vec![(ok(), Placement { width: 0.0, ..at })])]));
let mut short = ok();
short.data = ImageData::Raw { bits: 16, samples: vec![0; 5] };
assert!(bad(vec![page(vec![(short, at)])]));
let mut empty = ok();
empty.width = 0;
assert!(bad(vec![page(vec![(empty, at)])]));
let mut odd = ok();
odd.channels = 2;
assert!(bad(vec![page(vec![(odd, at)])]));
let mut not_jpeg = ok();
not_jpeg.data = ImageData::Jpeg(vec![1, 2, 3]);
assert!(bad(vec![page(vec![(not_jpeg, at)])]));
let too_many = (0..MAX_PAGES + 1).map(|_| page(vec![])).collect();
assert!(bad(too_many));
let info = PdfInfo {
producer: String::new(),
title: None,
output_intent: Some(OutputIntent { icc: srgb(), channels: 2, condition: String::new() }),
};
assert!(write(&[page(vec![])], &info).is_err());
}
#[test]
fn numbers_and_strings() {
assert_eq!(num(612.0), "612");
assert_eq!(num(72.5), "72.5");
assert_eq!(num(-0.00001), "0");
assert_eq!(num(f64::NAN), "0");
assert_eq!(num(1.0 / 3.0), "0.3333");
assert_eq!(text("a(b)c\\"), "(a\\(b\\)c\\\\)");
assert_eq!(text("é"), "<FEFF00E9>");
}
#[test]
fn distinct_profiles_and_component_counts_are_not_conflated() {
let common = Arc::new(b"opaque profile A".to_vec());
let different = Arc::new(b"opaque profile B".to_vec());
let image = |channels, profile| PdfImage {
width: 1,
height: 1,
channels,
data: ImageData::Raw { bits: 8, samples: vec![7; channels as usize] },
icc: Some(profile),
};
let pages = [page(vec![
(image(3, common.clone()), full(1.0, 1.0)),
(image(4, common.clone()), full(1.0, 1.0)),
(image(3, different.clone()), full(1.0, 1.0)),
(image(3, common.clone()), full(1.0, 1.0)),
])];
let pdf = write(&pages, &PdfInfo::default()).unwrap();
let refs: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
assert_ne!(refs[0], refs[1]);
assert_ne!(refs[0], refs[2]);
assert_eq!(refs[0], refs[3]);
for (i, channels, bytes) in [(0, 3, common.as_slice()), (1, 4, common.as_slice()), (2, 3, different.as_slice())] {
let (dict, data) = object(&pdf, refs[i]);
assert!(dict.contains(&format!("/N {channels}")));
assert_eq!(inflate(&data), bytes);
}
}
#[test]
fn gray_rgb_cmyk_raw_depths_preserve_samples() {
for channels in [1, 3, 4] {
for bits in [8, 16] {
let bytes: Vec<u8> = (0..channels * (bits / 8)).collect();
let image = PdfImage { width: 1, height: 1, channels, data: ImageData::Raw { bits, samples: bytes.clone() }, icc: None };
let pdf = write(&[page(vec![(image, full(1.0, 1.0))])], &PdfInfo::default()).unwrap();
let (dict, data) = object(&pdf, images(&pdf)[0]);
assert!(dict.contains(&format!("/BitsPerComponent {bits}")));
assert_eq!(inflate(&data), bytes);
}
}
}
#[test]
fn input_and_output_budgets_fail_without_partial_success() {
let image = PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
let pages = [page(vec![(image, full(1.0, 1.0))])];
let info = PdfInfo::default();
let expected = write(&pages, &info).unwrap();
assert_eq!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() }).unwrap(), expected);
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 5, output_bytes: expected.len() }).is_err());
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() - 1 }).is_err());
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: 0 }).is_err());
}
}
+1 -1
View File
@@ -33,7 +33,7 @@ The format layer ends where arithmetic on the picture begins.
|---|---|
| Recognising containers (`probe`), walking TIFF/IFD structure on `craft-tiff` | Black subtraction, white scaling, cropping to the active area |
| Decompressing the samples: unpacking, lossless JPEG, lossy JPEG, Deflate, JPEG XL | Demosaicing, binned previews, hot-pixel and highlight handling |
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values; left unapplied when `OpcodeList1` is present, so the develop pipeline can run that list on the stored codes first) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
| Black and white levels, CFA layout, active area, default crop, orientation, pixel aspect | The DNG colour model's arithmetic (illuminant interpolation, camera to working space) |
| Colour tags as data: matrices, calibration illuminants, as-shot neutral / white | White balance, Kelvin scales, spectral or fitted camera colour |
| Opcode lists parsed and written (`opcodes`), profile tables and gain table maps as data | Applying opcodes (gain maps, warps, bad pixels), profile look tables, tone curves |
+1 -1
View File
@@ -187,7 +187,7 @@ the six verified fixtures are unchanged. Partial-subband and
rounded-plane coding, vertical C-RAW tiling and adaptive low-frequency gains
remain unsupported (see the variant table above; `cr3_variants.rs` pins each rejection). The decoder bounds image dimensions, sample count,
tile count, marker headers and temporary allocations. The full sensor buffer is
allocated only after the first plane decodes, and plane buffers grow row by row,
allocated only after every plane has decoded, and plane buffers grow row by row,
so a small crafted file fails before reserving memory for the image it claims.
## Further bodies: one CC0 file each
+1
View File
@@ -852,6 +852,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// makes a neutral surface magenta. WB edits remain relative to this as-shot RGB.
wb_multipliers: if linear_rgb { Some([1.0; 3]) } else { wb },
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
// only validated components become opcodes (see `arw_lens`, and `arw_lens_rm6` for the ILCE-7RM6)
opcodes: OpcodeLists { list3: warp.into_iter().chain(rm6).collect(), ..Default::default() },
+1
View File
@@ -311,6 +311,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData { kelvin_presets, ..Default::default() },
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -388,6 +388,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode, limits: &
// the camera's white balance is already applied, as in Sony's linear YCbCr
wb_multipliers: Some([1.0; 3]),
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+2
View File
@@ -269,6 +269,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: levels.wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: ifd0.map(Camera::from_ifd0).unwrap_or_default(),
@@ -824,6 +825,7 @@ mod tests {
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Camera::default(),
+1
View File
@@ -59,6 +59,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, caller: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: h.wb_multipliers,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera,
+518 -101
View File
@@ -95,60 +95,90 @@ pub(crate) fn decode(sample: &[u8], config: &Cr3Compression) -> Result<Vec<u16>>
let count = image_size(config)?;
let tiles = layout(sample, config)?;
validate_planes(&tiles, config)?;
// Allocated once the first plane decodes: a tiny crafted file can claim a huge image.
let mut output = Vec::new();
// A tile's QP map error is reported by its first plane, so errors keep tile, then plane order.
let qp: Vec<Result<Option<QpMap>>> = tiles.iter().map(|tile| tile_qp(sample, tile)).collect();
let midpoint = 1i32 << (config.bit_depth - 1);
let maximum = (1i32 << config.bit_depth) - 1;
for tile in tiles {
let qp = tile_qp(sample, &tile)?;
// Each plane is its own entropy stream: decode a tile's planes in parallel.
let planes: Vec<Result<Vec<i32>>> = tile
.planes
.par_iter()
.enumerate()
.map(|(p, plane)| {
if config.levels == 0 {
let band = plane.bands.first().ok_or_else(|| corrupt("missing lossless band"))?;
lossless_plane(band_bytes(sample, band)?, tile.width / 2, tile.height / 2)
} else {
let edges = TileEdges { left: tile.x > 0, right: tile.x + tile.width < width, top: false, bottom: false };
wavelet_plane(sample, plane, tile.width / 2, tile.height / 2, edges, qp.as_ref())
}
.map_err(|error| RawError::Corrupt(format!("Canon CRX tile ({}, {}), plane {p}: {error}", tile.x, tile.y)))
})
.collect();
for (p, decoded) in planes.into_iter().enumerate() {
let decoded = decoded?;
if output.is_empty() {
output.try_reserve_exact(count).map_err(|_| RawError::Limit("CRX sample allocation"))?;
output.resize(count, 0);
}
for (i, &value) in decoded.iter().enumerate() {
let sensor_value = sensor_sample(value, midpoint, maximum, config.levels != 0)?;
let (x, y) = (tile.x + 2 * (i % (tile.width / 2)) + p % 2, tile.y + 2 * (i / (tile.width / 2)) + p / 2);
let slot = y
.checked_mul(width)
.and_then(|n| n.checked_add(x))
.and_then(|n| output.get_mut(n))
.ok_or_else(|| corrupt("tile writes outside image"))?;
*slot = sensor_value;
}
}
}
// Every plane of every tile is its own entropy stream: decode them all in parallel, each straight
// to sensor samples, so a task holds only its own plane as i32. The first error in tile and plane
// order is reported, as when the tiles were decoded one after another.
let jobs: Vec<(&Tile, &Result<Option<QpMap>>, usize, &Plane)> =
tiles.iter().zip(&qp).flat_map(|(tile, qp)| tile.planes.iter().enumerate().map(move |(p, plane)| (tile, qp, p, plane))).collect();
let planes: Vec<Result<Vec<u16>>> = jobs
.par_iter()
.map(|&(tile, qp, p, plane)| {
let qp = qp.as_ref().map_err(Clone::clone)?.as_ref();
let decoded = decode_plane(sample, config, width, tile, plane, qp)
.map_err(|error| RawError::Corrupt(format!("Canon CRX tile ({}, {}), plane {p}: {error}", tile.x, tile.y)))?;
sensor_plane(&decoded, midpoint, maximum, config.levels != 0)
})
.collect();
let planes = planes.into_iter().collect::<Result<Vec<_>>>()?;
// Allocated once every plane has decoded: a tiny crafted file can claim a huge image.
let mut output = Vec::new();
output.try_reserve_exact(count).map_err(|_| RawError::Limit("CRX sample allocation"))?;
output.resize(count, 0);
interleave(&mut output, width, &tiles, &planes)?;
Ok(output)
}
/// A decoded plane value as a sensor sample of the CMP1 bit depth.
fn decode_plane(sample: &[u8], config: &Cr3Compression, width: usize, tile: &Tile, plane: &Plane, qp: Option<&QpMap>) -> Result<Vec<i32>> {
if config.levels == 0 {
let band = plane.bands.first().ok_or_else(|| corrupt("missing lossless band"))?;
lossless_plane(band_bytes(sample, band)?, tile.width / 2, tile.height / 2)
} else {
let edges = TileEdges { left: tile.x > 0, right: tile.x + tile.width < width, top: false, bottom: false };
wavelet_plane(sample, plane, tile.width / 2, tile.height / 2, edges, qp)
}
}
/// Decoded plane values as sensor samples of the CMP1 bit depth.
///
/// Lossless planes reproduce the sensor exactly, so a value outside the bit depth is corrupt data.
/// Wavelet (C-RAW) planes are quantized: next to clipped highlights the reconstruction overshoots
/// the sensor's range by up to a few quantization steps (measured on 17 CC0 files: at most 36 codes
/// above 16383 at 14 bits, never below 0), while lossless files of the same bodies clip at exactly
/// 16383. Those samples are clamped to the range the sensor can record.
fn sensor_sample(value: i32, midpoint: i32, maximum: i32, quantized: bool) -> Result<u16> {
let sample = value.checked_add(midpoint).ok_or_else(|| corrupt("decoded sample outside its bit depth"))?;
let sample = if quantized { sample.clamp(0, maximum) } else { sample };
u16::try_from(sample).ok().filter(|_| sample <= maximum).ok_or_else(|| corrupt("decoded sample outside its bit depth"))
/// 16383. Those samples are clamped to the range the sensor can record. The range is checked once,
/// on the plane's extremes; `midpoint` is positive, so only the largest sum can overflow.
fn sensor_plane(values: &[i32], midpoint: i32, maximum: i32, quantized: bool) -> Result<Vec<u16>> {
let outside = || corrupt("decoded sample outside its bit depth");
let (Some(&low), Some(&high)) = (values.iter().min(), values.iter().max()) else { return Ok(Vec::new()) };
let high = high.checked_add(midpoint).ok_or_else(outside)?;
if !quantized && (low.saturating_add(midpoint) < 0 || high > maximum) {
return Err(outside());
}
let mut samples = Vec::new();
samples.try_reserve_exact(values.len()).map_err(|_| RawError::Limit("CRX sample allocation"))?;
// Every sum fits (the largest did), and the clamp leaves in-range lossless samples unchanged.
samples.extend(values.iter().map(|&value| (value + midpoint).clamp(0, maximum) as u16));
Ok(samples)
}
/// Writes each tile's four planes (`planes`: four per tile, in tile order) into the Bayer mosaic.
/// Plane `p` holds the samples at row parity `p / 2` and column parity `p % 2`. Runs in parallel
/// over pairs of output rows.
fn interleave(output: &mut [u16], width: usize, tiles: &[Tile], planes: &[Vec<u16>]) -> Result<()> {
let outside = || corrupt("tile writes outside image");
if width == 0 || planes.len() != tiles.len() * 4 {
return Err(outside());
}
output.par_chunks_mut(2 * width).enumerate().try_for_each(|(pair, rows)| -> Result<()> {
let y = 2 * pair;
for (tile, planes) in tiles.iter().zip(planes.as_chunks::<4>().0).filter(|(tile, _)| (tile.y..tile.y + tile.height).contains(&y)) {
let half = tile.width / 2;
let start = (y - tile.y) / 2 * half;
for (row, [even, odd]) in rows.chunks_exact_mut(width).zip(planes.as_chunks::<2>().0) {
let target = row.get_mut(tile.x..tile.x + tile.width).ok_or_else(outside)?;
let even = even.get(start..start + half).ok_or_else(outside)?;
let odd = odd.get(start..start + half).ok_or_else(outside)?;
for ([a, b], (&left, &right)) in target.as_chunks_mut::<2>().0.iter_mut().zip(even.iter().zip(odd)) {
*a = left;
*b = right;
}
}
}
Ok(())
})
}
/// Header-only probing must reject the same unsupported coding variants as full decoding.
@@ -413,24 +443,58 @@ fn wavelet_plane(sample: &[u8], plane: &Plane, width: usize, height: usize, edge
for high in high_dimensions.iter().rev() {
shapes.extend_from_slice(high);
}
let mut bands = Vec::with_capacity(10);
for (i, (&(w, h), band)) in shapes.iter().zip(&plane.bands).enumerate() {
let bytes = band_bytes(sample, band)?;
let mut data = if i == 0 { lossless_plane(bytes, w, h)? } else { coded_high_frequency_band(bytes, w, coded_band_width(i, w, edges), h)? };
for (position, value) in data.iter_mut().enumerate() {
let step = match band.quant {
Quantization::Uniform(value) => quant_step(value, MAX_UNIFORM_QUANT)?,
Quantization::Adaptive { base, gain } => {
adaptive_step(qp.ok_or_else(|| corrupt("missing QP map"))?, i, position % w, position / w, base, gain)?
}
};
*value = value.checked_mul(step).ok_or_else(|| corrupt("dequantized coefficient overflow"))?;
}
bands.push(WaveletBand { width: w, height: h, data });
}
// Each subband is its own entropy stream: decode and dequantize them in parallel. The first error
// in band order is reported, as when they were decoded one after another.
let bands: Vec<Result<WaveletBand>> = shapes
.par_iter()
.zip(plane.bands.par_iter())
.enumerate()
.map(|(i, (&(w, h), band))| {
let bytes = band_bytes(sample, band)?;
let mut data = if i == 0 { lossless_plane(bytes, w, h)? } else { coded_high_frequency_band(bytes, w, coded_band_width(i, w, edges), h)? };
dequantize(&mut data, i, w, &band.quant, qp)?;
Ok(WaveletBand { width: w, height: h, data })
})
.collect();
let bands = bands.into_iter().collect::<Result<Vec<_>>>()?;
crx_wavelet::reconstruct(&bands, width, height, 3, edges)
}
/// Multiplies the coefficients of subband `band` (`width` columns) by their quantization step.
///
/// The step is constant per subband, except for adaptive quantization with a gain, where it is
/// constant per QP cell: 2 coefficients wide in bands 4-6 and 4 wide in bands 7-9 (see
/// [`adaptive_step`]), so it is looked up once per cell rather than per coefficient.
fn dequantize(data: &mut [i32], band: usize, width: usize, quant: &Quantization, qp: Option<&QpMap>) -> Result<()> {
let (map, base, gain) = match *quant {
Quantization::Uniform(value) => return scale(data, quant_step(value, MAX_UNIFORM_QUANT)?),
Quantization::Adaptive { base, gain } => (qp.ok_or_else(|| corrupt("missing QP map"))?, base, gain),
};
if gain == 0 {
return scale(data, adaptive_step(map, band, 0, 0, base, gain)?);
}
if width == 0 {
return Err(corrupt("empty subband row"));
}
let cell = if (4..=6).contains(&band) { 2 } else { 4 };
for (y, row) in data.chunks_exact_mut(width).enumerate() {
for (column, values) in row.chunks_mut(cell).enumerate() {
scale(values, adaptive_step(map, band, column * cell, y, base, gain)?)?;
}
}
Ok(())
}
fn scale(values: &mut [i32], step: i32) -> Result<()> {
let mut overflow = false;
for value in values.iter_mut() {
let (product, wrapped) = value.overflowing_mul(step);
*value = product;
overflow |= wrapped;
}
if overflow { Err(corrupt("dequantized coefficient overflow")) } else { Ok(()) }
}
fn adaptive_step(map: &QpMap, band: usize, x: usize, y: usize, base: u32, gain: u32) -> Result<i32> {
let step = if gain == 0 {
u64::from(base)
@@ -502,17 +566,60 @@ fn be32(src: &[u8], at: usize) -> Result<u32> {
Ok(u32::from_be_bytes(bytes))
}
/// An MSB-first bit reader over one entropy stream.
///
/// Up to 64 stream bits are held in `cache`, left-aligned: the top `count` bits are the next unread
/// ones. A refill loads 8 bytes at once and keeps whole bytes only, so the bits below `count` are
/// either zero or the stream's own next bits at their final positions, which a later refill ORs in
/// again unchanged. Reads never trust those bits: they are checked against `count`.
struct Bits<'a> {
src: &'a [u8],
at: usize,
/// The next byte to load into `cache`.
next: usize,
cache: u64,
count: u32,
}
impl Bits<'_> {
/// The next 32 bits, MSB first, when at least that many remain before the end of the stream.
fn peek32(&self) -> Option<u32> {
let bytes = self.src.get(self.at / 8..self.at / 8 + 5)?;
let word = u64::from_be_bytes([0, 0, 0, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4]]);
Some((word >> (8 - self.at % 8)) as u32)
impl<'a> Bits<'a> {
fn new(src: &'a [u8]) -> Self {
Self { src, next: 0, cache: 0, count: 0 }
}
/// The number of bits consumed so far.
fn position(&self) -> usize {
self.next * 8 - self.count as usize
}
/// Loads whole bytes until more than 56 bits are cached or the stream ends.
fn refill(&mut self) {
if self.count > 56 {
return;
}
if let Some(bytes) = self.src.get(self.next..self.next + 8).and_then(|s| <[u8; 8]>::try_from(s).ok()) {
self.cache |= u64::from_be_bytes(bytes) >> self.count;
let loaded = (63 - self.count) / 8;
self.next += loaded as usize;
self.count += loaded * 8;
return;
}
// The last 7 bytes: one at a time.
while self.count <= 56 {
let Some(&byte) = self.src.get(self.next) else { break };
self.cache |= u64::from(byte) << (56 - self.count);
self.next += 1;
self.count += 8;
}
}
/// Drops `count` bits (at most `self.count`, below 64).
fn consume(&mut self, count: u32) {
self.cache <<= count;
self.count -= count;
}
/// The top `count` cached bits (0 to 32) as a number; `>> 1 >> (63 - count)` is 0 for `count` 0.
fn top(&self, count: u32) -> u32 {
(self.cache >> 1 >> (63 - count)) as u32
}
fn take(&mut self, count: u32) -> Result<u32> {
@@ -522,27 +629,34 @@ impl Bits<'_> {
if count == 0 {
return Ok(0);
}
if let Some(window) = self.peek32() {
self.at += count as usize;
return Ok(window >> (32 - count));
}
// Near the end of the stream: bit by bit, so truncation is reported exactly.
let mut value = 0;
for _ in 0..count {
let byte = self.src.get(self.at / 8).ok_or_else(|| corrupt("truncated entropy stream"))?;
value = (value << 1) | u32::from((byte >> (7 - self.at % 8)) & 1);
self.at += 1;
if self.count < count {
self.refill();
if self.count < count {
return Err(corrupt("truncated entropy stream"));
}
}
let value = self.top(count);
self.consume(count);
Ok(value)
}
fn rice(&mut self, k: u32) -> Result<u32> {
// Common case: the unary prefix and its terminating 1 lie within the next 32 bits.
if let Some(window) = self.peek32().filter(|&w| w != 0) {
let zeros = window.leading_zeros();
self.at += zeros as usize + 1;
return if k <= 20 { Ok((zeros << k) | self.take(k)?) } else { Err(corrupt("Rice parameter exceeds sample precision")) };
// Common case: the unary prefix (shorter than the 41-zero escape), its terminating 1 and the k
// low bits are all cached.
if k <= 20 {
let mut zeros = self.cache.leading_zeros();
if zeros + 1 + k > self.count {
self.refill();
zeros = self.cache.leading_zeros();
}
if zeros < 41 && zeros + 1 + k <= self.count {
self.consume(zeros + 1);
let low = self.top(k);
self.consume(k);
return Ok((zeros << k) | low);
}
}
// Escapes, k above 20 and the end of the stream: bit by bit, so truncation is reported exactly.
let mut zeros = 0;
while self.take(1)? == 0 {
zeros += 1;
@@ -580,11 +694,12 @@ impl Bits<'_> {
fn finish(&self) -> Result<()> {
let bits = self.src.len().checked_mul(8).ok_or(RawError::Limit("CRX entropy length"))?;
if bits.saturating_sub(self.at) > 7 {
let at = self.position();
if bits.saturating_sub(at) > 7 {
return Err(corrupt("unconsumed entropy data"));
}
if let Some(&last) = self.src.last() {
let remaining = (bits - self.at) as u32;
let remaining = (bits - at) as u32;
if u32::from(last) & ((1 << remaining) - 1) != 0 {
return Err(corrupt("nonzero entropy alignment bits"));
}
@@ -653,7 +768,7 @@ fn lossless_plane(src: &[u8], width: usize, height: usize) -> Result<Vec<i32>> {
current.try_reserve_exact(width).map_err(|_| RawError::Limit("CRX row allocation"))?;
current.resize(width, 0i32);
let (mut k, mut run_index) = (0u32, 0usize);
let mut bits = Bits { src, at: 0 };
let mut bits = Bits::new(src);
for y in 0..height {
let mut x = 0;
let mut left = previous.first().copied().ok_or_else(|| corrupt("empty plane row"))?;
@@ -716,7 +831,7 @@ fn high_frequency_band(src: &[u8], width: usize, height: usize) -> Result<Vec<i3
let (mut previous, mut current) = (row()?, row()?);
let (mut previous_k, mut current_k) = (row()?, row()?);
let (mut k, mut run_index) = (0u32, 0usize);
let mut bits = Bits { src, at: 0 };
let mut bits = Bits::new(src);
for y in 0..height {
let (mut x, mut left) = (0, 0);
while x < width {
@@ -780,7 +895,7 @@ fn qp_map(src: &[u8], width: usize, height: usize) -> Result<QpMap> {
Ok(values)
};
let (mut previous, mut current) = (row()?, row()?);
let mut bits = Bits { src, at: 0 };
let mut bits = Bits::new(src);
let mut k = 0;
for y in 0..height {
let mut left = *previous.first().ok_or_else(|| corrupt("empty QP row"))?;
@@ -879,11 +994,11 @@ mod tests {
#[test]
fn partial_final_run_segment_keeps_its_index() {
let mut bits = Bits { src: &[0xc0], at: 0 };
let mut bits = Bits::new(&[0xc0]);
let mut index = 11;
assert_eq!(run(&mut bits, &mut index, 6).unwrap(), 6);
assert_eq!(index, 12);
assert_eq!(bits.at, 2);
assert_eq!(bits.position(), 2);
}
#[test]
@@ -1043,14 +1158,22 @@ mod tests {
#[test]
fn only_quantized_planes_clamp_out_of_range_samples() {
assert_eq!(sensor_sample(8200, 8192, 16383, true).unwrap(), 16383);
assert_eq!(sensor_sample(-8300, 8192, 16383, true).unwrap(), 0);
assert_eq!(sensor_sample(8191, 8192, 16383, false).unwrap(), 16383);
assert_eq!(sensor_sample(-8192, 8192, 16383, false).unwrap(), 0);
assert_eq!(sensor_plane(&[8200], 8192, 16383, true).unwrap(), [16383]);
assert_eq!(sensor_plane(&[-8300], 8192, 16383, true).unwrap(), [0]);
assert_eq!(sensor_plane(&[-8300, 0, 8200], 8192, 16383, true).unwrap(), [0, 8192, 16383]);
assert_eq!(sensor_plane(&[8191], 8192, 16383, false).unwrap(), [16383]);
assert_eq!(sensor_plane(&[-8192], 8192, 16383, false).unwrap(), [0]);
assert_eq!(sensor_plane(&[-8192, 5, 8191], 8192, 16383, false).unwrap(), [0, 8197, 16383]);
// Lossless planes reproduce the sensor exactly: outside the bit depth is corrupt data.
assert!(matches!(sensor_sample(8192, 8192, 16383, false), Err(RawError::Corrupt(_))));
assert!(matches!(sensor_sample(-8193, 8192, 16383, false), Err(RawError::Corrupt(_))));
assert!(matches!(sensor_sample(i32::MAX, 8192, 16383, true), Err(RawError::Corrupt(_))));
assert!(matches!(sensor_plane(&[8192], 8192, 16383, false), Err(RawError::Corrupt(_))));
assert!(matches!(sensor_plane(&[-8193], 8192, 16383, false), Err(RawError::Corrupt(_))));
assert!(matches!(sensor_plane(&[0, 8192, 0], 8192, 16383, false), Err(RawError::Corrupt(_))));
assert!(matches!(sensor_plane(&[0, -8193, 0], 8192, 16383, false), Err(RawError::Corrupt(_))));
assert!(matches!(sensor_plane(&[i32::MAX], 8192, 16383, true), Err(RawError::Corrupt(_))));
assert!(matches!(sensor_plane(&[0, i32::MAX], 8192, 16383, true), Err(RawError::Corrupt(_))));
// The lowest i32 cannot overflow below: it is clamped in a quantized plane and corrupt in a lossless one.
assert_eq!(sensor_plane(&[i32::MIN], 8192, 16383, true).unwrap(), [0]);
assert!(matches!(sensor_plane(&[i32::MIN], 8192, 16383, false), Err(RawError::Corrupt(_))));
}
#[test]
@@ -1081,14 +1204,14 @@ mod tests {
#[test]
fn qp_escape_and_average_precede_the_quantization_table() {
// 23 zeros, the terminating 1, then the symbol in 8 bits.
let mut bits = Bits { src: &[0, 0, 1, 0x18], at: 0 };
let mut bits = Bits::new(&[0, 0, 1, 0x18]);
assert_eq!(bits.qp_rice(0).unwrap(), 24);
assert_eq!(bits.at, 32);
assert_eq!(bits.position(), 32);
// 22 zeros are an ordinary Rice prefix.
let mut bits = Bits { src: &[0, 0, 2, 0], at: 0 };
let mut bits = Bits::new(&[0, 0, 2, 0]);
assert_eq!(bits.qp_rice(0).unwrap(), 22);
assert_eq!(bits.at, 23);
assert!(Bits { src: &[0, 0, 0, 0x80, 0], at: 0 }.qp_rice(0).is_err());
assert_eq!(bits.position(), 23);
assert!(Bits::new(&[0, 0, 0, 0x80, 0]).qp_rice(0).is_err());
// The map's first symbol is a 32-bit literal (300 = QP 150 from prediction 0).
let map = qp_map(&[0, 0, 1, 0x2c, 0x94], 1, 4).unwrap();
assert_eq!(map.data, [150; 4]);
@@ -1187,11 +1310,17 @@ mod tests {
for end in 0..sample.len() {
assert!(decode(&sample[..end], &config).is_err());
}
// A corrupt QP map (nonzero alignment bits) is reported as such, not as a plane error.
let at = sample.windows(5).position(|bytes| bytes == [0, 0, 1, 0x2c, 0x94]).unwrap() + 4;
sample[at] ^= 1;
let error = decode(&sample, &config).unwrap_err().to_string();
assert!(error.contains("alignment") && !error.contains("plane"), "{error}");
}
#[test]
fn optional_cc0_samples_match_external_oracle() {
let Some(directory) = std::env::var_os("LIGHTCRAFT_CR3_CORPUS") else { return };
let mut checked = 0;
for (name, expected) in [
("cr3-canon-r100-raw.cr3", 0xcbe5299ab9c52630u64),
("cr3-canon-m50-raw.cr3", 0x62261f0ba81cfcd2u64),
@@ -1203,9 +1332,18 @@ mod tests {
("cr3-canon-rp-craw.cr3", 0xf48df06ca1d67382u64),
("cr3-canon-rp-raw-crop.cr3", 0x46808a9de3b3dcc7u64),
("cr3-canon-rp-craw-crop.cr3", 0x39a60c4efed4e1f4u64),
// EOS R5 Mark II (version 0x200 C-RAW and its lossless pair): pinned to this decoder's
// own output, see docs/cr3.md.
("cr3-canon-r5m2-raw.cr3", 0xff716fb30248496fu64),
("cr3-canon-r5m2-craw.cr3", 0xaf32b45357b534e5u64),
("cr3-canon-r5m2-raw-crop.cr3", 0x31cf630afe868958u64),
("cr3-canon-r5m2-craw-crop.cr3", 0x4721605b20339332u64),
] {
let path = std::path::Path::new(&directory).join(name);
let bytes = std::fs::read(path).unwrap();
let Ok(bytes) = std::fs::read(&path) else {
eprintln!("skipped {}: not in LIGHTCRAFT_CR3_CORPUS", path.display());
continue;
};
let container = craft_metadata::cr3::parse_cr3(&bytes).unwrap();
let mut candidates = container
.tracks
@@ -1224,6 +1362,285 @@ mod tests {
.flat_map(|value| value.to_le_bytes())
.fold(0xcbf29ce484222325u64, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3));
assert_eq!(hash, expected, "{name}");
checked += 1;
}
assert!(checked > 0, "LIGHTCRAFT_CR3_CORPUS holds none of the pinned files");
}
/// The bit reader before the 64-bit cache: a checked 5-byte window per read, bit by bit near the end.
struct ReferenceBits<'a> {
src: &'a [u8],
at: usize,
}
impl ReferenceBits<'_> {
fn peek32(&self) -> Option<u32> {
let bytes = self.src.get(self.at / 8..self.at / 8 + 5)?;
let word = u64::from_be_bytes([0, 0, 0, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4]]);
Some((word >> (8 - self.at % 8)) as u32)
}
fn take(&mut self, count: u32) -> Result<u32> {
if count > 24 {
return Err(corrupt("invalid Rice parameter"));
}
if count == 0 {
return Ok(0);
}
if let Some(window) = self.peek32() {
self.at += count as usize;
return Ok(window >> (32 - count));
}
let mut value = 0;
for _ in 0..count {
let byte = self.src.get(self.at / 8).ok_or_else(|| corrupt("truncated entropy stream"))?;
value = (value << 1) | u32::from((byte >> (7 - self.at % 8)) & 1);
self.at += 1;
}
Ok(value)
}
fn rice(&mut self, k: u32) -> Result<u32> {
if let Some(window) = self.peek32().filter(|&w| w != 0) {
let zeros = window.leading_zeros();
self.at += zeros as usize + 1;
return if k <= 20 { Ok((zeros << k) | self.take(k)?) } else { Err(corrupt("Rice parameter exceeds sample precision")) };
}
let mut zeros = 0;
while self.take(1)? == 0 {
zeros += 1;
if zeros > 41 {
return Err(corrupt("Rice escape prefix exceeds 41 bits"));
}
}
if zeros == 41 {
self.take(21)
} else if k <= 20 {
Ok((zeros << k) | self.take(k)?)
} else {
Err(corrupt("Rice parameter exceeds sample precision"))
}
}
fn qp_rice(&mut self, k: u32) -> Result<u32> {
let mut zeros = 0;
while self.take(1)? == 0 {
zeros += 1;
if zeros > QP_ESCAPE_ZEROS {
return Err(corrupt("QP Rice escape prefix exceeds 23 bits"));
}
}
if zeros == QP_ESCAPE_ZEROS {
self.take(8)
} else if k <= 20 {
Ok((zeros << k) | self.take(k)?)
} else {
Err(corrupt("QP Rice parameter exceeds precision"))
}
}
fn finish(&self) -> bool {
let bits = self.src.len() * 8;
bits.saturating_sub(self.at) <= 7 && self.src.last().is_none_or(|&last| u32::from(last) & ((1u32 << (bits - self.at)) - 1) == 0)
}
}
/// A xorshift generator: deterministic pseudo-random test input without a dependency.
fn xorshift(seed: &mut u64) -> u64 {
*seed ^= *seed << 13;
*seed ^= *seed >> 7;
*seed ^= *seed << 17;
*seed
}
#[test]
fn cached_bit_reader_matches_the_bitwise_reference() {
let mut seed = 0x2545_f491_4f6c_dd1d;
for round in 0..400 {
let length = (xorshift(&mut seed) % 48) as usize;
// Mostly zero and small bytes, so long unary prefixes and both escapes occur.
let stream: Vec<u8> = (0..length)
.map(|_| match xorshift(&mut seed) % 4 {
0 | 1 => 0,
2 => (xorshift(&mut seed) & 0x0f) as u8,
_ => xorshift(&mut seed) as u8,
})
.collect();
let ops: Vec<(u64, u32)> = (0..48).map(|_| (xorshift(&mut seed) % 3, (xorshift(&mut seed) % 26) as u32)).collect();
// Every truncation of the stream, so reads meet the end at every offset.
for end in 0..=stream.len() {
let src = &stream[..end];
let (mut cached, mut reference) = (Bits::new(src), ReferenceBits { src, at: 0 });
let mut complete = true;
for &(op, n) in &ops {
let (a, b) = match op {
0 => (cached.take(n), reference.take(n)),
1 => (cached.rice(n), reference.rice(n)),
_ => (cached.qp_rice(n % 22), reference.qp_rice(n % 22)),
};
assert_eq!(a.as_ref().ok(), b.as_ref().ok(), "round {round}, end {end}, op {op} {n}, stream {stream:02x?}");
assert_eq!(
a.as_ref().err().map(ToString::to_string),
b.as_ref().err().map(ToString::to_string),
"round {round}, end {end}, op {op} {n}"
);
if a.is_err() {
complete = false;
break;
}
assert_eq!(cached.position(), reference.at, "round {round}, end {end}, op {op} {n}");
}
if complete {
assert_eq!(cached.finish().is_ok(), reference.finish(), "round {round}, end {end}");
}
}
}
}
#[test]
fn rice_escape_is_an_error_when_cut_at_any_byte() {
// 41 zeros, the terminating 1 and the 21-bit literal 16400 (+8200): the whole stream is one escape.
let src = [0, 0, 0, 0, 0, 0x40, 0x80, 0x20];
assert_eq!(lossless_plane(&src, 1, 1).unwrap(), [8200]);
for end in 0..src.len() {
assert!(lossless_plane(&src[..end], 1, 1).is_err(), "prefix {end}");
}
// The same escape read from the cache, 9 bytes into a longer stream.
let mut long = vec![0xff; 9];
long.extend(src);
let mut bits = Bits::new(&long);
for _ in 0..72 {
assert_eq!(bits.rice(0).unwrap(), 0);
}
assert_eq!(bits.rice(0).unwrap(), 16400);
bits.finish().unwrap();
}
#[test]
fn dequantization_per_cell_matches_a_step_per_coefficient() {
let mut seed = 0x9e37_79b9_7f4a_7c15;
// Odd map heights as on the EOS R7, and band sizes that fit the map and that overrun it by one.
for (map_width, map_height) in [(1, 1), (2, 3), (3, 4), (5, 7)] {
let map = QpMap {
width: map_width,
height: map_height,
data: (0..map_width * map_height).map(|_| 131 + (xorshift(&mut seed) % 45) as i32).collect(),
};
for band in 0..10 {
for (width, height) in [
(1, 1),
(3, 2),
(2 * map_width - 1, map_height.div_ceil(2)),
(2 * map_width + 1, map_height),
(4 * map_width, map_height),
(4 * map_width + 1, map_height + 1),
] {
for quant in [
Quantization::Uniform(4 + 4 * band as u8),
Quantization::Adaptive { base: 0, gain: 0 },
Quantization::Adaptive { base: 3, gain: 0 },
Quantization::Adaptive { base: 1, gain: 8 },
Quantization::Adaptive { base: 0, gain: 16 },
] {
let data: Vec<i32> = (0..width * height).map(|_| (xorshift(&mut seed) % 2001) as i32 - 1000).collect();
let mut cells = data.clone();
let cells_result = dequantize(&mut cells, band, width, &quant, Some(&map));
let mut reference = data;
let reference_result = reference.iter_mut().enumerate().try_for_each(|(position, value)| -> Result<()> {
let step = match quant {
Quantization::Uniform(uniform) => quant_step(uniform, MAX_UNIFORM_QUANT)?,
Quantization::Adaptive { base, gain } => adaptive_step(&map, band, position % width, position / width, base, gain)?,
};
*value = value.checked_mul(step).ok_or_else(|| corrupt("dequantized coefficient overflow"))?;
Ok(())
});
let case = format!("map {map_width}x{map_height}, band {band}, {width}x{height}, {quant:?}");
assert_eq!(cells_result.is_ok(), reference_result.is_ok(), "{case}");
assert_eq!(
cells_result.as_ref().err().map(ToString::to_string),
reference_result.as_ref().err().map(ToString::to_string),
"{case}"
);
if cells_result.is_ok() {
assert_eq!(cells, reference, "{case}");
}
}
}
}
}
assert!(dequantize(&mut [1 << 30], 1, 1, &Quantization::Uniform(10), None).is_err());
assert!(dequantize(&mut [1], 4, 1, &Quantization::Adaptive { base: 0, gain: 0 }, None).is_err());
}
#[test]
fn interleave_places_every_plane_of_every_tile_like_a_per_sample_scatter() {
// A 2 x 2 tile grid whose right column is narrower, as at the right edge of a real image.
let (width, height) = (12, 8);
let tile = |x, y, width, height| Tile { x, y, width, height, planes: Vec::new(), qp: None };
let tiles = [tile(0, 0, 8, 4), tile(8, 0, 4, 4), tile(0, 4, 8, 4), tile(8, 4, 4, 4)];
let planes: Vec<Vec<u16>> = tiles
.iter()
.enumerate()
.flat_map(|(t, tile)| {
(0..4).map(move |p| (0..tile.width * tile.height / 4).map(|i| ((t * 4 + p) * 100 + i) as u16).collect::<Vec<u16>>())
})
.collect();
let mut expected = vec![0u16; width * height];
for (t, tile) in tiles.iter().enumerate() {
for p in 0..4 {
for (i, &value) in planes[t * 4 + p].iter().enumerate() {
let (x, y) = (tile.x + 2 * (i % (tile.width / 2)) + p % 2, tile.y + 2 * (i / (tile.width / 2)) + p / 2);
expected[y * width + x] = value;
}
}
}
let mut output = vec![0u16; width * height];
interleave(&mut output, width, &tiles, &planes).unwrap();
assert_eq!(output, expected);
// A plane shorter than its tile is an error, not a partial image.
let mut short = planes.clone();
short[5].pop();
assert!(interleave(&mut output, width, &tiles, &short).is_err());
assert!(interleave(&mut output, width, &tiles, &planes[..12]).is_err());
}
#[test]
fn lossless_tile_grid_decodes_each_plane_to_its_place() {
// Four 4 x 4 tiles in a 2 x 2 grid (horizontal and vertical tile boundaries). Each 2 x 2 plane
// is flat at value 1, 2 or 3, chosen per tile and plane. Bitstreams per value: row 0 has a
// zero-run flag 0, the Rice-coded first sample and a 0 symbol; row 1 is one run to the row end.
let stream = |value: usize| -> &'static [u8] {
match value {
1 => &[0x1e], // 0, 001 (k 0), 1 (k 0), 1, 1
2 => &[0x06, 0xc0], // 0, 00001 (k 0), 10 (k 1), 1, 1
_ => &[0x01, 0x98], // 0, 0000001 (k 0), 100 (k 2), 1, 1
}
};
let value = |t: usize, p: usize| 1 + (t + p) % 3;
let mut config = coding();
config.width = 8;
config.height = 8;
config.header_size = 432;
let mut sample = Vec::new();
let mut data = Vec::new();
for t in 0..4 {
let sizes: Vec<u32> = (0..4).map(|p| stream(value(t, p)).len() as u32).collect();
marker(&mut sample, 0xff01, sizes.iter().sum(), (t as u32) << 16);
for (p, &size) in sizes.iter().enumerate() {
marker(&mut sample, 0xff02, size, ((p as u32) << 28) | 0x0800_0000);
marker(&mut sample, 0xff03, size, 0x0020_0000);
data.extend_from_slice(stream(value(t, p)));
}
}
assert_eq!(sample.len(), 432);
sample.extend(data);
validate(&sample, &config).unwrap();
let expected: Vec<u16> =
(0..64).map(|i| (i % 8, i / 8)).map(|(x, y)| 8192 + value((y / 4) * 2 + x / 4, (y % 2) * 2 + x % 2) as u16).collect();
assert_eq!(decode(&sample, &config).unwrap(), expected);
// A corrupt plane in the last tile still fails the whole image.
let mut broken = sample.clone();
*broken.last_mut().unwrap() ^= 1;
assert!(decode(&broken, &config).is_err());
}
}
+236 -46
View File
@@ -6,6 +6,7 @@
//! neighbouring low/high coefficients; only outer image edges are symmetric.
use crate::{RawError, Result};
use rayon::prelude::*;
#[derive(Clone, Debug)]
pub(crate) struct Band {
@@ -18,10 +19,6 @@ fn corrupt() -> RawError {
RawError::Corrupt("invalid CRX wavelet subband geometry".into())
}
fn checked_sample(v: i64) -> Result<i32> {
i32::try_from(v).map_err(|_| RawError::Corrupt("CRX wavelet coefficient overflow".into()))
}
#[derive(Clone, Copy, Debug, Default)]
pub(crate) struct TileEdges {
pub left: bool,
@@ -34,40 +31,112 @@ fn sizes(length: usize, before: bool, after: bool) -> (usize, usize) {
(length.div_ceil(2) + usize::from(after && length.is_multiple_of(2)), length / 2 + usize::from(before) + usize::from(after))
}
/// Whether a lifted value still fits the i32 coefficient range.
fn fits(value: i64) -> bool {
i32::try_from(value).is_ok()
}
fn overflow() -> RawError {
RawError::Corrupt("CRX wavelet coefficient overflow".into())
}
fn inverse_line(low: &[i32], high: &[i32], out: &mut [i32], before: bool, after: bool) -> Result<()> {
let (low_count, high_count) = sizes(out.len(), before, after);
if out.is_empty() || low.len() != low_count || high.len() != high_count {
return Err(corrupt());
}
if high.is_empty() {
let Some(&last_high) = high.last() else {
let value = low.first().copied().ok_or_else(corrupt)?;
*out.first_mut().ok_or_else(corrupt)? = value;
return Ok(());
}
};
// The high-pass storage starts at -1 when there is a preceding tile.
// Only absent external neighbours are reflected at the image boundary.
let high_at = |index: usize| -> Result<i32> { high.get(index.min(high.len().saturating_sub(1))).copied().ok_or_else(corrupt) };
let even = |i: usize| -> Result<i32> {
let left = high_at((i + usize::from(before)).saturating_sub(1))?;
let right = high_at(i + usize::from(before))?;
let update = (i64::from(left) + i64::from(right) + 2).div_euclid(4);
checked_sample(i64::from(*low.get(i).ok_or_else(corrupt)?) - update)
let shift = usize::from(before);
let high_at = |index: usize| i64::from(high.get(index).copied().unwrap_or(last_high));
let even = |i: usize, value: i32| i64::from(value) - ((high_at((i + shift).saturating_sub(1)) + high_at(i + shift) + 2) >> 2);
let mut in_range = true;
for (i, (slot, &value)) in out.iter_mut().step_by(2).zip(low).enumerate() {
let lifted = even(i, value);
in_range &= fits(lifted);
*slot = lifted as i32;
}
// With a following tile, the last odd sample's right neighbour is the even sample past the end.
let length = out.len();
let past_end = if after && length.is_multiple_of(2) { low.get(length / 2).map(|&value| even(length / 2, value)) } else { None };
in_range &= past_end.is_none_or(fits);
for i in 0..length / 2 {
let left = out.get(2 * i).copied().map_or(0, i64::from);
let right = out.get(2 * i + 2).copied().map(i64::from).or(past_end).unwrap_or(left);
let lifted = high_at(i + shift) + ((left + right) >> 1);
in_range &= fits(lifted);
if let Some(slot) = out.get_mut(2 * i + 1) {
*slot = lifted as i32;
}
}
if in_range { Ok(()) } else { Err(overflow()) }
}
/// `out = low - floor((left + right + 2) / 4)` per column: one even row of the vertical lifting.
fn even_row(out: &mut [i32], low: &[i32], left: &[i32], right: &[i32]) -> bool {
let mut in_range = true;
for (((slot, &l), &a), &b) in out.iter_mut().zip(low).zip(left).zip(right) {
let lifted = i64::from(l) - ((i64::from(a) + i64::from(b) + 2) >> 2);
in_range &= fits(lifted);
*slot = lifted as i32;
}
in_range
}
/// `out = high + floor((above + below) / 2)` per column: one odd row of the vertical lifting.
fn odd_row(out: &mut [i32], high: &[i32], above: &[i32], below: &[i32]) -> bool {
let mut in_range = true;
for (((slot, &h), &a), &b) in out.iter_mut().zip(high).zip(above).zip(below) {
let lifted = i64::from(h) + ((i64::from(a) + i64::from(b)) >> 1);
in_range &= fits(lifted);
*slot = lifted as i32;
}
in_range
}
/// [`inverse_line`] applied to every column of `out`, on whole rows instead of strided columns.
///
/// `rows` holds the low-pass rows, then the high-pass rows, each `width` wide. Runs in parallel over
/// pairs of output rows: each pair lifts its even row, recomputes the next even row (the following
/// pair's, or the one a following tile supplies past the end) and lifts its odd row from both.
fn inverse_columns(rows: &[i32], out: &mut [i32], width: usize, before: bool, after: bool) -> Result<()> {
let height = out.len().checked_div(width).filter(|&h| h > 0 && h * width == out.len()).ok_or_else(corrupt)?;
let (low_count, high_count) = sizes(height, before, after);
if rows.len() != (low_count + high_count) * width {
return Err(corrupt());
}
let (low, high) = (rows.get(..low_count * width).ok_or_else(corrupt)?, rows.get(low_count * width..).ok_or_else(corrupt)?);
let low_row = |i: usize| low.get(i * width..(i + 1) * width).ok_or_else(corrupt);
let Some(last_high) = high_count.checked_sub(1) else {
let first = out.get_mut(..width).ok_or_else(corrupt)?;
first.copy_from_slice(low_row(0)?);
return Ok(());
};
for i in 0..out.len().div_ceil(2) {
*out.get_mut(i * 2).ok_or_else(corrupt)? = even(i)?;
}
for i in 0..out.len() / 2 {
let left = out.get(i * 2).copied().ok_or_else(corrupt)?;
let right = match out.get(i * 2 + 2).copied() {
Some(value) => value,
None if after => even(i + 1)?,
None => left,
};
let prediction = (i64::from(left) + i64::from(right)).div_euclid(2);
let value = high_at(i + usize::from(before))?;
*out.get_mut(i * 2 + 1).ok_or_else(corrupt)? = checked_sample(i64::from(value) + prediction)?;
}
Ok(())
let high_row = |i: usize| {
let i = i.min(last_high);
high.get(i * width..(i + 1) * width).ok_or_else(corrupt)
};
let shift = usize::from(before);
out.par_chunks_mut(2 * width).enumerate().try_for_each_init(Vec::new, |next, (i, pair)| -> Result<()> {
let (even, odd) = pair.split_at_mut(width);
let mut in_range = even_row(even, low_row(i)?, high_row((i + shift).saturating_sub(1))?, high_row(i + shift)?);
if !odd.is_empty() {
let below: &[i32] = if 2 * i + 2 < height || after {
next.resize(width, 0);
in_range &= even_row(next, low_row(i + 1)?, high_row(i + shift)?, high_row(i + shift + 1)?);
next.as_slice()
} else {
&*even
};
in_range &= odd_row(odd, high_row(i + shift)?, even, below);
}
if in_range { Ok(()) } else { Err(overflow()) }
})
}
fn valid_band(band: &Band, width: usize, height: usize) -> bool {
@@ -97,32 +166,18 @@ pub(crate) fn inverse_level(ll: &Band, hl: &Band, lh: &Band, hh: &Band, width: u
let mut temporary = zeroed(intermediate_count)?;
// CRX analysis is vertical then horizontal. Integer rounding means that
// synthesis must reverse that order, including for quantized subbands.
for y in 0..lhgt + hhgt {
temporary.par_chunks_mut(width).enumerate().try_for_each(|(y, row)| {
let (lo, hi, by) = if y < lhgt { (ll, hl, y) } else { (lh, hh, y - lhgt) };
inverse_line(
lo.data.get(by * lw..(by + 1) * lw).ok_or_else(corrupt)?,
hi.data.get(by * hw..(by + 1) * hw).ok_or_else(corrupt)?,
temporary.get_mut(y * width..(y + 1) * width).ok_or_else(corrupt)?,
row,
edges.left,
edges.right,
)?;
}
)
})?;
let mut data = zeroed(count)?;
let mut low = zeroed(lhgt)?;
let mut high = zeroed(hhgt)?;
let mut column = zeroed(height)?;
for x in 0..width {
for (y, value) in low.iter_mut().enumerate() {
*value = temporary.get(y * width + x).copied().ok_or_else(corrupt)?;
}
for (y, value) in high.iter_mut().enumerate() {
*value = temporary.get((y + lhgt) * width + x).copied().ok_or_else(corrupt)?;
}
inverse_line(&low, &high, &mut column, edges.top, edges.bottom)?;
for (y, &value) in column.iter().enumerate() {
*data.get_mut(y * width + x).ok_or_else(corrupt)? = value;
}
}
inverse_columns(&temporary, &mut data, width, edges.top, edges.bottom)?;
Ok(Band { width, height, data })
}
@@ -335,4 +390,139 @@ mod tests {
assert!(reconstruct(&bands, 20, 9, 3, TileEdges::default()).is_err());
}
}
/// The lifting before the row-wise rewrite: a checked closure per sample, as a reference.
fn reference_line(low: &[i32], high: &[i32], out: &mut [i32], before: bool, after: bool) -> Result<()> {
let (low_count, high_count) = sizes(out.len(), before, after);
if out.is_empty() || low.len() != low_count || high.len() != high_count {
return Err(corrupt());
}
if high.is_empty() {
out[0] = low[0];
return Ok(());
}
let checked = |v: i64| i32::try_from(v).map_err(|_| overflow());
let high_at = |index: usize| high[index.min(high.len() - 1)];
let even = |i: usize| -> Result<i32> {
let left = high_at((i + usize::from(before)).saturating_sub(1));
let right = high_at(i + usize::from(before));
checked(i64::from(low[i]) - (i64::from(left) + i64::from(right) + 2).div_euclid(4))
};
for i in 0..out.len().div_ceil(2) {
out[i * 2] = even(i)?;
}
for i in 0..out.len() / 2 {
let left = out[i * 2];
let right = match out.get(i * 2 + 2).copied() {
Some(value) => value,
None if after => even(i + 1)?,
None => left,
};
out[i * 2 + 1] = checked(i64::from(high_at(i + usize::from(before))) + (i64::from(left) + i64::from(right)).div_euclid(2))?;
}
Ok(())
}
/// One level before the row-wise rewrite: horizontal lines, then strided columns.
fn reference_level(bands: &[Band; 4], width: usize, height: usize, edges: TileEdges) -> Result<Vec<i32>> {
let [ll, hl, lh, hh] = bands;
let (lhgt, hhgt) = sizes(height, edges.top, edges.bottom);
let mut temporary = vec![0; width * (lhgt + hhgt)];
for y in 0..lhgt + hhgt {
let (lo, hi, by) = if y < lhgt { (ll, hl, y) } else { (lh, hh, y - lhgt) };
reference_line(
&lo.data[by * lo.width..(by + 1) * lo.width],
&hi.data[by * hi.width..(by + 1) * hi.width],
&mut temporary[y * width..(y + 1) * width],
edges.left,
edges.right,
)?;
}
let mut data = vec![0; width * height];
let mut column = vec![0; height];
for x in 0..width {
let low: Vec<i32> = (0..lhgt).map(|y| temporary[y * width + x]).collect();
let high: Vec<i32> = (lhgt..lhgt + hhgt).map(|y| temporary[y * width + x]).collect();
reference_line(&low, &high, &mut column, edges.top, edges.bottom)?;
for (y, &value) in column.iter().enumerate() {
data[y * width + x] = value;
}
}
Ok(data)
}
fn xorshift(seed: &mut u64) -> u64 {
*seed ^= *seed << 13;
*seed ^= *seed >> 7;
*seed ^= *seed << 17;
*seed
}
/// Small coefficients, or (with `extreme`) values near the i32 limits so lifting can overflow.
fn coefficients(seed: &mut u64, count: usize, extreme: bool) -> Vec<i32> {
(0..count)
.map(|_| {
let r = xorshift(seed);
if extreme && r.is_multiple_of(5) {
if r & 32 == 0 { i32::MAX - (r >> 40) as i32 % 4 } else { i32::MIN + (r >> 40) as i32 % 4 }
} else {
(r % 2001) as i32 - 1000
}
})
.collect()
}
const EDGE_CASES: [(bool, bool); 4] = [(false, false), (true, false), (false, true), (true, true)];
#[test]
fn line_lifting_matches_the_reference_for_every_length_and_edge() {
let mut seed = 0x0123_4567_89ab_cdef;
for length in 1..=40 {
for (before, after) in EDGE_CASES {
let (low_count, high_count) = sizes(length, before, after);
for extreme in [false, true] {
let low = coefficients(&mut seed, low_count, extreme);
let high = coefficients(&mut seed, high_count, extreme);
let (mut fast, mut reference) = (vec![0; length], vec![0; length]);
let fast_result = inverse_line(&low, &high, &mut fast, before, after);
let reference_result = reference_line(&low, &high, &mut reference, before, after);
let case = format!("length {length}, before {before}, after {after}, extreme {extreme}");
assert_eq!(fast_result.is_ok(), reference_result.is_ok(), "{case}");
assert_eq!(
fast_result.as_ref().err().map(ToString::to_string),
reference_result.as_ref().err().map(ToString::to_string),
"{case}"
);
if fast_result.is_ok() {
assert_eq!(fast, reference, "{case}");
}
}
}
}
}
#[test]
fn row_wise_level_matches_column_lifting_for_every_size_and_edge() {
let mut seed = 0xfeed_beef_dead_cafe;
for width in 1..=17 {
for height in 1..=17 {
for (left, right) in EDGE_CASES {
for (top, bottom) in EDGE_CASES {
let edges = TileEdges { left, right, top, bottom };
let (lw, hw) = sizes(width, left, right);
let (lhgt, hhgt) = sizes(height, top, bottom);
// Overflow cases on the smaller sizes only; they fail early and are cheap.
let extreme = width * height < 40 && xorshift(&mut seed).is_multiple_of(2);
let mut band = |w, h| Band { width: w, height: h, data: coefficients(&mut seed, w * h, extreme) };
let bands = [band(lw, lhgt), band(hw, lhgt), band(lw, hhgt), band(hw, hhgt)];
let fast = inverse_level(&bands[0], &bands[1], &bands[2], &bands[3], width, height, edges);
let reference = reference_level(&bands, width, height, edges);
let case = format!("{width}x{height}, {edges:?}, extreme {extreme}");
assert_eq!(fast.as_ref().ok().map(|band| &band.data), reference.as_ref().ok(), "{case}");
assert_eq!(fast.as_ref().err().map(ToString::to_string), reference.as_ref().err().map(ToString::to_string), "{case}");
}
}
}
}
}
}
+30 -14
View File
@@ -299,17 +299,11 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
deinterleave(&mut data, w, h, cpp, factor(t::ROW_INTERLEAVE_FACTOR), factor(t::COLUMN_INTERLEAVE_FACTOR))?;
}
// linearization table (integer data only)
let mut linearized = false;
// linearization table (integer data only) and opcode lists. OpcodeList1 runs on the stored
// codes first (DNG 1.7 chapter 7); see [`linearization_and_opcodes`].
let (linearized, linearization_table, opcodes) = linearization_and_opcodes(pixels, raw, &mut data);
// the largest value the table can give a sample: the white level when the file states none
let mut table_max = None;
if let (RawData::U16(v), Some(table)) = (&mut data, pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty())) {
let table: Vec<u16> = table.iter().map(|&x| x.min(65535) as u16).collect();
let last = table.len() - 1;
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
linearized = true;
table_max = table.iter().copied().max().filter(|&m| m > 0);
}
let table_max = linearization_table.as_deref().and_then(|t| t.iter().copied().max().filter(|&m| m > 0));
// a reduced copy's own levels, where it states them
let levels = |tag| if pixels.contains(tag) { pixels } else { raw };
@@ -442,10 +436,6 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
None
};
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
let opcodes =
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
let color = color_data(ifd0, raw, tiff.order);
let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2))
&& color.profile.gain_table_map.is_none();
@@ -465,6 +455,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
color,
wb_multipliers: None,
linearized,
linearization_table,
pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()),
opcodes,
camera: Camera::from_ifd0(ifd0),
@@ -477,6 +468,31 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
Ok(img)
}
/// Parse opcode lists and, for integer data, the LinearizationTable.
///
/// OpcodeList1 runs on the stored codes, then the table maps them to sensor values (DNG 1.7
/// chapter 7). Applying opcodes is the develop pipeline's work, so a non-empty list 1 leaves the
/// table unapplied (`linearized` is false) and available on the image.
fn linearization_and_opcodes(pixels: &Ifd, raw: &Ifd, data: &mut RawData) -> (bool, Option<Vec<u16>>, crate::OpcodeLists) {
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
let opcodes =
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
let linearization_table = if matches!(data, RawData::U16(_)) {
pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty()).map(|table| table.iter().map(|&x| x.min(65535) as u16).collect::<Vec<u16>>())
} else {
None
};
let mut linearized = false;
if opcodes.list1.is_empty()
&& let (RawData::U16(v), Some(table)) = (data, linearization_table.as_deref())
{
let last = table.len() - 1;
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
linearized = true;
}
(linearized, linearization_table, opcodes)
}
/// CinemaDNG `TimeCodes` (CinemaDNG Image Data Format Specification 1.0, tag 51043).
const TIME_CODES: u16 = 51043;
/// CinemaDNG `FrameRate` (tag 51044).
+10
View File
@@ -165,6 +165,14 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result<Vec<u8>> {
} else {
ifd.set(t::WHITE_LEVEL, Value::Long(white));
}
// A table still to be applied (OpcodeList1 has to run on the stored codes first) is written
// with the samples. Once the samples have been mapped, writing the tag would apply it twice
// on the next read of a file without list 1.
if !raw.linearized
&& let Some(table) = raw.linearization_table.as_deref().filter(|t| !t.is_empty())
{
ifd.set(t::LINEARIZATION_TABLE, Value::Short(table.to_vec()));
}
let a = raw.active_area;
ifd.set(t::ACTIVE_AREA, Value::Long(vec![a.y as u32, a.x as u32, (a.y + a.height) as u32, (a.x + a.width) as u32]));
ifd.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![raw.crop.x as u32, raw.crop.y as u32]));
@@ -457,6 +465,7 @@ mod tests {
color: crate::ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Default::default(),
@@ -514,6 +523,7 @@ mod tests {
color: crate::ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Default::default(),
+1
View File
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
+126 -13
View File
@@ -4,16 +4,19 @@
use crate::{RawError, Result};
/// A canonical-by-table Huffman decoder with a 12-bit lookup.
/// A canonical-by-table Huffman decoder with a 12-bit lookup that also holds the difference when code and
/// additional bits fit in 12 bits.
pub(crate) struct Huffman {
/// Indexed by the next 12 bits: (symbol, length); length 0 = invalid.
lut: Vec<(u8, u8)>,
/// Indexed by the next 12 bits. Per entry: bits 0-7 the symbol, bits 8-11 the code length (0 = invalid), bits
/// 12-15 the length of code plus additional bits when both fit in 12 bits (0 = they do not), bits 16-31 the
/// signed difference in that case.
lut: Box<[u32; 4096]>,
}
impl Huffman {
/// From `(left-aligned 12-bit code, length)` per symbol.
pub fn new(codes: &[(u16, u8)]) -> Result<Huffman> {
let mut lut = vec![(0u8, 0u8); 4096];
let mut lut = Box::new([0u32; 4096]);
for (sym, &(code, len)) in codes.iter().enumerate() {
if len == 0 {
continue;
@@ -21,10 +24,19 @@ impl Huffman {
if len > 12 || sym > 16 {
return Err(RawError::Unsupported(format!("PEF Huffman code of length {len}")));
}
let (len, s) = (len as u32, sym as u32);
let first = (code >> (12 - len) << (12 - len)) as usize;
for e in &mut lut[first..first + (1 << (12 - len))] {
if e.1 == 0 {
*e = (sym as u8, len);
for (i, e) in lut[first..first + (1 << (12 - len))].iter_mut().enumerate() {
if (*e >> 8) & 15 == 0 {
let mut entry = (len << 8) | s;
if s == 0 {
entry |= len << 12;
} else if len + s <= 12 {
let v = (((first + i) >> (12 - len - s)) & ((1 << s) - 1)) as i32;
let d = if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v };
entry |= ((len + s) << 12) | ((d as i16 as u16 as u32) << 16);
}
*e = entry;
}
}
}
@@ -85,21 +97,122 @@ impl<'a> Bits<'a> {
}
}
/// Decode one difference: Huffman category then T.81 additional bits.
/// Decode one difference: Huffman category then T.81 additional bits, in one lookup when both fit in 12 bits.
#[inline]
pub(crate) fn diff(bits: &mut Bits, h: &Huffman) -> Option<i32> {
let (sym, len) = h.lut[bits.peek(12) as usize];
let e = h.lut[(bits.peek(12) & 0xfff) as usize];
let len = (e >> 8) & 15;
if len == 0 {
return None;
}
bits.skip(len as u32);
let s = sym as u32;
if s == 0 {
return Some(0);
let total = (e >> 12) & 15;
if total > 0 {
bits.skip(total);
return Some((e as i32) >> 16);
}
bits.skip(len);
let s = e & 0xff;
if s > 16 {
return None;
}
let v = bits.get(s) as i32;
Some(if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v })
}
#[cfg(test)]
mod tests {
use super::*;
/// Canonical left-aligned 12-bit codes for symbols 0..lens.len() (shorter codes first, ties by symbol).
fn canonical(lens: &[u8]) -> Vec<(u16, u8)> {
let mut order: Vec<usize> = (0..lens.len()).collect();
order.sort_by_key(|&s| (lens[s], s));
let mut codes = vec![(0u16, 0u8); lens.len()];
let (mut code, mut prev) = (0u32, 0u8);
for s in order {
code <<= lens[s] - prev;
prev = lens[s];
codes[s] = ((code << (12 - lens[s])) as u16, lens[s]);
code += 1;
}
codes
}
fn bit(src: &[u8], i: usize) -> u32 {
src.get(i / 8).map_or(0, |b| ((b >> (7 - i % 8)) & 1) as u32)
}
/// Reference: match the code bit by bit, then read the category's additional bits (T.81 F.1.2.1).
fn reference(src: &[u8], pos: &mut usize, codes: &[(u16, u8)]) -> Option<i32> {
let window: u32 = (0..12).fold(0, |a, k| (a << 1) | bit(src, *pos + k));
let (sym, len) = codes
.iter()
.enumerate()
.find_map(|(s, &(c, l))| (l > 0 && (c as u32 >> (12 - l)) == window >> (12 - l)).then_some((s as u32, l as usize)))?;
*pos += len;
if sym == 0 {
return Some(0);
}
let v = (0..sym as usize).fold(0u32, |a, k| (a << 1) | bit(src, *pos + k)) as i32;
*pos += sym as usize;
Some(if v < 1 << (sym - 1) { v - (1 << sym) + 1 } else { v })
}
/// `diff` against the reference on random streams (invalid codes are stepped over) and on every (code, extra
/// bits) pair of every symbol.
fn check_table(lens: &[u8]) {
let codes = canonical(lens);
let h = Huffman::new(&codes).unwrap();
for (sym, &(code, len)) in codes.iter().enumerate() {
for v in 0..1u64 << sym {
let (len, n) = (len as u32, sym as u32);
let word = (((code >> (12 - len)) as u64) << n | v) << (64 - len - n);
let src = word.to_be_bytes();
let mut pos = 0;
let want = reference(&src, &mut pos, &codes);
let mut bits = Bits::new(&src);
assert_eq!(diff(&mut bits, &h), want, "symbol {sym} bits {v:#b}");
assert_eq!(bits.consumed_bits(), pos);
}
}
let mut s = 0x1234_5678_9abc_def0u64;
let (mut invalid, mut long) = (0, 0);
for _ in 0..40 {
let src: Vec<u8> = (0..400)
.map(|_| {
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
(s >> 40) as u8
})
.collect();
let mut bits = Bits::new(&src);
let mut pos = 0;
while pos < src.len() * 8 - 64 {
let before = pos;
let want = reference(&src, &mut pos, &codes);
assert_eq!(diff(&mut bits, &h), want);
if want.is_none() {
// step over the invalid code
invalid += 1;
pos += 1;
bits.skip(1);
} else {
long += (pos - before > 12) as u32;
}
assert_eq!(bits.consumed_bits(), pos);
}
}
assert!(long > 0, "{invalid} invalid codes, no symbol longer than the lookup");
}
#[test]
fn diff_matches_bitwise_reference_for_every_category() {
// categories 0..=16, code lengths up to 12 so that short codes combine with their bits and long ones do not
check_table(&[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12]);
}
#[test]
fn diff_matches_reference_when_code_and_bits_fill_the_lookup_exactly() {
// category 8 with a 4-bit code and category 7 with a 5-bit code: len + category == 12
check_table(&[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12]);
}
}
+1
View File
@@ -115,6 +115,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+32 -5
View File
@@ -10,11 +10,13 @@
//!
//! # Where this crate stops
//!
//! The samples come out as the file stores them (after a DNG `LinearizationTable`): black is not subtracted,
//! nothing is scaled, cropped, white balanced or demosaiced, opcode lists and profile tables are parsed but not
//! applied, and no colour transform is computed from the matrices. Normalisation, demosaicing, highlight
//! handling, the DNG colour model's arithmetic, lens corrections, tone and everything after belong to the
//! develop pipeline, which reads the public fields of the types here.
//! The samples come out as the file stores them (after a DNG `LinearizationTable`, except when the file
//! also has an `OpcodeList1`: that list runs on the stored codes first, and applying it is the develop
//! pipeline's work, so the table is left unapplied and carried on [`RawImage::linearization_table`]).
//! Black is not subtracted, nothing is scaled, cropped, white balanced or demosaiced, opcode lists and
//! profile tables are parsed but not applied, and no colour transform is computed from the matrices.
//! Normalisation, demosaicing, highlight handling, the DNG colour model's arithmetic, lens corrections,
//! tone and everything after belong to the develop pipeline, which reads the public fields of the types here.
//!
//! # Formats
//!
@@ -645,6 +647,8 @@ pub fn orientation(bytes: &[u8]) -> Option<u16> {
}
// A damaged/unsupported pixel IFD must not hide usable orientation metadata from preview
// fallback. CR3 keeps IFD0 in its CMT1 box; TIFF-based raws keep it at the file start.
// The `cr3` module exists only with the `cr3` feature: this gate is required so
// `cargo check -p craft-raw --no-default-features --features dng` builds (E0433 otherwise).
#[cfg(feature = "cr3")]
if probe(bytes) == Some(RawFormat::Cr3) {
return cr3::orientation(bytes);
@@ -1041,6 +1045,12 @@ pub struct RawImage {
pub wb_multipliers: Option<[f32; 3]>,
/// Whether a `LinearizationTable` was applied to `data`.
pub linearized: bool,
/// The file's `LinearizationTable`, when present. `data` has already been mapped through it when
/// [`Self::linearized`] is true. When `linearized` is false and this is `Some`, the table was left
/// unapplied so `OpcodeList1` can run on the stored samples first (DNG chapter 7); call
/// [`Self::apply_linearization`] after that list.
#[serde(default)]
pub linearization_table: Option<Vec<u16>>,
/// Height of a sensor pixel over its width (1.0 for square pixels). The D1X and D1H write 2.0: their frame is
/// 4028 x 1324 and the picture is 3:2 only once the rows are doubled. A developer stretches the rows by
/// this factor.
@@ -1119,6 +1129,23 @@ impl RawImage {
self.check(true)
}
/// Map integer `data` through [`Self::linearization_table`] when the table has not yet been applied.
/// No-op when [`Self::linearized`] is already true, the table is missing, or the samples are not
/// integers. Call this after `OpcodeList1`, which DNG applies to the stored codes (chapter 7).
pub fn apply_linearization(&mut self) {
if self.linearized {
return;
}
let Some(table) = self.linearization_table.as_deref().filter(|t| !t.is_empty()) else {
return;
};
if let RawData::U16(v) = &mut self.data {
let last = table.len() - 1;
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
self.linearized = true;
}
}
fn check(&self, samples: bool) -> Result<()> {
if self.width == 0 || self.height == 0 || !(1..=4).contains(&self.cpp) {
return Err(RawError::Corrupt("bad dimensions".into()));
+129 -39
View File
@@ -32,15 +32,17 @@ fn err(s: &str) -> RawError {
/// Huffman decoding table (T.81 Annex C "BITS"/"HUFFVAL").
#[derive(Clone, Debug)]
pub struct Huffman {
/// For peeks of `LOOKUP` bits: (code length, symbol); length 0 = slow path.
fast: Vec<(u8, u8)>,
/// Indexed by the next `LOOKUP` bits; length 0 = slow path. Per entry: bits 0-7 the symbol, bits 8-11 the code
/// length, bits 12-15 the length of code plus extra bits when both fit in the lookup (0 = they do not), and
/// bits 16-31 the signed difference in that case.
fast: Vec<u32>,
maxcode: [i32; 18],
valptr: [i32; 17],
mincode: [i32; 17],
values: Vec<u8>,
}
const LOOKUP: u32 = 10;
const LOOKUP: u32 = 12;
impl Huffman {
/// Build from the 16 code-length counts and the symbol list.
@@ -52,7 +54,7 @@ impl Huffman {
let mut maxcode = [-1i32; 18];
let mut valptr = [0i32; 17];
let mut mincode = [0i32; 17];
let mut fast = vec![(0u8, 0u8); 1 << LOOKUP];
let mut fast = vec![0u32; 1 << LOOKUP];
let mut code = 0i32;
let mut k = 0usize;
for len in 1..=16usize {
@@ -63,9 +65,10 @@ impl Huffman {
if len as u32 <= LOOKUP {
let shift = LOOKUP - len as u32;
let base = (code as usize) << shift;
let ssss = values[k] as u32;
for i in 0..(1usize << shift) {
if let Some(slot) = fast.get_mut(base + i) {
*slot = (len as u8, values[k]);
*slot = fast_entry(len as u32, ssss, base + i);
}
}
}
@@ -84,12 +87,41 @@ impl Huffman {
#[inline]
pub fn decode<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
let peek = br.peek(LOOKUP);
let (len, sym) = self.fast[peek as usize];
let e = self.fast[br.peek(LOOKUP) as usize];
let len = (e >> 8) & 15;
if len > 0 {
br.consume(len as u32);
return Ok(sym);
br.consume(len);
return Ok(e as u8);
}
self.decode_slow(br)
}
/// One symbol and its T.81 difference in a single lookup when code and extra bits fit in `LOOKUP` bits; the same
/// as [`Huffman::decode`] followed by [`diff_value`]. `strict`: a category above 16 is an error (subsampled
/// scans) instead of reading 16 bits.
#[inline]
pub fn decode_diff<B: Bits>(&self, br: &mut B, strict: bool) -> Result<i32, RawError> {
let e = self.fast[br.peek(LOOKUP) as usize];
let len = (e >> 8) & 15;
let total = (e >> 12) & 15;
if total > 0 {
br.consume(total);
return Ok((e as i32) >> 16);
}
let ssss = if len > 0 {
br.consume(len);
e as u8
} else {
self.decode_slow(br)?
};
if strict && ssss > 16 {
return Err(err("invalid difference category"));
}
Ok(diff_value(br, ssss))
}
#[cold]
fn decode_slow<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
let mut code = br.peek(LOOKUP + 1) as i32;
let mut len = LOOKUP as usize + 1;
while len <= 16 && code > self.maxcode[len] {
@@ -105,6 +137,21 @@ impl Huffman {
}
}
/// Lookup entry for a code of `len` bits (`len <= LOOKUP`) naming category `ssss`, at window position `index`
/// (the code is the top `len` bits of the `LOOKUP`-bit window). See [`Huffman::fast`].
fn fast_entry(len: u32, ssss: u32, index: usize) -> u32 {
let base = (len << 8) | ssss;
if ssss == 0 {
return base | (len << 12);
}
if ssss > 11 || len + ssss > LOOKUP {
return base;
}
let v = ((index >> (LOOKUP - len - ssss)) & ((1 << ssss) - 1)) as i32;
let d = if v < (1 << (ssss - 1)) { v - (1 << ssss) + 1 } else { v };
base | ((len + ssss) << 12) | ((d as i16 as u16 as u32) << 16)
}
/// A source of entropy-coded bits, first bit most significant.
pub trait Bits {
/// The next `n` (1..=32) bits without consuming them; zeros past the end.
@@ -614,11 +661,7 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
Prediction::Sequential if mx == 0 && dx == 0 && dy == 0 => row_first[c],
Prediction::Sequential => last[c],
};
let ssss = tables[c].decode(&mut br)?;
if ssss > 16 {
return Err(err("invalid difference category"));
}
let value = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16;
let value = ((pred + tables[c].decode_diff(&mut br, true)?) & 0xffff) as u16;
plane[i] = value;
last[c] = i32::from(value);
if mx == 0 && dx == 0 && dy == 0 {
@@ -642,6 +685,44 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
Ok(FrameSubsampled { width: h.width, height: h.height, vertical_subsampling: vertical, planes })
}
/// T.81 Table H.1 predictor `pred` (1..=7; 0 = none) from the left, above and above-left neighbours.
#[inline(always)]
fn predict(pred: u8, ra: i32, rb: i32, rc: i32) -> i32 {
match pred {
0 => 0,
1 => ra,
2 => rb,
3 => rc,
4 => ra + rb - rc,
5 => ra + ((rb - rc) >> 1),
6 => rb + ((ra - rc) >> 1),
_ => (ra + rb) >> 1,
}
}
/// Decode the pixels `x0..x1` (`x0 >= 1`) of one row with the fixed predictor `P`. `prev` is empty on the first
/// line, which only predicts from the left (`P = 1`).
#[inline(always)]
fn decode_run<const P: u8>(
tables: &[&Huffman],
br: &mut BitReader<'_>,
cur: &mut [u16],
prev: &[u16],
nc: usize,
x0: usize,
x1: usize,
) -> Result<(), RawError> {
for x in x0..x1 {
for (c, table) in tables.iter().enumerate() {
let i = x * nc + c;
let ra = cur[i - nc] as i32;
let p = if P <= 1 { predict(P, ra, 0, 0) } else { predict(P, ra, prev[i] as i32, prev[i - nc] as i32) };
cur[i] = ((p + table.decode_diff(br, false)?) & 0xffff) as u16;
}
}
Ok(())
}
/// Decode a lossless JPEG stream. `max_samples` bounds the allocation (hostile headers).
pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
let h = parse_header(d)?;
@@ -680,18 +761,23 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
let (prev_rows, cur_rows) = out.split_at_mut(y * row_len);
let cur = &mut cur_rows[..row_len];
let prev = if y > 0 { &prev_rows[(y - 1) * row_len..] } else { &[][..] };
for x in 0..w {
if h.restart > 0 {
if restart_left == 0 {
if !br.restart() {
return Err(err("missing restart marker"));
}
restart_left = h.restart;
first_line = true;
reset_pending = true;
let mut x = 0;
while x < w {
if h.restart > 0 && restart_left == 0 {
if !br.restart() {
return Err(err("missing restart marker"));
}
restart_left -= 1;
restart_left = h.restart;
first_line = true;
reset_pending = true;
}
// pixels up to the next restart marker or the end of the row
let mut end = w;
if h.restart > 0 {
end = w.min(x + restart_left);
restart_left -= end - x;
}
// the first pixel of the run has the special predictions (start of the interval, first column)
#[allow(clippy::needless_range_loop)] // `c` indexes the tables and the sample position alike
for c in 0..nc {
let i = x * nc + c;
@@ -702,24 +788,28 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
} else if x == 0 || pred == 0 {
if pred == 0 { 0 } else { prev[i] as i32 }
} else {
let ra = cur[i - nc] as i32;
let rb = prev[i] as i32;
let rc = prev[i - nc] as i32;
match pred {
1 => ra,
2 => rb,
3 => rc,
4 => ra + rb - rc,
5 => ra + ((rb - rc) >> 1),
6 => rb + ((ra - rc) >> 1),
_ => (ra + rb) >> 1,
}
predict(pred, cur[i - nc] as i32, prev[i] as i32, prev[i - nc] as i32)
};
let ssss = tables[c].decode(&mut br)?;
let diff = diff_value(&mut br, ssss);
cur[i] = ((p + diff) & mask) as u16;
cur[i] = ((p + tables[c].decode_diff(&mut br, false)?) & mask) as u16;
}
reset_pending = false;
// the rest of the run: the predictor is fixed, so it is chosen once here, not per sample
if first_line {
// the first line of an interval predicts from the left neighbour
decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?;
} else {
match pred {
0 => decode_run::<0>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
1 => decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
2 => decode_run::<2>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
3 => decode_run::<3>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
4 => decode_run::<4>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
5 => decode_run::<5>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
6 => decode_run::<6>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
_ => decode_run::<7>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
}
}
x = end;
}
first_line = false;
if br.overrun > 64 * 8 {
+115
View File
@@ -125,6 +125,121 @@ fn roundtrip_extremes_and_restarts() {
}
}
/// Re-label an encoded stream as `precision` bits with point transform `pt`: the encoder codes the samples at
/// `precision - pt` bits, which is what a transformed scan holds.
fn with_point_transform(mut enc: Vec<u8>, precision: u8, pt: u8) -> Vec<u8> {
enc[6] = precision; // SOF3 precision follows SOI, marker and length
let sos = enc.windows(2).position(|w| w == [0xff, 0xda]).unwrap();
let seg_len = u16::from_be_bytes([enc[sos + 2], enc[sos + 3]]) as usize;
enc[sos + 2 + seg_len - 1] = pt;
enc
}
#[test]
fn roundtrip_point_transform_all_predictors() {
for pred in 1..=7u8 {
for comps in [1usize, 2, 3] {
for (bits, pt) in [(12u8, 1u8), (14, 2), (16, 4), (8, 3)] {
let (w, h) = (11, 6);
let data = noise(w * h * comps, (bits - pt) as u32, pred as u64 * 17 + comps as u64);
let enc = with_point_transform(encode(&data, w, h, comps, bits - pt, pred, 0), bits, pt);
let f = decode(&enc, usize::MAX).unwrap();
assert_eq!((f.precision, f.point_transform, f.predictor), (bits, pt, pred));
let want: Vec<u16> = data.iter().map(|&v| v << pt).collect();
assert_eq!(f.data, want, "pred {pred} comps {comps} bits {bits} pt {pt}");
}
}
}
// with restart intervals that end mid-row
let data = noise(20 * 9 * 2, 10, 9);
for rst in [3usize, 7, 20] {
let enc = with_point_transform(encode(&data, 20, 9, 2, 10, 4, rst), 12, 2);
let want: Vec<u16> = data.iter().map(|&v| v << 2).collect();
assert_eq!(decode(&enc, usize::MAX).unwrap().data, want, "restart {rst}");
}
}
#[test]
fn roundtrip_skewed_categories_with_long_codes() {
// category k with probability 2^-(k+1): the table gets codes longer than the lookup, rare 16-bit differences
let mut s = 0x9e37_79b9_7f4a_7c15u64;
let mut last = 0u32;
let data: Vec<u16> = (0..256 * 256)
.map(|_| {
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
let r = (s >> 32) as u32;
let cat = r.trailing_zeros().min(16);
let mag = (((s >> 8) as u32) & ((1u32 << cat) - 1)) | (1 << cat) >> 1;
let d = if s & 1 == 0 { mag } else { 0u32.wrapping_sub(mag) };
last = last.wrapping_add(d) & 0xffff;
last as u16
})
.collect();
for pred in [1u8, 4, 7] {
let enc = encode(&data, 256, 256, 1, 16, pred, 0);
assert_eq!(decode(&enc, usize::MAX).unwrap().data, data, "pred {pred}");
}
}
/// Canonical decoding from the code lengths alone, bit by bit (T.81 Annex F, Figure F.16): the reference for the
/// lookup tables.
fn canonical_symbol(br: &mut BitReader, lens: &[u8]) -> Option<u8> {
let window = br.peek(16);
let (mut first, mut index) = (0u32, 0usize);
for len in 1..=16u8 {
let code = window >> (16 - len);
let n = lens.iter().filter(|&&l| l == len).count() as u32;
if code < first + n {
br.consume(len as u32);
return Some((index + (code - first) as usize) as u8);
}
index += n as usize;
first = (first + n) << 1;
}
None
}
/// `decode_diff` against canonical decoding and `diff_value`, for tables with codes inside and beyond the lookup
/// and with `len + category` equal to the lookup width.
#[test]
fn decode_diff_matches_canonical_decoding() {
let tables: [[u8; 17]; 3] = [
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12],
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 16],
[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12],
];
for lens in tables {
let mut counts = [0u8; 16];
for &l in &lens {
counts[l as usize - 1] += 1;
}
// symbols in code order: by length, ties by symbol
let mut values: Vec<u8> = (0..17).collect();
values.sort_by_key(|&v| (lens[v as usize], v));
let sorted: Vec<u8> = values.iter().map(|&v| lens[v as usize]).collect();
let t = Huffman::new(&counts, &values).unwrap();
let mut s = 77u64;
let src: Vec<u8> = (0..6000)
.map(|_| {
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
((s >> 40) as u8).min(0xfe)
})
.collect();
let (mut a, mut b) = (BitReader::new(&src), BitReader::new(&src));
for n in 0..8000 {
let want = canonical_symbol(&mut a, &sorted).map(|i| diff_value(&mut a, values[i as usize]));
let got = t.decode_diff(&mut b, false);
assert_eq!(got.ok(), want, "symbol {n}");
assert_eq!(a.peek(32), b.peek(32), "symbol {n}");
if want.is_none() {
// step past the invalid code so the walk goes on
a.consume(1);
b.consume(1);
}
}
}
}
#[test]
fn constant_image_single_symbol_table() {
let data = vec![1234u16; 16 * 4];
+1
View File
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
+1
View File
@@ -165,6 +165,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: tiff.as_ref().and_then(|t| t.ifds.first()).map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
+2
View File
@@ -329,6 +329,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect,
// Nikon's in-file distortion correction, about the optical centre (which follows the sensor shift)
opcodes: OpcodeLists {
@@ -671,6 +672,7 @@ fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::ima
// the stored colour is already white balanced (see above)
wb_multipliers: Some([1.0; 3]),
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -357,6 +357,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -238,6 +238,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -298,6 +298,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData { kelvin_presets, baseline_exposure: baseline_exposure(&h), ..Default::default() },
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists { list3: crate::raf_lens::corrections(&camera_name(bytes), &ifd, active), ..Default::default() },
camera: ifd0
+459 -118
View File
@@ -183,14 +183,15 @@ struct State {
}
impl State {
/// The smallest `k` with `count << k >= sum`. `count << k0` has the bit length of `sum` (for `k0` the bit-length
/// difference), so it is at most one step short.
#[inline]
fn k(self) -> u32 {
let mut k = 0;
while self.count << k < self.sum {
k += 1;
}
k
let k = self.count.leading_zeros().saturating_sub(self.sum.leading_zeros());
k + u32::from(self.count << k < self.sum)
}
#[inline]
fn update(&mut self, delta: i32) {
self.sum += delta.unsigned_abs();
if self.count == 64 {
@@ -202,8 +203,7 @@ impl State {
}
#[inline]
fn quant(v: i32, near: u32) -> i32 {
let t = [18 + 3 * near, 67 + 5 * near, 276 + 7 * near];
fn quant(v: i32, near: u32, t: &[u32; 3]) -> i32 {
let a = v.unsigned_abs();
let bucket = if a <= near {
0
@@ -232,124 +232,169 @@ fn even_prediction(nw: i32, n: i32, ne: i32, nn: i32) -> i32 {
(pair + 2 * n) >> 2
}
#[inline]
fn sample(rows: &[Vec<i32>], y: usize, x: usize) -> Result<i32> {
rows.get(y).and_then(|r| r.get(x)).copied().ok_or_else(|| corrupt("invalid predictor position"))
/// Residual coding parameters of one precision (`2 * precision + 1` is the quantizer step).
#[derive(Clone, Copy)]
struct Coding {
range: u32,
bits: u32,
limit: u32,
step: i32,
}
fn decode_stripe(stripe: &Stripe<'_>, height: usize, bits: u32, xtrans: bool) -> Result<Vec<u16>> {
let size = if xtrans { BLOCK * 2 / 3 } else { BLOCK / 2 };
let mut previous = vec![vec![vec![0; size + 2]; 2]; 3];
let max = (1u32 << bits) - 1;
let range = |near: u32| (max + 2 * near) / (2 * near + 1) + 1;
let seed = |near: u32| State { sum: ((range(near) + 32) / 64).max(2), count: 1 };
let mut states = [[[seed(0); 41]; 2]; 3];
// Smooth regions retain finer precision and independent state across block quantizer changes.
let mut fine = [[[[seed(0); 5]; 2]; 3], [[[seed(1); 5]; 2]; 3], [[[seed(2); 5]; 2]; 3]];
let mut last_near = None;
let mut reader = Bits::new(stripe.bytes);
let n = height.checked_mul(stripe.width).filter(|&n| n <= crate::MAX_SAMPLES).ok_or(RawError::Limit("image too large"))?;
// a capacity hint only: the stripe's declared height is the file's word, so the reservation is capped at a sample
// per bit of the stripe's real bytes (all stripes reserve in parallel; a one-byte stripe must not take 2 GiB)
let mut output = Vec::with_capacity(n.min(stripe.bytes.len().saturating_mul(8)));
for group in 0..height / 6 {
let near = match stripe.quantizers {
Some(q) => u32::from(*q.get(group).ok_or_else(|| corrupt("truncated quantizer table"))?),
None => 0,
impl Coding {
fn new(max: u32, sample_bits: u32, precision: u32) -> Self {
let range = (max + 2 * precision) / (2 * precision + 1) + 1;
let bits = 32 - (range - 1).leading_zeros();
Self { range, bits, limit: 4 * sample_bits - bits - 1, step: (2 * precision + 1) as i32 }
}
}
/// Everything that depends only on a six-row group's quantizer, computed when the quantizer changes.
struct Group {
near: u32,
thresholds: [u32; 3],
main: Coding,
/// Precisions 0, 1 and 2 for the low-gradient sites.
fine: [Coding; 3],
}
impl Group {
fn new(near: u32, max: u32, sample_bits: u32) -> Self {
Self {
near,
thresholds: [18 + 3 * near, 67 + 5 * near, 276 + 7 * near],
main: Coding::new(max, sample_bits, near),
fine: [0, 1, 2].map(|p| Coding::new(max, sample_bits, p)),
}
}
}
fn seed(max: u32, near: u32) -> State {
let range = (max + 2 * near) / (2 * near + 1) + 1;
State { sum: ((range + 32) / 64).max(2), count: 1 }
}
/// Colour vectors of one stripe, kept for the rows of a six-row group plus the two rows above it. Red and blue get
/// three rows per group, green six; they are stored one after the other in one buffer of rows `stride` wide.
const ROWS: [usize; 3] = [5, 8, 5];
const BASE: [usize; 3] = [0, 5, 13];
const TOTAL_ROWS: usize = 18;
/// The three lines a sample is predicted from: the two above and the one being decoded. Each line has one edge
/// cell on both sides, which the new line takes from the line above.
struct Lines<'a> {
above2: &'a [i32],
above: &'a [i32],
current: &'a mut [i32],
}
impl<'a> Lines<'a> {
fn new(plane: &'a mut [i32], y: usize, stride: usize) -> Self {
let (done, rest) = plane.split_at_mut(y * stride);
let done: &[i32] = done;
let current = &mut rest[..stride];
let above = &done[(y - 1) * stride..];
let above2 = &done[(y - 2) * stride..(y - 1) * stride];
current[0] = above[1];
current[stride - 1] = above[stride - 2];
Self { above2, above, current }
}
}
struct Coder<'a> {
reader: Bits<'a>,
states: [[[State; 41]; 2]; 3],
/// Smooth regions retain finer precision and independent state across block quantizer changes.
fine: [[[[State; 5]; 2]; 3]; 3],
max: u32,
xtrans: bool,
}
impl Coder<'_> {
/// Decodes one sample's residual with the context its gradients select and applies it to the prediction.
#[inline]
fn residual(&mut self, g: &Group, pair: usize, odd: usize, (d1, d2): (i32, i32), predict: i32) -> Result<i32> {
let near = g.near;
let contrast = d1.unsigned_abs() + d2.unsigned_abs();
let precision = match contrast {
0..=5 => 0,
6 => near.min(1),
7 => near.min(2),
_ => near,
};
if last_near != Some(near) {
states = [[[seed(near); 41]; 2]; 3];
last_near = Some(near);
let (context, state, coding) = if precision < near {
let sign = |v: i32| if v.unsigned_abs() <= precision { 0 } else { v.signum() };
let context = 3 * sign(d1) + sign(d2);
(context, &mut self.fine[precision as usize][pair % 3][odd][context.unsigned_abs() as usize], g.fine[precision as usize])
} else {
let context = 9 * quant(d1, near, &g.thresholds) + quant(d2, near, &g.thresholds);
(context, &mut self.states[pair % 3][odd][context.unsigned_abs() as usize], g.main)
};
let mut delta = self.reader.residual(state.k(), coding.bits, coding.range, coding.limit)?;
state.update(delta);
if context < 0 {
delta = -delta;
}
let mut rows = previous;
let mut value = predict + delta * coding.step;
if value < -(precision as i32) {
value += coding.range as i32 * coding.step;
} else if value > (self.max + precision) as i32 {
value -= coding.range as i32 * coding.step;
}
Ok(value.clamp(0, self.max as i32))
}
/// One position of the two colour lines of a pair: the even column (`odd == 0`) or the one seven behind it.
#[inline]
fn position(&mut self, g: &Group, pair: usize, odd: usize, pos: usize, lines: &mut [Lines<'_>; 2]) -> Result<()> {
let x = pos + 1;
for (which, l) in lines.iter_mut().enumerate() {
let (nw, north, ne) = (l.above[x - 1], l.above[x], l.above[x + 1]);
let real = !self.xtrans || (MASKS[pair][which] >> (pos % 4)) & 1 != 0;
l.current[x] = if odd == 0 {
let nn = l.above2[x];
let predict = even_prediction(nw, north, ne, nn);
if real { self.residual(g, pair, 0, (north - nn, nw - north), predict)? } else { predict }
} else {
let (west, east) = (l.current[x - 1], l.current[x + 1]);
let predict = if (nw.min(ne)..=nw.max(ne)).contains(&north) { (west + east) >> 1 } else { (west + east + 2 * north) >> 2 };
if real { self.residual(g, pair, 1, (north - nw, nw - west), predict)? } else { predict }
};
}
Ok(())
}
/// Decodes the six colour-vector pairs of one group into `buf`.
fn group(&mut self, g: &Group, buf: &mut [i32], size: usize) -> Result<()> {
let stride = size + 2;
let (red, rest) = buf.split_at_mut(ROWS[0] * stride);
let (green, blue) = rest.split_at_mut(ROWS[1] * stride);
for pair in 0..6 {
let colours = if pair % 2 == 0 { [0, 1] } else { [1, 2] };
for &colour in &colours {
let lines = rows.get_mut(colour).ok_or_else(|| corrupt("invalid colour vector"))?;
let mut row = vec![0; size + 2];
let prev = lines.len() - 1;
let left = sample(lines, prev, 1)?;
let right = sample(lines, prev, size)?;
if let Some(v) = row.first_mut() {
*v = left;
}
if let Some(v) = row.last_mut() {
*v = right;
}
lines.push(row);
}
let mut lines = if pair % 2 == 0 {
[Lines::new(&mut *red, 2 + pair / 2, stride), Lines::new(&mut *green, 2 + pair, stride)]
} else {
[Lines::new(&mut *green, 2 + pair, stride), Lines::new(&mut *blue, 2 + pair / 2, stride)]
};
for even in (0..size + 8).step_by(2) {
for (odd, position) in [(0, Some(even)), (1, even.checked_sub(7))] {
let Some(pos) = position.filter(|&x| x < size) else { continue };
let x = pos + 1;
for (which, &colour) in colours.iter().enumerate() {
let lines = rows.get_mut(colour).ok_or_else(|| corrupt("invalid colour vector"))?;
let y = lines.len() - 1;
let (nw, north, ne, nn) =
(sample(lines, y - 1, x - 1)?, sample(lines, y - 1, x)?, sample(lines, y - 1, x + 1)?, sample(lines, y - 2, x)?);
let west = sample(lines, y, x - 1)?;
let east = sample(lines, y, x + 1)?;
let (predict, d1, d2) = if odd == 0 {
(even_prediction(nw, north, ne, nn), north - nn, nw - north)
} else {
let p = if (nw.min(ne)..=nw.max(ne)).contains(&north) { (west + east) >> 1 } else { (west + east + 2 * north) >> 2 };
(p, north - nw, nw - west)
};
let real = !xtrans || MASKS.get(pair).and_then(|m| m.get(which)).is_some_and(|m| (m >> (pos % 4)) & 1 != 0);
let value = if real {
let contrast = d1.unsigned_abs() + d2.unsigned_abs();
let precision = match contrast {
0..=5 => 0,
6 => near.min(1),
7 => near.min(2),
_ => near,
};
let (context, state) = if precision < near {
let sign = |v: i32| if v.unsigned_abs() <= precision { 0 } else { v.signum() };
let context = 3 * sign(d1) + sign(d2);
let state = fine
.get_mut(precision as usize)
.and_then(|f| f.get_mut(pair % 3))
.and_then(|s| s.get_mut(odd))
.and_then(|s| s.get_mut(context.unsigned_abs() as usize))
.ok_or_else(|| corrupt("invalid fine coding context"))?;
(context, state)
} else {
let context = 9 * quant(d1, near) + quant(d2, near);
let state = states
.get_mut(pair % 3)
.and_then(|s| s.get_mut(odd))
.and_then(|s| s.get_mut(context.unsigned_abs() as usize))
.ok_or_else(|| corrupt("invalid coding context"))?;
(context, state)
};
let coded_range = range(precision);
let coded_bits = 32 - (coded_range - 1).leading_zeros();
let mut delta = reader.residual(state.k(), coded_bits, coded_range, 4 * bits - coded_bits - 1)?;
state.update(delta);
if context < 0 {
delta = -delta;
}
let step = (2 * precision + 1) as i32;
let mut value = predict + delta * step;
if value < -(precision as i32) {
value += coded_range as i32 * step;
} else if value > (max + precision) as i32 {
value -= coded_range as i32 * step;
}
value.clamp(0, max as i32)
} else {
predict
};
*lines.get_mut(y).and_then(|r| r.get_mut(x)).ok_or_else(|| corrupt("invalid predictor position"))? = value;
}
if even < size {
self.position(g, pair, 0, even, &mut lines)?;
}
if even >= 7 && even - 7 < size {
self.position(g, pair, 1, even - 7, &mut lines)?;
}
}
}
for y in 0..6 {
for x in 0..stripe.width {
Ok(())
}
}
/// For each of the six rows of a group, where each output sample sits in the colour-vector buffer.
fn placement(xtrans: bool, width: usize, stride: usize) -> [Vec<usize>; 6] {
std::array::from_fn(|y| {
(0..width)
.map(|x| {
let colour = if xtrans {
XTRANS.get(y).and_then(|r| r.get(x % 6)).copied().ok_or_else(|| corrupt("invalid CFA position"))?
XTRANS[y][x % 6]
} else {
match (y % 2, x % 2) {
(0, 0) => 0,
@@ -359,10 +404,49 @@ fn decode_stripe(stripe: &Stripe<'_>, height: usize, bits: u32, xtrans: bool) ->
};
let row = 2 + if colour == 1 { y } else { y / 2 };
let col = 1 + if xtrans { 2 * (x / 3) + (x % 3).min(1) } else { x / 2 };
output.push(sample(rows.get(colour).ok_or_else(|| corrupt("invalid colour vector"))?, row, col)? as u16);
}
(BASE[colour] + row) * stride + col
})
.collect()
})
}
fn decode_stripe(stripe: &Stripe<'_>, height: usize, bits: u32, xtrans: bool) -> Result<Vec<u16>> {
let size = if xtrans { BLOCK * 2 / 3 } else { BLOCK / 2 };
let stride = size + 2;
let max = (1u32 << bits) - 1;
let mut coder = Coder {
reader: Bits::new(stripe.bytes),
states: [[[seed(max, 0); 41]; 2]; 3],
fine: [[[[seed(max, 0); 5]; 2]; 3], [[[seed(max, 1); 5]; 2]; 3], [[[seed(max, 2); 5]; 2]; 3]],
max,
xtrans,
};
let mut last_near = None;
let mut group = Group::new(0, max, bits);
let n = height.checked_mul(stripe.width).filter(|&n| n <= crate::MAX_SAMPLES).ok_or(RawError::Limit("image too large"))?;
// a capacity hint only: the stripe's declared height is the file's word, so the reservation is capped at a sample
// per bit of the stripe's real bytes (all stripes reserve in parallel; a one-byte stripe must not take 2 GiB)
let mut output = Vec::with_capacity(n.min(stripe.bytes.len().saturating_mul(8)));
let place = placement(xtrans, stripe.width, stride);
let mut buf = vec![0i32; TOTAL_ROWS * stride];
for index in 0..height / 6 {
let near = match stripe.quantizers {
Some(q) => u32::from(*q.get(index).ok_or_else(|| corrupt("truncated quantizer table"))?),
None => 0,
};
if last_near != Some(near) {
coder.states = [[[seed(max, near); 41]; 2]; 3];
group = Group::new(near, max, bits);
last_near = Some(near);
}
coder.group(&group, &mut buf, size)?;
for row in &place {
output.extend(row.iter().map(|&at| buf[at] as u16));
}
// the last two rows of each colour are the next group's rows above
for (base, rows) in BASE.iter().zip(ROWS) {
buf.copy_within((base + rows - 2) * stride..(base + rows) * stride, base * stride);
}
previous = rows.into_iter().map(|mut r| r.split_off(r.len() - 2)).collect();
}
Ok(output)
}
@@ -590,3 +674,260 @@ mod tests {
}
}
}
/// The straightforward decoder this module started from (shift loop for `k`, an `Option`/`Result` neighbour read
/// per access, a `Vec` per colour line, one pixel placed at a time), kept as the oracle for the optimised one.
#[cfg(test)]
mod differential {
use super::*;
use sha2::{Digest, Sha256};
fn k_by_shifting(state: State) -> u32 {
let mut k = 0;
while state.count << k < state.sum {
k += 1;
}
k
}
fn quant_by_table(v: i32, near: u32) -> i32 {
let t = [18 + 3 * near, 67 + 5 * near, 276 + 7 * near];
quant(v, near, &t)
}
fn sample(rows: &[Vec<i32>], y: usize, x: usize) -> Result<i32> {
rows.get(y).and_then(|r| r.get(x)).copied().ok_or_else(|| corrupt("invalid predictor position"))
}
fn reference_stripe(stripe: &Stripe<'_>, height: usize, bits: u32, xtrans: bool) -> Result<Vec<u16>> {
let size = if xtrans { BLOCK * 2 / 3 } else { BLOCK / 2 };
let mut previous = vec![vec![vec![0; size + 2]; 2]; 3];
let max = (1u32 << bits) - 1;
let range = |near: u32| (max + 2 * near) / (2 * near + 1) + 1;
let seed = |near: u32| State { sum: ((range(near) + 32) / 64).max(2), count: 1 };
let mut states = [[[seed(0); 41]; 2]; 3];
let mut fine = [[[[seed(0); 5]; 2]; 3], [[[seed(1); 5]; 2]; 3], [[[seed(2); 5]; 2]; 3]];
let mut last_near = None;
let mut reader = Bits::new(stripe.bytes);
let n = height.checked_mul(stripe.width).filter(|&n| n <= crate::MAX_SAMPLES).ok_or(RawError::Limit("image too large"))?;
let mut output = Vec::with_capacity(n.min(stripe.bytes.len().saturating_mul(8)));
for group in 0..height / 6 {
let near = match stripe.quantizers {
Some(q) => u32::from(*q.get(group).ok_or_else(|| corrupt("truncated quantizer table"))?),
None => 0,
};
if last_near != Some(near) {
states = [[[seed(near); 41]; 2]; 3];
last_near = Some(near);
}
let mut rows = previous;
for pair in 0..6 {
let colours = if pair % 2 == 0 { [0, 1] } else { [1, 2] };
for &colour in &colours {
let lines = rows.get_mut(colour).ok_or_else(|| corrupt("invalid colour vector"))?;
let mut row = vec![0; size + 2];
let prev = lines.len() - 1;
let left = sample(lines, prev, 1)?;
let right = sample(lines, prev, size)?;
if let Some(v) = row.first_mut() {
*v = left;
}
if let Some(v) = row.last_mut() {
*v = right;
}
lines.push(row);
}
for even in (0..size + 8).step_by(2) {
for (odd, position) in [(0, Some(even)), (1, even.checked_sub(7))] {
let Some(pos) = position.filter(|&x| x < size) else { continue };
let x = pos + 1;
for (which, &colour) in colours.iter().enumerate() {
let lines = rows.get_mut(colour).ok_or_else(|| corrupt("invalid colour vector"))?;
let y = lines.len() - 1;
let (nw, north, ne, nn) =
(sample(lines, y - 1, x - 1)?, sample(lines, y - 1, x)?, sample(lines, y - 1, x + 1)?, sample(lines, y - 2, x)?);
let west = sample(lines, y, x - 1)?;
let east = sample(lines, y, x + 1)?;
let (predict, d1, d2) = if odd == 0 {
(even_prediction(nw, north, ne, nn), north - nn, nw - north)
} else {
let p = if (nw.min(ne)..=nw.max(ne)).contains(&north) { (west + east) >> 1 } else { (west + east + 2 * north) >> 2 };
(p, north - nw, nw - west)
};
let real = !xtrans || MASKS.get(pair).and_then(|m| m.get(which)).is_some_and(|m| (m >> (pos % 4)) & 1 != 0);
let value = if real {
let contrast = d1.unsigned_abs() + d2.unsigned_abs();
let precision = match contrast {
0..=5 => 0,
6 => near.min(1),
7 => near.min(2),
_ => near,
};
let (context, state) = if precision < near {
let sign = |v: i32| if v.unsigned_abs() <= precision { 0 } else { v.signum() };
let context = 3 * sign(d1) + sign(d2);
let state = fine
.get_mut(precision as usize)
.and_then(|f| f.get_mut(pair % 3))
.and_then(|s| s.get_mut(odd))
.and_then(|s| s.get_mut(context.unsigned_abs() as usize))
.ok_or_else(|| corrupt("invalid fine coding context"))?;
(context, state)
} else {
let context = 9 * quant_by_table(d1, near) + quant_by_table(d2, near);
let state = states
.get_mut(pair % 3)
.and_then(|s| s.get_mut(odd))
.and_then(|s| s.get_mut(context.unsigned_abs() as usize))
.ok_or_else(|| corrupt("invalid coding context"))?;
(context, state)
};
let coded_range = range(precision);
let coded_bits = 32 - (coded_range - 1).leading_zeros();
let mut delta = reader.residual(k_by_shifting(*state), coded_bits, coded_range, 4 * bits - coded_bits - 1)?;
state.update(delta);
if context < 0 {
delta = -delta;
}
let step = (2 * precision + 1) as i32;
let mut value = predict + delta * step;
if value < -(precision as i32) {
value += coded_range as i32 * step;
} else if value > (max + precision) as i32 {
value -= coded_range as i32 * step;
}
value.clamp(0, max as i32)
} else {
predict
};
*lines.get_mut(y).and_then(|r| r.get_mut(x)).ok_or_else(|| corrupt("invalid predictor position"))? = value;
}
}
}
}
for y in 0..6 {
for x in 0..stripe.width {
let colour = if xtrans {
XTRANS.get(y).and_then(|r| r.get(x % 6)).copied().ok_or_else(|| corrupt("invalid CFA position"))?
} else {
match (y % 2, x % 2) {
(0, 0) => 0,
(1, 1) => 2,
_ => 1,
}
};
let row = 2 + if colour == 1 { y } else { y / 2 };
let col = 1 + if xtrans { 2 * (x / 3) + (x % 3).min(1) } else { x / 2 };
output.push(sample(rows.get(colour).ok_or_else(|| corrupt("invalid colour vector"))?, row, col)? as u16);
}
}
previous = rows.into_iter().map(|mut r| r.split_off(r.len() - 2)).collect();
}
Ok(output)
}
/// Deterministic bytes: uniform (0), mostly zero (1: long unary prefixes and escapes) or one-heavy (2: short
/// prefixes and small residuals, which stay inside the range of every quantizer so the stream decodes to the end).
fn bytes(seed: u64, len: usize, kind: u8) -> Vec<u8> {
let mut x = seed | 1;
(0..len)
.map(|_| {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
let b = (x >> 24) as u8;
match kind {
1 if x & 0b11 != 0 => 0,
2 => b | (x >> 40) as u8,
_ => b,
}
})
.collect()
}
#[test]
fn k_without_the_shift_loop_matches_it() {
let mut sums: Vec<u32> = (0..=4200).collect();
for i in 0..=30 {
sums.extend([(1u32 << i) - 1, 1 << i, (1 << i) + 1]);
}
for count in 1..=64 {
for &sum in &sums {
let state = State { sum, count };
assert_eq!(state.k(), k_by_shifting(state), "count {count}, sum {sum}");
}
}
}
/// Every variant the stripe decoder handles (lossless and lossy, Bayer and X-Trans, 12/14/16 bits, full and
/// partial stripes, quantizer changes, fine contexts) gives the oracle's samples, or its error, on
/// uniform pseudo-random entropy streams that reach all contexts, one-heavy ones that every quantizer table decodes
/// to the end, and sparse ones that reach the escapes. Each quantizer table must have streams that finish.
#[test]
fn stripe_decoder_matches_the_straightforward_one() {
let quantizers: [Option<&[u8]>; 4] =
[None, Some([0, 1, 2, 3].as_slice()), Some([2, 2, 7, 0, 40, 255].as_slice()), Some([1, 1, 0, 5, 5, 9, 2].as_slice())];
// uniform and one-heavy streams that decode to the end, per quantizer table
let mut finished = [0usize; 4];
for bits in [12, 14, 16] {
for xtrans in [false, true] {
for (table, q) in quantizers.into_iter().enumerate() {
for (seed, width, kind) in [(1, BLOCK, 0), (2, 301, 0), (3, 12, 0), (4, BLOCK, 1), (5, BLOCK, 2), (6, 301, 2)] {
let data = bytes(seed * 31 + u64::from(bits), 32 << 10, kind);
let groups = q.map_or(3, <[u8]>::len);
let stripe = Stripe { bytes: &data, quantizers: q, width };
let want = reference_stripe(&stripe, groups * 6, bits, xtrans).map_err(|e| e.to_string());
let got = decode_stripe(&stripe, groups * 6, bits, xtrans).map_err(|e| e.to_string());
assert_eq!(got, want, "bits {bits}, xtrans {xtrans}, quantizers {q:?}, width {width}, stream kind {kind}");
finished[table] += usize::from(want.is_ok() && kind != 1);
}
}
}
}
eprintln!("dense streams decoded to the end per quantizer table (of 30): {finished:?}");
assert!(finished.iter().all(|&n| n > 0), "some quantizer table never decodes to the end: {finished:?}");
}
/// Optional: the CC0 files of docs/raf-corpus.sha256 in the directory named by `LIGHTCRAFT_RAF_CORPUS` (file
/// names as listed there, e.g. `raf-fuji-xt5-6122.raf`). Each present file's SHA-256 is checked, then the
/// decoded sensor samples are compared with the hash recorded from the decoder before the optimisation.
#[test]
fn optional_cc0_samples_decode_as_recorded() {
let Some(directory) = std::env::var_os("LIGHTCRAFT_RAF_CORPUS") else { return };
let list = include_str!("../docs/raf-corpus.sha256");
for line in list.lines().filter(|l| !l.starts_with('#') && !l.trim().is_empty()) {
let mut columns = line.split_whitespace();
let (Some(digest), Some(path)) = (columns.next(), columns.next()) else { panic!("bad corpus row {line:?}") };
let name = path.rsplit('/').next().unwrap();
let Ok(bytes) = std::fs::read(std::path::Path::new(&directory).join(name)) else { continue };
let sha = Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect::<String>();
assert_eq!(sha, digest, "{name}: not the published file");
let Some(&(_, expected)) = RECORDED.iter().find(|(n, _)| *n == name) else { panic!("{name}: no recorded hash") };
let image = crate::decode(&bytes).unwrap();
let crate::RawData::U16(data) = &image.data else { panic!("{name}: not 16-bit samples") };
let hash = data.iter().flat_map(|v| v.to_le_bytes()).fold(0xcbf29ce484222325u64, |h, b| (h ^ u64::from(b)).wrapping_mul(0x100000001b3));
assert_eq!(hash, expected, "{name}");
}
}
/// FNV-1a over the little-endian decoded samples, recorded from `upstream/main` before the change.
const RECORDED: &[(&str, u64)] = &[
("raf-fuji-gfx100-3773.raf", 0x1195d3ab70da7cb4),
("raf-fuji-gfx100-3775.raf", 0xb7c7c1a7717ef93c),
("raf-fuji-gfx100rf-8091.raf", 0x49110f1b80eefe2e),
("raf-fuji-gfx100s-4495.raf", 0x84ab46122f72796c),
("raf-fuji-gfx100s-4503.raf", 0x63debf0afde73e2e),
("raf-fuji-gfx50s-1435.raf", 0xb3e570a6c88455f5),
("raf-fuji-xe5-8509.raf", 0x6a7d5cde2832d4f3),
("raf-fuji-xh2-6001.raf", 0x466a5707ea484f9c),
("raf-fuji-xh2-6002.raf", 0xbbf043676476925a),
("raf-fuji-xm5-7748.raf", 0xedd17960a117a553),
("raf-fuji-xt2-865.raf", 0xe020816ec114436f),
("raf-fuji-xt20-compressed.raf", 0x85014025edc2cebd),
("raf-fuji-xt4-3914.raf", 0x7c98e527dbe01f23),
("raf-fuji-xt4-3918.raf", 0xfb4ba0df34c1c904),
("raf-fuji-xt5-6122.raf", 0x8f76abd57de79a94),
("raf-fuji-xt5-6123.raf", 0x0b2784853c31278d),
("raf-fuji-xt50-7807.raf", 0xa8e2be9cac387ebc),
];
}
+1
View File
@@ -672,6 +672,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists { list3: distortion(ifd0, active).into_iter().collect(), ..Default::default() },
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -186,6 +186,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -44,6 +44,7 @@ fn an_active_area_that_wraps_past_the_image_is_corrupt() {
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: Camera::default(),
cinema: Default::default(),
+154 -47
View File
@@ -2,7 +2,7 @@
#[cfg(feature = "dng")]
use crate::unpack::inflate;
use crate::unpack::{f16_to_f32, f24_to_f32, read_u16s, undo_diff_u16, undo_float_predictor, unpack_lsb, unpack_msb, unpack_words32_msb};
use crate::unpack::{f16_to_f32, f24_to_f32, read_u16s, undo_diff_u16, undo_float_predictor, unpack_lsb, unpack_msb, unpack_words32_msb_with};
use crate::{Limits, RawData, RawError, Result, ljpeg};
use craft_tiff::image::{Chunk, ImageInfo, chunk_bytes};
use craft_tiff::{ByteOrder, tags::compression as comp};
@@ -18,8 +18,8 @@ pub enum Packing {
Lsb,
/// Each sample stored in 16 bits in file byte order, regardless of `bits`.
Word16,
/// MSB-first bit stream inside 32-bit words stored in file byte order ([`unpack_words32_msb`]); rows start on a
/// word boundary.
/// MSB-first bit stream inside 32-bit words stored in file byte order ([`unpack_words32_msb`](crate::unpack::unpack_words32_msb));
/// rows start on a word boundary.
Words32Msb,
}
@@ -184,46 +184,72 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin
.par_iter()
.map(|c| decode_chunk(data, info, c, order, packing, ccpp, bits, float, limits).map(|p| (*c, p)))
.collect();
for r in decoded {
// a chunk that cannot be decoded is not a partial image: the file stays preview-only
let (c, px) = r?;
let (cw, ch) = (c.width as usize, c.height as usize);
let (x0, y0) = (c.x as usize, c.y as usize);
if x0 >= w || y0 >= h {
continue;
}
let copy_w = cw.min(w - x0);
let copy_h = ch.min(h - y0);
for y in 0..copy_h {
for x in 0..copy_w {
for s in 0..ccpp {
let src = (y * cw + x) * ccpp + s;
let dst_s = if planar { c.plane as usize } else { s };
if dst_s >= cpp {
continue;
}
let dst = ((y0 + y) * w + x0 + x) * cpp + dst_s;
match &px {
ChunkPx::U16(v) => {
if let (Some(o), Some(&s)) = (out_u16.get_mut(dst), v.get(src)) {
*o = s;
}
}
ChunkPx::F32(v) => {
if let (Some(o), Some(&s)) = (out_f32.get_mut(dst), v.get(src)) {
*o = s;
}
}
}
// a chunk that cannot be decoded is not a partial image: the file stays preview-only
let decoded = decoded.into_iter().collect::<Result<Vec<_>>>()?;
// The copy runs in parallel over bands of output rows: each band takes its rows of every chunk of the
// group that reaches it, in chunk order (so overlapping chunks resolve as a serial copy would).
let row_samples = w * cpp;
let threads = rayon::current_num_threads().max(1);
let band_rows = h.div_ceil(threads * 4).max(1);
if float {
out_f32.par_chunks_mut(band_rows * row_samples).enumerate().for_each(|(bi, band)| {
for (c, px) in &decoded {
if let ChunkPx::F32(v) = px {
place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar);
}
}
}
});
} else {
out_u16.par_chunks_mut(band_rows * row_samples).enumerate().for_each(|(bi, band)| {
for (c, px) in &decoded {
if let ChunkPx::U16(v) = px {
place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar);
}
}
});
}
decode_from = to;
}
Ok(if float { RawData::F32(out_f32) } else { RawData::U16(out_u16) })
}
/// Copy the rows of chunk `c` (samples `src`, `c.width` per row, `ccpp` samples per pixel) that fall into the output
/// band `band`, which holds the image rows from `band_y0` on (`w` pixels of `cpp` samples each, `h` rows in the whole
/// image). Pixels past the image edge and samples `src` does not hold are skipped; a planar chunk (`planar`) writes
/// its one sample per pixel into the channel `c.plane`, a chunk of a plane the image does not have is skipped.
#[allow(clippy::too_many_arguments)]
fn place_chunk<T: Copy>(band: &mut [T], band_y0: usize, c: &Chunk, src: &[T], w: usize, h: usize, cpp: usize, planar: bool) {
let (cw, ch) = (c.width as usize, c.height as usize);
let (x0, y0) = (c.x as usize, c.y as usize);
if x0 >= w || y0 >= h || cpp == 0 {
return;
}
let plane = c.plane as usize;
if planar && plane >= cpp {
return;
}
let ccpp = if planar { 1 } else { cpp };
let copy_w = cw.min(w - x0);
let copy_h = ch.min(h - y0);
let band_rows = band.len() / (w * cpp);
let first = y0.max(band_y0);
let last = (y0 + copy_h).min(band_y0 + band_rows);
for y in first..last {
let s0 = (y - y0) * cw * ccpp;
let Some(avail) = src.get(s0..) else { break };
let d0 = ((y - band_y0) * w + x0) * cpp;
if planar {
let row = &mut band[d0..d0 + copy_w * cpp];
for (o, &v) in row[plane..].iter_mut().step_by(cpp).zip(&avail[..copy_w.min(avail.len())]) {
*o = v;
}
} else {
let n = (copy_w * cpp).min(avail.len());
band[d0..d0 + n].copy_from_slice(&avail[..n]);
}
}
}
/// Framing precedes allocation even in header-only probes: missing source rows may not
/// be reported as a decodable image and later silently filled with zero samples.
fn check_uncompressed_chunks(data: &[u8], info: &ImageInfo, chunks: &[Chunk], packing: Packing) -> Result<()> {
@@ -512,13 +538,14 @@ fn unpack_chunk(
let row_bytes = row_n * bp;
let mut out = vec![0f32; row_n * ch];
let mut rowbuf = vec![0u8; row_bytes];
let mut scratch = Vec::new();
for y in 0..ch {
let s = src.get(y * row_bytes..).unwrap_or(&[]);
let len = s.len().min(row_bytes);
rowbuf[..len].copy_from_slice(&s[..len]);
rowbuf[len..].fill(0);
let big = if fp {
undo_float_predictor(&mut rowbuf, row_n, bp, cpp * factor);
undo_float_predictor(&mut rowbuf, row_n, bp, cpp * factor, &mut scratch);
true
} else {
if factor > 0 {
@@ -526,16 +553,15 @@ fn unpack_chunk(
}
order == ByteOrder::Big
};
for (i, o) in out[y * row_n..(y + 1) * row_n].iter_mut().enumerate() {
let b = &rowbuf[i * bp..(i + 1) * bp];
*o = match (bp, big) {
(2, true) => f16_to_f32(u16::from_be_bytes([b[0], b[1]])),
(2, false) => f16_to_f32(u16::from_le_bytes([b[0], b[1]])),
(3, true) => f24_to_f32(u32::from_be_bytes([0, b[0], b[1], b[2]])),
(3, false) => f24_to_f32(u32::from_le_bytes([b[0], b[1], b[2], 0])),
(4, true) => f32::from_be_bytes([b[0], b[1], b[2], b[3]]),
_ => f32::from_le_bytes([b[0], b[1], b[2], b[3]]),
};
let dst = &mut out[y * row_n..(y + 1) * row_n];
let samples = rowbuf.chunks_exact(bp);
match (bp, big) {
(2, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f16_to_f32(u16::from_be_bytes([b[0], b[1]]))),
(2, false) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f16_to_f32(u16::from_le_bytes([b[0], b[1]]))),
(3, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f24_to_f32(u32::from_be_bytes([0, b[0], b[1], b[2]]))),
(3, false) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f24_to_f32(u32::from_le_bytes([b[0], b[1], b[2], 0]))),
(4, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f32::from_be_bytes([b[0], b[1], b[2], b[3]])),
_ => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f32::from_le_bytes([b[0], b[1], b[2], b[3]])),
}
}
return Ok(ChunkPx::F32(out));
@@ -544,6 +570,7 @@ fn unpack_chunk(
return Err(RawError::Unsupported("floating-point predictor on integer data".into()));
}
let mut out = vec![0u16; row_n * ch];
let mut words = Vec::new();
let (row_bytes, kind) = match (bits, packing) {
(8, _) => (row_n, 0),
(16, _) | (_, Packing::Word16) => (row_n * 2, 1),
@@ -559,7 +586,7 @@ fn unpack_chunk(
0 => row.iter_mut().zip(s).for_each(|(o, &b)| *o = b as u16),
1 => read_u16s(s, order, row),
2 => unpack_msb(s, bits, row),
4 => unpack_words32_msb(s, order, bits, row),
4 => unpack_words32_msb_with(s, order, bits, row, &mut words),
_ => unpack_lsb(s, bits, row),
}
if factor > 0 {
@@ -581,6 +608,86 @@ fn unpack_chunk(
Ok(ChunkPx::U16(out))
}
#[cfg(test)]
mod place_tests {
use super::*;
/// The serial per-sample copy `place_chunk` replaced: the oracle.
#[allow(clippy::too_many_arguments)]
fn reference(out: &mut [u16], c: &Chunk, v: &[u16], w: usize, h: usize, cpp: usize, planar: bool) {
let ccpp = if planar { 1 } else { cpp };
let (cw, ch) = (c.width as usize, c.height as usize);
let (x0, y0) = (c.x as usize, c.y as usize);
if x0 >= w || y0 >= h {
return;
}
for y in 0..ch.min(h - y0) {
for x in 0..cw.min(w - x0) {
for s in 0..ccpp {
let src = (y * cw + x) * ccpp + s;
let dst_s = if planar { c.plane as usize } else { s };
if dst_s >= cpp {
continue;
}
let dst = ((y0 + y) * w + x0 + x) * cpp + dst_s;
if let (Some(o), Some(&s)) = (out.get_mut(dst), v.get(src)) {
*o = s;
}
}
}
}
}
fn chunk(x: u32, y: u32, width: u32, height: u32, plane: u16) -> Chunk {
Chunk { index: 0, x, y, width, height, plane, offset: 0, len: 0 }
}
/// Chunks that overlap (the later one wins), run past the image edge, hold fewer samples than their size, or name a
/// plane the image does not have, placed through bands of several heights, equal the serial copy.
#[test]
fn place_chunk_matches_the_serial_copy_for_odd_chunks() {
let (w, h) = (11, 9);
for cpp in [1usize, 3] {
for planar in [false, true] {
let ccpp = if planar { 1 } else { cpp };
let specs = [
(0, 0, 8, 6, 0, 1u16),
(4, 3, 8, 8, 0, 2), // overlaps the first and runs past both edges
(4, 3, 8, 8, 0, 3), // the same place again, later wins
(0, 5, 5, 4, 1, 4),
(2, 2, 6, 6, 2, 5),
(0, 0, 4, 4, 3, 6), // plane 3: not an image plane for cpp <= 3
(9, 0, 6, 12, 0, 7), // only two columns inside
(20, 0, 4, 4, 0, 8), // right of the image
(0, 0, 11, 9, 0, 9), // whole image, short source below
];
let chunks: Vec<(Chunk, Vec<u16>)> = specs
.iter()
.map(|&(x, y, cw, ch, plane, val)| {
let n = (cw * ch) as usize * ccpp;
// the last chunk holds only 40 samples
let n = if val == 9 { 40 } else { n };
(chunk(x, y, cw, ch, plane), (0..n).map(|i| val * 100 + i as u16).collect())
})
.collect();
let mut want = vec![0u16; w * h * cpp];
for (c, v) in &chunks {
reference(&mut want, c, v, w, h, cpp, planar);
}
for band_rows in [1usize, 2, 4, 9, 20] {
let mut got = vec![0u16; w * h * cpp];
for (bi, band) in got.chunks_mut(band_rows * w * cpp).enumerate() {
for (c, v) in &chunks {
place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar);
}
}
assert_eq!(got, want, "cpp {cpp} planar {planar} band rows {band_rows}");
}
}
}
}
}
#[cfg(test)]
#[cfg(feature = "dng")]
mod lossy_tests {
+31 -7
View File
@@ -46,12 +46,18 @@ pub fn unpack_lsb(src: &[u8], bits: u32, out: &mut [u16]) {
/// data, which is not the byte-wise MSB stream of [`unpack_msb`] unless `order` is big-endian. A short last word
/// is padded with zeros.
pub fn unpack_words32_msb(src: &[u8], order: ByteOrder, bits: u32, out: &mut [u16]) {
let words = src.chunks(4).flat_map(|c| {
unpack_words32_msb_with(src, order, bits, out, &mut Vec::new());
}
/// [`unpack_words32_msb`] with the byte-swapped copy of `src` kept in `scratch` (reused across rows).
pub fn unpack_words32_msb_with(src: &[u8], order: ByteOrder, bits: u32, out: &mut [u16], scratch: &mut Vec<u8>) {
scratch.clear();
scratch.extend(src.chunks(4).flat_map(|c| {
let mut w = [0u8; 4];
w[..c.len()].copy_from_slice(c);
order.u32(w).to_be_bytes()
});
unpack_msb(&words.collect::<Vec<u8>>(), bits, out);
}));
unpack_msb(scratch, bits, out);
}
pub fn read_u16s(src: &[u8], order: ByteOrder, out: &mut [u16]) {
@@ -126,10 +132,12 @@ pub fn undo_diff_u8(row: &mut [u8], stride: usize) {
/// Undo the floating-point predictor on one row: `row` holds `n` samples of `bytes_per` bytes, byte-differenced
/// with stride `stride` samples, then stored as byte planes (most significant first). Returns the samples as
/// big-endian byte groups in place.
pub fn undo_float_predictor(row: &mut [u8], n: usize, bytes_per: usize, stride: usize) {
/// big-endian byte groups in place. `scratch` holds the byte planes (reused across rows).
pub fn undo_float_predictor(row: &mut [u8], n: usize, bytes_per: usize, stride: usize, scratch: &mut Vec<u8>) {
undo_diff_u8(row, stride);
let tmp = row.to_vec();
scratch.clear();
scratch.extend_from_slice(row);
let tmp = &scratch[..];
for i in 0..n {
for b in 0..bytes_per {
if let (Some(dst), Some(&s)) = (row.get_mut(i * bytes_per + b), tmp.get(b * n + i)) {
@@ -167,6 +175,22 @@ mod tests {
assert_eq!(o, [0x000, 0x012]); // the word 0x00001234, top bits first
}
#[test]
fn words32_scratch_is_reusable() {
let long: Vec<u8> = (0u8..24).map(|i| i.wrapping_mul(37).wrapping_add(5)).collect();
let short = &long[3..10];
let mut scratch = Vec::new();
for order in [ByteOrder::Little, ByteOrder::Big] {
for (src, n) in [(&long[..], 16usize), (short, 4)] {
let mut want = vec![0u16; n];
unpack_words32_msb(src, order, 12, &mut want);
let mut got = vec![0u16; n];
unpack_words32_msb_with(src, order, 12, &mut got, &mut scratch);
assert_eq!(got, want);
}
}
}
#[test]
fn packing() {
// 12-bit MSB: 0xABC, 0xDEF -> AB CD EF
@@ -210,7 +234,7 @@ mod tests {
for i in (1..row.len()).rev() {
row[i] = row[i].wrapping_sub(row[i - 1]);
}
undo_float_predictor(&mut row, 2, 4, 1);
undo_float_predictor(&mut row, 2, 4, 1, &mut Vec::new());
assert_eq!(f32::from_be_bytes([row[0], row[1], row[2], row[3]]), 1.0);
assert_eq!(f32::from_be_bytes([row[4], row[5], row[6], row[7]]), 2.0);
}
+233 -1
View File
@@ -1,7 +1,7 @@
//! Synthetic DNG round trips: every storage variant we decode is written with our own TIFF writer (and LJ92
//! encoder) and must decode bit-exactly.
use craft_raw::opcodes::{Area, Opcode};
use craft_raw::opcodes::{self, Area, Opcode};
use craft_raw::{
BlackLevel, Cfa, ColorData, DngCompression, DngWriteOptions, OpcodeLists, Orientation, RawData, RawFormat, RawImage, Rect, decode,
embedded_preview, probe, write_dng,
@@ -58,6 +58,7 @@ fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
color,
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera,
@@ -403,6 +404,172 @@ fn floating_point_dngs() {
assert_eq!(decode(&dng(raw, ByteOrder::Big)).unwrap().data, RawData::F32(vals));
}
/// The chunks (tiles of `tw` x `th`, or strips when `tw == w`) of an image of `cpp` interleaved samples per pixel, each
/// nominal-sized (strips: clipped to the image height) with zero padding past the image edge: one chunk per tile, and for `planar` one set of chunks per
/// plane (one sample per pixel), plane after plane.
fn chunk_samples<T: Copy + Default>(px: &[T], w: usize, h: usize, cpp: usize, tw: usize, th: usize, planar: bool) -> Vec<Vec<T>> {
let planes = if planar { cpp } else { 1 };
let ccpp = if planar { 1 } else { cpp };
let mut out = Vec::new();
for plane in 0..planes {
for ty in 0..h.div_ceil(th) {
for tx in 0..w.div_ceil(tw) {
// a strip holds only the rows inside the image, a tile its nominal size
let rows = if tw == w { th.min(h - ty * th) } else { th };
let mut c = vec![T::default(); tw * rows * ccpp];
for y in 0..th.min(h - ty * th) {
for x in 0..tw.min(w - tx * tw) {
for s in 0..ccpp {
let src = ((ty * th + y) * w + tx * tw + x) * cpp + if planar { plane } else { s };
c[(y * tw + x) * ccpp + s] = px[src];
}
}
}
out.push(c);
}
}
}
out
}
fn layout_image(raw: &mut IfdBuilder, chunks: Vec<Vec<u8>>, w: usize, tile: (usize, usize)) {
raw.set_image(if tile.0 == w {
ImageData::Strips { rows_per_strip: tile.1 as u32, strips: chunks }
} else {
ImageData::Tiles { tile_width: tile.0 as u32, tile_height: tile.1 as u32, tiles: chunks }
});
}
/// Uncompressed integer DNGs in every chunk layout (chunky and planar, tiles and strips with edge chunks, several chunks
/// per image) at several bit depths and byte orders decode to the samples they were built from.
#[test]
fn uncompressed_chunk_layouts_place_every_sample() {
let (w, h) = (37, 23);
for cpp in [1usize, 3] {
for bits in [8u32, 12, 16] {
let px = pattern(w, h, cpp, bits);
// tiles overhanging both edges, strips with a shorter last strip
for tile in [(16, 16), (w, 5), (w, h), (16, 32)] {
for planar in [false, true] {
if planar && cpp == 1 {
continue;
}
for order in [ByteOrder::Little, ByteOrder::Big] {
let ccpp = if planar { 1 } else { cpp };
let chunks: Vec<Vec<u8>> = chunk_samples(&px, w, h, cpp, tile.0, tile.1, planar)
.iter()
.map(|c| match bits {
8 => c.iter().map(|&v| v as u8).collect(),
16 => u16_bytes(c, order),
_ => pack_msb(c, bits, tile.0 * ccpp),
})
.collect();
let mut raw = base_ifd(w, h, bits as u16, cpp, cpp == 1);
raw.set(t::COMPRESSION, Value::Short(vec![1]));
if planar {
raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
}
layout_image(&mut raw, chunks, w, tile);
let img = decode(&dng(raw, order)).unwrap();
assert_eq!(img.cpp, cpp);
assert_eq!(img.data, RawData::U16(px.clone()), "cpp {cpp} bits {bits} tile {tile:?} planar {planar} {order:?}");
}
}
}
}
}
}
/// Deflate with the integer and floating-point predictors, chunky and planar tiles and strips.
#[test]
fn deflate_chunk_layouts_place_every_sample() {
let (w, h) = (37, 23);
for planar in [false, true] {
let cpp = 3;
let ccpp = if planar { 1 } else { cpp };
let px = pattern(w, h, cpp, 16);
for tile in [(16, 16), (w, 5)] {
// integer predictor 2 on 16-bit samples
let chunks: Vec<Vec<u8>> = chunk_samples(&px, w, h, cpp, tile.0, tile.1, planar)
.into_iter()
.map(|mut c| {
for row in c.chunks_mut(tile.0 * ccpp) {
for i in (ccpp..row.len()).rev() {
row[i] = row[i].wrapping_sub(row[i - ccpp]);
}
}
miniz_oxide::deflate::compress_to_vec_zlib(&u16_bytes(&c, ByteOrder::Little), 6)
})
.collect();
let mut raw = base_ifd(w, h, 16, cpp, false);
raw.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE]));
raw.set(t::PREDICTOR, Value::Short(vec![2]));
if planar {
raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
}
layout_image(&mut raw, chunks, w, tile);
let img = decode(&dng(raw, ByteOrder::Little)).unwrap();
assert_eq!(img.data, RawData::U16(px.clone()), "integer predictor, tile {tile:?} planar {planar}");
}
}
}
/// Floating-point DNGs (uncompressed and with the floating-point predictor) in tiled and strip, chunky and planar
/// layouts with several chunks and edge chunks.
#[test]
fn float_chunk_layouts_place_every_sample() {
let (w, h) = (37, 23);
for cpp in [1usize, 3] {
let vals: Vec<f32> = (0..w * h * cpp).map(|i| ((i * 37) % 200) as f32 / 64.0).collect();
for tile in [(16, 16), (w, 5)] {
for planar in [false, true] {
if planar && cpp == 1 {
continue;
}
let ccpp = if planar { 1 } else { cpp };
let parts = chunk_samples(&vals, w, h, cpp, tile.0, tile.1, planar);
for bits in [16u16, 24, 32] {
let bp = bits as usize / 8;
for (pred, order) in [(1u16, ByteOrder::Little), (1, ByteOrder::Big), (3, ByteOrder::Little)] {
if pred == 1 && bits != 16 && bits != 32 {
continue; // uncompressed 24-bit floats are not written by this test
}
let chunks: Vec<Vec<u8>> = parts
.iter()
.map(|c| {
if pred == 3 {
let enc = fp_encode(c, bp, ccpp, tile.0 * ccpp);
miniz_oxide::deflate::compress_to_vec_zlib(&enc, 6)
} else if bits == 16 {
c.iter()
.flat_map(
|&v| if order == ByteOrder::Big { f32_to_f16(v).to_be_bytes() } else { f32_to_f16(v).to_le_bytes() },
)
.collect()
} else {
c.iter().flat_map(|&v| if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() }).collect()
}
})
.collect();
let mut raw = base_ifd(w, h, bits, cpp, cpp == 1);
raw.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp]));
raw.set(t::COMPRESSION, Value::Short(vec![if pred == 3 { compression::DEFLATE } else { 1 }]));
if pred == 3 {
raw.set(t::PREDICTOR, Value::Short(vec![3]));
}
if planar {
raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
}
layout_image(&mut raw, chunks, w, tile);
let img = decode(&dng(raw, order)).unwrap();
assert_eq!(img.data, RawData::F32(vals.clone()), "cpp {cpp} {bits}-bit pred {pred} tile {tile:?} planar {planar} {order:?}");
}
}
}
}
}
}
#[test]
fn non_finite_float_samples_decode_as_levels() {
// NaN samples in a float CFA DNG spread through any arithmetic downstream: the decoder hands on numbers only
@@ -1177,12 +1344,77 @@ fn white_level_defaults_to_the_linearization_tables_maximum() {
};
let img = build(None);
assert!(img.linearized);
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
assert_eq!(img.data, RawData::U16(codes.iter().map(|&c| table[c as usize]).collect()));
assert_eq!(img.white, vec![51000.0], "255 × 200, the table's top");
// a stated level still wins
assert_eq!(build(Some(40000)).white, vec![40000.0]);
}
/// DNG chapter 7: OpcodeList1 runs on the stored codes, then the LinearizationTable maps them to
/// sensor values. When list 1 is present, decode leaves the table unapplied so the develop pipeline
/// can keep that order. Applying the table first would give a different picture.
#[test]
fn opcode_list1_runs_on_stored_samples_before_linearization() {
let (w, h) = (4, 2);
let stored = vec![2u16; w * h];
// table: 2 → 20, 7 → 30, 20 → 99 (so the two orders disagree)
let mut table = vec![0u16; 32];
table[2] = 20;
table[7] = 30;
table[20] = 99;
// MapTable: 2 → 7, 20 → 8
let mut map: Vec<u16> = (0..32).collect();
map[2] = 7;
map[20] = 8;
let area = Area { top: 0, left: 0, bottom: h as u32, right: w as u32, plane: 0, planes: 1, row_pitch: 1, col_pitch: 1 };
let list1 = opcodes::write_list(&[Opcode::MapTable { area, table: map.clone() }]);
let mut raw = base_ifd(w, h, 16, 1, true);
raw.set(t::COMPRESSION, Value::Short(vec![compression::NONE]));
raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![u16_bytes(&stored, ByteOrder::Little)] });
raw.set(t::LINEARIZATION_TABLE, Value::Short(table.clone()));
raw.set(t::OPCODE_LIST_1, Value::Undefined(list1));
raw.set(t::WHITE_LEVEL, Value::Long(vec![100]));
let img = decode(&dng(raw, ByteOrder::Little)).unwrap();
assert!(!img.linearized, "list 1 is present, so the table must wait");
assert_eq!(img.data, RawData::U16(stored.clone()));
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
assert_eq!(img.opcodes.list1.len(), 1);
let apply_map = |samples: &mut [u16]| {
for s in samples {
let i = (*s as usize).min(map.len() - 1);
*s = map[i];
}
};
let mut correct = img.clone();
if let RawData::U16(v) = &mut correct.data {
apply_map(v);
}
correct.apply_linearization();
assert!(correct.linearized);
// 2 → 7 (MapTable) → 30 (linearization)
assert_eq!(correct.data, RawData::U16(vec![30; w * h]));
let mut wrong = img.clone();
wrong.apply_linearization();
if let RawData::U16(v) = &mut wrong.data {
apply_map(v);
}
// 2 → 20 (linearization) → 8 (MapTable)
assert_eq!(wrong.data, RawData::U16(vec![8; w * h]));
assert_ne!(correct.data, wrong.data);
// the unapplied table and list 1 survive a write / decode round trip
let back = decode(&write_dng(&img, &DngWriteOptions { compression: DngCompression::Uncompressed, ..Default::default() }).unwrap()).unwrap();
assert!(!back.linearized);
assert_eq!(back.data, RawData::U16(stored));
assert_eq!(back.linearization_table.as_deref(), Some(table.as_slice()));
assert_eq!(back.opcodes.list1.len(), 1);
}
/// PhotoCraft: `AsShotWhiteXY` is a chromaticity, so both coordinates lie inside (0, 1).
#[test]
fn an_as_shot_white_outside_the_chromaticity_range_is_ignored() {
+1
View File
@@ -22,6 +22,7 @@ fn sample(comp: DngCompression) -> Vec<u8> {
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Default::default(),
+4
View File
@@ -659,6 +659,10 @@ impl Writer {
Some(Align::Distribute) => out.push_str("\\qd"),
_ => out.push_str("\\ql"),
}
// The paragraph's text direction, as Word writes it (`\rtlpar`; `\ltrpar` is the default).
if p.rtl {
out.push_str("\\rtlpar");
}
if p.page_break {
out.push_str("\\pagebb");
}
+3
View File
@@ -148,6 +148,8 @@ pub(super) struct St {
pub(super) hidden: bool,
// Paragraph properties.
pub(super) align: Option<Align>,
/// `\rtlpar`: the paragraph reads right to left.
pub(super) rtldir: Option<bool>,
pub(super) style: i32,
pub(super) intbl: bool,
pub(super) ls: Option<i32>,
@@ -185,6 +187,7 @@ impl Default for St {
uc: 1,
hidden: false,
align: None,
rtldir: None,
style: 0,
intbl: false,
ls: None,
+9
View File
@@ -30,6 +30,7 @@ impl Reader {
out.italic = out.italic.or(d.italic);
out.size = out.size.or(d.size);
out.font = out.font.or(d.font);
out.rtl = out.rtl.or(d.rtl);
cur = d.based_on;
}
out
@@ -87,6 +88,8 @@ impl Reader {
pub(super) fn para_props(&self, st: &St) -> Para {
let mut p = Para::new(self.para_kind(st));
p.align = st.align;
// Direct `\rtlpar`/`\ltrpar` wins over the style's direction.
p.rtl = st.rtldir.or_else(|| self.file_style(st.style).rtl).unwrap_or(false);
p.page_break = st.pagebb;
p.spacing = spacing(st);
p.borders = st.borders.any_visible().then_some(st.borders);
@@ -135,6 +138,8 @@ pub(super) struct StyleDef {
pub(super) size: Option<f32>,
/// A `\fonttbl` index, named when it's used (the table may follow).
pub(super) font: Option<i32>,
/// `\rtlpar` (`Some(true)`) or `\ltrpar`: the paragraph direction the style sets.
pub(super) rtl: Option<bool>,
}
/// Clear from `f` the formatting a paragraph of kind `kind` gets from its style, whose
@@ -203,6 +208,10 @@ pub(super) fn is_format_word(w: &str) -> bool {
| "s"
| "outlinelevel"
| "pagebb"
| "rtlpar"
| "ltrpar"
| "rtldir"
| "ltrdir"
| "intbl"
| "ls"
| "ilvl"
+123
View File
@@ -38,6 +38,86 @@ fn basic_reader() {
assert!(matches!(&l.inlines[0], Inline::Text(_, f) if f.link.as_deref() == Some("http://a.b")));
}
#[test]
fn rtl_paragraphs_read_and_write_their_direction() {
// Word writes an RTL paragraph as `\rtlpar` (`\ltrpar` for LTR); reading it keeps the
// direction as w:bidi on the document paragraph, and export writes it back.
let rtf = br#"{\rtf1\ansi\pard\rtlpar RTL\par\pard\ltrpar LTR\par\pard plain\par}"#;
let flow = parse(rtf).unwrap();
let paras: Vec<&Para> = flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect();
assert!(paras[0].rtl, "rtlpar sets the direction");
assert!(!paras[1].rtl && !paras[2].rtl, "ltrpar and no word: not RTL");
// \rtldir and \ltrdir, which some writers used for paragraphs, are read too.
let old = parse(br#"{\rtf1\ansi\pard\rtldir RTL\par\pard\ltrdir LTR\par}"#).unwrap();
let old: Vec<&Para> = old.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect();
assert!(old[0].rtl && !old[1].rtl, "rtldir and ltrdir are read");
// \pard resets the direction.
let reset = parse(br#"{\rtf1\ansi\pard\rtlpar A\par\pard B\par}"#).unwrap();
let reset: Vec<&Para> = reset.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect();
assert!(reset[0].rtl && !reset[1].rtl, "pard resets the direction");
// Import keeps it as bidi on the document paragraph.
let bidi = |d: &Document| {
d.body.iter().filter_map(|b| if let Block::Para(p) = &**b { Some(p.props.bidi.unwrap_or(false)) } else { None }).collect::<Vec<_>>()
};
let doc = import(rtf).unwrap();
assert_eq!(bidi(&doc), [true, false, false]);
// Export writes \rtlpar once, and not the \rtldir.
let out = export(&doc);
assert_eq!(out.matches("\\rtlpar").count(), 1, "{out}");
assert!(!out.contains("rtldir"), "{out}");
assert_eq!(bidi(&import(out.as_bytes()).unwrap()), [true, false, false]);
}
#[test]
fn rtl_paragraph_styles_carry_their_direction() {
// Word often writes the direction in the style: `{\s1\rtlpar RTL;}` then `\pard\s1`.
let rtf = br#"{\rtf1\ansi{\stylesheet{\s0 Normal;}{\s1\rtlpar RTL;}{\s2\sbasedon1 Child;}{\s3\sbasedon1\ltrpar Back;}}
\pard\s1 A\par\pard\s2 B\par\pard\s3 C\par\pard\s1\ltrpar D\par\pard\s0\rtlpar E\par\pard F\par}"#;
let flow = parse(rtf).unwrap();
let rtl: Vec<bool> = flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.rtl) } else { None }).collect();
// A: style. B: inherited from the base style. C: the style overrides its base. D: direct
// \ltrpar beats the style. E: direct \rtlpar on Normal. F: \pard drops direct formatting.
assert_eq!(rtl, [true, true, false, false, true, false]);
}
#[test]
fn oldpprops_keep_the_old_paragraph_direction() {
// A tracked change of direction: now LTR, before RTL (`\rtlpar` in the old properties).
let rtf = br"{\rtf1\ansi{\*\revtbl {Unknown;}{Alice;}}
\pard\ltrpar\prauth1\prdate-2014675576{\*\oldpprops \pard \rtlpar}A\par}";
let flow = parse(rtf).unwrap();
let Some(FBlock::Para(p)) = flow.blocks.first() else { panic!("no paragraph") };
assert!(!p.rtl);
assert!(p.fmt_change.as_deref().expect("paragraph change").old.rtl, "old direction is RTL");
let doc = import(rtf).unwrap();
let c = doc_paras(&doc)[0].props.fmt_change.as_deref().expect("change");
assert_eq!(c.old.bidi, Some(true));
}
#[test]
fn tracked_rtl_to_ltr_change_from_a_document_keeps_the_old_direction() {
use craft_wordmodel::props::{CharProps, ParaProps, PropChange};
use craft_wordmodel::{Revision, RevisionKind};
// Document origin (not RTF): now LTR, before RTL.
let mut p = craft_wordmodel::Paragraph::with_text("A", CharProps::default());
p.props.bidi = Some(false);
p.props.fmt_change = PropChange::boxed(0, ParaProps { bidi: Some(true), ..Default::default() });
let doc = doc_with(vec![p], vec![Revision::new(RevisionKind::Format, "Bob", "2025-01-02T08:05:00Z")]);
let flow = model::from_doc_tracked(&doc);
let Some(FBlock::Para(fp)) = flow.blocks.first() else { panic!("no paragraph") };
assert!(!fp.rtl);
assert!(fp.fmt_change.as_deref().expect("change").old.rtl, "snapshot keeps RTL");
let rtf = export(&doc);
assert!(rtf.contains("{\\*\\oldpprops"), "{rtf}");
let back = import(rtf.as_bytes()).unwrap();
let p = doc_paras(&back)[0];
assert_ne!(p.props.bidi, Some(true), "current direction is LTR");
let change = p.props.fmt_change.as_deref().expect("change read back");
assert_eq!(change.old.bidi, Some(true));
// Reject restores RTL.
assert_eq!(change.old.formatting().bidi, Some(true));
}
#[test]
fn stylesheet_names_take_unicode_escapes() {
// \u116?=t \u105?=i \u116?=t \u108?=l \u101?=e — how Word writes a non-Latin style name.
@@ -1863,3 +1943,46 @@ fn compound_borders_round_trip() {
let back = import(export(&d).as_bytes()).unwrap();
assert_eq!(tops(&back), want);
}
fn para_aligns(flow: &Flow) -> Vec<(bool, Option<Align>)> {
flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some((p.rtl, p.align)) } else { None }).collect()
}
#[test]
fn alignment_words_after_rtlpar_mean_start_and_end() {
// Word reads `\ql`/`\qr` that follow `\rtlpar` as start/end (the model's alignment is
// logical), so `\rtlpar\qr` sits at the right edge and `\rtlpar\ql` at the left edge.
// Words written before `\rtlpar` keep their plain meaning (checked in Word by saving to DOCX:
// `\qr\rtlpar` gives jc=right, i.e. end, in the bidi paragraph; `\ql\rtlpar` gives no jc).
let rtf = br#"{\rtf1\ansi\deff0{\fonttbl{\f0 Arial;}}\pard LTR one.\par \pard\rtlpar\qr A.\par \pard\rtlpar B.\par \pard\ltrpar Back.\par \pard\rtlpar\ql C.\par \pard\rtlpar\qc D.\par \pard\ql\rtlpar E.\par \pard\qr\rtlpar F.\par \pard\rtlpar\ltrpar\qr G.\par}"#;
let got = para_aligns(&parse(rtf).unwrap());
assert_eq!(
got,
[
(false, None),
(true, Some(Align::Left)),
(true, None),
(false, None),
(true, Some(Align::Right)),
(true, Some(Align::Center)),
(true, Some(Align::Left)),
(true, Some(Align::Right)),
(false, Some(Align::Right)),
]
);
}
#[test]
fn alignment_survives_an_export_import_round_trip() {
let rtf = br#"{\rtf1\ansi\pard\ql L.\par \pard\qr R.\par \pard\qc C.\par \pard\rtlpar\qr start.\par \pard\rtlpar\ql end.\par \pard\rtlpar\qc mid.\par \pard\rtlpar plain.\par}"#;
let first = para_aligns(&parse(rtf).unwrap());
let doc = import(rtf).unwrap();
let out = export(&doc);
let second = para_aligns(&parse(out.as_bytes()).unwrap());
let norm = |v: Vec<(bool, Option<Align>)>| v.into_iter().map(|(r, a)| (r, a.filter(|a| *a != Align::Left))).collect::<Vec<_>>();
assert_eq!(norm(first), norm(second.clone()), "{out}");
// `\rtlpar\qr` is start (no explicit alignment in the model) and `\rtlpar\ql` is end.
assert_eq!(norm(second)[3..5], [(true, None), (true, Some(Align::Right))], "{out}");
// The export writes the alignment before the direction, which Word reads as logical start/end.
assert!(out.contains("\\qr\\rtlpar"), "{out}");
}
+20 -2
View File
@@ -3,6 +3,18 @@ use super::styles::{StyleDef, is_char_format_word, is_format_word};
use super::{Align, Fmt, Reader, VAlign, model};
impl Reader {
/// The logical alignment for a side word (`Left` for `\ql`, `Right` for `\qr`). The model's
/// alignment is logical (start/end), and Word reads a side word that comes after `\rtlpar` as
/// start/end: the sides swap in a paragraph already marked right to left. Before `\rtlpar`
/// the side keeps its plain meaning, which is why the writer puts the alignment first.
fn side_align(&self, side: Align) -> Align {
match (self.st.rtldir, side) {
(Some(true), Align::Left) => Align::Right,
(Some(true), Align::Right) => Align::Left,
_ => side,
}
}
pub(super) fn word(&mut self, w: &str, p: Option<i32>, starred: bool, first_in_group: bool) {
// Every control word ends a multibyte run: the code page it is read in may change here
// (`\f`, `\fcharset`, `\cpg`, `\ansicpg`, `\pard`), and `\par` must see the text before it.
@@ -166,11 +178,15 @@ impl Reader {
"cf" => self.st.fmt.color = self.colors.get(pv.max(0) as usize).copied().flatten(),
"highlight" | "chcbpat" | "cb" => self.st.fmt.background = self.colors.get(pv.max(0) as usize).copied().flatten(),
"v" => self.st.hidden = on,
"ql" => self.st.align = Some(Align::Left),
"ql" => self.st.align = Some(self.side_align(Align::Left)),
"qc" => self.st.align = Some(Align::Center),
"qr" => self.st.align = Some(Align::Right),
"qr" => self.st.align = Some(self.side_align(Align::Right)),
"qj" => self.st.align = Some(Align::Justify),
"qd" => self.st.align = Some(Align::Distribute),
// `\rtldir`/`\ltrdir` are not paragraph words (they set a table's or section's
// direction in RTF); some writers used them for paragraphs, so they are read too.
"rtlpar" | "rtldir" => self.st.rtldir = Some(true),
"ltrpar" | "ltrdir" => self.st.rtldir = Some(false),
"s" => self.st.style = pv,
"outlinelevel" => self.st.outline = Some(pv.clamp(0, 9) as u8),
"pagebb" => self.st.pagebb = on,
@@ -417,6 +433,8 @@ impl Reader {
"i" => def.italic = Some(on),
"fs" => def.size = (value > 0).then(|| (value as f32 / 2.0).clamp(1.0, 1638.0)),
"f" => def.font = Some(value),
"rtlpar" | "rtldir" => def.rtl = Some(true),
"ltrpar" | "ltrdir" => def.rtl = Some(false),
_ => {}
}
true
+1 -1
View File
@@ -2,6 +2,6 @@
Bounded wire primitives for USD Crate, independent of application scene and material models.
Provides bounded integer-delta array decoding and Crate-framed LZ4 decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and export remain outside this extraction. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation.
Provides bounded integer-delta array decoding and Crate-framed LZ4 compression/decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and full scene export remain outside this extraction. Compression uses a caller cap on worst-case reserved output and fallible allocation. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation.
Adapted from TextureCraft `crates/formats/src/mesh_import/usd/codec.rs`, introduced by bflatastic in `bdbb52c9b66b221a55b370105b42f485a76e6fc7`. The LZ4 backend is MIT-licensed `lz4_flex`.
+76
View File
@@ -0,0 +1,76 @@
//! Bounded single-block USD Crate LZ4 writing.
/// Compression admission, allocation or backend failure.
#[derive(Debug, thiserror::Error)]
pub enum Error {
/// Input or worst-case output exceeds the supported/caller budget.
#[error("USD Crate LZ4 compression exceeds budget")]
Limit,
/// The bounded output reservation failed.
#[error("cannot allocate USD Crate LZ4 output")]
Allocation,
/// The backend could not encode into the preflighted output buffer.
#[error("USD Crate LZ4 compression: {0}")]
Encode(#[from] lz4_flex::block::CompressError),
}
fn capacity(input: usize, max_output_bytes: usize) -> Result<usize, Error> {
// Keep inputs below the LZ4 single-block limit and preflight the backend's
// 20 + input*110/100 sizing arithmetic before calling it, including the count byte.
if input > 0x7e00_0000 {
return Err(Error::Limit);
}
input
.checked_mul(110)
.map(|n| n / 100)
.and_then(|n| n.checked_add(21))
.filter(|n| *n <= max_output_bytes && *n <= isize::MAX as usize)
.ok_or(Error::Limit)
}
/// Write one Crate-framed LZ4 block, beginning with a zero chunk-count byte.
///
/// `max_output_bytes` bounds the reserved output including framing. Admission uses
/// the backend's worst-case size, so even highly compressible input must fit that bound.
/// Allocation is fallible; backend scratch memory is not included in the output cap.
/// Uses TextureCraft's original LZ4 backend/configuration and produces the same block bytes.
pub fn compress(input: &[u8], max_output_bytes: usize) -> Result<Vec<u8>, Error> {
let capacity = capacity(input.len(), max_output_bytes)?;
let mut output = Vec::new();
output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?;
output.resize(capacity, 0);
let target = output.get_mut(1..).ok_or(Error::Limit)?;
let written = lz4_flex::block::compress_into(input, target)?;
output.truncate(written.checked_add(1).ok_or(Error::Limit)?);
Ok(output)
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use super::*;
#[test]
fn original_wire_bytes_and_roundtrip() {
for size in [0, 1, 15, 16, 255, 4096, 65536] {
for repeated in [false, true] {
let input: Vec<u8> = (0..size).map(|i| if repeated { 17 } else { (i % 251) as u8 }).collect();
let mut old = vec![0];
old.extend(lz4_flex::block::compress(&input));
let encoded = compress(&input, 1 << 20).unwrap();
assert_eq!(encoded, old);
assert_eq!(crate::decompress(&encoded, size).unwrap(), input);
}
}
}
#[test]
fn checks_worst_case_before_allocation() {
let input = [0; 100];
assert!(matches!(compress(&input, 130), Err(Error::Limit)));
assert!(compress(&input, 131).is_ok());
assert!(capacity(usize::MAX, usize::MAX).is_err());
assert!(capacity(0x7e00_0001, usize::MAX).is_err());
assert_eq!(capacity(0, 21).unwrap(), 21);
}
}
+144
View File
@@ -80,6 +80,72 @@ pub fn decode(data: &[u8], count: usize, width: Width, max_output_bytes: usize)
Ok(output)
}
/// Encode integers with their original signed wrapping and largest-delta tie-break policy.
///
/// `max_bytes` bounds the combined worst-case output reservation and sorted delta scratch.
/// Both reservations are fallible. Sorting uses no extra heap storage; input storage and
/// allocator bookkeeping are outside this budget. Empty input emits no bytes.
pub fn encode(values: &[i64], width: Width, max_bytes: usize) -> Result<Vec<u8>, Error> {
if values.is_empty() {
return Ok(Vec::new());
}
let size = width.bytes();
let codes_len = values.len().div_ceil(4);
let capacity = values.len().checked_mul(size).and_then(|n| n.checked_add(size)).and_then(|n| n.checked_add(codes_len)).ok_or(Error::Limit)?;
let scratch = values.len().checked_mul(8).ok_or(Error::Limit)?;
if capacity > isize::MAX as usize || scratch > isize::MAX as usize || capacity.checked_add(scratch).is_none_or(|n| n > max_bytes) {
return Err(Error::Limit);
}
let wrap = |v: i64| match width {
Width::I32 => i64::from(v as i32),
Width::I64 => v,
};
let mut deltas = Vec::new();
deltas.try_reserve_exact(values.len()).map_err(|_| Error::Allocation)?;
let mut previous = 0i64;
for &value in values {
deltas.push(wrap(value.wrapping_sub(previous)));
previous = value;
}
deltas.sort_unstable();
let mut best = (0usize, 0i64);
let mut at = 0;
while at < deltas.len() {
let value = deltas[at];
let start = at;
while at < deltas.len() && deltas[at] == value {
at += 1;
}
best = best.max((at - start, value));
}
let common = best.1;
let mut output = Vec::new();
output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?;
output.extend_from_slice(&common.to_le_bytes()[..size]);
output.resize(size + codes_len, 0);
previous = 0;
for (i, &value) in values.iter().enumerate() {
let delta = wrap(value.wrapping_sub(previous));
previous = value;
let small = |bits: u32| delta >= -(1i64 << (bits - 1)) && delta < (1i64 << (bits - 1));
let code: u8 = if delta == common {
0
} else if small(size as u32 * 2) {
1
} else if small(size as u32 * 4) {
2
} else {
3
};
if code != 0 {
let bytes = (size >> (3 - usize::from(code))).max(1);
output.extend_from_slice(&delta.to_le_bytes()[..bytes]);
}
output[size + i / 4] |= code << (2 * (i % 4));
}
Ok(output)
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
@@ -125,3 +191,81 @@ mod tests {
}
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod encode_tests {
use super::*;
fn oracle(values: &[i64], width: usize) -> Vec<u8> {
if values.is_empty() {
return Vec::new();
}
let wrap = |v: i64| if width == 4 { i64::from(v as i32) } else { v };
let mut prev = 0i64;
let deltas: Vec<i64> = values
.iter()
.map(|&v| {
let d = wrap(v.wrapping_sub(prev));
prev = v;
d
})
.collect();
let mut counts = std::collections::HashMap::new();
for d in &deltas {
*counts.entry(*d).or_insert(0usize) += 1;
}
let common = counts.into_iter().max_by_key(|&(v, c)| (c, v)).map_or(0, |(v, _)| v);
let mut out = Vec::with_capacity(width + values.len().div_ceil(4) + values.len() * width);
out.extend_from_slice(&common.to_le_bytes()[..width]);
let codes_at = out.len();
out.resize(codes_at + values.len().div_ceil(4), 0);
for (i, &d) in deltas.iter().enumerate() {
let small = |bits: u32| d >= -(1i64 << (bits - 1)) && d < (1i64 << (bits - 1));
let code: u8 = if d == common {
0
} else if small(width as u32 * 2) {
1
} else if small(width as u32 * 4) {
2
} else {
3
};
if code > 0 {
let w = (width >> (3 - code as usize)).max(1);
out.extend_from_slice(&d.to_le_bytes()[..w]);
}
if let Some(b) = out.get_mut(codes_at + i / 4) {
*b |= code << (2 * (i % 4));
}
}
out
}
#[test]
fn matches_original_bytes_and_wrapping() {
for width in [Width::I32, Width::I64] {
for n in [0, 1, 2, 3, 4, 5, 127, 1024] {
let mut seed = 17u64;
let values: Vec<i64> = (0..n)
.map(|_| {
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
seed as i64
})
.collect();
let bytes = encode(&values, width, 32768).unwrap();
assert_eq!(bytes, oracle(&values, width.bytes()));
let expected: Vec<i64> = values.iter().map(|&v| if width == Width::I32 { i64::from(v as i32) } else { v }).collect();
assert_eq!(decode(&bytes, n, width, 32768).unwrap(), expected);
}
for values in [vec![1, 3, 4, 6], vec![i64::MAX, i64::MIN, 0], vec![7; 32]] {
assert_eq!(encode(&values, width, 4096).unwrap(), oracle(&values, width.bytes()));
}
}
}
#[test]
fn combined_reservation_budget_is_enforced() {
assert_eq!(encode(&[], Width::I64, 0).unwrap(), Vec::<u8>::new());
assert!(matches!(encode(&[1], Width::I32, 16), Err(Error::Limit)));
assert!(encode(&[1], Width::I32, 17).is_ok());
}
}
+3
View File
@@ -5,6 +5,9 @@
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
/// Bounded single-block Crate LZ4 compression.
pub mod compress;
/// Bounded Crate integer delta arrays.
pub mod integers;
+49 -5
View File
@@ -73,11 +73,17 @@ with libdav1d: **bit-exact** on all 6 fixtures.
`Decoder::new()` uses all cores (`Decoder::with_threads(1)` decodes on the calling thread; on
wasm32, or without the default `threads` feature, everything runs on the calling thread). Headers are parsed in order; each frame is
then decoded by a frame worker once the reference frames it reads are finished, so independent
frames (hierarchical GOP layers) decode concurrently; tiles of a frame decode in parallel into
private buffers that are merged afterwards; the in-loop filters (deblocking, CDEF, loop
restoration) each run in parallel bands of 64 rows (`postfilter.rs` explains why the bands are
independent). Results never depend on scheduling (the oracle tests compare 1 thread, many threads
then decoded by a frame worker. With worker threads it first reads the symbols of all its tiles,
which needs only the parse results of its references (CDFs, segment ids, saved motion vectors),
publishes its own, and then reconstructs from the recorded blocks. Frames with several tiles
read the symbols of their tiles in parallel (one after another with intra block copy). A frame reconstructs superblock row by superblock
row (that row of every tile), each row waiting only for the reference rows its motion vectors
reach (row wavefront), and the in-loop filters (deblocking, CDEF, loop restoration) follow a few
64-row bands behind and publish each final band for the frames predicting from it
(`decoder.rs` and `postfilter.rs` give the dependency rules). Super-resolution frames wait for
their whole references and filter the whole frame. There is one frame worker per thread (at most
16), and the frames in flight are bounded by their size. Independent frames (hierarchical GOP
layers) decode concurrently. Results never depend on scheduling (the oracle tests compare 1 thread, many threads
and libdav1d). With frame threads, pictures may be returned by a
later `decode_pts` call or by `flush`; each carries the `pts` of the temporal unit that showed it.
@@ -130,6 +136,44 @@ motion vectors nor CDFs reach another frame — skip deblocking, CDEF and loop r
(super-resolution still runs, it sets the output size) and are flagged `Picture::draft`. In
SVT-AV1's hierarchical GOPs that is the top layer, half of the frames.
Frame threading on a Ryzen 7800X3D (8 cores / 16 threads, release `examples/av1_av1dec`, median of 7
alternating runs against Filmcraft `6909e461`, 2026-11-08), wall time for the whole file:
| Fixture | Threads | Before | After | Speed-up |
|---|---|---|---|---|
| `perf_1080p_gop` (60 frames) | 1 | 69.2 fps | 69.1 fps | 1.00x |
| `perf_1080p_gop` | 16 | 159.6 fps | 227.3 fps | 1.42x |
| `perf_2160p_gop` (30 frames) | 1 | 17.9 fps | 17.8 fps | 1.00x |
| `perf_2160p_gop` | 16 | 40.2 fps | 57.7 fps | 1.43x |
Peak RSS at 16 threads goes from 278 to 365 MB (1080p) and from 1031 to 1217 MB (2160p, +18%):
every frame in flight holds its parse record and its final samples are kept in 64-row bands.
With one frame worker per thread, the row wavefront for frames with several tiles and parse
waits limited to what parsing reads (same machine, median of 5 alternating runs, 2026-11-08):
| Fixture | Threads | Before (`6909e461`) | After | Speed-up | Peak RSS before / after |
|---|---|---|---|---|---|
| `perf_1080p_gop` | 4 | 95.7 fps | 185.2 fps | 1.94x | 148 / 236 MB |
| `perf_1080p_gop` | 16 | 155.0 fps | 259.7 fps | 1.68x | 274 / 560 MB |
| `perf_2160p_gop` | 4 | 23.6 fps | 47.1 fps | 2.00x | 512 / 798 MB |
| `perf_2160p_gop` | 16 | 39.9 fps | 65.8 fps | 1.65x | 994 / 1353 MB |
| `perf_1080p_gop_tiles` (4x2 tiles) | 16 | 186.3 fps | 237.2 fps | 1.27x | 328 / 617 MB |
| SVT-AV1 2160p, 4x2 tiles (30 frames) | 16 | 57.1 fps | 70.8 fps | 1.24x | 1197 / 1502 MB |
Known limitations of the frame threading (the output is the same in every case, only the
overlap is smaller):
- A frame whose parse record would pass 32 bytes per luma sample of a tile (pathological streams:
tiny blocks, large palettes, every coefficient coded), or that cannot reserve its record from
the decoder-wide budget of 1 GiB shared by the frames in flight, is decoded again in one pass.
- A superblock row using local warped motion, global motion beyond translation or a scaled
reference waits for the whole reference instead of the rows its motion vectors reach.
- Super-resolution frames do not use the row wavefront: they wait for their whole references
and filter the whole frame.
- The tile columns of a superblock row are reconstructed one after another on the frame's worker
(their symbols are still read in parallel).
## Limits
Frames beyond the AV1 level 6.x limits (Annex A: at most 16384 wide, 8704 high and 35 651 584 luma samples
+21 -9
View File
@@ -10,19 +10,33 @@ use crate::av1::decode::state::FrameState;
/// equals CurrFrame). Bands of 64 luma rows (whole 64x64 filter blocks) are filtered in
/// parallel: every block reads only CurrFrame and writes only its own samples.
pub(crate) fn apply(fs: &FrameState, pool: &Pool) -> Option<FrameBuf> {
if !active(fs) {
return None;
}
let mut out = fs.cur.clone();
let ranges = bands(pool, fs.fh.mi_rows as usize * 4);
filter_bands(fs, pool, out.row_bands(&ranges), &ranges);
Some(out)
}
/// Whether CDEF filters any block of the frame (it reads only parse results).
pub(crate) fn active(fs: &FrameState) -> bool {
let fh = &fs.fh;
if fh.coded_lossless || fh.allow_intrabc || !fs.seq.enable_cdef {
return None;
return false;
}
let (mi_rows, mi_cols) = (fh.mi_rows as usize, fh.mi_cols as usize);
let fb_rows = mi_rows.div_ceil(16);
let fb_cols = mi_cols.div_ceil(16);
if !(0..fb_rows).any(|fr| (0..fb_cols).any(|fc| fb_skip_mask(fs, fr * 16, fc * 16).is_some())) {
return None;
}
let mut out = fs.cur.clone();
let ranges = bands(pool, mi_rows * 4);
let mut planes = out.row_bands(&ranges);
(0..fb_rows).any(|fr| (0..fb_cols).any(|fc| fb_skip_mask(fs, fr * 16, fc * 16).is_some()))
}
/// Filter the bands `planes` covering luma rows `ranges` (whole 64x64 filter block rows), which
/// hold CurrFrame; reads CurrFrame up to 2 rows beyond each band.
pub(crate) fn filter_bands(fs: &FrameState, pool: &Pool, mut planes: Vec<[RowBand; 3]>, ranges: &[(usize, usize)]) {
let (mi_rows, mi_cols) = (fs.fh.mi_rows as usize, fs.fh.mi_cols as usize);
let fb_rows = mi_rows.div_ceil(16);
let fb_cols = mi_cols.div_ceil(16);
pool.map_mut(&mut planes, |b, band| {
let (start, end) = ranges[b];
let mut buf = [0i16; BUF_LEN];
@@ -34,8 +48,6 @@ pub(crate) fn apply(fs: &FrameState, pool: &Pool) -> Option<FrameBuf> {
}
}
});
drop(planes);
Some(out)
}
/// The filtered 8x8 blocks of the 64x64 filter block at (r, c): bit `(i << 3) | j` is set for
File diff suppressed because it is too large. Load diff
+166
View File
@@ -197,6 +197,50 @@ impl FrameBuf {
}
}
/// Copy luma rows [y0, y1) (chroma rows scaled down; clipped to the stored rows) of `src`, a
/// frame of the same geometry.
pub fn copy_rows_from(&mut self, src: &FrameBuf, y0: usize, y1: usize) {
for p in 0..self.num_planes {
let sy = if p == 0 { 0 } else { self.subsampling_y };
let (d, s) = (&mut self.planes[p], &src.planes[p]);
if d.stride != s.stride {
continue;
}
let a = (y0 >> sy).min(d.rows).min(s.rows) * d.stride;
let b = (y1 >> sy).min(d.rows).min(s.rows) * d.stride;
if let (Some(dst), Some(src)) = (d.data.get_mut(a..b), s.data.get(a..b)) {
dst.copy_from_slice(src);
}
}
}
/// Copies of luma rows [y0, y1) (chroma rows scaled down; clipped to the stored rows), one
/// plane region each.
pub fn rows_copy(&self, y0: usize, y1: usize) -> [Plane; 3] {
std::array::from_fn(|p| {
if p >= self.num_planes {
return Plane::default();
}
let sy = if p == 0 { 0 } else { self.subsampling_y };
let s = &self.planes[p];
let (a, b) = ((y0 >> sy).min(s.rows), (y1 >> sy).min(s.rows));
let mut out = Plane::region(0, a, s.stride, b - a);
if let Some(src) = s.data.get(a * s.stride..b * s.stride) {
out.data.copy_from_slice(src);
}
out
})
}
/// A frame of this geometry (samples not copied: zero).
pub fn same_geometry(&self) -> FrameBuf {
let planes = std::array::from_fn(|p| {
let pl = &self.planes[p];
Plane { data: zeroed(pl.data.len()), stride: pl.stride, rows: pl.rows, ox: pl.ox, oy: pl.oy }
});
FrameBuf { planes, ..*self }
}
/// [`Plane::row_bands`] of all three planes for ranges of luma rows (chroma rows scaled
/// down): element `i` holds the bands of range `i`.
pub fn row_bands(&mut self, luma: &[(usize, usize)]) -> Vec<[RowBand<'_>; 3]> {
@@ -208,6 +252,128 @@ impl FrameBuf {
}
}
/// Luma rows per published band of a [`BandFrame`] (log2).
pub const BAND_LOG2: u32 = 6;
/// One plane of a frame's final samples, published band by band while the rest of the frame
/// is still being decoded (see [`BandFrame`]). A band is written once and then only read.
pub struct BandPlane {
bands: Vec<std::sync::OnceLock<Plane>>,
stride: usize,
rows: usize,
log2: u32,
/// Returned for rows of a band that is not published: zeros instead of a panic. Readers wait
/// for the row progress first and references of another chroma format are refused, so
/// this only happens through a scheduling bug.
missing: Box<[u16]>,
}
impl BandPlane {
fn new(stride: usize, rows: usize, log2: u32) -> BandPlane {
let n = rows.div_ceil(1 << log2);
BandPlane { bands: (0..n).map(|_| std::sync::OnceLock::new()).collect(), stride, rows, log2, missing: vec![0; stride].into_boxed_slice() }
}
/// Row `y` (`stride` samples) of a published band.
#[inline(always)]
pub fn row(&self, y: usize) -> &[u16] {
let o = (y & ((1 << self.log2) - 1)) * self.stride;
match self.bands.get(y >> self.log2).and_then(|b| b.get()).and_then(|b| b.data.get(o..o + self.stride)) {
Some(r) => r,
None => {
debug_assert!(false, "row {y} read before its band was published");
&self.missing
}
}
}
/// Publish band `band` copied from the rows of the whole plane `src` (same geometry); a
/// band is published once.
fn publish(&self, band: usize, src: &Plane) {
let y0 = band << self.log2;
let y1 = ((band + 1) << self.log2).min(self.rows);
let mut p = Plane::region(0, y0, self.stride, y1.saturating_sub(y0));
let w = self.stride.min(src.stride);
for (y, out) in (y0..y1).zip(p.data.chunks_exact_mut(self.stride.max(1))) {
if let Some(r) = src.data.get(y * src.stride..y * src.stride + w) {
out[..w].copy_from_slice(r);
}
}
self.publish_owned(band, p);
}
/// Publish band `band` as `p`, whose rows start at the band's first row (stride `stride`;
/// rows past the band are ignored).
fn publish_owned(&self, band: usize, p: Plane) {
if let Some(slot) = self.bands.get(band)
&& p.stride == self.stride
{
let _ = slot.set(p);
}
}
}
/// A frame's final samples (every in-loop filter applied) in bands of 64 luma rows, so later
/// frames can predict from the top of a reference while its bottom is still being decoded.
/// Each band is published once, before the frame's row progress says it is final.
pub struct BandFrame {
pub planes: [BandPlane; 3],
pub num_planes: usize,
pub subsampling_x: usize,
pub subsampling_y: usize,
}
impl BandFrame {
/// Bands for the visible rows of a `width` x `height` frame stored like [`FrameBuf::new`].
pub fn new(width: usize, height: usize, num_planes: usize, subsampling_x: usize, subsampling_y: usize) -> BandFrame {
let aw = width.div_ceil(128) * 128 + 160;
let planes = std::array::from_fn(|p| {
if p >= num_planes {
return BandPlane::new(0, 0, BAND_LOG2);
}
let (sx, sy) = if p == 0 { (0, 0) } else { (subsampling_x, subsampling_y) };
BandPlane::new(aw >> sx, (height + sy) >> sy, BAND_LOG2 - sy as u32)
});
BandFrame { planes, num_planes, subsampling_x, subsampling_y }
}
/// Number of bands (of 64 luma rows).
pub fn bands(&self) -> usize {
self.planes[0].bands.len()
}
/// Publish luma band `band` (and the chroma rows it covers) copied from `src`.
pub fn publish(&self, band: usize, src: &FrameBuf) {
for p in 0..self.num_planes {
self.planes[p].publish(band, &src.planes[p]);
}
}
/// Publish luma band `band` from per-plane buffers starting at the band's first row.
pub fn publish_owned(&self, band: usize, planes: [Plane; 3]) {
for (p, pl) in planes.into_iter().enumerate().take(self.num_planes) {
self.planes[p].publish_owned(band, pl);
}
}
}
/// A plane to predict from: a whole stored plane or the published bands of a reference.
#[derive(Clone, Copy)]
pub enum PlaneRef<'a> {
Whole(&'a Plane),
Bands(&'a BandPlane),
}
impl<'a> PlaneRef<'a> {
#[inline(always)]
pub fn row(&self, y: usize) -> &'a [u16] {
match *self {
PlaneRef::Whole(p) => p.row(y),
PlaneRef::Bands(b) => b.row(y),
}
}
}
/// A motion vector (row, col) in 1/8 sample units.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Mv {
+17 -12
View File
@@ -3,7 +3,7 @@
use std::sync::OnceLock;
use crate::av1::decode::frame::Plane;
use crate::av1::decode::frame::{Plane, PlaneRef};
use crate::av1::decode::mvpred::round2_signed64;
use crate::av1::decode::spec_tables::{
BLOCK_8X8, BLOCK_SIZES, COMPOUND_AVERAGE, COMPOUND_DIFFWTD, COMPOUND_DISTANCE, COMPOUND_INTRA, COMPOUND_WEDGE, DIV_LUT, DIV_LUT_BITS,
@@ -227,12 +227,9 @@ impl TileDecoder<'_, '_> {
(start_x as i32, start_y as i32, step_x as i32, step_y as i32)
}
fn ref_plane(&self, ref_idx: i32, plane: usize) -> &Plane {
fn ref_plane(&self, ref_idx: i32, plane: usize) -> PlaneRef<'_> {
// A missing reference (rejected earlier for valid streams) predicts from the current frame.
match usize::try_from(ref_idx).ok().and_then(|i| self.fs.refs.get(i)).and_then(|r| r.as_ref()) {
Some(r) => &r.buf.planes[plane],
None => &self.t.cur.planes[plane],
}
self.fs.ref_samples(ref_idx, plane).unwrap_or(PlaneRef::Whole(&self.t.cur.planes[plane]))
}
/// Block inter prediction process (7.11.3.4) into `pred` (stride PRED_STRIDE); `tmp` holds
@@ -512,12 +509,12 @@ impl TileDecoder<'_, '_> {
return false;
}
let fs = self.fs;
let Some(r) = usize::try_from(ref_idx).ok().and_then(|i| fs.refs.get(i)).and_then(|r| r.as_ref()) else {
let Some(refp) = fs.ref_samples(ref_idx, plane) else {
return false;
};
let dims = self.ref_plane_dims(ref_idx, plane);
let max = ((1u32 << fs.bit_depth) - 1) as u16;
copy_fullpel(&r.buf.planes[plane], dims, McBlock { x: sx, y: sy, w, h }, &mut self.t.cur.planes[plane], (x, y), max);
copy_fullpel(refp, dims, McBlock { x: sx, y: sy, w, h }, &mut self.t.cur.planes[plane], (x, y), max);
true
}
@@ -763,7 +760,7 @@ impl TileDecoder<'_, '_> {
/// Unscaled single-reference prediction at a whole-sample position: both filter phases are the
/// identity and Round2(128 * Round2(128 * v, InterRound0), InterRound1) == v for every bit
/// depth, so the prediction is the (clamped) reference sample.
fn copy_fullpel(refp: &Plane, (last_x, last_y): (i32, i32), src_blk: McBlock, dst: &mut Plane, (dx, dy): (usize, usize), max: u16) {
fn copy_fullpel(refp: PlaneRef, (last_x, last_y): (i32, i32), src_blk: McBlock, dst: &mut Plane, (dx, dy): (usize, usize), max: u16) {
let McBlock { x, y, w, h } = src_blk;
let x0 = x >> 10;
let y0 = y >> 10;
@@ -831,7 +828,7 @@ fn vpass<const W: usize>(rows: &[i32], vf: &[i16; 8], out: &mut [i32], half: i32
/// Block inter prediction without reference scaling: every column / row uses the same filter
/// phase, so the filters run over whole rows (bit-identical to the per-sample process).
fn mc_unscaled(
refp: &Plane,
refp: PlaneRef,
(last_x, last_y): (i32, i32),
blk: McBlock,
(hf, vf): (&[i16; 8], &[i16; 8]),
@@ -982,9 +979,17 @@ mod tests {
[(0, 0), (-5, -3), (7, 9), (last_x - w as i32 / 2, last_y - h as i32 / 2), (last_x + 3, last_y + 2), (-(w as i32) - 4, 3)]
{
let (x, y) = ((px << 10) | 32, (py << 10) | 32);
mc_unscaled(&refp, (last_x, last_y), McBlock { x, y, w, h }, (&IDENTITY_TAPS, &IDENTITY_TAPS), rnd, &mut pred, &mut tmp);
mc_unscaled(
PlaneRef::Whole(&refp),
(last_x, last_y),
McBlock { x, y, w, h },
(&IDENTITY_TAPS, &IDENTITY_TAPS),
rnd,
&mut pred,
&mut tmp,
);
let mut dst = Plane::new(w + 3, h);
copy_fullpel(&refp, (last_x, last_y), McBlock { x, y, w, h }, &mut dst, (3, 0), max);
copy_fullpel(PlaneRef::Whole(&refp), (last_x, last_y), McBlock { x, y, w, h }, &mut dst, (3, 0), max);
for i in 0..h {
for j in 0..w {
let want = spec_phase0(&refp, (last_x, last_y), (px, py), (j, i), (w * 3 + h) % SUBPEL_FILTERS.len(), rnd)
+146 -14
View File
@@ -1,7 +1,7 @@
//! In-loop post filters run by decode_frame_wrapup: loop filter (7.14), CDEF (7.15),
//! super-resolution upscaling (7.16) and loop restoration (7.17).
use crate::av1::decode::frame::RowBand;
use crate::av1::decode::frame::{FrameBuf, Plane, RowBand};
use crate::av1::decode::spec_tables::{
GLOBAL_GLOBALMV, GLOBALMV, INTRA_FRAME, MAX_LOOP_FILTER, NEARESTMV, NUM_4X4_BLOCKS_HIGH, NUM_4X4_BLOCKS_WIDE, SEG_LVL_ALT_LF_Y_V, TX_HEIGHT,
TX_WIDTH,
@@ -48,6 +48,121 @@ pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, pool: &crate::
stats.add(Stage::Restoration, t.lap());
}
/// The in-loop filters run band by band behind reconstruction (row wavefront): each stage of a
/// band starts once the stage before has finished the band below it, which bounds what the
/// stage reads beyond the band (the loop filter changes up to 7 rows above a band and intra
/// prediction of the next superblock row reads the unfiltered row above it; CDEF reads 2 rows
/// and loop restoration 3 rows beyond a band). Bands are 64 luma rows. The result is the same
/// as running each filter over the whole frame. Super-resolution frames use [`apply`].
///
/// The last active stage writes each band into its own buffers, which are handed out as the
/// band's final samples without a copy.
pub(crate) struct Pipeline {
bands: Vec<(usize, usize)>,
/// Luma rows of the coded frame (mi_rows * 4).
rows: usize,
lf_on: bool,
cdef_on: bool,
lr_on: bool,
/// Bands finished by each stage.
lf: usize,
cdef: usize,
lr: usize,
pending: [Vec<u16>; 3],
/// CdefFrame when loop restoration reads it.
cdef_out: Option<FrameBuf>,
}
/// Final bands from [`Pipeline::advance`]: (band, its samples starting at the band's first row).
pub(crate) type FinalBands = Vec<(usize, [Plane; 3])>;
impl Pipeline {
pub fn new(fs: &FrameState, draft: bool) -> Pipeline {
let rows = fs.fh.mi_rows as usize * 4;
let n = rows.div_ceil(BAND_ROWS).max(1);
let bands = (0..n).map(|b| (b * BAND_ROWS, if b + 1 == n { usize::MAX } else { (b + 1) * BAND_ROWS })).collect();
let lvl = fs.fh.lf.level;
let cdef_on = !draft && crate::av1::decode::cdef::active(fs);
let lr_on = fs.fh.lr.uses_lr && !draft;
let cdef_out = (cdef_on && lr_on).then(|| fs.cur.same_geometry());
Pipeline {
bands,
rows,
lf_on: (lvl[0] != 0 || lvl[1] != 0) && !draft,
cdef_on,
lr_on,
lf: 0,
cdef: 0,
lr: 0,
pending: Default::default(),
cdef_out,
}
}
/// Filter what luma rows [0, `recon_rows`) of reconstructed samples allow (everything once
/// they cover the frame); returns the bands whose samples became final.
pub fn advance(&mut self, fs: &mut FrameState, stats: &mut DecodeStats, pool: &crate::av1::decode::par::Pool, recon_rows: usize) -> FinalBands {
let n = self.bands.len();
let all = recon_rows >= self.rows;
let mut out = FinalBands::new();
let mut t = Timer::start();
// The loop filter of a band waits for the band below it to be reconstructed.
let lf_to = if all { n } else { (recon_rows / BAND_ROWS).saturating_sub(1) };
if lf_to > self.lf {
if self.lf_on {
loop_filter_bands(fs, pool, &self.bands, self.lf..lf_to, &mut self.pending);
}
self.lf = lf_to;
}
stats.add(Stage::LoopFilter, t.lap());
let cdef_to = if all { n } else { self.lf.saturating_sub(1) };
if cdef_to > self.cdef {
let r = &self.bands[self.cdef..cdef_to];
if self.cdef_on {
if let Some(cdef) = &mut self.cdef_out {
cdef.copy_rows_from(&fs.cur, r[0].0, r[r.len() - 1].1);
crate::av1::decode::cdef::filter_bands(fs, pool, cdef.row_bands(r), r);
} else {
let mut own: Vec<[Plane; 3]> = r.iter().map(|&(a, b)| fs.cur.rows_copy(a, b)).collect();
crate::av1::decode::cdef::filter_bands(fs, pool, own_bands(&mut own), r);
out.extend((self.cdef..cdef_to).zip(own));
}
} else if !self.lr_on {
// The loop filter is the last stage: the band is final once the band below it
// is filtered.
out.extend((self.cdef..cdef_to).zip(r.iter().map(|&(a, b)| fs.cur.rows_copy(a, b))));
}
self.cdef = cdef_to;
}
stats.add(Stage::Cdef, t.lap());
let lr_to = if all { n } else { self.cdef.saturating_sub(1) };
if self.lr_on && lr_to > self.lr {
let r = &self.bands[self.lr..lr_to];
let cdef = self.cdef_out.as_ref().unwrap_or(&fs.cur);
let mut own: Vec<[Plane; 3]> = r.iter().map(|&(a, b)| cdef.rows_copy(a, b)).collect();
crate::av1::decode::restoration::restore_bands(fs, &fs.cur, cdef, own_bands(&mut own), pool, r);
out.extend((self.lr..lr_to).zip(own));
self.lr = lr_to;
}
stats.add(Stage::Restoration, t.lap());
out
}
}
/// Row bands over per-band buffers made by [`FrameBuf::rows_copy`].
fn own_bands(own: &mut [[Plane; 3]]) -> Vec<[RowBand<'_>; 3]> {
fn band(p: &mut Plane) -> RowBand<'_> {
let (stride, y0) = (p.stride.max(1), p.oy);
RowBand { data: p.data.as_mut_slice(), stride, y0 }
}
own.iter_mut()
.map(|planes| {
let [a, b, c] = planes;
[band(a), band(b), band(c)]
})
.collect()
}
// ---------------------------------------------------------------------------------------------
// Loop filter (7.14)
@@ -69,23 +184,40 @@ const LF_MARGIN: usize = 8;
/// written back; the band above never touches them.
fn loop_filter(fs: &mut FrameState, pool: &crate::av1::decode::par::Pool) {
let luma = bands(pool, fs.fh.mi_rows as usize * 4);
loop_filter_bands(fs, pool, &luma, 0..luma.len(), &mut Default::default());
}
/// The loop filter on bands `range` of `all` (the frame's bands). The bands above the range
/// are filtered already: `pending` holds the unfiltered rows above the first band of the range
/// (empty for band 0), saved before the band above was filtered, and is set to those of the
/// band after the range.
fn loop_filter_bands(
fs: &mut FrameState,
pool: &crate::av1::decode::par::Pool,
all: &[(usize, usize)],
range: std::ops::Range<usize>,
pending: &mut [Vec<u16>; 3],
) {
let Some(luma) = all.get(range.clone()) else { return };
let planes: Vec<usize> = (0..fs.num_planes).filter(|&p| p == 0 || fs.fh.lf.level[1 + p] != 0).collect();
let mut cur = std::mem::take(&mut fs.cur);
let f: &FrameState = fs;
// Unfiltered copies of the rows above each band but the first.
let margins: Vec<[Vec<u16>; 3]> = luma
.iter()
.map(|&(start, _)| {
std::array::from_fn(|p| {
let sub_y = if p == 0 { 0 } else { f.ssy };
let pl = &cur.planes[p];
let (y0, y1) = ((start.saturating_sub(LF_MARGIN)) >> sub_y, start >> sub_y);
let rows = pl.data.get(y0 * pl.stride..y1 * pl.stride);
if start == 0 || !planes.contains(&p) { Vec::new() } else { rows.map(<[u16]>::to_vec).unwrap_or_default() }
})
// Unfiltered copies of the rows above a band (none above the first).
let margin = |cur: &crate::av1::decode::frame::FrameBuf, start: usize| -> [Vec<u16>; 3] {
std::array::from_fn(|p| {
let sub_y = if p == 0 { 0 } else { f.ssy };
let pl = &cur.planes[p];
let (y0, y1) = ((start.saturating_sub(LF_MARGIN)) >> sub_y, start >> sub_y);
let rows = pl.data.get(y0 * pl.stride..y1 * pl.stride);
if start == 0 || !planes.contains(&p) { Vec::new() } else { rows.map(<[u16]>::to_vec).unwrap_or_default() }
})
.collect();
let mut bands: Vec<([RowBand; 3], [Vec<u16>; 3])> = cur.row_bands(&luma).into_iter().zip(margins).collect();
};
let margins: Vec<[Vec<u16>; 3]> =
luma.iter().enumerate().map(|(i, &(start, _))| if i == 0 { std::mem::take(pending) } else { margin(&cur, start) }).collect();
if let Some(&(next, _)) = all.get(range.end) {
*pending = margin(&cur, next);
}
let mut bands: Vec<([RowBand; 3], [Vec<u16>; 3])> = cur.row_bands(luma).into_iter().zip(margins).collect();
let changed = pool.map_mut(&mut bands, |b, (band, margin)| {
let mut changed: [Vec<(usize, u16)>; 3] = Default::default();
for &p in &planes {
+9 -5
View File
@@ -70,10 +70,16 @@ struct LrPlane<'a> {
/// Bands of 64 luma rows are restored in parallel: every unit reads only the two input frames
/// and writes only its own samples.
pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf, pool: &Pool) -> FrameBuf {
let fh = &fs.fh;
let mut lr = cdef.clone();
let ranges = bands(pool, fh.frame_height as usize);
let mut planes = lr.row_bands(&ranges);
let ranges = bands(pool, fs.fh.frame_height as usize);
restore_bands(fs, cur, cdef, lr.row_bands(&ranges), pool, &ranges);
lr
}
/// Restore the bands `planes` covering luma rows `ranges`, which hold UpscaledCdefFrame; reads
/// both inputs up to 3 rows beyond each band.
pub(crate) fn restore_bands(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf, mut planes: Vec<[RowBand; 3]>, pool: &Pool, ranges: &[(usize, usize)]) {
let fh = &fs.fh;
pool.map_mut(&mut planes, |b, band| {
let (start, end) = ranges[b];
let mut scratch = Scratch::default();
@@ -100,8 +106,6 @@ pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf,
p.restore(out, start >> sub_y, end >> sub_y, &mut scratch);
}
});
drop(planes);
lr
}
/// Row buffers reused across stripes.
+27 -2
View File
@@ -6,8 +6,8 @@
//! with several, each tile decodes into private buffers covering its area (tiles never read
//! outside themselves), which are then copied into the frame.
use crate::av1::decode::decoder::RefFrame;
use crate::av1::decode::frame::{FrameBuf, MiInfo};
use crate::av1::decode::decoder::{Parsed, RefFrame};
use crate::av1::decode::frame::{BandFrame, FrameBuf, MiInfo, PlaneRef};
use crate::av1::decode::header::{FrameHeader, RefInfo, SequenceHeader};
use crate::av1::decode::spec_tables::{DEFAULT_INTRA_FRAME_Y_MODE_CDF, NUM_REF_FRAMES, RESTORE_NONE, SUBSAMPLED_SIZE};
use std::ops::{Deref, DerefMut};
@@ -26,6 +26,11 @@ pub(crate) struct FrameShared {
pub cdef_cols: usize,
/// Reference slots (FrameStore and saved state) used by this frame.
pub refs: [Option<Arc<RefFrame>>; NUM_REF_FRAMES],
/// Parse results (CDFs, segment ids, saved motion vectors) of the same slots.
pub parsed: [Option<Arc<Parsed>>; NUM_REF_FRAMES],
/// The published bands of references that are still being decoded (row wavefront): read
/// only below the row progress that was waited for.
pub ref_rows: [Option<Arc<BandFrame>>; NUM_REF_FRAMES],
pub ref_info: [RefInfo; NUM_REF_FRAMES],
/// MotionFieldMvs[ ref ][ y8 * w8 + x8 ] (7.9); empty when unused.
pub motion_field: [Vec<[i32; 2]>; 8],
@@ -34,6 +39,16 @@ pub(crate) struct FrameShared {
}
impl FrameShared {
/// The samples of reference slot `ref_idx` to predict from (None for a missing reference).
#[inline]
pub fn ref_samples(&self, ref_idx: i32, plane: usize) -> Option<PlaneRef<'_>> {
let i = usize::try_from(ref_idx).ok()?;
if let Some(b) = self.ref_rows.get(i)?.as_ref() {
return b.planes.get(plane).map(PlaneRef::Bands);
}
self.refs.get(i)?.as_ref().map(|r| r.plane(plane))
}
pub fn plane_residual_size(&self, subsize: usize, plane: usize) -> usize {
let sx = if plane > 0 { self.ssx } else { 0 };
let sy = if plane > 0 { self.ssy } else { 0 };
@@ -210,6 +225,8 @@ impl FrameState {
lr_unit_cols,
cdef_cols,
refs: Default::default(),
parsed: Default::default(),
ref_rows: Default::default(),
ref_info: Default::default(),
motion_field: Default::default(),
prev_segment_ids: vec![0; mi_cols * mi_rows],
@@ -283,6 +300,14 @@ impl FrameState {
pub fn merge_tile_state(&mut self, t: &TileState) {
for p in 0..self.num_planes {
t.cur.planes[p].copy_into(&mut self.cur.planes[p]);
}
self.merge_tile_modes(t);
}
/// Copy a tile's private buffers except the samples (after parsing, before reconstruction)
/// into the frame.
pub fn merge_tile_modes(&mut self, t: &TileState) {
for p in 0..self.num_planes {
t.lf_tx_size[p].copy_into(&mut self.lf_tx_size[p]);
for &u in &t.lr_written[p] {
self.lr.lr_type[p][u] = t.lr.lr_type[p][u];
+303 -11
View File
@@ -94,6 +94,191 @@ pub(crate) struct Block {
pub local_warp_params: [i32; 6],
}
/// What a tile decoder does with the blocks it visits.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(crate) enum Pass {
/// Read the symbols and reconstruct each block right away (single-threaded decoding).
Full,
/// Read the symbols only, recording what reconstruction needs in a [`TileRecord`].
Parse,
/// Reconstruct from a [`TileRecord`], reading no symbols.
Reconstruct,
}
/// The per-block values reconstruction needs beyond the mode info (which is final after
/// parsing): the block variables as parsed, its quantizer index and the warp samples / palette
/// color maps when used.
#[derive(Clone)]
pub(crate) struct BlockRecord {
b: Block,
current_q_index: i32,
warp_samples: Option<Box<(usize, [[i32; 4]; crate::av1::decode::spec_tables::LEAST_SQUARES_SAMPLES_MAX])>>,
color_maps: Option<Box<(Vec<u8>, Vec<u8>)>>,
}
/// A parsed tile: its blocks in decoding order, how many of them each superblock holds and the
/// coefficients of every coded transform block as a stream of
/// `eob, tx_type, count, (position, value) * count` (only non-zero values).
#[derive(Default)]
pub(crate) struct TileRecord {
blocks: Vec<BlockRecord>,
sb_blocks: Vec<u32>,
coeffs: Vec<i32>,
/// Read positions while reconstructing.
next_block: usize,
next_coeff: usize,
next_sb: usize,
next_sb_row: usize,
/// Bytes recorded and the most allowed.
pub bytes: usize,
pub limit: usize,
/// The decoder-wide budget the recorded bytes are reserved from, and how much is reserved.
pub budget: Option<std::sync::Arc<RecordBudget>>,
reserved: usize,
}
impl Drop for TileRecord {
fn drop(&mut self) {
if let Some(b) = &self.budget {
b.release(self.reserved);
}
}
}
/// The parse record bytes all frames of a decoder may hold at once (frames in flight decode in
/// one pass once it is used up).
pub(crate) struct RecordBudget {
used: std::sync::atomic::AtomicUsize,
max: usize,
/// Most bytes ever reserved at once (tests).
#[cfg(test)]
pub peak: std::sync::atomic::AtomicUsize,
}
/// Bytes reserved from a [`RecordBudget`] at a time.
const RESERVE_STEP: usize = 1 << 20;
impl RecordBudget {
pub fn new(max: usize) -> RecordBudget {
RecordBudget {
used: Default::default(),
max,
#[cfg(test)]
peak: Default::default(),
}
}
/// Reserve `n` bytes; false (nothing reserved) if that would go over the budget.
fn reserve(&self, n: usize) -> bool {
use std::sync::atomic::Ordering;
let ok = self.used.try_update(Ordering::AcqRel, Ordering::Acquire, |u| u.checked_add(n).filter(|&v| v <= self.max)).is_ok();
#[cfg(test)]
if ok {
self.peak.fetch_max(self.used.load(Ordering::Acquire), Ordering::AcqRel);
}
ok
}
fn release(&self, n: usize) {
self.used.fetch_sub(n, std::sync::atomic::Ordering::AcqRel);
}
/// Bytes reserved now (tests).
#[cfg(test)]
pub fn used(&self) -> usize {
self.used.load(std::sync::atomic::Ordering::Acquire)
}
}
/// The motion of a recorded block (what a reference row wait depends on).
pub(crate) struct BlockMotion {
pub mi_row: usize,
pub bh4: usize,
/// Inter prediction from reference frames (not intra block copy).
pub inter: bool,
pub ref_frame: [i8; 2],
/// Mv[ list ] (row, col) in 1/8 samples.
pub mv: [[i32; 2]; 2],
pub motion_mode: u8,
pub y_mode: usize,
}
impl TileRecord {
/// Whether recording stopped at [`TileRecord::limit`] (the record is unusable).
pub fn over_limit(&self) -> bool {
self.bytes > self.limit
}
/// Account for `n` more recorded bytes; false once over the limit (nothing more is kept).
fn grow(&mut self, n: usize) -> bool {
self.bytes = self.bytes.saturating_add(n);
if self.bytes > self.limit {
return false;
}
if self.bytes > self.reserved
&& let Some(b) = &self.budget
{
let step = (self.bytes - self.reserved).max(RESERVE_STEP);
if !b.reserve(step) {
// The decoder-wide budget is used up: this frame is decoded in one pass.
self.limit = 0;
return false;
}
self.reserved += step;
}
true
}
/// The recorded blocks of superblock rows [first, end), for a tile `sbs_per_row` superblocks wide.
pub fn row_blocks(&self, sbs_per_row: usize, first: usize, end: usize) -> impl Iterator<Item = BlockMotion> + '_ {
let start: usize = self.sb_blocks.iter().take(first * sbs_per_row).map(|&n| n as usize).sum();
let count: usize = self.sb_blocks.iter().skip(first * sbs_per_row).take((end - first) * sbs_per_row).map(|&n| n as usize).sum();
self.blocks.iter().skip(start).take(count).map(|r| BlockMotion {
mi_row: r.b.mi_row,
bh4: r.b.bh4,
inter: r.b.is_inter && !r.b.use_intrabc,
ref_frame: r.b.ref_frame,
mv: r.b.mv_i,
motion_mode: r.b.motion_mode,
y_mode: r.b.y_mode,
})
}
fn next_coeff(&mut self) -> Result<i32> {
let v = *self.coeffs.get(self.next_coeff).ok_or(crate::av1::decode::Error::Invalid("coefficient record exhausted"))?;
self.next_coeff += 1;
Ok(v)
}
}
/// The work buffers of a [`TileDecoder`] (`quant` all zero between transform blocks).
pub(crate) struct Buffers {
quant: Vec<i32>,
dequant: Vec<i32>,
residual: Vec<i32>,
color_map_y: Vec<u8>,
color_map_uv: Vec<u8>,
pred_buf: [Vec<i32>; 2],
mask: Vec<i32>,
mc_tmp: Vec<i32>,
levels: Vec<u8>,
}
impl Buffers {
pub fn new() -> Buffers {
Buffers {
quant: vec![0; 1024],
dequant: vec![0; 64 * 64],
residual: vec![0; 64 * 64],
color_map_y: vec![0; 64 * 64],
color_map_uv: vec![0; 64 * 64],
pred_buf: [vec![0; 128 * 128], vec![0; 128 * 128]],
mask: vec![0; 128 * 128],
mc_tmp: vec![0; (2 * 128 + 8) * 128],
levels: vec![0; (32 + TX_PAD) * (32 + TX_PAD)],
}
}
}
/// Decoder for one tile.
pub(crate) struct TileDecoder<'a, 'f> {
pub fs: &'f FrameShared,
@@ -128,6 +313,9 @@ pub(crate) struct TileDecoder<'a, 'f> {
pub mc_tmp: Vec<i32>,
/// Coefficient levels of the current transform block, padded (see TX_PAD).
levels: Vec<u8>,
pub pass: Pass,
/// Written by [`Pass::Parse`], read by [`Pass::Reconstruct`].
pub record: TileRecord,
}
const SB_MAX: usize = 32; // superblock size in 4x4 units (128 / 4)
@@ -141,6 +329,21 @@ fn block_height(bs: usize) -> usize {
impl<'a, 'f> TileDecoder<'a, 'f> {
pub fn new(fs: &'f FrameShared, t: &'f mut TileState, data: &'a [u8], cdf: Box<Cdfs>, tile_row: usize, tile_col: usize) -> Self {
Self::with_buffers(fs, t, data, cdf, tile_row, tile_col, Buffers::new())
}
/// [`TileDecoder::new`] reusing the work buffers of an earlier decoder of the same frame
/// (see [`TileDecoder::into_buffers`]).
pub fn with_buffers(
fs: &'f FrameShared,
t: &'f mut TileState,
data: &'a [u8],
cdf: Box<Cdfs>,
tile_row: usize,
tile_col: usize,
bufs: Buffers,
) -> Self {
let Buffers { quant, dequant, residual, color_map_y, color_map_uv, pred_buf, mask, mc_tmp, levels } = bufs;
let ti = &fs.fh.tile_info;
let (rs, re) = (ti.mi_row_starts[tile_row] as usize, ti.mi_row_starts[tile_row + 1] as usize);
let (cs, ce) = (ti.mi_col_starts[tile_col] as usize, ti.mi_col_starts[tile_col + 1] as usize);
@@ -164,19 +367,27 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
b: Block::default(),
mvs: Default::default(),
nb: Default::default(),
quant: vec![0; 1024],
dequant: vec![0; 64 * 64],
residual: vec![0; 64 * 64],
color_map_y: vec![0; 64 * 64],
color_map_uv: vec![0; 64 * 64],
quant,
dequant,
residual,
color_map_y,
color_map_uv,
plane_tx_type: 0,
pred_buf: [vec![0; 128 * 128], vec![0; 128 * 128]],
mask: vec![0; 128 * 128],
mc_tmp: vec![0; (2 * 128 + 8) * 128],
levels: vec![0; (32 + TX_PAD) * (32 + TX_PAD)],
pred_buf,
mask,
mc_tmp,
levels,
pass: Pass::Full,
record: TileRecord::default(),
}
}
/// The work buffers and CDFs, for [`TileDecoder::with_buffers`].
pub fn into_buffers(self) -> (Buffers, Box<Cdfs>) {
let TileDecoder { quant, dequant, residual, color_map_y, color_map_uv, pred_buf, mask, mc_tmp, levels, cdf, .. } = self;
(Buffers { quant, dequant, residual, color_map_y, color_map_uv, pred_buf, mask, mc_tmp, levels }, cdf)
}
fn seq(&self) -> &SequenceHeader {
&self.fs.seq
}
@@ -189,8 +400,11 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
c >= self.mi_col_start as isize && c < self.mi_col_end as isize && r >= self.mi_row_start as isize && r < self.mi_row_end as isize
}
/// decode_tile( )
/// decode_tile( ), or its reconstruction from [`TileDecoder::record`] in [`Pass::Reconstruct`].
pub fn decode_tile(&mut self) -> Result<()> {
if self.pass == Pass::Reconstruct {
return self.reconstruct_tile();
}
// clear_above_context( )
let t = &mut *self.t;
for p in 0..3 {
@@ -223,7 +437,11 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
self.clear_cdef(r, c);
self.clear_block_decoded_flags(r, c, sb4);
self.read_lr(r, c, sb_size);
let before = self.record.blocks.len();
self.decode_partition(r, c, sb_size)?;
if self.pass == Pass::Parse {
self.record.sb_blocks.push((self.record.blocks.len() - before) as u32);
}
c += sb4;
}
r += sb4;
@@ -231,6 +449,76 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
Ok(())
}
/// The reconstruction half of decode_tile( ): the superblocks in the same order, each block
/// predicted and reconstructed from its record.
fn reconstruct_tile(&mut self) -> Result<()> {
self.reconstruct_sb_rows(usize::MAX)
}
/// Reconstruct the next `rows` superblock rows of the tile (the record keeps the position).
pub fn reconstruct_sb_rows(&mut self, rows: usize) -> Result<()> {
let sb4 = if self.seq().use_128x128_superblock { 32 } else { 16 };
let mut r = self.mi_row_start + self.record.next_sb_row * sb4;
let mut done = 0;
while r < self.mi_row_end && done < rows {
let mut c = self.mi_col_start;
while c < self.mi_col_end {
self.clear_block_decoded_flags(r, c, sb4);
let n = *self.record.sb_blocks.get(self.record.next_sb).ok_or(crate::av1::decode::Error::Invalid("block record exhausted"))?;
self.record.next_sb += 1;
for _ in 0..n {
self.reconstruct_block()?;
}
c += sb4;
}
r += sb4;
done += 1;
self.record.next_sb_row += 1;
}
Ok(())
}
fn reconstruct_block(&mut self) -> Result<()> {
let rec = self.record.blocks.get(self.record.next_block).ok_or(crate::av1::decode::Error::Invalid("block record exhausted"))?;
self.record.next_block += 1;
self.b = rec.b.clone();
self.current_q_index = rec.current_q_index;
if let Some(w) = &rec.warp_samples {
self.mvs.num_samples = w.0;
self.mvs.cand_list = w.1;
}
if let Some(maps) = &rec.color_maps {
if let Some(d) = self.color_map_y.get_mut(..maps.0.len()) {
d.copy_from_slice(&maps.0);
}
if let Some(d) = self.color_map_uv.get_mut(..maps.1.len()) {
d.copy_from_slice(&maps.1);
}
}
self.compute_prediction()?;
self.residual()
}
/// Records the block for [`Pass::Reconstruct`] (its variables as parsed).
fn record_block(&mut self) {
let b = &self.b;
// The rows of the color maps (stride 64) the block's palette prediction reads.
let map_rows = |n: usize, h: usize| if n > 0 { (h + 2).min(64) * 64 } else { 0 };
let rows_y = map_rows(b.palette_size_y, block_height(b.mi_size));
let rows_uv = map_rows(b.palette_size_uv, block_height(b.mi_size) >> self.fs.ssy);
let size = std::mem::size_of::<BlockRecord>()
+ rows_y
+ rows_uv
+ if b.motion_mode as usize == crate::av1::decode::spec_tables::LOCALWARP { 160 } else { 0 };
if !self.record.grow(size) {
return;
}
let warp_samples =
(b.motion_mode as usize == crate::av1::decode::spec_tables::LOCALWARP).then(|| Box::new((self.mvs.num_samples, self.mvs.cand_list)));
let color_maps = (rows_y + rows_uv > 0).then(|| Box::new((self.color_map_y[..rows_y].to_vec(), self.color_map_uv[..rows_uv].to_vec())));
self.record.blocks.push(BlockRecord { b: b.clone(), current_q_index: self.current_q_index, warp_samples, color_maps });
}
fn clear_cdef(&mut self, r: usize, c: usize) {
let t = &mut *self.t;
t.set_cdef_idx(r, c, -1);
@@ -461,7 +749,11 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
}
}
}
self.compute_prediction()?;
if self.pass == Pass::Parse {
self.record_block();
} else {
self.compute_prediction()?;
}
self.residual()?;
let b = &self.b;
let delta_lf = self.delta_lf;
@@ -459,6 +459,58 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
}
}
/// Moves the coefficients `coeffs` left in `quant` into the record (leaving `quant` zero, as
/// reconstruction does).
pub(super) fn record_coeffs(&mut self, eob: usize, tx_sz: usize) {
let scan = self.get_scan(tx_sz);
let scan = scan.get(..eob).unwrap_or(scan);
if !self.record.grow(4 * (3 + 2 * scan.len())) {
// Over the limit: the record is dropped (the frame is decoded again in one pass);
// `quant` must still be left zero.
for &pos in scan {
if let Some(q) = self.quant.get_mut(pos as usize) {
*q = 0;
}
}
return;
}
let rec = &mut self.record.coeffs;
rec.push(eob as i32);
if eob == 0 {
return;
}
rec.push(self.plane_tx_type as i32);
let count_at = rec.len();
rec.push(0);
let mut count = 0;
for &pos in scan {
let v = std::mem::take(&mut self.quant[pos as usize]);
if v != 0 {
rec.push(pos as i32);
rec.push(v);
count += 1;
}
}
rec[count_at] = count;
}
/// The next transform block's coefficients from the record into `quant` (and its type);
/// returns its eob.
pub(super) fn replay_coeffs(&mut self) -> crate::av1::decode::Result<usize> {
let eob = self.record.next_coeff()?;
if eob <= 0 {
return Ok(0);
}
self.plane_tx_type = usize::try_from(self.record.next_coeff()?).map_err(|_| crate::av1::decode::Error::Invalid("coefficient record"))?;
let count = self.record.next_coeff()?;
for _ in 0..count {
let pos = self.record.next_coeff()? as usize;
let v = self.record.next_coeff()?;
*self.quant.get_mut(pos).ok_or(crate::av1::decode::Error::Invalid("coefficient record"))? = v;
}
Ok(eob as usize)
}
/// Reconstruct process (7.12.3).
pub(super) fn reconstruct(&mut self, plane: usize, x: usize, y: usize, tx_sz: usize, dc_only: bool) {
// dqDenom (7.12.3) is 1, 2 or 4 and the dividend is non-negative.
+19 -4
View File
@@ -213,7 +213,8 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
if start_x >= max_x || start_y >= max_y {
return Ok(());
}
if !self.b.is_inter {
let parse_only = self.pass == super::Pass::Parse;
if !self.b.is_inter && !parse_only {
let palette = if plane == 0 { self.b.palette_size_y } else { self.b.palette_size_uv };
if palette > 0 {
self.predict_palette(plane, start_x, start_y, x, y, tx_sz);
@@ -266,9 +267,23 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
}
}
if !self.b.skip {
let eob = self.coeffs(plane, start_x, start_y, tx_sz);
if eob > 0 {
self.reconstruct(plane, start_x, start_y, tx_sz, eob == 1);
match self.pass {
super::Pass::Full => {
let eob = self.coeffs(plane, start_x, start_y, tx_sz);
if eob > 0 {
self.reconstruct(plane, start_x, start_y, tx_sz, eob == 1);
}
}
super::Pass::Parse => {
let eob = self.coeffs(plane, start_x, start_y, tx_sz);
self.record_coeffs(eob, tx_sz);
}
super::Pass::Reconstruct => {
let eob = self.replay_coeffs()?;
if eob > 0 {
self.reconstruct(plane, start_x, start_y, tx_sz, eob == 1);
}
}
}
}
for i in 0..step_y {
+15 -18
View File
@@ -152,38 +152,35 @@ impl<'a, S: crate::hevc::decode::picture::Sample> SliceDecoder<'a, S> {
col.ref_idx[1] = 0;
col_avail = true;
}
let mut cands: Vec<MvField> = list[..n].to_vec();
if col_avail {
cands.push(col);
list[n] = col;
n += 1;
}
if merge_idx < cands.len() {
return self.finish_merge(cands[merge_idx], w0, h0);
if merge_idx < n {
return self.finish_merge(list[merge_idx], w0, h0);
}
// combined bi-predictive candidates (8.5.3.2.4)
let num_orig = cands.len();
let num_orig = n;
if self.sh.is_b() && num_orig > 1 && num_orig < max {
let mut comb = 0;
while comb < num_orig * (num_orig - 1) && cands.len() < max {
while comb < num_orig * (num_orig - 1) && n < max {
let (i0, i1) = COMB[comb];
let (c0, c1) = (cands[i0], cands[i1]);
let (c0, c1) = (list[i0], list[i1]);
if c0.pred(0) && c1.pred(1) && (self.ref_pic(0, c0.ref_idx[0]).poc != self.ref_pic(1, c1.ref_idx[1]).poc || c0.mv[0] != c1.mv[1]) {
cands.push(MvField { mv: [c0.mv[0], c1.mv[1]], ref_idx: [c0.ref_idx[0], c1.ref_idx[1]] });
list[n] = MvField { mv: [c0.mv[0], c1.mv[1]], ref_idx: [c0.ref_idx[0], c1.ref_idx[1]] };
n += 1;
}
comb += 1;
}
}
if merge_idx < cands.len() {
return self.finish_merge(cands[merge_idx], w0, h0);
if merge_idx < n {
return self.finish_merge(list[merge_idx], w0, h0);
}
// zero candidates (8.5.3.2.5)
// zero candidates (8.5.3.2.5): candidate merge_idx is the (merge_idx - n)-th one
let num_ref = if self.sh.is_b() { self.sh.num_ref_idx[0].min(self.sh.num_ref_idx[1]) } else { self.sh.num_ref_idx[0] } as usize;
let mut zero_idx = 0;
while cands.len() <= merge_idx {
let r = if zero_idx < num_ref { zero_idx as i8 } else { 0 };
cands.push(MvField { mv: [[0; 2]; 2], ref_idx: [r, if self.sh.is_b() { r } else { -1 }] });
zero_idx += 1;
}
self.finish_merge(cands[merge_idx], w0, h0)
let zero_idx = merge_idx - n;
let r = if zero_idx < num_ref { zero_idx as i8 } else { 0 };
self.finish_merge(MvField { mv: [[0; 2]; 2], ref_idx: [r, if self.sh.is_b() { r } else { -1 }] }, w0, h0)
}
fn finish_merge(&self, mut f: MvField, w: i32, h: i32) -> MvField {
+31 -1
View File
@@ -16,7 +16,7 @@ use crate::hevc::decode::spec_tables::{
INTRA_CHROMA, LAST_X_PREFIX, LAST_Y_PREFIX, MERGE_FLAG, MERGE_IDX, MVP_FLAG, PART_MODE, PRED_MODE, PREV_INTRA_LUMA, REF_IDX, RQT_ROOT_CBF,
SAO_MERGE, SAO_TYPE, SIG_COEFF, SPLIT_CU, SPLIT_TRANSFORM, TRANSFORM_SKIP,
};
use crate::hevc::decode::tables::{CTX_IDX_MAP, INTRA_CHROMA_MODE_422, LEVEL_SCALE, SCAN_4, qpc, scan_order};
use crate::hevc::decode::tables::{INTRA_CHROMA_MODE_422, LEVEL_SCALE, SCAN_4, SIG_CTX, qpc, scan_order, scan_pos};
use crate::hevc::decode::transform;
use std::sync::Arc;
@@ -340,6 +340,8 @@ pub struct SliceDecoder<'a, S> {
pub cu_part: PartMode,
// scratch
pub coeffs: Vec<i32>,
/// Scaled non-zero coefficients of the current transform block: (index in `coeffs`, value).
pub nz: Vec<(u16, i32)>,
pub pred0: Vec<i16>,
pub pred1: Vec<i16>,
pub mc: crate::hevc::decode::inter::McScratch,
@@ -461,6 +463,7 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
intra_split: false,
cu_part: PartMode::P2Nx2N,
coeffs: vec![0; 32 * 32],
nz: vec![(0, 0); 32 * 32],
pred0: vec![0; 64 * 64],
pred1: vec![0; 64 * 64],
mc: Default::default(),
@@ -632,3 +635,30 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use crate::hevc::decode::transform;
/// `reconstruct` clears only the coefficient bounding box, so the inverse transform must not
/// read the stale values left outside it by the previous block.
#[test]
fn inverse_transform_reads_only_the_coefficient_box() {
for &(n, dst) in &[(4usize, true), (4, false), (8, false), (16, false), (32, false)] {
for (max_x, max_y) in [(0, 1), (1, 0), (2, 3), (n / 2, 1), (n - 1, 0), (n - 1, n - 1)] {
let (mut clean, mut stale) = (vec![0i32; n * n], vec![0i32; n * n]);
for (i, (c, s)) in clean.iter_mut().zip(stale.iter_mut()).enumerate() {
let (x, y) = (i % n, i / n);
*s = 12345 - i as i32;
if x <= max_x && y <= max_y {
*c = (i as i32 * 37) % 201 - 100;
*s = *c;
}
}
transform::inverse_transform(&mut clean, n, dst, max_x, max_y, 12);
transform::inverse_transform(&mut stale, n, dst, max_x, max_y, 12);
assert_eq!(clean, stale, "n {n} dst {dst} box ({max_x}, {max_y})");
}
}
}
}
@@ -1,7 +1,7 @@
use super::{
CBF_CHROMA, CBF_LUMA, CODED_SUB_BLOCK, CTX_IDX_MAP, CU_QP_DELTA_ABS, ChromaFormat, E_TU_H, E_TU_V, F_CBF, F_INTRA, GT1, GT2, LAST_X_PREFIX,
LAST_Y_PREFIX, LEVEL_SCALE, PartMode, Result, SCAN_4, SIG_COEFF, SPLIT_TRANSFORM, Sample, SliceDecoder, TRANSFORM_SKIP, ensure, for_runs, intra,
scan_order, transform,
CBF_CHROMA, CBF_LUMA, CODED_SUB_BLOCK, CU_QP_DELTA_ABS, ChromaFormat, E_TU_H, E_TU_V, F_CBF, F_INTRA, GT1, GT2, LAST_X_PREFIX, LAST_Y_PREFIX,
LEVEL_SCALE, PartMode, Result, SCAN_4, SIG_COEFF, SIG_CTX, SPLIT_TRANSFORM, Sample, SliceDecoder, TRANSFORM_SKIP, ensure, for_runs, intra,
scan_order, scan_pos, transform,
};
/// Significant scan positions of one sub-block, in decoding (descending) order.
@@ -44,54 +44,26 @@ struct GreaterFlags {
g1ctx: u32,
}
/// sigCtx increment of sig_coeff_flag (9.3.4.2.5) for the position `(xp, yp)` inside the sub-block.
fn sig_ctx_inc(p: &SigMapParams, (xp, yp): (usize, usize)) -> usize {
let (xc, yc) = (p.xs * 4 + xp, p.ys * 4 + yp);
/// sigCtx increments (9.3.4.2.5) of the 16 scan positions of one sub-block.
fn sig_ctx_table(p: &SigMapParams) -> [u8; 16] {
if p.log2 == 2 {
return CTX_IDX_MAP[(yc << 2) + xc] as usize;
return SIG_CTX[p.scan_idx][4];
}
if xc + yc == 0 {
return 0;
}
let mut s = match p.prev_csbf {
0 => {
if xp + yp == 0 {
2
} else if xp + yp < 3 {
1
} else {
0
}
}
1 => {
if yp == 0 {
2
} else if yp == 1 {
1
} else {
0
}
}
2 => {
if xp == 0 {
2
} else if xp == 1 {
1
} else {
0
}
}
_ => 2,
};
if p.c == 0 {
if p.xs + p.ys > 0 {
s += 3;
}
s += if p.log2 == 3 { if p.scan_idx == 0 { 9 } else { 15 } } else { 21 };
let off = if p.c == 0 {
(if p.xs + p.ys > 0 { 3 } else { 0 }) + if p.log2 == 3 { if p.scan_idx == 0 { 9 } else { 15 } } else { 21 }
} else if p.log2 == 3 {
9
} else {
s += if p.log2 == 3 { 9 } else { 12 };
12
};
let mut t = SIG_CTX[p.scan_idx][p.prev_csbf as usize];
for v in t.iter_mut() {
*v += off;
}
s
if p.xs + p.ys == 0 {
t[0] = 0;
}
t
}
/// A node of the transform tree: the block at (`x0`, `y0`) of size 2^`log2`, its parent's origin
@@ -333,14 +305,13 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
/// sig_coeff_flag decoding of one coded sub-block, appending the significant scan positions
/// (descending) to `list`.
fn read_sig_map(&mut self, p: &SigMapParams, list: &mut SigList) {
let pos_scan = &SCAN_4[p.scan_idx];
let c_off_sig = if p.c == 0 { 0 } else { 27 };
let ctx_tab = sig_ctx_table(p);
let mut infer_dc = p.infer_dc;
let mut np = p.start;
while np >= 0 {
let (xp, yp) = (pos_scan[np as usize].0 as usize, pos_scan[np as usize].1 as usize);
if np > 0 || !infer_dc {
let sig_ctx = sig_ctx_inc(p, (xp, yp));
let sig_ctx = ctx_tab[np as usize] as usize;
if self.c.decode_decision(SIG_COEFF + c_off_sig + sig_ctx) == 1 {
list.push(np as u8);
infer_dc = false;
@@ -438,9 +409,23 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
let sb_scan = scan_order(log2sb, scan_idx);
let pos_scan = &SCAN_4[scan_idx];
let sb_w = 1usize << log2sb;
let last_sb = sb_scan.iter().position(|&(x, y)| x as u32 == lx >> 2 && y as u32 == ly >> 2).unwrap_or(0);
let last_pos = pos_scan.iter().position(|&(x, y)| x as u32 == lx & 3 && y as u32 == ly & 3).unwrap_or(0);
self.coeffs[..n * n].fill(0);
let last_sb = scan_pos(log2sb, scan_idx, lx >> 2, ly >> 2);
let last_pos = scan_pos(2, scan_idx, lx & 3, ly & 3);
// scaling (8.6.3), applied as each coefficient is decoded
let bd = if c == 0 { self.pic.bd_y } else { self.pic.bd_c };
let bd_shift = bd + log2 - 5;
let round = 1i64 << (bd_shift - 1);
let scale = (LEVEL_SCALE[(qp % 6) as usize] as i64) << (qp / 6);
let matrix = match &self.pic.scaling {
Some(s) if !(bypass || (ts && n > 4)) => {
let size_id = (log2 - 2) as usize;
let matrix_id = if self.cu_intra { c } else { 3 + c };
Some((s.clone(), size_id * 6 + matrix_id))
}
_ => None,
};
let m: Option<&[u8]> = matrix.as_ref().map(|(s, i)| &s[*i][..]);
let mut nnz = 0;
let mut csbf = [0u8; 64];
let mut prev_g1: Option<u32> = None;
let sdh = self.pps.sign_data_hiding && !bypass;
@@ -485,12 +470,18 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
for k in 0..list.n {
let np = list.pos[k] as usize;
let (xc, yc) = (xs * 4 + pos_scan[np].0 as usize, ys * 4 + pos_scan[np].1 as usize);
self.coeffs[yc * n + xc] = values[k];
let mut v = values[k];
if !bypass {
let f = m.map_or(16, |m| m[yc * n + xc] as i64);
v = ((v as i64 * f * scale + round) >> bd_shift).clamp(-32768, 32767) as i32;
}
self.nz[nnz] = ((yc * n + xc) as u16, v);
nnz += 1;
max_x = max_x.max(xc);
max_y = max_y.max(yc);
}
}
self.reconstruct((x0, y0), log2, c, qp, ts, (max_x, max_y))
self.reconstruct((x0, y0), log2, c, ts, nnz, (max_x, max_y))
}
fn coeff_abs_level_remaining(&mut self, rice: u32) -> Result<i32> {
@@ -508,55 +499,70 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
}
}
/// Scaling, transform and addition to the prediction.
fn reconstruct(&mut self, (x0, y0): (i32, i32), log2: u32, c: usize, qp: i32, ts: bool, (max_x, max_y): (usize, usize)) -> Result<()> {
/// Transform of the scaled coefficients `nz[..nnz]` and addition to the prediction.
fn reconstruct(&mut self, (x0, y0): (i32, i32), log2: u32, c: usize, ts: bool, nnz: usize, (max_x, max_y): (usize, usize)) -> Result<()> {
let n = 1usize << log2;
let bd = if c == 0 { self.pic.bd_y } else { self.pic.bd_c };
let coeffs = &mut self.coeffs[..n * n];
let mut dc_only = false;
if !self.cu_transquant_bypass {
// scaling (8.6.3)
let bd_shift = bd + log2 - 5;
let round = 1i64 << (bd_shift - 1);
let scale = (LEVEL_SCALE[(qp % 6) as usize] as i64) << (qp / 6);
let m: Option<&[u8]> = match &self.pic.scaling {
Some(s) if !(ts && n > 4) => {
let size_id = (log2 - 2) as usize;
let matrix_id = if self.cu_intra { c } else { 3 + c };
Some(&s[size_id * 6 + matrix_id])
}
_ => None,
};
for y in 0..=max_y {
for x in 0..=max_x {
let v = coeffs[y * n + x];
if v != 0 {
let f = m.map_or(16, |m| m[y * n + x] as i64);
coeffs[y * n + x] = ((v as i64 * f * scale + round) >> bd_shift).clamp(-32768, 32767) as i32;
}
}
let bypass = self.cu_transquant_bypass;
let dst = self.cu_intra && c == 0 && n == 4;
let (stride, max) = (if c == 0 { self.pic.width } else { self.pic.cwidth }, (1i32 << bd) - 1);
let off = y0 as usize * stride + x0 as usize;
let plane = &mut self.pic.planes[c];
let bd_shift = 20 - bd;
if !bypass && !ts && max_x == 0 && max_y == 0 && !dst {
let v = transform::inverse_dc(self.nz[0].1, bd_shift);
match n {
4 => add_dc::<S, 4>(plane, off, stride, v, max),
8 => add_dc::<S, 8>(plane, off, stride, v, max),
16 => add_dc::<S, 16>(plane, off, stride, v, max),
_ => add_dc::<S, 32>(plane, off, stride, v, max),
}
let bd_shift = 20 - bd;
return Ok(());
}
// the inverse transform reads only the bounding box of the coefficients
let coeffs = &mut self.coeffs[..n * n];
if bypass || ts {
coeffs.fill(0);
} else {
for row in coeffs.chunks_exact_mut(n).take(max_y + 1) {
row[..=max_x].fill(0);
}
}
for &(p, v) in &self.nz[..nnz] {
coeffs[p as usize] = v;
}
if !bypass {
if ts {
transform::transform_skip(coeffs, n, bd_shift);
} else if max_x == 0 && max_y == 0 && !(self.cu_intra && c == 0 && n == 4) {
let v = transform::inverse_dc(coeffs[0], bd_shift);
coeffs.fill(v);
dc_only = true;
} else {
let dst = self.cu_intra && c == 0 && n == 4;
transform::inverse_transform(coeffs, n, dst, max_x, max_y, bd_shift);
}
}
let _ = dc_only;
let (stride, max) = (if c == 0 { self.pic.width } else { self.pic.cwidth }, (1i32 << bd) - 1);
let plane = &mut self.pic.planes[c];
for y in 0..n {
let row = &mut plane[(y0 as usize + y) * stride + x0 as usize..][..n];
for (x, s) in row.iter_mut().enumerate() {
*s = S::from_i32((s.to_i32() + coeffs[y * n + x]).clamp(0, max));
}
match n {
4 => add_residual::<S, 4>(plane, off, stride, coeffs, max),
8 => add_residual::<S, 8>(plane, off, stride, coeffs, max),
16 => add_residual::<S, 16>(plane, off, stride, coeffs, max),
_ => add_residual::<S, 32>(plane, off, stride, coeffs, max),
}
Ok(())
}
}
#[inline(always)]
fn add_residual<S: Sample, const N: usize>(plane: &mut [S], off: usize, stride: usize, res: &[i32], max: i32) {
for (y, r) in res.as_chunks::<N>().0.iter().take(N).enumerate() {
let row = &mut plane[off + y * stride..][..N];
for (s, &d) in row.iter_mut().zip(r) {
*s = S::from_i32((s.to_i32() + d).max(0).min(max));
}
}
}
#[inline(always)]
fn add_dc<S: Sample, const N: usize>(plane: &mut [S], off: usize, stride: usize, v: i32, max: i32) {
for y in 0..N {
for s in &mut plane[off + y * stride..][..N] {
*s = S::from_i32((s.to_i32() + v).max(0).min(max));
}
}
}
+78 -1
View File
@@ -51,6 +51,72 @@ pub fn scan_order(log2: u32, scan_idx: usize) -> &'static [(u8, u8)] {
}
}
/// Inverse of a scan of an `N x N` block: scan position at `[y * N + x]`.
const fn inverse<const L: usize>(scan: [(u8, u8); L], n: usize) -> [u8; L] {
let mut out = [0u8; L];
let mut i = 0;
while i < L {
out[scan[i].1 as usize * n + scan[i].0 as usize] = i as u8;
i += 1;
}
out
}
static INV_SCAN_1: [[u8; 1]; 3] = [[0], [0], [0]];
static INV_SCAN_2: [[u8; 4]; 3] = [inverse(diag::<4>(2), 2), inverse(hv::<4>(2, false), 2), inverse(hv::<4>(2, true), 2)];
static INV_SCAN_4: [[u8; 16]; 3] = [inverse(diag::<16>(4), 4), inverse(hv::<16>(4, false), 4), inverse(hv::<16>(4, true), 4)];
static INV_SCAN_8: [[u8; 64]; 3] = [inverse(diag::<64>(8), 8), inverse(hv::<64>(8, false), 8), inverse(hv::<64>(8, true), 8)];
/// Position of (x, y) in ScanOrder[log2BlockSize][scanIdx] (log2BlockSize 0..=3).
#[inline]
pub fn scan_pos(log2: u32, scan_idx: usize, x: u32, y: u32) -> usize {
let i = ((y << log2) + x) as usize;
(match log2 {
0 => INV_SCAN_1[scan_idx][i],
1 => INV_SCAN_2[scan_idx][i],
2 => INV_SCAN_4[scan_idx][i],
_ => INV_SCAN_8[scan_idx][i],
}) as usize
}
/// sig_coeff_flag ctxInc (9.3.4.2.5) by scan position n inside a 4x4 sub-block, `[scanIdx][k][n]`:
/// k = prevCsbf (0..=3) gives sigCtx of blocks larger than 4x4 before the position and size
/// offsets; k = 4 gives ctxIdxMap of 4x4 blocks.
pub static SIG_CTX: [[[u8; 16]; 5]; 3] = [sig_ctx(diag::<16>(4)), sig_ctx(hv::<16>(4, false)), sig_ctx(hv::<16>(4, true))];
const fn sig_ctx(scan: [(u8, u8); 16]) -> [[u8; 16]; 5] {
let mut out = [[0u8; 16]; 5];
let mut n = 0;
while n < 16 {
let (x, y) = (scan[n].0, scan[n].1);
out[0][n] = if x + y == 0 {
2
} else if x + y < 3 {
1
} else {
0
};
out[1][n] = if y == 0 {
2
} else if y == 1 {
1
} else {
0
};
out[2][n] = if x == 0 {
2
} else if x == 1 {
1
} else {
0
};
out[3][n] = 2;
out[4][n] = CTX_IDX_MAP[(y as usize) * 4 + x as usize];
n += 1;
}
out
}
/// Up-right diagonal scan for log2 size 2 or 3.
pub fn scan_diag(log2: u32) -> &'static [(u8, u8)] {
scan_order(log2, 0)
@@ -81,7 +147,7 @@ pub fn qpc(qpi: i32, chroma_array_type_1: bool) -> i32 {
}
/// Table 9-50 ctxIdxMap for 4x4 sig_coeff_flag contexts.
pub static CTX_IDX_MAP: [u8; 16] = [0, 1, 4, 5, 2, 3, 4, 5, 6, 6, 8, 8, 7, 7, 8, 8];
pub const CTX_IDX_MAP: [u8; 16] = [0, 1, 4, 5, 2, 3, 4, 5, 6, 6, 8, 8, 7, 7, 8, 8];
/// levelScale (8-309).
pub static LEVEL_SCALE: [i32; 6] = [40, 45, 51, 57, 64, 72];
@@ -137,6 +203,17 @@ mod tests {
assert_eq!(SCAN_4[1][1], (1, 0));
}
#[test]
fn scan_pos_inverts_scan_order() {
for log2 in 0..4 {
for scan_idx in 0..3 {
for (i, &(x, y)) in scan_order(log2, scan_idx).iter().enumerate() {
assert_eq!(scan_pos(log2, scan_idx, x as u32, y as u32), i);
}
}
}
}
#[test]
fn dct_matrix_properties() {
// rows are nearly orthogonal with norm ~ 64^2 * 32
+45 -25
View File
@@ -1,7 +1,6 @@
//! Hostile-input robustness: truncation, header-region sweeps and random mutation of small
//! ffmpeg-generated AV1 streams (frame by frame) must never panic or run away. Skips without ffmpeg.
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -20,8 +19,15 @@ impl Rng {
}
}
/// (name, encoder args, pix_fmt)
const SEEDS: &[(&str, &str, &str)] = &[("av1_fz_default", "-c:v libsvtav1 -g 4", "yuv420p"), ("av1_fz_10bit", "-c:v libsvtav1 -g 6", "yuv420p10le")];
// Seeds are at least 64x64: SVT-AV1 before 3.0 (the Ubuntu runners have 2.3) refuses a source under 64 lines.
/// (name, encoder args, pix_fmt, size, decoder thread counts)
const SEEDS: &[(&str, &str, &str, &str, &[usize])] = &[
("av1_fz_default", "-c:v libsvtav1 -g 4", "yuv420p", "64x64", &[1, 3]),
("av1_fz_10bit", "-c:v libsvtav1 -g 6", "yuv420p10le", "64x64", &[1, 3]),
// Several superblock rows, so corrupt references meet frames already predicting from their
// top rows (row wavefront with frame threads).
("av1_fz_rows", "-c:v libsvtav1 -g 6", "yuv420p", "64x136", &[1, 8]),
];
fn ivf_frames(data: &[u8]) -> Vec<Vec<u8>> {
let mut out = Vec::new();
@@ -37,14 +43,13 @@ fn ivf_frames(data: &[u8]) -> Vec<Vec<u8>> {
out
}
// Seeds are 64x64: SVT-AV1 before 3.0 (the Ubuntu runners have 2.3) refuses a source under 64 lines.
fn make(ff: &std::path::Path, name: &str, args: &str, pix: &str) -> Option<Vec<Vec<u8>>> {
fn make(ff: &std::path::Path, name: &str, args: &str, pix: &str, size: &str) -> Option<Vec<Vec<u8>>> {
let out = craft_media_testkit::fixtures_dir("av1").join(format!("{name}.ivf"));
let p = craft_media_testkit::fixtures::generate(&out, |tmp| {
let a: Vec<&str> = args.split_whitespace().collect();
matches!(
Command::new(ff)
.args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=64x64:rate=25", "-frames:v", "6", "-pix_fmt", pix])
.args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", &format!("testsrc2=size={size}:rate=25"), "-frames:v", "6", "-pix_fmt", pix])
.args(a)
.args(["-f", "ivf"])
.arg(tmp)
@@ -69,20 +74,29 @@ fn count_panics() {
});
}
fn run_all(frames: &[Vec<u8>], label: &str) {
fn run_all(frames: &[Vec<u8>], threads: &[usize], label: &str) {
craft_media_testkit::set_progress(label);
count_panics();
let before = PANICS.load(Ordering::SeqCst);
let res = catch_unwind(AssertUnwindSafe(|| {
for threads in [1, 3] {
let mut dec = Decoder::with_threads(threads);
for (i, f) in frames.iter().enumerate() {
let _ = dec.decode_pts(f, i as i64);
}
let _ = dec.flush_result();
}
}));
assert!(res.is_ok() && PANICS.load(Ordering::SeqCst) == before, "panic: {label}");
// Every thread count decodes the stream at the same time, each on a thread of its own.
let ok = std::thread::scope(|s| {
let runs: Vec<_> = threads
.iter()
.map(|&threads| {
s.spawn(move || {
let mut dec = Decoder::with_threads(threads);
for (i, f) in frames.iter().enumerate() {
let _ = dec.decode_pts(f, i as i64);
}
let _ = dec.flush_result();
})
})
.collect();
// Join every run (no short cut), then check them all.
let joined: Vec<bool> = runs.into_iter().map(|r| r.join().is_ok()).collect();
joined.into_iter().all(|ok| ok)
});
assert!(ok && PANICS.load(Ordering::SeqCst) == before, "panic: {label}");
}
/// Random-mutation rounds per seed (raise with FILMCRAFT_FUZZ_ITERS for a long hunt).
@@ -95,17 +109,23 @@ const DEADLINE_SECS: u64 = 600;
#[test]
fn hostile_streams_never_panic() {
craft_media_testkit::with_deadline(DEADLINE_SECS, sweep);
craft_media_testkit::with_deadline(DEADLINE_SECS, || sweep(&SEEDS[..2]));
}
fn sweep() {
/// The seed with several superblock rows, in a test of its own so it runs beside the others.
#[test]
fn hostile_streams_with_superblock_rows_never_panic() {
craft_media_testkit::with_deadline(DEADLINE_SECS, || sweep(&SEEDS[2..]));
}
fn sweep(seeds: &[(&str, &str, &str, &str, &[usize])]) {
let Some(ff) = craft_media_testkit::ffmpeg_or_skip("AV1 fuzz") else { return };
for &(name, args, pix) in SEEDS {
for &(name, args, pix, size, threads) in seeds {
if !craft_media_testkit::ffmpeg_has_encoder(&ff, "libsvtav1") {
craft_media_testkit::skip_test("AV1 fuzz", &format!("this ffmpeg has no libsvtav1 encoder (seed {name})"));
continue;
}
let Some(seed) = make(&ff, name, args, pix) else {
let Some(seed) = make(&ff, name, args, pix, size) else {
panic!("ffmpeg has libsvtav1 but could not encode the {name} seed");
};
if seed.is_empty() {
@@ -118,7 +138,7 @@ fn sweep() {
for n in (0..len).step_by((len / 200).max(1)) {
let mut d = seed.clone();
d[fi].truncate(n);
run_all(&d, &format!("{name} frame {fi} truncate {n}"));
run_all(&d, threads, &format!("{name} frame {fi} truncate {n}"));
}
}
// header region of every frame: every byte to a few values, every bit
@@ -127,12 +147,12 @@ fn sweep() {
for v in [0u8, 0xff, 0x80, 0x01, 0x7f] {
let mut d = seed.clone();
d[fi][p] = v;
run_all(&d, &format!("{name} frame {fi} byte {p}={v:#x}"));
run_all(&d, threads, &format!("{name} frame {fi} byte {p}={v:#x}"));
}
for b in 0..8 {
let mut d = seed.clone();
d[fi][p] ^= 1 << b;
run_all(&d, &format!("{name} frame {fi} flip {p}.{b}"));
run_all(&d, threads, &format!("{name} frame {fi} flip {p}.{b}"));
}
}
}
@@ -151,7 +171,7 @@ fn sweep() {
d[fi][p] = rng.next() as u8;
}
}
run_all(&d, &format!("{name} random {i}"));
run_all(&d, threads, &format!("{name} random {i}"));
}
}
}
+1
View File
@@ -9,3 +9,4 @@ mod hostile_dims;
mod oracle_inter;
mod oracle_intra;
mod perf;
mod wavefront;
@@ -0,0 +1,93 @@
//! Stress tests of the frame threading (parse / reconstruct split and the row wavefront across
//! frames): every thread count gives the single-thread pictures, run after run, and dropping
//! or flushing a busy decoder never hangs. A hang fails the test through the deadline.
use crate::common;
use common::*;
use craft_video::av1::decode::{Decoder, Picture};
/// A GOP with many superblock rows (the wavefront has rows to overlap), hierarchical
/// references and motion reaching across rows.
fn spec() -> Spec {
Spec::new("wavefront_640x480", "testsrc2=s=640x480:r=25,rotate=a=t*0.5:c=black", 24, "yuv420p", &["-preset", "8", "-crf", "38"])
}
/// Seconds a test may take before it counts as a hang (override: FILMCRAFT_HOSTILE_TIMEOUT_SECS).
const DEADLINE_SECS: u64 = 300;
fn decode(tus: &[Vec<u8>], threads: usize) -> Vec<Picture> {
let mut dec = Decoder::with_threads(threads);
let mut pics = Vec::new();
for (i, tu) in tus.iter().enumerate() {
pics.extend(dec.decode_pts(tu, i as i64).unwrap_or_else(|e| panic!("threads {threads}, unit {i}: {e}")));
}
pics.extend(dec.flush_result().unwrap_or_else(|e| panic!("threads {threads}, flush: {e}")));
pics
}
#[test]
fn every_thread_count_gives_the_single_thread_pictures_repeatedly() {
let Some(ff) = ffmpeg() else { return };
let tus = ivf_frames(&make(&ff, &spec()));
craft_media_testkit::with_deadline(DEADLINE_SECS, move || {
let want = decode(&tus, 1);
assert_eq!(want.len(), 24);
for threads in [2, 3, 4, 5, 8, 16] {
for round in 0..4 {
assert!(decode(&tus, threads) == want, "threads {threads}, round {round}: pictures differ");
}
}
});
}
/// 128x128 superblocks in two tile columns (libaom), with motion field estimation: the
/// wavefront over the superblock rows of every tile column.
fn spec_sb128_tiles() -> Spec {
let enc = ["-cpu-used", "8", "-crf", "40", "-tile-columns", "1", "-lag-in-frames", "16", "-aom-params", "sb-size=128:enable-ref-frame-mvs=1"];
Spec::new("wavefront_sb128_tiles", "testsrc2=s=640x400:r=25,rotate=a=t*0.5:c=black", 16, "yuv420p", &enc).encoder("libaom-av1")
}
#[test]
fn superblocks_of_128_in_tile_columns_give_the_single_thread_pictures() {
let Some(ff) = ffmpeg_libaom() else { return };
let tus = ivf_frames(&make(&ff, &spec_sb128_tiles()));
craft_media_testkit::with_deadline(DEADLINE_SECS, move || {
let want = decode(&tus, 1);
assert_eq!(want.len(), 16);
for threads in [2, 3, 4, 8, 16] {
for round in 0..2 {
assert!(decode(&tus, threads) == want, "threads {threads}, round {round}: pictures differ");
}
}
});
}
#[test]
fn dropping_or_flushing_a_busy_decoder_never_hangs() {
let Some(ff) = ffmpeg() else { return };
let tus = ivf_frames(&make(&ff, &spec()));
craft_media_testkit::with_deadline(DEADLINE_SECS, move || {
let want = decode(&tus, 1);
for threads in [2, 3, 8, 16] {
for cut in [1, 2, 7, 12, 23] {
// Dropped with frames in flight (no flush).
let mut dec = Decoder::with_threads(threads);
for (i, tu) in tus.iter().take(cut).enumerate() {
dec.decode_pts(tu, i as i64).expect("decode");
}
drop(dec);
// Flushed mid-stream (as before a seek), then continued: the same pictures.
let mut dec = Decoder::with_threads(threads);
let mut pics = Vec::new();
for (i, tu) in tus.iter().enumerate() {
if i == cut {
pics.extend(dec.flush_result().expect("flush"));
}
pics.extend(dec.decode_pts(tu, i as i64).expect("decode"));
}
pics.extend(dec.flush_result().expect("flush"));
assert!(pics == want, "threads {threads}, flush before unit {cut}: pictures differ");
}
}
});
}
+194 -5
View File
@@ -11,6 +11,8 @@
use serde::{Deserialize, Serialize};
use crate::props::Rgb;
/// Most series in a chart.
pub const MAX_SERIES: usize = 255;
/// Most categories (points per series) in a chart.
@@ -123,17 +125,111 @@ impl LegendPos {
}
}
/// One series: its name and a value per category (`None` = no value, a gap).
/// One series: its name, a value per category (`None` = no value, a gap) and how it is drawn
/// (Chart Design ▸ Format Series): fill, line, markers, smooth lines, its own data labels and
/// number format. Unset fields follow the chart's defaults.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ChartSeries {
pub name: String,
#[serde(default)]
pub values: Vec<Option<f64>>,
/// Fill colour; `None`: the chart's accent colour for the series.
#[serde(default)]
pub fill: Option<Rgb>,
/// "None" fill: the series draws no fill.
#[serde(default)]
pub no_fill: bool,
/// Line width in points; `None`: the chart's default.
#[serde(default)]
pub line_w: Option<f32>,
/// "None" line: the series draws no line.
#[serde(default)]
pub no_line: bool,
/// Markers on points; `None`: unset (scatter charts draw them; a saved scatter series then has no `c:marker`, so Word picks its own symbols).
#[serde(default)]
pub markers: Option<bool>,
/// Draw the line as a smooth curve.
#[serde(default)]
pub smooth: bool,
/// Data labels for this series; `None`: follow the chart's setting.
#[serde(default)]
pub labels: Option<bool>,
/// Number format of this series's labels and values; `None`: linked to the source.
#[serde(default)]
pub fmt: Option<String>,
}
/// One chart axis as Word formats it (Format Axis): title, scale bounds, tick-label number
/// format, gridlines, tick labels and the axis line.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ChartAxis {
/// `None`: no axis title; text cut to [`MAX_TEXT`] characters.
#[serde(default)]
pub title: Option<String>,
/// Fixed lower bound; `None`: automatic.
#[serde(default)]
pub min: Option<f64>,
/// Fixed upper bound; `None`: automatic.
#[serde(default)]
pub max: Option<f64>,
/// Tick-label number format (`#,##0`, `0.00`, `0%`…); `None`: linked to the source data.
#[serde(default)]
pub fmt: Option<String>,
/// Major gridlines drawn along this axis.
#[serde(default)]
pub gridlines: bool,
/// Tick labels; `false` hides the axis the way Word's Axis Labels ▸ None does.
#[serde(default = "the_true")]
pub labels: bool,
/// The axis line.
#[serde(default = "the_true")]
pub line: bool,
}
fn the_true() -> bool {
true
}
impl Default for ChartAxis {
fn default() -> Self {
ChartAxis { title: None, min: None, max: None, fmt: None, gridlines: false, labels: true, line: true }
}
}
/// The value axis as a new chart has it: gridlines, labels, no axis line.
pub fn value_axis() -> ChartAxis {
ChartAxis { gridlines: true, line: false, ..ChartAxis::default() }
}
impl ChartAxis {
/// Make the axis safe to store: texts cleaned, bounds finite, `min` under `max` (an
/// impossible pair drops both), the format code cut to [`MAX_FMT`] characters.
pub fn sanitize(&mut self) {
self.title = self.title.as_deref().map(clean_text).filter(|t| !t.is_empty());
self.min = self.min.filter(|v| v.is_finite());
self.max = self.max.filter(|v| v.is_finite());
if matches!((self.min, self.max), (Some(lo), Some(hi)) if lo >= hi) {
self.min = None;
self.max = None;
}
self.fmt = self.fmt.as_deref().map(clean_fmt).filter(|f| !f.is_empty());
}
}
/// Longest axis number format kept, characters.
pub const MAX_FMT: usize = 128;
/// A number format code: control characters as spaces, ends trimmed, at most [`MAX_FMT`]
/// characters.
pub fn clean_fmt(s: &str) -> String {
let s: String = s.chars().take(MAX_FMT * 4).map(|c| if c.is_control() { ' ' } else { c }).collect();
s.trim().chars().take(MAX_FMT).collect::<String>().trim_end().to_string()
}
/// A chart the app can edit.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ChartSpec {
#[serde(rename = "type")]
@@ -151,6 +247,29 @@ pub struct ChartSpec {
/// Show each point's value next to it.
#[serde(default)]
pub data_labels: bool,
/// The category axis (scatter charts: the x axis).
#[serde(default)]
pub cat_axis: ChartAxis,
/// The value axis (scatter charts: the y axis); gridlines on, no axis line, as a new chart has them.
#[serde(default = "value_axis")]
pub val_axis: ChartAxis,
}
impl Default for ChartSpec {
/// An empty chart whose value axis is the one a new chart (and a missing JSON field) has:
/// gridlines on, no axis line.
fn default() -> Self {
ChartSpec {
kind: ChartType::default(),
categories: Vec::new(),
series: Vec::new(),
title: None,
legend: None,
data_labels: false,
cat_axis: ChartAxis::default(),
val_axis: value_axis(),
}
}
}
impl ChartSpec {
@@ -168,10 +287,23 @@ impl ChartSpec {
ChartSpec {
kind,
categories: categories.into_iter().map(str::to_string).collect(),
series: series.into_iter().map(|(n, v)| ChartSeries { name: n.to_string(), values: v.into_iter().map(Some).collect() }).collect(),
series: series
.into_iter()
.map(|(n, v)| ChartSeries {
name: n.to_string(),
values: v.into_iter().map(Some).collect(),
// Scatter series are drawn as markers without a line.
no_line: kind == ChartType::Scatter,
..Default::default()
})
.collect(),
title: None,
legend: Some(LegendPos::Bottom),
data_labels: false,
cat_axis: ChartAxis::default(),
// Pie and doughnut charts have no axes: their axes stay plain, so the chart reads back
// as the same model.
val_axis: if kind.is_round() { ChartAxis::default() } else { value_axis() },
}
}
@@ -203,8 +335,12 @@ impl ChartSpec {
*v = v.filter(|x| x.is_finite());
}
s.values.resize(n, None);
s.fmt = s.fmt.as_deref().map(clean_fmt).filter(|f| !f.is_empty());
s.line_w = s.line_w.filter(|w| w.is_finite()).map(|w| w.clamp(0.0, 20.0));
}
self.title = self.title.as_deref().map(clean_text).filter(|t| !t.is_empty());
self.cat_axis.sanitize();
self.val_axis.sanitize();
}
/// The chart after `sanitize`.
@@ -229,7 +365,11 @@ mod tests {
let mut c = ChartSpec {
categories: vec!["a".into(); MAX_POINTS + 10],
series: (0..MAX_SERIES + 5)
.map(|i| ChartSeries { name: format!("{}\n{}", "x".repeat(400), i), values: vec![Some(f64::NAN), Some(f64::INFINITY), Some(1.0)] })
.map(|i| ChartSeries {
name: format!("{}\n{}", "x".repeat(400), i),
values: vec![Some(f64::NAN), Some(f64::INFINITY), Some(1.0)],
..Default::default()
})
.collect(),
title: Some(" ".into()),
..Default::default()
@@ -250,7 +390,7 @@ mod tests {
fn short_categories_are_padded_to_the_longest_series() {
let c = ChartSpec {
categories: vec!["a".into()],
series: vec![ChartSeries { name: "s".into(), values: vec![Some(1.0), Some(2.0)] }],
series: vec![ChartSeries { name: "s".into(), values: vec![Some(1.0), Some(2.0)], ..Default::default() }],
..Default::default()
}
.sanitized();
@@ -258,4 +398,53 @@ mod tests {
assert_eq!(ChartType::from_id("BARSTACKED"), Some(ChartType::BarStacked));
assert_eq!(LegendPos::from_id("top"), Some(LegendPos::Top));
}
#[test]
fn default_and_missing_json_axis_fields_agree() {
let from_json: ChartSpec = serde_json::from_str(r#"{"type":"column"}"#).unwrap();
let d = ChartSpec::default();
assert_eq!((d.cat_axis.clone(), d.val_axis.clone()), (from_json.cat_axis, from_json.val_axis));
assert!(d.val_axis.gridlines && !d.val_axis.line);
}
#[test]
fn axis_bounds_and_formats_are_cleaned() {
let mut a = ChartAxis {
title: Some(" Sales\n".into()),
min: Some(f64::NAN),
max: Some(50.0),
fmt: Some("x".repeat(MAX_FMT + 40)),
gridlines: true,
labels: false,
line: true,
};
a.sanitize();
assert_eq!(a.title.as_deref(), Some("Sales"));
assert_eq!(a.min, None);
assert_eq!(a.max, Some(50.0));
assert_eq!(a.fmt.as_deref().map(|f| f.chars().count()), Some(MAX_FMT));
// min not under max drops the pair; a format of only spaces is no format.
let mut b = ChartAxis { min: Some(50.0), max: Some(50.0), fmt: Some(" ".into()), ..Default::default() };
b.sanitize();
assert_eq!((b.min, b.max), (None, None));
assert_eq!(b.fmt, None);
let mut spec = ChartSpec {
series: vec![ChartSeries { name: "s".into(), values: vec![Some(1.0)], ..Default::default() }],
categories: vec!["a".into()],
..Default::default()
};
spec.series[0].line_w = Some(f32::NAN);
spec.series[0].fmt = Some("0.00".into());
spec.sanitize();
assert_eq!(spec.series[0].line_w, None);
assert_eq!(spec.series[0].fmt.as_deref(), Some("0.00"));
let mut spec = ChartSpec {
series: vec![ChartSeries { name: "s".into(), values: vec![], line_w: Some(100.0), ..Default::default() }],
..Default::default()
};
spec.sanitize();
assert_eq!(spec.series[0].line_w, Some(20.0), "a wild width clamps to 20 pt");
}
}
+5 -1
View File
@@ -115,7 +115,9 @@ pub enum GraphicItem {
#[serde(default)]
stroke_width: f32,
},
/// One line of text in `rect`, aligned horizontally by `align`, centred vertically.
/// One line of text in `rect`, aligned horizontally by `align`, centred vertically; `angle`
/// degrees clockwise turns the text about `rect`'s centre as it is drawn (chart axis titles
/// turn the vertical axis's title -90°, reading upwards, as Word does).
Text {
rect: [f32; 4],
text: String,
@@ -127,6 +129,8 @@ pub enum GraphicItem {
align: TextAlign,
#[serde(default)]
font: Option<String>,
#[serde(default)]
angle: f32,
},
/// A picture (`Document::media` key) stretched into `rect`.
Image { rect: [f32; 4], media: String },
+16 -11
View File
@@ -83,17 +83,22 @@ for new pictures (`vm = i + 1`) and the `XLDAPR` dynamic-array metadata (`cm = 1
The parts this crate writes use the transitional namespaces; a strict file's kept parts stay as
they were. A binary workbook is not written.
A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not
read (each cell keeps its format index). Its cell formulas are decoded from their tokens into the
text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four
ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in
the formula, numbers as Excel spells them), with
`FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell
metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the
file keeps the tokens once. A formula whose decoding needs something the reader does not read
keeps only its saved result, with a warning saying why: user defined names, table references,
other workbooks, Python code stored outside the cell, data tables. The XLSB function table
(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s.
A binary workbook gives the same `Workbook`, strings, rows and merged cells, and the same `Styles`:
number formats, fonts, fills, borders, cell and cell style formats, cell styles, differential
formats and the indexed colours, as Excel writes them in `styles.xml` (default fill colours and
empty settings left out, a cell style format's `apply*="0"`, a number format with a newer locale tag
in its Excel 2007 spelling, as `formatCode` has it). Custom table styles and recent colours are not
read; a damaged styles part is a warning and no styles, while one past a cap (65,536 entries in a
table, 1,024 gradient stops) or past `binary_text` is too large. Its cell formulas are decoded from
their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from
the four ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks
typed in the formula, numbers as Excel spells them), with `FormulaKind::Shared` or
`FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a
shared formula gets its own text, decoded at the cell, since the file keeps the tokens once. A
formula whose decoding needs something the reader does not read keeps only its saved result, with a
warning saying why: user defined names, table references, other workbooks, Python code stored
outside the cell, data tables. The XLSB function table (`xlsb/ftab.rs`) and the expression printer
are this crate's own, apart from `craft-xls`'s.
## Hostile input
+2 -2
View File
@@ -209,8 +209,8 @@ pub struct Document {
}
impl Document {
/// Reads every part. A binary workbook is refused: its formulas and styles are not read, so
/// it cannot be written back as SpreadsheetML without loss.
/// Reads every part. A binary workbook is refused: only some of its parts are read, so it
/// cannot be written back as SpreadsheetML without loss.
pub fn read(bytes: &[u8], limits: &Limits) -> Result<Document, Error> {
let mut p = Package::open(bytes, limits)?;
if p.is_binary() {
+16 -4
View File
@@ -220,16 +220,28 @@ impl<'a> Package<'a> {
Ok(items)
}
/// The styles part, `None` when the workbook has none (or is binary: its styles are not read).
/// The styles part, `None` when the workbook has none. A damaged one is a warning and `None`;
/// only a limit is an error.
pub fn styles(&mut self) -> Result<Option<Styles>, Error> {
if self.binary.is_some() {
return Ok(None);
}
let rels = self.workbook_rels()?;
let Some(rel) = rels.first("styles").filter(|r| !r.external) else {
self.warn("no styles part");
return Ok(None);
};
if let Some(types) = &self.binary {
let read = xlsb::check_type(types, &rel.target, xlsb::STYLES).and_then(|()| {
let bytes = self.opc.read(&rel.target)?.ok_or_else(|| xlsb::invalid(format!("missing part {}", rel.target)))?;
xlsb::styles(&bytes, &self.limits, &self.budget, &mut self.warnings)
});
return match read {
Ok(styles) => Ok(Some(styles)),
Err(Error::TooLarge(m)) => Err(Error::TooLarge(m)),
Err(e) => {
self.warn(format!("skipped {}: {e}", rel.target));
Ok(None)
}
};
}
Ok(self.optional_xml(&rel.target.clone())?.map(Styles::read))
}
+27 -8
View File
@@ -1,11 +1,13 @@
//! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheet cells, from
//! their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an
//! `.xlsx` file holds; one that can't be keeps only its saved result, with a warning saying why.
//! Styles are not read. A record that does not parse fails the read.
//! The styles part is read into the same [`crate::styles::Styles`] ([`styles`]). A record that
//! does not parse fails the read.
mod expr;
mod formula;
mod ftab;
mod styles;
use std::collections::HashMap;
use std::sync::Arc;
@@ -19,11 +21,13 @@ use crate::workbook::{SheetEntry, SheetKind, Workbook};
use crate::{Error, Limits, Warnings};
pub(crate) use formula::Links;
use formula::{Book, Mode, Name, Xti};
pub(crate) use styles::styles;
pub(crate) const MAIN: &str = "application/vnd.ms-excel.main";
const BINARY_MAIN: &str = "application/vnd.ms-excel.sheet.binary.macroEnabled.main";
pub(crate) const WORKSHEET: &str = "application/vnd.ms-excel.worksheet";
pub(crate) const STRINGS: &str = "application/vnd.ms-excel.sharedStrings";
pub(crate) const STYLES: &str = "application/vnd.ms-excel.styles";
/// Most sheets in the workbook.
const MAX_SHEETS: usize = 1024;
/// Supporting links (this workbook, add-ins, other workbooks) read at most; past it, formulas
@@ -174,12 +178,16 @@ struct Records<'a, 'l> {
bytes: Bytes<'a>,
limits: &'l Limits,
budget: &'l Budget,
blocks: Vec<u16>,
/// Each open block's end record, and whether its records are read.
blocks: Vec<(u16, bool)>,
/// Read the alternate content meant for Excel 2007 (version 12), as an `.xlsx` reader sees
/// the main attribute where Excel keeps a newer spelling in an extension.
excel_2007: bool,
}
impl<'a, 'l> Records<'a, 'l> {
fn new(bytes: &'a [u8], limits: &'l Limits, budget: &'l Budget) -> Self {
Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![] }
Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![], excel_2007: false }
}
fn next(&mut self, warnings: &mut Warnings) -> Result<Option<(u16, &'a [u8])>, Error> {
@@ -208,6 +216,7 @@ impl<'a, 'l> Records<'a, 'l> {
// BrtFRTBegin, BrtACBegin.
35 | 37 => {
let mut b = Bytes::new(payload);
let mut read = false;
if id == 35 {
b.take(4)?;
} else {
@@ -215,22 +224,32 @@ impl<'a, 'l> Records<'a, 'l> {
if count == 0 {
return Err(invalid("empty alternate-content version list"));
}
b.take(usize::from(count) * 4)?;
// ACProductVersion: fileVersion, then fileProduct (15 bits) and
// fileExtension (the version and later).
for _ in 0..count {
let (version, product) = (b.u16()?, b.u16()?);
let later = product & 0x8000 != 0;
read |= self.excel_2007 && product & 0x7FFF == 0 && (version == 12 || later && version <= 12);
}
}
b.finish()?;
if self.blocks.len() >= 64 {
return Err(invalid("extension blocks nested too deeply"));
}
self.blocks.push(id + 1);
warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped");
// Inside a skipped block, everything is skipped.
let read = read && self.blocks.iter().all(|(_, r)| *r);
self.blocks.push((id + 1, read));
if !read {
warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped");
}
}
// BrtFRTEnd, BrtACEnd.
36 | 38 => {
if self.blocks.pop() != Some(id) || !payload.is_empty() {
if self.blocks.pop().map(|(end, _)| end) != Some(id) || !payload.is_empty() {
return Err(invalid("mismatched extension block end"));
}
}
_ if !self.blocks.is_empty() => {}
_ if self.blocks.iter().any(|(_, read)| !read) => {}
_ => return Ok(Some((id, payload))),
}
}
+484
View File
@@ -0,0 +1,484 @@
//! The styles part of a binary workbook (`styles.bin`, [MS-XLSB] 2.1.7.50), read into the same
//! [`Styles`] the XML part gives: what Excel writes in `styles.xml` for the same workbook. Table
//! styles, recent colours and future records are not read.
use craft_sheet::{BorderStyle, HAlign, Pattern, Script, Underline, VAlign};
use craft_xml::El;
use super::{Budget, Bytes, Records, invalid};
use crate::styles::{
Alignment, Border, BorderEdge, CellStyle, Color, ColorSource, Dxf, Fill, Font, FontScheme, GradientFill, NumFmt, PatternFill, Protection, Styles,
Xf,
};
use crate::{Error, Limits, Warnings};
/// Characters of a font name or number format kept, as the XML side keeps of a font name.
const MAX_NAME_CHARS: usize = 255;
/// Longest string read from the part (Excel's own limits are 31 and 255 characters).
const MAX_STRING_UNITS: u32 = 32_767;
/// Most stops in one gradient fill (Excel writes a handful).
const MAX_STOPS: u32 = 1024;
/// Most entries in one table of the part (Excel allows about 64,000 cell formats). Each entry is
/// an allocation, and the record allowance alone would let a small part hold millions.
const MAX_ENTRIES: usize = 65_536;
/// `BrtColor` and `XFPropColor`: the same eight bytes. `None` for "not set" (`XFPropColor` type 4).
fn color(b: &mut Bytes<'_>) -> Result<Option<Color>, Error> {
let kind = b.u8()? >> 1;
let index = b.u8()?;
let tint = i16::from_le_bytes(<[u8; 2]>::try_from(b.take(2)?).map_err(|_| invalid("truncated colour"))?);
let [r, g, bl, a] = <[u8; 4]>::try_from(b.take(4)?).map_err(|_| invalid("truncated colour"))?;
let source = match kind {
0 => ColorSource::Auto,
1 => ColorSource::Indexed(u32::from(index)),
2 => ColorSource::Rgb(u32::from_be_bytes([a, r, g, bl])),
3 => ColorSource::Theme(u32::from(index)),
_ => return Ok(None),
};
Ok(Some(Color { source, tint: (f64::from(tint) / 32767.0).clamp(-1.0, 1.0) }))
}
fn pattern(fls: u32) -> Result<Pattern, Error> {
const PATTERNS: [Pattern; 19] = [
Pattern::None,
Pattern::Solid,
Pattern::Gray50,
Pattern::Gray75,
Pattern::Gray25,
Pattern::DarkHorizontal,
Pattern::DarkVertical,
Pattern::DarkDown,
Pattern::DarkUp,
Pattern::DarkGrid,
Pattern::DarkTrellis,
Pattern::LightHorizontal,
Pattern::LightVertical,
Pattern::LightDown,
Pattern::LightUp,
Pattern::LightGrid,
Pattern::LightTrellis,
Pattern::Gray125,
Pattern::Gray0625,
];
PATTERNS.get(fls as usize).copied().ok_or_else(|| invalid("unknown fill pattern"))
}
fn border_style(dg: u16) -> Result<BorderStyle, Error> {
const STYLES: [BorderStyle; 14] = [
BorderStyle::None,
BorderStyle::Thin,
BorderStyle::Medium,
BorderStyle::Dashed,
BorderStyle::Dotted,
BorderStyle::Thick,
BorderStyle::Double,
BorderStyle::Hair,
BorderStyle::MediumDashed,
BorderStyle::DashDot,
BorderStyle::MediumDashDot,
BorderStyle::DashDotDot,
BorderStyle::MediumDashDotDot,
BorderStyle::SlantDashDot,
];
STYLES.get(usize::from(dg)).copied().ok_or_else(|| invalid("unknown border style"))
}
fn halign(alc: u32) -> Option<HAlign> {
[HAlign::General, HAlign::Left, HAlign::Center, HAlign::Right, HAlign::Fill, HAlign::Justify, HAlign::CenterAcross, HAlign::Distributed]
.get(alc as usize)
.copied()
}
fn valign(alcv: u32) -> Option<VAlign> {
[VAlign::Top, VAlign::Center, VAlign::Bottom, VAlign::Justify, VAlign::Distributed].get(alcv as usize).copied()
}
fn underline(uls: u32) -> Underline {
match uls {
0 => Underline::None,
2 => Underline::Double,
0x21 => Underline::SingleAccounting,
0x22 => Underline::DoubleAccounting,
_ => Underline::Single,
}
}
fn script(sss: u32) -> Script {
match sss {
1 => Script::Superscript,
2 => Script::Subscript,
_ => Script::Baseline,
}
}
fn scheme(s: u32) -> Option<FontScheme> {
match s {
0 => Some(FontScheme::None),
1 => Some(FontScheme::Major),
2 => Some(FontScheme::Minor),
_ => None,
}
}
fn name(text: String) -> String {
text.chars().take(MAX_NAME_CHARS).collect()
}
/// `BrtFont`: only what Excel's `.xlsx` states for the same font (no `<b val="0"/>`, no scheme
/// `none`, no family or character set 0).
fn font(b: &mut Bytes<'_>) -> Result<Font, Error> {
let height = b.u16()?;
let flags = b.u16()?;
let weight = b.u16()?;
let sss = b.u16()?;
let uls = b.u8()?;
let family = b.u8()?;
let charset = b.u8()?;
b.u8()?;
let color = color(b)?;
let font_scheme = b.u8()?;
let font_name = name(b.text(MAX_STRING_UNITS)?);
let on = |bit: u16| (flags & bit != 0).then_some(true);
Ok(Font {
name: Some(font_name),
size: Some(f64::from(height) / 20.0),
bold: (weight >= 0x2BC).then_some(true),
italic: on(0x02),
strike: on(0x08),
outline: on(0x10),
shadow: on(0x20),
condense: on(0x40),
extend: on(0x80),
underline: (uls != 0).then(|| underline(u32::from(uls))),
vert_align: (sss != 0).then(|| script(u32::from(sss))),
family: (family != 0).then_some(u32::from(family)),
charset: (charset != 0).then_some(u32::from(charset)),
scheme: scheme(u32::from(font_scheme)).filter(|s| *s != FontScheme::None),
// An automatic colour is a font with no `<color>` in `.xlsx`.
color: color.filter(|c| c.source != ColorSource::Auto),
})
}
/// A gradient's angle and inner rectangle, as `BrtFill` and `XFPropGradient` hold them.
fn gradient(b: &mut Bytes<'_>, path: bool) -> Result<GradientFill, Error> {
Ok(GradientFill { path, degree: b.f64()?, left: b.f64()?, right: b.f64()?, top: b.f64()?, bottom: b.f64()?, stops: vec![] })
}
/// `BrtFill`. Excel's `.xlsx` leaves out the default colours: system foreground (indexed 64) as
/// the pattern colour and system background (indexed 65) as the background, unless the pattern
/// colour has a tint (as the built-in "20% - Accent1" styles do), when it states the background.
fn fill(b: &mut Bytes<'_>) -> Result<Fill, Error> {
let fls = b.u32()?;
let fg = color(b)?;
let bg = color(b)?;
let kind = b.u32()?;
let mut g = gradient(b, kind == 1)?;
let count = b.u32()?;
if fls == 0x28 {
if count > MAX_STOPS {
return Err(Error::TooLarge(format!("more than {MAX_STOPS} gradient stops")));
}
for _ in 0..count {
let c = color(b)?.unwrap_or_default();
g.stops.push((b.f64()?, c));
}
return Ok(Fill::Gradient(g));
}
let system = |c: Color, i: u32| c.source == ColorSource::Indexed(i) && c.tint == 0.0;
let tinted = fg.is_some_and(|c| c.tint != 0.0);
Ok(Fill::Pattern(PatternFill { pattern: Some(pattern(fls)?), fg: fg.filter(|c| !system(*c, 64)), bg: bg.filter(|c| tinted || !system(*c, 65)) }))
}
/// `BrtBorder`: the five edges a cell border has, each stated (`<left/>` when it has no line).
fn border(b: &mut Bytes<'_>) -> Result<Border, Error> {
let flags = b.u8()?;
let mut edge = || -> Result<BorderEdge, Error> {
let style = border_style(u16::from(b.u8()?))?;
b.u8()?;
let c = color(b)?;
Ok(BorderEdge { style, color: c.filter(|_| style != BorderStyle::None) })
};
let (top, bottom, left, right, diagonal) = (edge()?, edge()?, edge()?, edge()?, edge()?);
Ok(Border {
left: Some(left),
right: Some(right),
top: Some(top),
bottom: Some(bottom),
diagonal: Some(diagonal),
vertical: None,
horizontal: None,
diagonal_up: flags & 2 != 0,
diagonal_down: flags & 1 != 0,
outline: true,
})
}
/// `BrtXF`. A cell style format (`style`) marks what it leaves out (`apply*="0"`); a cell format
/// marks what it sets apart from its style (`apply*="1"`).
fn xf(b: &mut Bytes<'_>, style: bool) -> Result<Xf, Error> {
const APPLY: [(u8, &str); 6] = [
(0x01, "applyNumberFormat"),
(0x02, "applyFont"),
(0x10, "applyFill"),
(0x08, "applyBorder"),
(0x04, "applyAlignment"),
(0x20, "applyProtection"),
];
let parent = b.u16()?;
let num_fmt_id = u32::from(b.u16()?);
let font_id = u32::from(b.u16()?);
let fill_id = u32::from(b.u16()?);
let border_id = u32::from(b.u16()?);
let rotation = u32::from(b.u8()?);
let indent = u32::from(b.u8()?);
let flags = u32::from(b.u16()?);
let apply = b.u8()?;
b.u8()?;
let alignment = Alignment {
horizontal: halign(flags & 7).filter(|h| *h != HAlign::General),
vertical: valign((flags >> 3) & 7).filter(|v| *v != VAlign::Bottom),
text_rotation: (rotation != 0).then_some(rotation),
wrap_text: flags & 0x40 != 0,
justify_last_line: flags & 0x80 != 0,
shrink_to_fit: flags & 0x100 != 0,
indent: (indent != 0).then_some(indent),
reading_order: Some((flags >> 10) & 3).filter(|r| *r != 0),
};
let protection = Protection { locked: (flags & 0x1000 == 0).then_some(false), hidden: (flags & 0x2000 != 0).then_some(true) };
let mut el = El::new("xf");
for (bit, attr) in APPLY {
if apply & bit != 0 {
el = el.with_attr(attr, if style { "0" } else { "1" });
}
}
Ok(Xf {
num_fmt_id,
font_id,
fill_id,
border_id,
xf_id: (!style).then_some(u32::from(parent)),
alignment: (alignment != Alignment::default()).then_some(alignment),
protection: (protection != Protection::default()).then_some(protection),
quote_prefix: flags & 0x8000 != 0,
el,
})
}
/// `BrtStyle`.
fn cell_style(b: &mut Bytes<'_>) -> Result<CellStyle, Error> {
let xf_id = b.u32()?;
let flags = b.u16()?;
let builtin = b.u8()?;
let level = b.u8()?;
let style_name = b.string(MAX_STRING_UNITS, true)?.map(name).unwrap_or_default();
let is_builtin = flags & 1 != 0;
Ok(CellStyle {
name: style_name,
xf_id,
builtin_id: is_builtin.then_some(u32::from(builtin)),
level: (is_builtin && matches!(builtin, 1 | 2)).then_some(u32::from(level)),
hidden: flags & 2 != 0,
custom_builtin: flags & 4 != 0,
})
}
/// An unsigned integer of the size a property has (1, 2 or 4 bytes).
fn uint(data: &[u8]) -> Result<u32, Error> {
match *data {
[a] => Ok(u32::from(a)),
[a, b] => Ok(u32::from(u16::from_le_bytes([a, b]))),
[a, b, c, d] => Ok(u32::from_le_bytes([a, b, c, d])),
_ => Err(invalid("unexpected formatting property size")),
}
}
/// A string with a 2-byte length (`LPWideString`, and the number format of `XFProp` 0x26).
fn short_string(b: &mut Bytes<'_>) -> Result<String, Error> {
let count = usize::from(b.u16()?);
let mut text = String::new();
for ch in char::decode_utf16(b.take(count * 2)?.as_chunks::<2>().0.iter().copied().map(u16::from_le_bytes)) {
text.push(ch.map_err(|_| invalid("invalid UTF-16 string"))?);
}
Ok(name(text))
}
fn dxf_font(d: &mut Dxf) -> &mut Font {
d.font.get_or_insert_with(Font::default)
}
/// `BrtDXF`: its `XFProps`, each stating one property.
fn dxf(b: &mut Bytes<'_>) -> Result<Dxf, Error> {
b.u16()?;
b.u16()?;
let count = b.u16()?;
let mut d = Dxf::default();
let mut gradient_fill: Option<GradientFill> = None;
let mut pattern_fill: Option<PatternFill> = None;
for _ in 0..count {
let kind = b.u16()?;
let size = usize::from(b.u16()?).checked_sub(4).ok_or_else(|| invalid("formatting property too short"))?;
let data = b.take(size)?;
let mut p = Bytes::new(data);
match kind {
0x00 => pattern_fill.get_or_insert_with(PatternFill::default).pattern = Some(pattern(uint(data)?)?),
0x01 => pattern_fill.get_or_insert_with(PatternFill::default).fg = color(&mut p)?,
0x02 => pattern_fill.get_or_insert_with(PatternFill::default).bg = color(&mut p)?,
0x03 => {
let path = p.u32()? == 1;
gradient_fill = Some(gradient(&mut p, path)?);
}
0x04 => {
let g = gradient_fill.as_mut().ok_or_else(|| invalid("gradient stop without a gradient"))?;
if g.stops.len() >= MAX_STOPS as usize {
return Err(Error::TooLarge(format!("more than {MAX_STOPS} gradient stops")));
}
p.u16()?;
let position = p.f64()?;
g.stops.push((position, color(&mut p)?.unwrap_or_default()));
}
0x05 => dxf_font(&mut d).color = color(&mut p)?,
0x06..=0x0C => {
let c = color(&mut p)?;
let edge = Some(BorderEdge { style: border_style(p.u16()?)?, color: c });
let border = d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() });
match kind {
0x06 => border.top = edge,
0x07 => border.bottom = edge,
0x08 => border.left = edge,
0x09 => border.right = edge,
0x0A => border.diagonal = edge,
0x0B => border.vertical = edge,
_ => border.horizontal = edge,
}
}
0x0D => d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }).diagonal_up = uint(data)? != 0,
0x0E => d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }).diagonal_down = uint(data)? != 0,
0x0F..=0x16 => {
let v = uint(data)?;
let a = d.alignment.get_or_insert_with(Alignment::default);
match kind {
0x0F => a.horizontal = halign(v),
0x10 => a.vertical = valign(v),
0x11 => a.text_rotation = Some(v),
0x12 => a.indent = Some(v),
0x13 => a.reading_order = Some(v),
0x14 => a.wrap_text = v != 0,
0x15 => a.justify_last_line = v != 0,
_ => a.shrink_to_fit = v != 0,
}
}
0x18 => dxf_font(&mut d).name = Some(short_string(&mut p)?),
0x19 => dxf_font(&mut d).bold = Some(uint(data)? >= 0x2BC),
0x1A => dxf_font(&mut d).underline = Some(underline(uint(data)?)),
0x1B => dxf_font(&mut d).vert_align = Some(script(uint(data)?)),
0x1C => dxf_font(&mut d).italic = Some(uint(data)? != 0),
0x1D => dxf_font(&mut d).strike = Some(uint(data)? != 0),
0x1E => dxf_font(&mut d).outline = Some(uint(data)? != 0),
0x1F => dxf_font(&mut d).shadow = Some(uint(data)? != 0),
0x20 => dxf_font(&mut d).condense = Some(uint(data)? != 0),
0x21 => dxf_font(&mut d).extend = Some(uint(data)? != 0),
0x22 => dxf_font(&mut d).charset = Some(uint(data)?),
0x23 => dxf_font(&mut d).family = Some(uint(data)?),
0x24 => dxf_font(&mut d).size = Some(f64::from(uint(data)?) / 20.0),
0x25 => dxf_font(&mut d).scheme = scheme(uint(data)?),
0x26 => d.num_fmt.get_or_insert_with(NumFmt::default).code = Some(short_string(&mut p)?),
0x29 => d.num_fmt.get_or_insert_with(NumFmt::default).id = Some(uint(data)?),
0x2B => d.protection.get_or_insert_with(Protection::default).locked = Some(uint(data)? != 0),
0x2C => d.protection.get_or_insert_with(Protection::default).hidden = Some(uint(data)? != 0),
// Merged cells (0x17) and relative indents (0x2A) have no place in `<dxf>`.
_ => {}
}
}
d.fill = gradient_fill.map(Fill::Gradient).or(pattern_fill.map(Fill::Pattern));
Ok(d)
}
/// The styles part: `BrtBeginStyleSheet` to `BrtEndStyleSheet`.
pub(crate) fn styles(bytes: &[u8], limits: &Limits, budget: &Budget, warnings: &mut Warnings) -> Result<Styles, Error> {
let mut records = Records::new(bytes, limits, budget);
// Excel keeps a number format with a newer locale tag in two alternate-content blocks: the
// newer spelling for itself, and for Excel 2007 the one `.xlsx` has as `formatCode`.
records.excel_2007 = true;
records.first(278, warnings)?;
let mut s = Styles::default();
// Inside BrtBeginCellStyleXFs (true) or BrtBeginCellXFs (false).
let mut xfs: Option<bool> = None;
let mut palette = false;
while let Some((id, payload)) = records.next(warnings)? {
let mut b = Bytes::new(payload);
let entries = match id {
44 => s.num_fmts.len(),
43 => s.fonts.len(),
45 => s.fills.len(),
46 => s.borders.len(),
47 => s.cell_style_xfs.len().max(s.cell_xfs.len()),
48 => s.cell_styles.len(),
507 => s.dxfs.len(),
475 => s.indexed_colors.len(),
_ => 0,
};
if entries >= MAX_ENTRIES {
return Err(Error::TooLarge(format!("more than {MAX_ENTRIES} entries in a styles table")));
}
match id {
279 => {
records.end(payload, warnings)?;
return Ok(s);
}
44 => {
let id = u32::from(b.u16()?);
let code = name(b.text(MAX_STRING_UNITS)?);
budget.text(code.len(), limits)?;
s.num_fmts.push((id, code));
}
43 => {
let f = font(&mut b)?;
budget.text(f.name.as_ref().map_or(0, String::len), limits)?;
s.fonts.push(f);
}
45 => s.fills.push(fill(&mut b)?),
46 => s.borders.push(border(&mut b)?),
626 => xfs = Some(true),
617 => xfs = Some(false),
627 | 618 => xfs = None,
47 => match xfs {
Some(true) => s.cell_style_xfs.push(xf(&mut b, true)?),
Some(false) => s.cell_xfs.push(xf(&mut b, false)?),
None => return Err(invalid("cell format outside its list")),
},
48 => {
let c = cell_style(&mut b)?;
budget.text(c.name.len(), limits)?;
s.cell_styles.push(c);
}
507 => {
let d = dxf(&mut b)?;
// What it keeps of its strings, after a later property has replaced an earlier.
let font = d.font.as_ref().and_then(|f| f.name.as_ref()).map_or(0, String::len);
let format = d.num_fmt.as_ref().and_then(|n| n.code.as_ref()).map_or(0, String::len);
budget.text(font + format, limits)?;
s.dxfs.push(d);
}
// BrtBeginIndexedColors, BrtEndIndexedColors.
565 => palette = true,
566 => palette = false,
475 if palette => {
let [r, g, bl, _] = <[u8; 4]>::try_from(b.take(4)?).map_err(|_| invalid("truncated colour"))?;
s.indexed_colors.push(u32::from_be_bytes([0, r, g, bl]));
}
// BrtBeginTableStyles: the count of custom table styles, which are not read.
508 => {
if b.u32()? > 0 {
warnings.warn("custom table styles in a binary workbook were not read");
}
continue;
}
_ => continue,
}
// The records read above are read whole; the rest are skipped.
if matches!(id, 44 | 43 | 45 | 46 | 47 | 48 | 507 | 475) {
b.finish()?;
}
}
Err(invalid("missing styles end record"))
}
+288 -1
View File
@@ -5,7 +5,10 @@ mod common;
use common::{read, read_with};
use craft_xlsx::testing::zip;
use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Range, SheetKind, Value, Visibility, is_xlsb};
use craft_xlsx::{
CellError, CellRef, Color, ColorSource, Error, Fill, FormulaKind, Limits, Package, Pattern, PatternFill, Protection, Range, SheetKind, Styles,
Value, Visibility, is_xlsb,
};
const ROOT: &str = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="book" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="records/book.bin"/></Relationships>"#;
const RELS: &str = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="one" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="../data/one.bin"/><Relationship Id="two" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="../data/two.bin"/><Relationship Id="strings" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" Target="../text/strings.bin"/></Relationships>"#;
@@ -931,6 +934,280 @@ fn a_dynamic_array_kept_as_its_value_has_no_cell_metadata() {
assert!(warned(&wb, "missing from the file"), "{:?}", wb.warnings);
}
// Styles ([MS-XLSB] 2.1.7.50): a styles part assembled from records, and the `styles.xml` Excel
// writes for the same formats when it saves the workbook as `.xlsx`.
const STYLES_RELS: &str = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="one" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="../data/one.bin"/><Relationship Id="two" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="../data/two.bin"/><Relationship Id="styles" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="../look/styles.bin"/></Relationships>"#;
const STYLES_TYPES: &str = r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Override PartName="/records/book.bin" ContentType="application/vnd.ms-excel.sheet.binary.macroEnabled.main"/><Override PartName="/data/one.bin" ContentType="application/vnd.ms-excel.worksheet"/><Override PartName="/data/two.bin" ContentType="application/vnd.ms-excel.worksheet"/><Override PartName="/look/styles.bin" ContentType="application/vnd.ms-excel.styles"/></Types>"#;
fn styled_file(styles: &[u8], types: &str) -> Vec<u8> {
let sheet = worksheet(&[row(0), cell(5, 0, &1.0f64.to_le_bytes())].concat(), &[]);
zip(&[
("[Content_Types].xml", types.as_bytes()),
("_rels/.rels", ROOT.as_bytes()),
("records/book.bin", &book(false)),
("records/_rels/book.bin.rels", STYLES_RELS.as_bytes()),
("data/one.bin", &sheet),
("data/two.bin", &worksheet(&[], &[])),
("look/styles.bin", styles),
])
}
/// The styles part of a package, and the warnings reading it gave.
fn styles_of(bytes: &[u8]) -> (Option<Styles>, Vec<String>) {
let mut p = Package::open(bytes, &Limits::default()).unwrap();
let styles = p.styles().unwrap();
(styles, p.warnings().to_vec())
}
/// `BrtColor`: auto 0, indexed 1, RGB 2, theme 3.
fn color(kind: u8, index: u8, tint: i16, rgba: [u8; 4]) -> Vec<u8> {
[&[kind << 1 | 1, index][..], &tint.to_le_bytes(), &rgba].concat()
}
fn indexed(i: u8) -> Vec<u8> {
color(1, i, 0, [0, 0, 0, 255])
}
fn theme(i: u8, tint: i16) -> Vec<u8> {
color(3, i, tint, [0, 0, 0, 255])
}
fn rgb(r: u8, g: u8, b: u8) -> Vec<u8> {
color(2, 255, 0, [r, g, b, 255])
}
#[allow(clippy::too_many_arguments)]
fn font(twips: u16, flags: u16, weight: u16, script: u16, underline: u8, family: u8, charset: u8, c: &[u8], scheme: u8, name: &str) -> Vec<u8> {
let head =
[&twips.to_le_bytes()[..], &flags.to_le_bytes(), &weight.to_le_bytes(), &script.to_le_bytes(), &[underline, family, charset, 0]].concat();
record(43, &[&head[..], c, &[scheme], &wide(name)].concat())
}
fn pattern_fill(fls: u32, fg: &[u8], bg: &[u8]) -> Vec<u8> {
record(45, &[&fls.to_le_bytes()[..], fg, bg, &[0; 48]].concat())
}
fn edge(dg: u8, c: &[u8]) -> Vec<u8> {
[&[dg, 0][..], c].concat()
}
#[allow(clippy::too_many_arguments)]
fn brt_xf(parent: u16, fmt: u16, font: u16, fill: u16, border: u16, rotation: u8, indent: u8, flags: u16, apply: u8) -> Vec<u8> {
let ids = [parent, fmt, font, fill, border].map(u16::to_le_bytes).concat();
record(47, &[&ids[..], &[rotation, indent], &flags.to_le_bytes(), &[apply, 0]].concat())
}
fn style(xf: u32, flags: u16, builtin: u8, level: u8, name: &str) -> Vec<u8> {
record(48, &[&xf.to_le_bytes()[..], &flags.to_le_bytes(), &[builtin, level], &wide(name)].concat())
}
fn prop(kind: u16, data: &[u8]) -> Vec<u8> {
[&kind.to_le_bytes()[..], &(data.len() as u16 + 4).to_le_bytes(), data].concat()
}
fn short(text: &str) -> Vec<u8> {
let units: Vec<u16> = text.encode_utf16().collect();
[(units.len() as u16).to_le_bytes().to_vec(), units.iter().flat_map(|u| u.to_le_bytes()).collect()].concat()
}
fn brt_dxf(props: &[Vec<u8>]) -> Vec<u8> {
record(507, &[&[0, 0x80, 0, 0][..], &(props.len() as u16).to_le_bytes(), &props.concat()].concat())
}
fn begin(id: u16, count: u32) -> Vec<u8> {
record(id, &count.to_le_bytes())
}
/// Every kind of record the styles part has, as Excel writes them.
fn styles_bin() -> Vec<u8> {
let normal = font(220, 0, 400, 0, 0, 2, 0, &theme(1, 0), 2, "Aptos Narrow");
// Bold, italic, struck out, superscript, double accounting underline; family and
// character set 0 and scheme "none" are not written in `.xlsx`.
let fancy = font(290, 0x0A, 700, 1, 0x22, 0, 2, &rgb(0, 128, 255), 0, "Wingdings");
let heading = font(220, 0, 400, 0, 0, 2, 0, &theme(4, 13106), 1, "Aptos Display");
// An automatic colour: no `<color>` in `.xlsx`.
let auto = font(220, 0x02, 400, 0, 0, 2, 0, &color(0, 64, 0, [0, 0, 0, 255]), 2, "Aptos Narrow");
let gradient = [
&0x28u32.to_le_bytes()[..],
&indexed(64),
&indexed(65),
&1u32.to_le_bytes(),
&[0.0f64, 0.5, 0.5, 0.5, 0.5].map(f64::to_le_bytes).concat(),
&2u32.to_le_bytes(),
&theme(0, 0),
&0.0f64.to_le_bytes(),
&theme(4, 0),
&1.0f64.to_le_bytes(),
]
.concat();
let none = edge(0, &color(0, 0, 0, [0, 0, 0, 0]));
let parts: Vec<Vec<u8>> = vec![
record(278, &[]),
begin(615, 2),
record(44, &[&164u16.to_le_bytes()[..], &wide(r"yyyy\-mm\-dd")].concat()),
// A format in two alternate-content blocks: the newer spelling for version 13 and later
// (skipped), and the one for versions 12 and 13, which `.xlsx` has as `formatCode`.
record(37, &[1, 0, 13, 0, 0, 0x80]),
record(44, &[&165u16.to_le_bytes()[..], &wide("[$-ja-JP-x-gannen]ge")].concat()),
record(38, &[]),
record(37, &[2, 0, 12, 0, 0, 0, 13, 0, 0, 0]),
record(44, &[&165u16.to_le_bytes()[..], &wide("[$]ge")].concat()),
record(38, &[]),
record(616, &[]),
begin(611, 4),
normal,
fancy,
heading,
auto,
record(612, &[]),
begin(603, 6),
pattern_fill(0, &indexed(64), &indexed(65)),
pattern_fill(17, &indexed(64), &indexed(65)),
pattern_fill(1, &rgb(255, 255, 0), &indexed(64)),
pattern_fill(9, &theme(5, -8191), &indexed(65)),
record(45, &gradient),
// A tinted pattern colour: `.xlsx` states the system background, as for darkGrid above.
pattern_fill(1, &theme(4, 26213), &indexed(65)),
record(604, &[]),
begin(613, 2),
record(46, &[&[0][..], &none, &none, &none, &none, &none].concat()),
record(
46,
&[&[3][..], &edge(2, &rgb(200, 0, 0)), &none, &edge(7, &indexed(64)), &edge(13, &color(0, 0, 0, [0, 0, 0, 0])), &edge(1, &theme(4, 0))]
.concat(),
),
record(614, &[]),
begin(626, 3),
brt_xf(0xFFFF, 0, 0, 0, 0, 0, 0, 0x1010, 0),
brt_xf(0xFFFF, 9, 0, 0, 0, 0, 0, 0x1010, 0x3E),
brt_xf(0xFFFF, 0, 2, 0, 0, 0, 0, 0x1010, 0),
record(627, &[]),
begin(617, 4),
brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0),
brt_xf(0, 164, 0, 0, 0, 0, 0, 0x1010, 0x01),
// Left, top, wrapped, shrunk, right to left, unlocked, hidden, quote prefix.
brt_xf(0, 0, 0, 0, 0, 180, 2, 0x01 | 0x40 | 0x100 | 0x800 | 0x2000 | 0x8000, 0x24),
brt_xf(1, 0, 1, 4, 1, 0, 0, 0x1010, 0x1A),
record(618, &[]),
begin(619, 3),
style(0, 1, 0, 0, "Normal"),
// Excel wraps each style in alternate content, which is skipped.
record(37, &[1, 0, 2, 0x0e, 0, 0x80]),
record(3072, &[0; 16]),
record(38, &[]),
style(1, 1 | 4, 1, 1, "RowLevel_2"),
style(2, 2, 0, 0, "Probe Style"),
record(620, &[]),
begin(505, 2),
brt_dxf(&[
prop(0x05, &rgb(0x9C, 0, 6)),
prop(0x19, &700u16.to_le_bytes()),
prop(0x1C, &[0]),
prop(0x1A, &0x21u16.to_le_bytes()),
prop(0x24, &240u32.to_le_bytes()),
prop(0x18, &short("Arial")),
prop(0x29, &170u16.to_le_bytes()),
prop(0x26, &short("0.0")),
prop(0x02, &rgb(0xFF, 0xC7, 0xCE)),
prop(0x0F, &[2]),
prop(0x14, &[1]),
prop(0x2B, &[0]),
prop(0x07, &[&color(0, 0, 0, [0, 0, 0, 0])[..], &1u16.to_le_bytes()].concat()),
]),
brt_dxf(&[
prop(0x03, &[&0u32.to_le_bytes()[..], &[90.0f64, 0.0, 0.0, 0.0, 0.0].map(f64::to_le_bytes).concat()].concat()),
prop(0x04, &[&[0, 0][..], &0.0f64.to_le_bytes(), &rgb(255, 255, 255)].concat()),
prop(0x04, &[&[0, 0][..], &1.0f64.to_le_bytes(), &theme(4, 0)].concat()),
]),
record(506, &[]),
record(508, &[&0u32.to_le_bytes()[..], &wide("TableStyleMedium2"), &wide("PivotStyleLight16")].concat()),
record(509, &[]),
record(473, &[]),
record(565, &[]),
record(475, &[0x11, 0x22, 0x33, 0]),
record(475, &[0xFF, 0xFF, 0xFF, 0]),
record(566, &[]),
record(474, &[]),
record(279, &[]),
];
parts.concat()
}
const STYLES_XML: &str = r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><numFmts count="2"><numFmt numFmtId="164" formatCode="yyyy\-mm\-dd"/><numFmt numFmtId="165" formatCode="[$]ge"/></numFmts><fonts count="3"><font><sz val="11"/><color theme="1"/><name val="Aptos Narrow"/><family val="2"/><scheme val="minor"/></font><font><b/><i/><strike/><u val="doubleAccounting"/><vertAlign val="superscript"/><sz val="14.5"/><color rgb="FF0080FF"/><name val="Wingdings"/><charset val="2"/></font><font><sz val="11"/><color theme="4" tint="0.39997558519241921"/><name val="Aptos Display"/><family val="2"/><scheme val="major"/></font><font><i/><sz val="11"/><name val="Aptos Narrow"/><family val="2"/><scheme val="minor"/></font></fonts><fills count="5"><fill><patternFill patternType="none"/></fill><fill><patternFill patternType="gray125"/></fill><fill><patternFill patternType="solid"><fgColor rgb="FFFFFF00"/><bgColor indexed="64"/></patternFill></fill><fill><patternFill patternType="darkGrid"><fgColor theme="5" tint="-0.249977111117893"/><bgColor indexed="65"/></patternFill></fill><fill><gradientFill type="path" left="0.5" right="0.5" top="0.5" bottom="0.5"><stop position="0"><color theme="0"/></stop><stop position="1"><color theme="4"/></stop></gradientFill></fill><fill><patternFill patternType="solid"><fgColor theme="4" tint="0.79998168889431442"/><bgColor indexed="65"/></patternFill></fill></fills><borders count="2"><border><left/><right/><top/><bottom/><diagonal/></border><border diagonalUp="1" diagonalDown="1"><left style="hair"><color indexed="64"/></left><right style="slantDashDot"><color auto="1"/></right><top style="medium"><color rgb="FFC80000"/></top><bottom/><diagonal style="thin"><color theme="4"/></diagonal></border></borders><cellStyleXfs count="3"><xf numFmtId="0" fontId="0" fillId="0" borderId="0"/><xf numFmtId="9" fontId="0" fillId="0" borderId="0" applyFont="0" applyFill="0" applyBorder="0" applyAlignment="0" applyProtection="0"/><xf numFmtId="0" fontId="2" fillId="0" borderId="0"/></cellStyleXfs><cellXfs count="4"><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0"/><xf numFmtId="164" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0" quotePrefix="1" applyAlignment="1" applyProtection="1"><alignment horizontal="left" vertical="top" textRotation="180" wrapText="1" indent="2" shrinkToFit="1" readingOrder="2"/><protection locked="0" hidden="1"/></xf><xf numFmtId="0" fontId="1" fillId="4" borderId="1" xfId="1" applyFont="1" applyFill="1" applyBorder="1"/></cellXfs><cellStyles count="3"><cellStyle name="Normal" xfId="0" builtinId="0"/><cellStyle name="RowLevel_2" xfId="1" builtinId="1" iLevel="1" customBuiltin="1"/><cellStyle name="Probe Style" xfId="2" hidden="1"/></cellStyles><dxfs count="2"><dxf><font><b/><i val="0"/><u val="singleAccounting"/><sz val="12"/><color rgb="FF9C0006"/><name val="Arial"/></font><numFmt numFmtId="170" formatCode="0.0"/><fill><patternFill><bgColor rgb="FFFFC7CE"/></patternFill></fill><alignment horizontal="center" wrapText="1"/><protection locked="0"/><border><bottom style="thin"><color auto="1"/></bottom></border></dxf><dxf><fill><gradientFill degree="90"><stop position="0"><color rgb="FFFFFFFF"/></stop><stop position="1"><color theme="4"/></stop></gradientFill></fill></dxf></dxfs><colors><indexedColors><rgbColor rgb="00112233"/><rgbColor rgb="00FFFFFF"/></indexedColors></colors></styleSheet>"#;
/// The same styles from a SpreadsheetML package.
fn xml_styles(xml: &str) -> Styles {
let types = r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/></Types>"#;
let root = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="r" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#;
let rels = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="s" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/></Relationships>"#;
let bytes = zip(&[
("[Content_Types].xml", types.as_bytes()),
("_rels/.rels", root.as_bytes()),
("xl/workbook.xml", br#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"/>"#),
("xl/_rels/workbook.xml.rels", rels.as_bytes()),
("xl/styles.xml", xml.as_bytes()),
]);
styles_of(&bytes).0.unwrap()
}
/// What both readers lift out: an `<xf>`'s element keeps only its `apply*` flags, and the XML
/// part's own element is dropped.
fn lifted(mut s: Styles) -> Styles {
for xf in s.cell_style_xfs.iter_mut().chain(&mut s.cell_xfs) {
let attrs = xf.el.attrs.iter().filter(|(k, _)| k.starts_with("apply")).cloned().collect();
xf.el = craft_xml::El::new("xf");
xf.el.attrs = attrs;
}
s.root = craft_xml::El::default();
s
}
#[test]
fn styles_are_read_as_their_xlsx_spelling() {
let (binary, warnings) = styles_of(&styled_file(&styles_bin(), STYLES_TYPES));
let binary = binary.expect("styles are read");
assert_eq!(lifted(binary.clone()), lifted(xml_styles(STYLES_XML)));
// A few of the conversions, spelled out.
assert_eq!(binary.fonts[1].size, Some(14.5));
assert_eq!(binary.fonts[1].family, None);
let solid = Fill::Pattern(PatternFill {
pattern: Some(Pattern::Solid),
fg: Some(Color { source: ColorSource::Rgb(0xFFFF_FF00), tint: 0.0 }),
bg: Some(Color { source: ColorSource::Indexed(64), tint: 0.0 }),
});
assert_eq!(binary.fills[2], solid);
assert_eq!(binary.cell_xfs[2].protection, Some(Protection { locked: Some(false), hidden: Some(true) }));
assert_eq!(binary.palette()[0], 0x11_2233);
// Written back as SpreadsheetML, a cell style format says what it leaves out, as Excel's does.
let written = String::from_utf8(binary.to_xml()).unwrap();
assert!(written.contains(r#"applyFont="0" applyFill="0" applyBorder="0" applyAlignment="0" applyProtection="0""#), "{written}");
assert!(warnings.iter().any(|w| w.contains("alternate-content")), "{warnings:?}");
assert!(!warnings.iter().any(|w| w.contains("table styles")), "{warnings:?}");
}
#[test]
fn damaged_styles_are_a_warning_and_the_workbook_opens() {
let good = styles_bin();
let end = record(279, &[]);
let body = &good[..good.len() - end.len()];
let with = |extra: &[u8]| [body, extra, &end].concat();
let cases: Vec<(&str, Vec<u8>, &str)> = vec![
("truncated", good[..good.len() - 1].to_vec(), STYLES_TYPES),
("no end record", body.to_vec(), STYLES_TYPES),
("unknown fill pattern", with(&pattern_fill(99, &indexed(64), &indexed(65))), STYLES_TYPES),
("unknown border style", with(&record(46, &[&[0][..], &edge(14, &indexed(64)), &[0; 40]].concat())), STYLES_TYPES),
("cell format outside its list", with(&brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0)), STYLES_TYPES),
("short property", with(&brt_dxf(&[vec![0x19, 0, 2, 0]])), STYLES_TYPES),
("wrong content type", good.clone(), TYPES),
];
for (why, part, types) in cases {
let bytes = styled_file(&part, types);
let (styles, warnings) = styles_of(&bytes);
assert!(styles.is_none(), "{why}");
assert!(warnings.iter().any(|w| w.contains("look/styles.bin")), "{why}: {warnings:?}");
assert_eq!(read(&bytes).unwrap().sheets[0].value(CellRef::new(0, 0)), Value::Number(1.0), "{why}");
}
// A table past 65,536 entries is past a fixed cap: refused before it is held, as too large.
let xfs = brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0).repeat(65_537);
let bytes = styled_file(&with(&[&begin(617, 65_537)[..], &xfs, &record(618, &[])].concat()), STYLES_TYPES);
let mut p = Package::open(&bytes, &Limits::default()).unwrap();
assert!(matches!(p.styles(), Err(Error::TooLarge(m)) if m.contains("65536 entries")));
// Custom table styles are not read, and say so.
let table_styles = [body, &record(508, &[&1u32.to_le_bytes()[..], &wide("Probe Table"), &wide("")].concat()), &end].concat();
let (styles, warnings) = styles_of(&styled_file(&table_styles, STYLES_TYPES));
assert!(styles.is_some());
assert!(warnings.iter().any(|w| w.contains("table styles")), "{warnings:?}");
}
/// `PtgAttrSpace` ([MS-XLSB] 2.5.98.31): its type (0 spaces before the next token, 1 line breaks
/// before it, 2 spaces before a `(`, 4 spaces before a `)`) and count.
fn space(kind: u8, n: u8) -> Vec<u8> {
@@ -1063,6 +1340,16 @@ fn a_spent_formula_allowance_skips_decoding() {
assert!(!warned(&wb, "unknown formula token"), "{:?}", wb.warnings);
}
#[test]
fn differential_format_strings_are_charged_to_the_text_allowance() {
// A DXF's font name and number format are text the styles part keeps, charged like a font's.
let dxf = |name: &str| [record(278, &[]), begin(505, 1), brt_dxf(&[prop(0x18, &short(name))]), record(506, &[]), record(279, &[])].concat();
let limits = Limits { binary_text: 100, ..Limits::default() };
let open = |part: &[u8]| Package::open(&styled_file(part, STYLES_TYPES), &limits).unwrap().styles();
assert!(open(&dxf("Arial")).is_ok_and(|s| s.is_some_and(|s| s.dxfs.len() == 1)));
assert!(matches!(open(&dxf(&"x".repeat(200))), Err(Error::TooLarge(_))));
}
#[test]
fn module_sheets_and_supporting_links_are_bounded() {
// 1,025 very hidden module sheets (no relationship) before the worksheets: each keeps a place