mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:42:27 +00:00
Merge remote-tracking branch 'origin/main' into w-r-codex/legacy-exif-orientation
This commit is contained in:
commit
3d09cb9ab9
81 files changed
+6668
-648
No files matched your search
Generated
+4
@@ -1903,6 +1903,10 @@ dependencies = [
|
||||
"proptest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-http-url"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "craft-i18n"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -11,6 +11,7 @@ rust-version = "1.95"
|
||||
repository = "https://github.com/storytold/craft-libs"
|
||||
|
||||
[workspace.dependencies]
|
||||
craft-http-url = { path = "crates/http-url" }
|
||||
craft-usd = { path = "crates/usd" }
|
||||
craft-smime = { path = "crates/smime" }
|
||||
craft-ber = { path = "crates/ber" }
|
||||
|
||||
+171
-42
@@ -150,62 +150,142 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
|
||||
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
|
||||
let mut title = String::new();
|
||||
let (mut size3, mut size1) = (0usize, 0usize);
|
||||
let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]);
|
||||
let mut rows: Vec<f32> = Vec::new();
|
||||
for line in text.lines() {
|
||||
// Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an
|
||||
// unquoted title's comment must not turn that comment into part of the title.
|
||||
if let Some(quoted) = line
|
||||
.trim_start()
|
||||
.strip_prefix("TITLE")
|
||||
.filter(|rest| rest.starts_with(char::is_whitespace))
|
||||
.and_then(|rest| rest.trim_start().strip_prefix('"'))
|
||||
/// A bounded, three-dimensional Cube profile with RGB rows in file order.
|
||||
/// Unlike [`parse_cube`], this preserves the optional title and does not resample 1D curves.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Cube3d {
|
||||
pub title: Option<String>,
|
||||
pub size: usize,
|
||||
pub data: Vec<[f32; 3]>,
|
||||
pub domain_min: [f32; 3],
|
||||
pub domain_max: [f32; 3],
|
||||
}
|
||||
|
||||
/// Read the simple 3D profile dialect used by camera-profile consumers.
|
||||
///
|
||||
/// `max_size` must be in 2..=256; the row budget is checked before each allocation.
|
||||
/// Whole-line comments and literal titles are preserved. Unknown alphabetic directives
|
||||
/// are ignored; 1D tables are refused. Finite domains are returned unchanged so the
|
||||
/// consumer retains its existing policy for equal or reversed endpoints.
|
||||
/// Numeric fields must consist of exactly three finite numbers (never silently skip tokens).
|
||||
pub fn parse_cube_3d(text: &str, max_size: usize) -> Result<Cube3d, LutError> {
|
||||
let p = read_cube(text, Some(max_size))?;
|
||||
if p.size3 == 0 {
|
||||
return err("not a 3D .cube LUT (no LUT_3D_SIZE)");
|
||||
}
|
||||
let expected = p.size3.checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
|
||||
if p.rows.len() != expected {
|
||||
return err(format!("expected {expected} entries, found {}", p.rows.len()));
|
||||
}
|
||||
Ok(Cube3d { title: p.title, size: p.size3, data: p.rows, domain_min: p.min, domain_max: p.max })
|
||||
}
|
||||
|
||||
struct CubeRows {
|
||||
title: Option<String>,
|
||||
size3: usize,
|
||||
size1: usize,
|
||||
min: [f32; 3],
|
||||
max: [f32; 3],
|
||||
rows: Vec<[f32; 3]>,
|
||||
}
|
||||
|
||||
fn cube_triplet(text: &str) -> Result<[f32; 3], LutError> {
|
||||
let mut it = text.split_whitespace();
|
||||
let mut row = [0.0; 3];
|
||||
for value in &mut row {
|
||||
*value = it.next().and_then(|s| s.parse::<f32>().ok()).filter(|v| v.is_finite()).ok_or_else(|| LutError(format!("bad line `{text}`")))?;
|
||||
}
|
||||
if it.next().is_some() {
|
||||
return err(format!("bad line `{text}`"));
|
||||
}
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
fn read_cube(text: &str, profile_max: Option<usize>) -> Result<CubeRows, LutError> {
|
||||
if profile_max.is_some_and(|n| !(2..=256).contains(&n)) {
|
||||
return err("3D profile size limit must be in 2..=256");
|
||||
}
|
||||
let mut p = CubeRows { title: None, size3: 0, size1: 0, min: [0.0; 3], max: [1.0; 3], rows: Vec::new() };
|
||||
let row_limit = profile_max.unwrap_or(256).checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
|
||||
for raw in text.lines() {
|
||||
let profile = profile_max.is_some();
|
||||
let line = if profile { raw.trim() } else { raw.split('#').next().unwrap_or("").trim() };
|
||||
// The standard dialect protects '#' inside quoted titles; profile titles are literal.
|
||||
if !profile
|
||||
&& let Some(quoted) =
|
||||
raw.trim_start().strip_prefix("TITLE").filter(|r| r.starts_with(char::is_whitespace)).and_then(|r| r.trim_start().strip_prefix('"'))
|
||||
{
|
||||
let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
|
||||
title = value.to_string();
|
||||
let Some((title, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
|
||||
p.title = Some(title.to_string());
|
||||
continue;
|
||||
}
|
||||
let line = line.split('#').next().unwrap_or("").trim();
|
||||
if line.is_empty() {
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let mut it = line.split_whitespace();
|
||||
let key = it.next().unwrap_or("");
|
||||
let nums = |it: std::str::SplitWhitespace<'_>| -> Result<[f32; 3], LutError> {
|
||||
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
|
||||
if v.len() == 3 { Ok([v[0], v[1], v[2]]) } else { err(format!("bad line `{line}`")) }
|
||||
let (key, rest) = if profile {
|
||||
["TITLE", "LUT_3D_SIZE", "LUT_1D_SIZE", "DOMAIN_MIN", "DOMAIN_MAX"]
|
||||
.into_iter()
|
||||
.find_map(|key| line.strip_prefix(key).map(|rest| (key, rest.trim())))
|
||||
.unwrap_or_else(|| line.split_once(char::is_whitespace).unwrap_or((line, "")))
|
||||
} else {
|
||||
line.split_once(char::is_whitespace).map_or((line, ""), |(key, rest)| (key, rest.trim()))
|
||||
};
|
||||
match key {
|
||||
"TITLE" => title = line[5..].trim().trim_matches('"').to_string(),
|
||||
"LUT_3D_SIZE" => size3 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_3D_SIZE".into()))?,
|
||||
"LUT_1D_SIZE" => size1 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?,
|
||||
"DOMAIN_MIN" => dmin = nums(it)?,
|
||||
"DOMAIN_MAX" => dmax = nums(it)?,
|
||||
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" => {
|
||||
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
|
||||
if v.len() == 2 {
|
||||
dmin = [v[0]; 3];
|
||||
dmax = [v[1]; 3];
|
||||
"TITLE" => p.title = Some(rest.trim_matches('"').to_string()),
|
||||
"LUT_3D_SIZE" => {
|
||||
let value = if profile { rest } else { rest.split_whitespace().next().unwrap_or("") };
|
||||
p.size3 = value.parse().map_err(|_| LutError(format!("bad LUT_3D_SIZE `{value}`")))?;
|
||||
let limit = profile_max.unwrap_or(MAX_SIZE);
|
||||
if !(2..=limit).contains(&p.size3) {
|
||||
return err(format!("LUT_3D_SIZE {} out of range", p.size3));
|
||||
}
|
||||
}
|
||||
k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => {
|
||||
let mut v = vec![k.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?];
|
||||
for value in it {
|
||||
v.push(value.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?);
|
||||
"LUT_1D_SIZE" => {
|
||||
if profile {
|
||||
return err("1D LUTs are not supported (only 3D .cube files)");
|
||||
}
|
||||
if v.len() != 3 {
|
||||
return err(format!("bad data line `{line}`"));
|
||||
p.size1 = rest.split_whitespace().next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?;
|
||||
if p.size1 > row_limit {
|
||||
return err("1D LUT exceeds row budget");
|
||||
}
|
||||
}
|
||||
"DOMAIN_MIN" => p.min = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MIN".into()))?,
|
||||
"DOMAIN_MAX" => p.max = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MAX".into()))?,
|
||||
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" if !profile => {
|
||||
let mut it = rest.split_whitespace();
|
||||
let min = it.next().and_then(|s| s.parse::<f32>().ok());
|
||||
let max = it.next().and_then(|s| s.parse::<f32>().ok());
|
||||
if let (Some(min), Some(max), None) = (min, max, it.next()) {
|
||||
p.min = [min; 3];
|
||||
p.max = [max; 3];
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let numeric = key.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.');
|
||||
let nonfinite = key.parse::<f32>().is_ok();
|
||||
if numeric || nonfinite || (profile && !key.starts_with(|c: char| c.is_ascii_alphabetic())) {
|
||||
let row = cube_triplet(line)?;
|
||||
if p.rows.len() >= row_limit {
|
||||
return err("LUT exceeds row budget");
|
||||
}
|
||||
// Explicit fallible growth prevents capacity growth past the configured budget.
|
||||
if p.rows.len() == p.rows.capacity() {
|
||||
let grow = (p.rows.capacity().max(1)).min(row_limit - p.rows.len());
|
||||
p.rows.try_reserve_exact(grow).map_err(|_| LutError("LUT allocation failed".into()))?;
|
||||
}
|
||||
p.rows.push(row);
|
||||
}
|
||||
rows.extend(v);
|
||||
}
|
||||
// Unknown keywords (e.g. Resolve's LUT_IN_VIDEO_RANGE) are ignored, as the spec allows.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
|
||||
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
|
||||
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
|
||||
let p = read_cube(text, None)?;
|
||||
let (title, size3, size1, dmin, dmax) = (p.title.unwrap_or_default(), p.size3, p.size1, p.min, p.max);
|
||||
let rows: Vec<f32> = p.rows.into_iter().flatten().collect();
|
||||
let domain_ok = (0..3).all(|k| dmin[k] < dmax[k] && dmin[k].is_finite() && dmax[k].is_finite());
|
||||
if (size3 > 0 || size1 >= 2) && !domain_ok {
|
||||
return err(format!("DOMAIN_MIN {:?} must lie below DOMAIN_MAX {:?}", dmin, dmax));
|
||||
@@ -1386,3 +1466,52 @@ LUT_3D_SIZE 2
|
||||
assert_eq!(back.data, [0.0, 0.25, 1.0].repeat(8));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod profile_cube_tests {
|
||||
use super::*;
|
||||
|
||||
fn cube(prefix: &str) -> String {
|
||||
format!("{prefix}\nLUT_3D_SIZE 2\n{}", "0.25 0.5 0.75\n".repeat(8))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_preserves_optional_literal_title_and_domains() {
|
||||
let no_title = parse_cube_3d(&cube(""), 256).unwrap();
|
||||
assert_eq!(no_title.title, None);
|
||||
let value = parse_cube_3d(&cube("TITLE \"look # literal\"\nDOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1"), 256).unwrap();
|
||||
assert_eq!(value.title.as_deref(), Some("look # literal"));
|
||||
assert_eq!(value.domain_min, [1.0, 0.0, 2.0]);
|
||||
assert_eq!(value.domain_max, [0.0, 0.0, 1.0]);
|
||||
assert_eq!(value.data, vec![[0.25, 0.5, 0.75]; 8]);
|
||||
assert_eq!(parse_cube_3d(&cube("TITLE \"\""), 256).unwrap().title.as_deref(), Some(""));
|
||||
assert!(parse_cube(&cube("DOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_limits_are_checked_without_large_allocations() {
|
||||
assert!(parse_cube_3d("LUT_3D_SIZE 256", 256).unwrap_err().0.contains("16777216"));
|
||||
assert!(parse_cube_3d("LUT_3D_SIZE 257", 256).unwrap_err().0.contains("out of range"));
|
||||
assert!(parse_cube_3d("LUT_3D_SIZE 256", 129).unwrap_err().0.contains("out of range"));
|
||||
assert!(parse_cube_3d(&cube(""), usize::MAX).is_err());
|
||||
assert!(parse_cube_3d(&cube(""), 1).is_err());
|
||||
assert!(parse_cube_3d(&format!("{}0 0 0\n", cube("")), 2).unwrap_err().0.contains("budget"));
|
||||
assert!(parse_cube_3d("LUT_1D_SIZE 2\n0 0 0\n1 1 1", 256).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_numeric_token_must_be_valid_and_finite() {
|
||||
for row in ["0 ignored 0 0", "0 0 0 ignored", "NaN 0 0", "inf 0 0", "0 0 -inf", "1e39 0 0", "0 0", "0 0 0 0"] {
|
||||
assert!(parse_cube_3d(&format!("LUT_3D_SIZE 2\n{row}\n{}", "0 0 0\n".repeat(7)), 256).is_err(), "{row}");
|
||||
assert!(parse_cube_3d(&cube(&format!("DOMAIN_MIN {row}")), 256).is_err(), "{row}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_reader_keeps_quoted_comments_and_1d_expansion() {
|
||||
assert_eq!(parse_cube(&cube("TITLE \"look # preserved\" # comment")).unwrap().title, "look # preserved");
|
||||
let value = parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1").unwrap();
|
||||
assert_eq!(value.size, 33);
|
||||
assert!(parse_cube_3d(&cube("DOMAIN_MIN 0 0 0 # not a whole-line comment"), 256).is_err());
|
||||
}
|
||||
}
|
||||
+362
-65
@@ -6,7 +6,7 @@
|
||||
//! [`spec_of`] turns a chart part back into the model only when writing that model again gives
|
||||
//! the same markup, so a chart with anything the model leaves out stays as it was read.
|
||||
|
||||
use craft_wordmodel::chart::{ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES};
|
||||
use craft_wordmodel::chart::{ChartAxis, ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES};
|
||||
use craft_wordmodel::graphic::GraphicItem;
|
||||
use craft_wordmodel::props::Rgb;
|
||||
|
||||
@@ -102,11 +102,24 @@ fn bounded(spec: &ChartSpec) -> ChartSpec {
|
||||
.series
|
||||
.iter()
|
||||
.take(MAX_SERIES)
|
||||
.map(|s| ChartSeries { name: s.name.clone(), values: s.values.iter().take(MAX_POINTS).copied().collect() })
|
||||
.map(|s| ChartSeries {
|
||||
name: s.name.clone(),
|
||||
values: s.values.iter().take(MAX_POINTS).copied().collect(),
|
||||
fill: s.fill,
|
||||
no_fill: s.no_fill,
|
||||
line_w: s.line_w,
|
||||
no_line: s.no_line,
|
||||
markers: s.markers,
|
||||
smooth: s.smooth,
|
||||
labels: s.labels,
|
||||
fmt: s.fmt.clone(),
|
||||
})
|
||||
.collect(),
|
||||
title: spec.title.clone(),
|
||||
legend: spec.legend,
|
||||
data_labels: spec.data_labels,
|
||||
cat_axis: spec.cat_axis.clone(),
|
||||
val_axis: spec.val_axis.clone(),
|
||||
}
|
||||
.sanitized()
|
||||
}
|
||||
@@ -185,17 +198,19 @@ fn plot(w: &mut W, spec: &ChartSpec) {
|
||||
match k {
|
||||
ChartType::Pie | ChartType::Doughnut => {}
|
||||
ChartType::Scatter => {
|
||||
value_axis(w, CAT_AX, VAL_AX, "b", false, "midCat");
|
||||
value_axis(w, VAL_AX, CAT_AX, "l", true, "midCat");
|
||||
value_axis(w, CAT_AX, VAL_AX, "b", "midCat", &spec.cat_axis);
|
||||
value_axis(w, VAL_AX, CAT_AX, "l", "midCat", &spec.val_axis);
|
||||
}
|
||||
_ => {
|
||||
category_axis(w, if horizontal { "l" } else { "b" });
|
||||
value_axis(w, VAL_AX, CAT_AX, if horizontal { "b" } else { "l" }, true, "between");
|
||||
category_axis(w, if horizontal { "l" } else { "b" }, &spec.cat_axis);
|
||||
value_axis(w, VAL_AX, CAT_AX, if horizontal { "b" } else { "l" }, "between", &spec.val_axis);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One `c:ser` (ECMA-376 element order: idx, order, tx, spPr, marker, cat/xVal, val/yVal, smooth).
|
||||
/// One `c:ser` (ECMA-376 element order: idx, order, tx, spPr, marker, invertIfNegative, dLbls,
|
||||
/// cat/xVal, val/yVal, smooth): the series' formatting (fill, line, markers, its own labels) sits
|
||||
/// in `c:spPr`, `c:marker` and `c:dLbls`; the values carry their number format.
|
||||
fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) {
|
||||
let k = spec.kind;
|
||||
let i = i.to_string();
|
||||
@@ -207,25 +222,26 @@ fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) {
|
||||
w.leaf("c:v", &[], &s.name);
|
||||
w.close("c:tx");
|
||||
}
|
||||
match k {
|
||||
// Markers only: the series has no line.
|
||||
ChartType::Scatter => {
|
||||
w.open("c:spPr", &[]);
|
||||
w.open("a:ln", &[("w", "19050")]);
|
||||
w.empty("a:noFill", &[]);
|
||||
w.close("a:ln");
|
||||
w.close("c:spPr");
|
||||
}
|
||||
ChartType::Column | ChartType::ColumnStacked | ChartType::Bar | ChartType::BarStacked => {
|
||||
w.empty("c:invertIfNegative", &[("val", "0")]);
|
||||
}
|
||||
ChartType::Line => {
|
||||
w.open("c:marker", &[]);
|
||||
w.empty("c:symbol", &[("val", "none")]);
|
||||
w.close("c:marker");
|
||||
}
|
||||
_ => {}
|
||||
series_props(w, s);
|
||||
series_marker(w, k, s);
|
||||
if matches!(k, ChartType::Column | ChartType::ColumnStacked | ChartType::Bar | ChartType::BarStacked) {
|
||||
w.empty("c:invertIfNegative", &[("val", "0")]);
|
||||
}
|
||||
if let Some(on) = s.labels {
|
||||
w.open("c:dLbls", &[]);
|
||||
for (name, show) in [
|
||||
("c:showLegendKey", false),
|
||||
("c:showVal", on),
|
||||
("c:showCatName", false),
|
||||
("c:showSerName", false),
|
||||
("c:showPercent", false),
|
||||
("c:showBubbleSize", false),
|
||||
] {
|
||||
w.empty(name, &[("val", if show { "1" } else { "0" })]);
|
||||
}
|
||||
w.close("c:dLbls");
|
||||
}
|
||||
let code = s.fmt.as_deref().unwrap_or("General");
|
||||
if k == ChartType::Scatter {
|
||||
let xs: Option<Vec<Option<f64>>> = spec
|
||||
.categories
|
||||
@@ -234,27 +250,85 @@ fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) {
|
||||
.collect();
|
||||
w.open("c:xVal", &[]);
|
||||
match xs {
|
||||
Some(xs) => num_lit(w, &xs),
|
||||
Some(xs) => num_lit(w, &xs, "General"),
|
||||
None => str_lit(w, &spec.categories),
|
||||
}
|
||||
w.close("c:xVal");
|
||||
w.open("c:yVal", &[]);
|
||||
num_lit(w, &s.values);
|
||||
num_lit(w, &s.values, code);
|
||||
w.close("c:yVal");
|
||||
} else {
|
||||
w.open("c:cat", &[]);
|
||||
str_lit(w, &spec.categories);
|
||||
w.close("c:cat");
|
||||
w.open("c:val", &[]);
|
||||
num_lit(w, &s.values);
|
||||
num_lit(w, &s.values, code);
|
||||
w.close("c:val");
|
||||
}
|
||||
if matches!(k, ChartType::Line | ChartType::Scatter) {
|
||||
w.empty("c:smooth", &[("val", "0")]);
|
||||
w.empty("c:smooth", &[("val", if s.smooth { "1" } else { "0" })]);
|
||||
}
|
||||
w.close("c:ser");
|
||||
}
|
||||
|
||||
/// `c:spPr`: the series' fill (`a:solidFill` / `a:noFill`) and line (`a:ln`), written when the
|
||||
/// model sets them. `a:ln` without `w` is the default width, `w` a fixed one, and `a:noFill`
|
||||
/// inside it "no line" (a scatter series keeps its markers with no line).
|
||||
fn series_props(w: &mut W, s: &ChartSeries) {
|
||||
let fill_set = s.no_fill || s.fill.is_some();
|
||||
let ln_set = s.no_line || s.line_w.is_some();
|
||||
if !(fill_set || ln_set) {
|
||||
return;
|
||||
}
|
||||
w.open("c:spPr", &[]);
|
||||
if fill_set {
|
||||
// "No fill" wins over a stored colour, so the pair reads back as it was written.
|
||||
match &s.fill {
|
||||
Some(c) if !s.no_fill => {
|
||||
w.open("a:solidFill", &[]);
|
||||
w.empty("a:srgbClr", &[("val", &c.hex())]);
|
||||
w.close("a:solidFill");
|
||||
}
|
||||
_ => w.empty("a:noFill", &[]),
|
||||
}
|
||||
}
|
||||
if ln_set {
|
||||
let w_attr = s.line_w.map(emu).unwrap_or_default();
|
||||
let attrs = [("w", w_attr.as_str())];
|
||||
w.open("a:ln", if w_attr.is_empty() { &[] } else { &attrs });
|
||||
if s.no_line {
|
||||
w.empty("a:noFill", &[]);
|
||||
}
|
||||
w.close("a:ln");
|
||||
}
|
||||
w.close("c:spPr");
|
||||
}
|
||||
|
||||
/// `c:marker`: line and scatter series say which symbol each point gets; other types have none.
|
||||
fn series_marker(w: &mut W, k: ChartType, s: &ChartSeries) {
|
||||
match k {
|
||||
ChartType::Line => {
|
||||
w.open("c:marker", &[]);
|
||||
w.empty("c:symbol", &[("val", if s.markers == Some(true) { "circle" } else { "none" })]);
|
||||
w.close("c:marker");
|
||||
}
|
||||
// Unset: no `c:marker`, so Word picks its automatic symbol for each series.
|
||||
ChartType::Scatter => {
|
||||
if let Some(on) = s.markers {
|
||||
w.open("c:marker", &[]);
|
||||
w.empty("c:symbol", &[("val", if on { "circle" } else { "none" })]);
|
||||
w.close("c:marker");
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// A line width in points as the `w` attribute DrawingML wants (EMU).
|
||||
fn emu(pt: f32) -> String {
|
||||
((pt * 12700.0).round().max(0.0) as i64).to_string()
|
||||
}
|
||||
|
||||
fn str_lit(w: &mut W, items: &[String]) {
|
||||
w.open("c:strLit", &[]);
|
||||
w.empty("c:ptCount", &[("val", &items.len().to_string())]);
|
||||
@@ -266,9 +340,9 @@ fn str_lit(w: &mut W, items: &[String]) {
|
||||
w.close("c:strLit");
|
||||
}
|
||||
|
||||
fn num_lit(w: &mut W, values: &[Option<f64>]) {
|
||||
fn num_lit(w: &mut W, values: &[Option<f64>], code: &str) {
|
||||
w.open("c:numLit", &[]);
|
||||
w.leaf("c:formatCode", &[], "General");
|
||||
w.leaf("c:formatCode", &[], code);
|
||||
w.empty("c:ptCount", &[("val", &values.len().to_string())]);
|
||||
for (i, v) in values.iter().enumerate() {
|
||||
if let Some(v) = v.filter(|v| v.is_finite()) {
|
||||
@@ -285,17 +359,21 @@ fn num(v: f64) -> String {
|
||||
if v == 0.0 { "0".to_string() } else { format!("{v}") }
|
||||
}
|
||||
|
||||
fn category_axis(w: &mut W, pos: &str) {
|
||||
fn category_axis(w: &mut W, pos: &str, ax: &ChartAxis) {
|
||||
w.open("c:catAx", &[]);
|
||||
w.empty("c:axId", &[("val", CAT_AX)]);
|
||||
w.open("c:scaling", &[]);
|
||||
w.empty("c:orientation", &[("val", "minMax")]);
|
||||
w.close("c:scaling");
|
||||
scaling(w, ax);
|
||||
w.empty("c:delete", &[("val", "0")]);
|
||||
w.empty("c:axPos", &[("val", pos)]);
|
||||
if ax.gridlines {
|
||||
w.empty("c:majorGridlines", &[]);
|
||||
}
|
||||
axis_title(w, ax);
|
||||
num_fmt(w, ax);
|
||||
w.empty("c:majorTickMark", &[("val", "none")]);
|
||||
w.empty("c:minorTickMark", &[("val", "none")]);
|
||||
w.empty("c:tickLblPos", &[("val", "nextTo")]);
|
||||
w.empty("c:tickLblPos", &[("val", if ax.labels { "nextTo" } else { "none" })]);
|
||||
axis_line(w, ax);
|
||||
w.empty("c:crossAx", &[("val", VAL_AX)]);
|
||||
w.empty("c:crosses", &[("val", "autoZero")]);
|
||||
w.empty("c:auto", &[("val", "1")]);
|
||||
@@ -305,34 +383,81 @@ fn category_axis(w: &mut W, pos: &str) {
|
||||
w.close("c:catAx");
|
||||
}
|
||||
|
||||
/// A value axis: with gridlines and no axis line (`grid`), else with its line and no gridlines.
|
||||
fn value_axis(w: &mut W, id: &str, cross: &str, pos: &str, grid: bool, between: &str) {
|
||||
/// A value axis (scatter x axes included): its scale, gridlines, title, labels and line, as the
|
||||
/// model sets them.
|
||||
fn value_axis(w: &mut W, id: &str, cross: &str, pos: &str, between: &str, ax: &ChartAxis) {
|
||||
w.open("c:valAx", &[]);
|
||||
w.empty("c:axId", &[("val", id)]);
|
||||
w.open("c:scaling", &[]);
|
||||
w.empty("c:orientation", &[("val", "minMax")]);
|
||||
w.close("c:scaling");
|
||||
scaling(w, ax);
|
||||
w.empty("c:delete", &[("val", "0")]);
|
||||
w.empty("c:axPos", &[("val", pos)]);
|
||||
if grid {
|
||||
if ax.gridlines {
|
||||
w.empty("c:majorGridlines", &[]);
|
||||
}
|
||||
axis_title(w, ax);
|
||||
num_fmt(w, ax);
|
||||
w.empty("c:majorTickMark", &[("val", "none")]);
|
||||
w.empty("c:minorTickMark", &[("val", "none")]);
|
||||
w.empty("c:tickLblPos", &[("val", "nextTo")]);
|
||||
if grid {
|
||||
w.open("c:spPr", &[]);
|
||||
w.open("a:ln", &[]);
|
||||
w.empty("a:noFill", &[]);
|
||||
w.close("a:ln");
|
||||
w.close("c:spPr");
|
||||
}
|
||||
w.empty("c:tickLblPos", &[("val", if ax.labels { "nextTo" } else { "none" })]);
|
||||
axis_line(w, ax);
|
||||
w.empty("c:crossAx", &[("val", cross)]);
|
||||
w.empty("c:crosses", &[("val", "autoZero")]);
|
||||
w.empty("c:crossBetween", &[("val", between)]);
|
||||
w.close("c:valAx");
|
||||
}
|
||||
|
||||
/// `c:scaling`: the orientation, then the fixed bounds when the model sets them.
|
||||
fn scaling(w: &mut W, ax: &ChartAxis) {
|
||||
w.open("c:scaling", &[]);
|
||||
w.empty("c:orientation", &[("val", "minMax")]);
|
||||
for (name, v) in [("c:max", ax.max), ("c:min", ax.min)] {
|
||||
if let Some(v) = v.filter(|v| v.is_finite()) {
|
||||
w.empty(name, &[("val", &num(v))]);
|
||||
}
|
||||
}
|
||||
w.close("c:scaling");
|
||||
}
|
||||
|
||||
/// The axis title, in the same rich-text shape the chart title uses.
|
||||
fn axis_title(w: &mut W, ax: &ChartAxis) {
|
||||
let Some(t) = &ax.title else { return };
|
||||
w.open("c:title", &[]);
|
||||
w.open("c:tx", &[]);
|
||||
w.open("c:rich", &[]);
|
||||
w.empty("a:bodyPr", &[]);
|
||||
w.empty("a:lstStyle", &[]);
|
||||
w.open("a:p", &[]);
|
||||
w.open("a:r", &[]);
|
||||
w.leaf("a:t", &[], t);
|
||||
w.close("a:r");
|
||||
w.close("a:p");
|
||||
w.close("c:rich");
|
||||
w.close("c:tx");
|
||||
w.empty("c:overlay", &[("val", "0")]);
|
||||
w.close("c:title");
|
||||
}
|
||||
|
||||
/// Tick-label number format: the model's code, unlinked from the source, else the default.
|
||||
fn num_fmt(w: &mut W, ax: &ChartAxis) {
|
||||
match &ax.fmt {
|
||||
Some(code) => w.empty("c:numFmt", &[("formatCode", code), ("sourceLinked", "0")]),
|
||||
None => w.empty("c:numFmt", &[("formatCode", "General"), ("sourceLinked", "1")]),
|
||||
}
|
||||
}
|
||||
|
||||
/// `c:spPr`: an axis line only when the model keeps it (the default writes nothing, which reads
|
||||
/// as a line); `a:noFill` when the axis line is off.
|
||||
fn axis_line(w: &mut W, ax: &ChartAxis) {
|
||||
if ax.line {
|
||||
return;
|
||||
}
|
||||
w.open("c:spPr", &[]);
|
||||
w.open("a:ln", &[]);
|
||||
w.empty("a:noFill", &[]);
|
||||
w.close("a:ln");
|
||||
w.close("c:spPr");
|
||||
}
|
||||
|
||||
/// The chart a chart part holds, when the part is exactly what [`chart_xml`] writes for it (so
|
||||
/// editing and saving it loses nothing). `None` for any other chart.
|
||||
pub(crate) fn spec_of(space: &El) -> Option<ChartSpec> {
|
||||
@@ -365,15 +490,40 @@ pub(crate) fn spec_of(space: &El) -> Option<ChartSpec> {
|
||||
let categories = sers.first().and_then(|s| s.child(if scatter { "c:xVal" } else { "c:cat" })).map(literal).unwrap_or_default();
|
||||
let series = sers
|
||||
.iter()
|
||||
.map(|s| ChartSeries {
|
||||
name: s.child("c:tx").and_then(|t| t.child("c:v")).map(|v| v.text()).unwrap_or_default(),
|
||||
values: s
|
||||
.child(if scatter { "c:yVal" } else { "c:val" })
|
||||
.map(literal)
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|v| v.parse::<f64>().ok().filter(|v| v.is_finite()))
|
||||
.collect(),
|
||||
.map(|s| {
|
||||
let ln = s.child("c:spPr").and_then(|p| p.child("a:ln"));
|
||||
let symbol = s.child("c:marker").and_then(|m| m.child("c:symbol")).and_then(|m| m.attr("val"));
|
||||
ChartSeries {
|
||||
name: s.child("c:tx").and_then(|t| t.child("c:v")).map(|v| v.text()).unwrap_or_default(),
|
||||
values: s
|
||||
.child(if scatter { "c:yVal" } else { "c:val" })
|
||||
.map(literal)
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|v| v.parse::<f64>().ok().filter(|v| v.is_finite()))
|
||||
.collect(),
|
||||
fill: s
|
||||
.child("c:spPr")
|
||||
.and_then(|p| p.child("a:solidFill"))
|
||||
.and_then(|f| f.child("a:srgbClr"))
|
||||
.and_then(|c| c.attr("val"))
|
||||
.and_then(Rgb::parse),
|
||||
no_fill: s.child("c:spPr").and_then(|p| p.child("a:noFill")).is_some(),
|
||||
line_w: ln.and_then(|l| l.attr("w")).and_then(|w| w.parse::<f64>().ok()).filter(|w| w.is_finite()).map(|w| (w / 12700.0) as f32),
|
||||
no_line: ln.and_then(|l| l.child("a:noFill")).is_some(),
|
||||
// Line series show no markers by default: only a setting that differs from that
|
||||
// belongs in the model. A scatter series keeps what the file says, so a missing
|
||||
// `c:marker` (Word's automatic symbols) stays unset and is not written back.
|
||||
markers: symbol.map(|v| v != "none").filter(|m| scatter || *m),
|
||||
smooth: s.child("c:smooth").and_then(|v| v.attr("val")).is_some_and(|v| v == "1"),
|
||||
labels: s.child("c:dLbls").and_then(|d| d.child("c:showVal")).and_then(|v| v.attr("val")).map(|v| v == "1"),
|
||||
fmt: s
|
||||
.child(if scatter { "c:yVal" } else { "c:val" })
|
||||
.and_then(|v| v.els().find(|e| e.name == "c:numLit"))
|
||||
.and_then(|l| l.child("c:formatCode"))
|
||||
.map(|f| f.text())
|
||||
.filter(|f| *f != "General"),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let title = chart.child("c:title").and_then(|t| t.find("a:t")).map(|t| t.text());
|
||||
@@ -384,12 +534,48 @@ pub(crate) fn spec_of(space: &El) -> Option<ChartSpec> {
|
||||
_ => LegendPos::Right,
|
||||
});
|
||||
let data_labels = ty.child("c:dLbls").and_then(|d| d.child("c:showVal")).and_then(|v| v.attr("val")) == Some("1");
|
||||
let spec = ChartSpec { kind, categories, series, title, legend, data_labels }.sanitized();
|
||||
// Editable only when nothing would be lost: writing the model again gives the same part.
|
||||
let again = xml::parse(chart_xml(&spec).as_bytes()).ok()?;
|
||||
let (cat_a, val_a) = crate::read::chart::chart_axes(plot, scatter);
|
||||
let axis = |a: crate::read::chart::Axis| ChartAxis {
|
||||
title: a.title,
|
||||
min: a.min,
|
||||
max: a.max,
|
||||
fmt: a.fmt,
|
||||
gridlines: a.grid,
|
||||
labels: a.labels,
|
||||
line: a.line,
|
||||
};
|
||||
let spec = ChartSpec { kind, categories, series, title, legend, data_labels, cat_axis: axis(cat_a), val_axis: axis(val_a) }.sanitized();
|
||||
// Editable only when nothing would be lost: writing the model again gives the same part, as
|
||||
// this writer or the earlier one wrote it.
|
||||
let mut again = xml::parse(chart_xml(&spec).as_bytes()).ok()?;
|
||||
if same(space, &again, 0) {
|
||||
return Some(spec);
|
||||
}
|
||||
as_previous_writer(&mut again);
|
||||
same(space, &again, 0).then_some(spec)
|
||||
}
|
||||
|
||||
/// `part` (what [`chart_xml`] writes) as the earlier WordCraft writer wrote the same model: it
|
||||
/// had no axis `c:numFmt`. Only that element goes, and only with the value its absence reads as
|
||||
/// (General, linked to the source), so a part that matches still holds nothing the model leaves out.
|
||||
fn as_previous_writer(part: &mut El) {
|
||||
fn els_mut(e: &mut El) -> impl Iterator<Item = &mut El> {
|
||||
e.kids.iter_mut().filter_map(|n| match n {
|
||||
Node::El(e) => Some(e),
|
||||
Node::Text(_) => None,
|
||||
})
|
||||
}
|
||||
fn drop_els(e: &mut El, drop: impl Fn(&El) -> bool) {
|
||||
e.kids.retain(|n| !matches!(n, Node::El(c) if drop(c)));
|
||||
}
|
||||
let Some(plot) = els_mut(part).find(|e| e.name == "c:chart").and_then(|c| els_mut(c).find(|e| e.name == "c:plotArea")) else { return };
|
||||
for e in els_mut(plot) {
|
||||
if matches!(e.name.as_str(), "c:catAx" | "c:valAx") {
|
||||
drop_els(e, |c| c.name == "c:numFmt" && c.attr("sourceLinked") == Some("1"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The points of a `c:strLit` / `c:numLit` (or a cache) as texts by index, `ptCount` long.
|
||||
fn literal(e: &El) -> Vec<String> {
|
||||
let Some(lit) = e.els().find(|c| matches!(c.name.as_str(), "c:strLit" | "c:numLit")) else { return Vec::new() };
|
||||
@@ -461,10 +647,11 @@ mod tests {
|
||||
let spec = ChartSpec {
|
||||
kind: ChartType::Column,
|
||||
categories: (0..4500).map(|i| i.to_string()).collect(),
|
||||
series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500] }).collect(),
|
||||
series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500], ..Default::default() }).collect(),
|
||||
title: None,
|
||||
legend: None,
|
||||
data_labels: false,
|
||||
..Default::default()
|
||||
};
|
||||
let x = chart_xml(&spec);
|
||||
let series = x.matches("<c:ser>").count();
|
||||
@@ -491,6 +678,33 @@ mod tests {
|
||||
assert_eq!(spec_of(&parse(&plain)), Some(plain));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_fill_wins_over_a_stored_colour() {
|
||||
let mut spec = ChartSpec::sample(ChartType::Column);
|
||||
spec.series[0].fill = Some(Rgb(255, 0, 0));
|
||||
spec.series[0].no_fill = true;
|
||||
let x = chart_xml(&spec);
|
||||
assert!(!x.contains("FF0000"), "colour written next to no_fill");
|
||||
let back = spec_of(&parse(&spec)).expect("stays editable");
|
||||
assert!(back.series[0].no_fill);
|
||||
assert_eq!(back.series[0].fill, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_axis_format_is_limited_in_characters_not_bytes() {
|
||||
use craft_wordmodel::chart::MAX_FMT;
|
||||
let cjk = format!("0\"{}\"", "\u{5143}".repeat(90));
|
||||
let ascii = format!("0\"{}\"", "x".repeat(90));
|
||||
for fmt in [cjk, ascii] {
|
||||
assert!(fmt.chars().count() <= MAX_FMT);
|
||||
let mut spec = ChartSpec::sample(ChartType::Column);
|
||||
spec.val_axis.fmt = Some(fmt.clone());
|
||||
let spec = spec.sanitized();
|
||||
assert_eq!(spec.val_axis.fmt.as_deref(), Some(fmt.as_str()));
|
||||
assert_eq!(spec_of(&parse(&spec)).as_ref(), Some(&spec), "{} chars", fmt.chars().count());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chart_with_more_than_the_model_stays_read_only() {
|
||||
let spec = ChartSpec::sample(ChartType::Column);
|
||||
@@ -500,6 +714,61 @@ mod tests {
|
||||
assert_eq!(spec_of(&xml::parse(x.as_bytes()).unwrap()), None);
|
||||
}
|
||||
|
||||
/// Chart parts as the previous WordCraft chart writer saved them (its output for
|
||||
/// `ChartSpec::sample`): no axis `c:numFmt`, and scatter series with no `c:marker`.
|
||||
const PREVIOUS_COLUMN: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?><c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><c:roundedCorners val="0"/><c:chart><c:autoTitleDeleted val="1"/><c:plotArea><c:layout/><c:barChart><c:barDir val="col"/><c:grouping val="clustered"/><c:varyColors val="0"/><c:ser><c:idx val="0"/><c:order val="0"/><c:tx><c:v>Series 1</c:v></c:tx><c:invertIfNegative val="0"/><c:cat><c:strLit><c:ptCount val="4"/><c:pt idx="0"><c:v>Q1</c:v></c:pt><c:pt idx="1"><c:v>Q2</c:v></c:pt><c:pt idx="2"><c:v>Q3</c:v></c:pt><c:pt idx="3"><c:v>Q4</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>3.2</c:v></c:pt><c:pt idx="1"><c:v>4.1</c:v></c:pt><c:pt idx="2"><c:v>3.6</c:v></c:pt><c:pt idx="3"><c:v>5</c:v></c:pt></c:numLit></c:val></c:ser><c:ser><c:idx val="1"/><c:order val="1"/><c:tx><c:v>Series 2</c:v></c:tx><c:invertIfNegative val="0"/><c:cat><c:strLit><c:ptCount val="4"/><c:pt idx="0"><c:v>Q1</c:v></c:pt><c:pt idx="1"><c:v>Q2</c:v></c:pt><c:pt idx="2"><c:v>Q3</c:v></c:pt><c:pt idx="3"><c:v>Q4</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>2</c:v></c:pt><c:pt idx="1"><c:v>2.7</c:v></c:pt><c:pt idx="2"><c:v>3.9</c:v></c:pt><c:pt idx="3"><c:v>3.3</c:v></c:pt></c:numLit></c:val></c:ser><c:ser><c:idx val="2"/><c:order val="2"/><c:tx><c:v>Series 3</c:v></c:tx><c:invertIfNegative val="0"/><c:cat><c:strLit><c:ptCount val="4"/><c:pt idx="0"><c:v>Q1</c:v></c:pt><c:pt idx="1"><c:v>Q2</c:v></c:pt><c:pt idx="2"><c:v>Q3</c:v></c:pt><c:pt idx="3"><c:v>Q4</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1.5</c:v></c:pt><c:pt idx="1"><c:v>2.2</c:v></c:pt><c:pt idx="2"><c:v>2.4</c:v></c:pt><c:pt idx="3"><c:v>3.8</c:v></c:pt></c:numLit></c:val></c:ser><c:gapWidth val="150"/><c:axId val="500000001"/><c:axId val="500000002"/></c:barChart><c:catAx><c:axId val="500000001"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="b"/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:crossAx val="500000002"/><c:crosses val="autoZero"/><c:auto val="1"/><c:lblAlgn val="ctr"/><c:lblOffset val="100"/><c:noMultiLvlLbl val="0"/></c:catAx><c:valAx><c:axId val="500000002"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="l"/><c:majorGridlines/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:spPr><a:ln><a:noFill/></a:ln></c:spPr><c:crossAx val="500000001"/><c:crosses val="autoZero"/><c:crossBetween val="between"/></c:valAx></c:plotArea><c:legend><c:legendPos val="b"/><c:overlay val="0"/></c:legend><c:plotVisOnly val="1"/><c:dispBlanksAs val="gap"/></c:chart><c:spPr><a:solidFill><a:schemeClr val="bg1"/></a:solidFill><a:ln><a:noFill/></a:ln></c:spPr></c:chartSpace>"#;
|
||||
const PREVIOUS_SCATTER: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?><c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><c:roundedCorners val="0"/><c:chart><c:autoTitleDeleted val="1"/><c:plotArea><c:layout/><c:scatterChart><c:scatterStyle val="lineMarker"/><c:varyColors val="0"/><c:ser><c:idx val="0"/><c:order val="0"/><c:tx><c:v>Series 1</c:v></c:tx><c:spPr><a:ln w="19050"><a:noFill/></a:ln></c:spPr><c:xVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1</c:v></c:pt><c:pt idx="1"><c:v>2</c:v></c:pt><c:pt idx="2"><c:v>3</c:v></c:pt><c:pt idx="3"><c:v>4</c:v></c:pt></c:numLit></c:xVal><c:yVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>2.1</c:v></c:pt><c:pt idx="1"><c:v>3.4</c:v></c:pt><c:pt idx="2"><c:v>3</c:v></c:pt><c:pt idx="3"><c:v>4.6</c:v></c:pt></c:numLit></c:yVal><c:smooth val="0"/></c:ser><c:ser><c:idx val="1"/><c:order val="1"/><c:tx><c:v>Series 2</c:v></c:tx><c:spPr><a:ln w="19050"><a:noFill/></a:ln></c:spPr><c:xVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1</c:v></c:pt><c:pt idx="1"><c:v>2</c:v></c:pt><c:pt idx="2"><c:v>3</c:v></c:pt><c:pt idx="3"><c:v>4</c:v></c:pt></c:numLit></c:xVal><c:yVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1.2</c:v></c:pt><c:pt idx="1"><c:v>1.9</c:v></c:pt><c:pt idx="2"><c:v>2.8</c:v></c:pt><c:pt idx="3"><c:v>3.1</c:v></c:pt></c:numLit></c:yVal><c:smooth val="0"/></c:ser><c:axId val="500000001"/><c:axId val="500000002"/></c:scatterChart><c:valAx><c:axId val="500000001"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="b"/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:crossAx val="500000002"/><c:crosses val="autoZero"/><c:crossBetween val="midCat"/></c:valAx><c:valAx><c:axId val="500000002"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="l"/><c:majorGridlines/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:spPr><a:ln><a:noFill/></a:ln></c:spPr><c:crossAx val="500000001"/><c:crosses val="autoZero"/><c:crossBetween val="midCat"/></c:valAx></c:plotArea><c:legend><c:legendPos val="b"/><c:overlay val="0"/></c:legend><c:plotVisOnly val="1"/><c:dispBlanksAs val="gap"/></c:chart><c:spPr><a:solidFill><a:schemeClr val="bg1"/></a:solidFill><a:ln><a:noFill/></a:ln></c:spPr></c:chartSpace>"#;
|
||||
|
||||
#[test]
|
||||
fn charts_saved_by_the_previous_writer_stay_editable() {
|
||||
for (x, kind) in [(PREVIOUS_COLUMN, ChartType::Column), (PREVIOUS_SCATTER, ChartType::Scatter)] {
|
||||
let spec = spec_of(&xml::parse(x.as_bytes()).unwrap()).unwrap_or_else(|| panic!("{kind:?}: the earlier part is read-only"));
|
||||
assert_eq!((spec.kind, spec.series.len()), (kind, if kind == ChartType::Scatter { 2 } else { 3 }));
|
||||
assert_eq!((spec.cat_axis.fmt.as_deref(), spec.val_axis.fmt.as_deref()), (None, None), "{kind:?}");
|
||||
assert!(spec.series.iter().all(|s| s.markers.is_none()), "{kind:?}: markers unset");
|
||||
// Saved again, it is in the current shape and still reads back as the same model.
|
||||
assert_eq!(spec_of(&parse(&spec)).as_ref(), Some(&spec), "{kind:?}");
|
||||
}
|
||||
// What neither writer writes still keeps such a part read-only: an axis format linked to
|
||||
// the source that isn't General, a scatter marker other than the default, a marker size.
|
||||
let edited = |x: &str, from: &str, to: &str| {
|
||||
assert!(x.contains(from), "{from}");
|
||||
spec_of(&xml::parse(x.replacen(from, to, 1).as_bytes()).unwrap())
|
||||
};
|
||||
let ax = r#"<c:axPos val="l"/><c:majorGridlines/>"#;
|
||||
assert_eq!(edited(PREVIOUS_COLUMN, ax, &format!(r#"{ax}<c:numFmt formatCode="0.0" sourceLinked="1"/>"#)), None);
|
||||
let ln = r#"<a:ln w="19050"><a:noFill/></a:ln></c:spPr>"#;
|
||||
assert_eq!(edited(PREVIOUS_SCATTER, ln, &format!(r#"{ln}<c:marker><c:symbol val="square"/></c:marker>"#)), None);
|
||||
assert_eq!(edited(PREVIOUS_SCATTER, ln, &format!(r#"{ln}<c:marker><c:symbol val="circle"/><c:size val="9"/></c:marker>"#)), None);
|
||||
assert_eq!(edited(PREVIOUS_COLUMN, "<c:roundedCorners val=\"0\"/>", "<c:roundedCorners val=\"0\"/><c:style val=\"2\"/>"), None);
|
||||
}
|
||||
|
||||
/// A scatter series with no `c:marker` (Word's automatic symbols) stays without one after an
|
||||
/// unrelated edit and a save; an explicit circle or "none" stays explicit.
|
||||
#[test]
|
||||
fn scatter_series_without_a_marker_stay_without_after_a_save() {
|
||||
let marker_of = |x: &El| -> Vec<Option<String>> {
|
||||
let sc = x.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:scatterChart").unwrap();
|
||||
sc.children("c:ser").map(|s| s.child("c:marker").map(|m| m.child("c:symbol").unwrap().attr("val").unwrap().to_string())).collect()
|
||||
};
|
||||
let mut spec = spec_of(&xml::parse(PREVIOUS_SCATTER.as_bytes()).unwrap()).unwrap();
|
||||
assert_eq!(marker_of(&xml::parse(PREVIOUS_SCATTER.as_bytes()).unwrap()), [None, None]);
|
||||
spec.title = Some("Edited".into());
|
||||
spec.series[0].name = "Renamed".into();
|
||||
let saved = parse(&spec);
|
||||
assert_eq!(marker_of(&saved), [None, None]);
|
||||
let again = spec_of(&saved).unwrap();
|
||||
assert_eq!(again, spec);
|
||||
assert!(again.series.iter().all(|s| s.markers.is_none()));
|
||||
// Explicit choices are kept as written.
|
||||
spec.series[0].markers = Some(true);
|
||||
spec.series[1].markers = Some(false);
|
||||
let saved = parse(&spec);
|
||||
assert_eq!(marker_of(&saved), [Some("circle".to_string()), Some("none".to_string())]);
|
||||
let again = spec_of(&saved).unwrap();
|
||||
assert_eq!((again.series[0].markers, again.series[1].markers), (Some(true), Some(false)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elements_come_in_schema_order() {
|
||||
let mut spec = ChartSpec::sample(ChartType::ColumnStacked);
|
||||
@@ -517,6 +786,11 @@ mod tests {
|
||||
["c:barDir", "c:grouping", "c:varyColors", "c:ser", "c:ser", "c:ser", "c:dLbls", "c:gapWidth", "c:overlap", "c:axId", "c:axId"]
|
||||
);
|
||||
assert_eq!(names(bar.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:invertIfNegative", "c:cat", "c:val"]);
|
||||
// A series with its own labels keeps `c:invertIfNegative` ahead of `c:dLbls` (CT_BarSer).
|
||||
spec.series[0].labels = Some(true);
|
||||
let labelled = parse(&spec);
|
||||
let ser = labelled.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:barChart").unwrap().child("c:ser").unwrap();
|
||||
assert_eq!(names(ser), ["c:idx", "c:order", "c:tx", "c:invertIfNegative", "c:dLbls", "c:cat", "c:val"]);
|
||||
let val_ax = plot.child("c:valAx").unwrap();
|
||||
assert_eq!(
|
||||
names(val_ax),
|
||||
@@ -526,6 +800,7 @@ mod tests {
|
||||
"c:delete",
|
||||
"c:axPos",
|
||||
"c:majorGridlines",
|
||||
"c:numFmt",
|
||||
"c:majorTickMark",
|
||||
"c:minorTickMark",
|
||||
"c:tickLblPos",
|
||||
@@ -535,13 +810,35 @@ mod tests {
|
||||
"c:crossBetween"
|
||||
]
|
||||
);
|
||||
let cat_ax = plot.child("c:catAx").unwrap();
|
||||
assert_eq!(
|
||||
names(cat_ax),
|
||||
[
|
||||
"c:axId",
|
||||
"c:scaling",
|
||||
"c:delete",
|
||||
"c:axPos",
|
||||
"c:numFmt",
|
||||
"c:majorTickMark",
|
||||
"c:minorTickMark",
|
||||
"c:tickLblPos",
|
||||
"c:crossAx",
|
||||
"c:crosses",
|
||||
"c:auto",
|
||||
"c:lblAlgn",
|
||||
"c:lblOffset",
|
||||
"c:noMultiLvlLbl"
|
||||
]
|
||||
);
|
||||
let line = parse(&ChartSpec { data_labels: true, ..ChartSpec::sample(ChartType::Line) });
|
||||
let lc = line.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:lineChart").unwrap();
|
||||
assert_eq!(names(lc), ["c:grouping", "c:varyColors", "c:ser", "c:ser", "c:ser", "c:dLbls", "c:marker", "c:axId", "c:axId"]);
|
||||
assert_eq!(names(lc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:marker", "c:cat", "c:val", "c:smooth"]);
|
||||
let sc = parse(&ChartSpec::sample(ChartType::Scatter));
|
||||
let mut scatter = ChartSpec::sample(ChartType::Scatter);
|
||||
scatter.series.iter_mut().for_each(|s| s.markers = Some(true));
|
||||
let sc = parse(&scatter);
|
||||
let sc = sc.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:scatterChart").unwrap();
|
||||
assert_eq!(names(sc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:spPr", "c:xVal", "c:yVal", "c:smooth"]);
|
||||
assert_eq!(names(sc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:spPr", "c:marker", "c:xVal", "c:yVal", "c:smooth"]);
|
||||
let dn = parse(&ChartSpec::sample(ChartType::Doughnut));
|
||||
let dn = dn.child("c:chart").unwrap().child("c:plotArea").unwrap();
|
||||
assert_eq!(names(dn), ["c:layout", "c:doughnutChart"]);
|
||||
|
||||
@@ -29,7 +29,9 @@ pub(super) use canvas::text_w;
|
||||
use canvas::{Rect, clip_text, legend, text};
|
||||
#[cfg(test)]
|
||||
use format::{civil, excel_stamp, format_value};
|
||||
use model::{Axis, Grouping, Kind, Pos, Series};
|
||||
pub(crate) use model::Axis;
|
||||
use model::{Grouping, Kind, Pos, Series};
|
||||
pub(crate) use parse::chart_axes;
|
||||
#[cfg(test)]
|
||||
use parse::{axis, category_text, labels, numbers};
|
||||
use parse::{read_series, text_size, title_of};
|
||||
|
||||
@@ -41,8 +41,13 @@ pub(super) fn widest(items: &[String], size: f32) -> f32 {
|
||||
items.iter().map(|s| text_w(s, size)).fold(0.0, f32::max)
|
||||
}
|
||||
|
||||
/// Text in `rect`; `angle` degrees clockwise turns it about the rect's centre (axis titles).
|
||||
pub(super) fn text(out: &mut Vec<GraphicItem>, rect: Rect, s: &str, size: f32, align: TextAlign, bold: bool) {
|
||||
out.push(GraphicItem::Text { rect: rect.arr(), text: s.to_string(), size, color: TEXT, bold, align, font: None });
|
||||
text_angle(out, rect, s, size, align, bold, 0.0);
|
||||
}
|
||||
|
||||
pub(super) fn text_angle(out: &mut Vec<GraphicItem>, rect: Rect, s: &str, size: f32, align: TextAlign, bold: bool, angle: f32) {
|
||||
out.push(GraphicItem::Text { rect: rect.arr(), text: s.to_string(), size, color: TEXT, bold, align, font: None, angle });
|
||||
}
|
||||
|
||||
/// Text centred on `cx` with its top at `y`.
|
||||
|
||||
@@ -7,6 +7,33 @@ pub(super) struct DataLabel {
|
||||
pub(super) y: f32,
|
||||
pub(super) text: String,
|
||||
pub(super) inside: bool,
|
||||
/// Where a label that is not `inside` sits relative to (`x`, `y`).
|
||||
pub(super) pos: LabelPos,
|
||||
/// Distance from (`x`, `y`) to the label for `pos` other than `Center`, points (half the marker).
|
||||
pub(super) off: f32,
|
||||
}
|
||||
|
||||
/// Where a data label sits beside its point (`c:dLblPos`: `t`, `b`, `l`, `r`, `ctr`).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum LabelPos {
|
||||
Above,
|
||||
Below,
|
||||
Left,
|
||||
Right,
|
||||
Center,
|
||||
}
|
||||
|
||||
impl LabelPos {
|
||||
pub(super) fn parse(v: &str) -> Option<LabelPos> {
|
||||
Some(match v.trim() {
|
||||
"t" => LabelPos::Above,
|
||||
"b" => LabelPos::Below,
|
||||
"l" => LabelPos::Left,
|
||||
"r" => LabelPos::Right,
|
||||
"ctr" => LabelPos::Center,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -52,16 +79,56 @@ pub(super) enum Pos {
|
||||
|
||||
/// One value axis (or category axis) as the file states it.
|
||||
#[derive(Default)]
|
||||
pub(super) struct Axis {
|
||||
pub(super) deleted: bool,
|
||||
pub(super) grid: bool,
|
||||
pub(super) line: bool,
|
||||
pub(crate) struct Axis {
|
||||
pub(crate) deleted: bool,
|
||||
pub(crate) grid: bool,
|
||||
pub(crate) line: bool,
|
||||
/// `c:crossBetween val="midCat"`: categories sit on the tick marks, not between them.
|
||||
pub(super) mid: bool,
|
||||
pub(super) min: Option<f64>,
|
||||
pub(super) max: Option<f64>,
|
||||
pub(crate) min: Option<f64>,
|
||||
pub(crate) max: Option<f64>,
|
||||
/// Number format of the tick labels, when not linked to the source data.
|
||||
pub(super) fmt: Option<String>,
|
||||
pub(crate) fmt: Option<String>,
|
||||
/// The axis title, when the file gives it one.
|
||||
pub(crate) title: Option<String>,
|
||||
/// The title's `a:bodyPr/@rot` in degrees clockwise (`-5400000` is 90° counter-clockwise),
|
||||
/// when the file gives one.
|
||||
pub(crate) title_rot: Option<f32>,
|
||||
/// Tick labels, as `c:tickLblPos` says (`none` hides them; a file that says nothing shows
|
||||
/// them, as Word does).
|
||||
pub(crate) labels: bool,
|
||||
}
|
||||
|
||||
/// The shape of a series' point markers (`ST_MarkerStyle`, ECMA-376 §21.2.3.28).
|
||||
#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
|
||||
pub(super) enum Symbol {
|
||||
#[default]
|
||||
Circle,
|
||||
Square,
|
||||
Diamond,
|
||||
Triangle,
|
||||
X,
|
||||
Plus,
|
||||
Star,
|
||||
Dash,
|
||||
Dot,
|
||||
}
|
||||
|
||||
impl Symbol {
|
||||
/// The symbol a `c:symbol` value names; anything unknown (`auto`, `picture`) is a circle.
|
||||
pub(super) fn parse(name: &str) -> Symbol {
|
||||
match name {
|
||||
"square" => Symbol::Square,
|
||||
"diamond" => Symbol::Diamond,
|
||||
"triangle" => Symbol::Triangle,
|
||||
"x" => Symbol::X,
|
||||
"plus" => Symbol::Plus,
|
||||
"star" => Symbol::Star,
|
||||
"dash" => Symbol::Dash,
|
||||
"dot" => Symbol::Dot,
|
||||
_ => Symbol::Circle,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct Series {
|
||||
@@ -70,9 +137,21 @@ pub(super) struct Series {
|
||||
/// when missing or not a valid number.
|
||||
pub(super) idx: Option<usize>,
|
||||
pub(super) color: Option<Rgb>,
|
||||
pub(super) line_w: f32,
|
||||
/// Fixed line width in points, `None`: the chart's default.
|
||||
pub(super) line_w: Option<f32>,
|
||||
/// Whether the series is drawn as a line (scatter series without one get markers).
|
||||
pub(super) line: bool,
|
||||
/// The series' own fill: `Some` colour, `None` colour with `no_fill` set, else the chart's
|
||||
/// accent for the series.
|
||||
pub(super) no_fill: bool,
|
||||
/// Markers on the series' points, when the file says.
|
||||
pub(super) markers: Option<bool>,
|
||||
/// The marker shape (`c:marker/c:symbol`; a circle when missing or unknown) and its size in
|
||||
/// points (`c:marker/c:size`), when the file gives one.
|
||||
pub(super) symbol: Symbol,
|
||||
pub(super) marker_size: Option<f32>,
|
||||
/// Draw the series' line as a smooth curve (`c:smooth`).
|
||||
pub(super) smooth: bool,
|
||||
pub(super) values: Vec<Option<f64>>,
|
||||
/// Scatter x values (index + 1 when the file has none).
|
||||
pub(super) xs: Vec<Option<f64>>,
|
||||
@@ -83,6 +162,13 @@ pub(super) struct Series {
|
||||
pub(super) x_fmt: Option<String>,
|
||||
/// Data labels on or off for this series (`c:dLbls` of its own), when it says.
|
||||
pub(super) labels: Option<bool>,
|
||||
/// The labels' number format (`c:dLbls/c:numFmt`): `None` when not given (the chart type's
|
||||
/// applies), `Some(None)` when linked to the source, else `Some(Some(code))`.
|
||||
pub(super) label_fmt: Option<Option<String>>,
|
||||
/// The data labels' `c:dLblPos`, when the file gives a known one.
|
||||
pub(super) label_pos: Option<LabelPos>,
|
||||
/// The marker's own fill (`c:marker/c:spPr`); without it markers take the automatic colour.
|
||||
pub(super) marker_color: Option<Rgb>,
|
||||
/// The values as the file has them, for data labels (`values` become shares in a 100%
|
||||
/// stacked chart); empty when the series shows no labels.
|
||||
pub(super) raw: Vec<Option<f64>>,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::format::{date_text, excel_stamp, format_value, is_date_code};
|
||||
use super::model::{Axis, Grouping, Kind, Pos, Series};
|
||||
use super::model::{Axis, Grouping, Kind, LabelPos, Pos, Series, Symbol};
|
||||
use super::plot::label;
|
||||
use super::{Chart, El, FONT, LINE_W, MAX_CELLS, MAX_POINTS, MAX_RADAR_SERIES, MAX_SERIES, MAX_SPOKES, Rgb, fill_colour, int, measure, slot};
|
||||
use super::{Chart, El, FONT, MAX_CELLS, MAX_POINTS, MAX_RADAR_SERIES, MAX_SERIES, MAX_SPOKES, Rgb, fill_colour, int, measure, slot};
|
||||
|
||||
impl Chart {
|
||||
pub(super) fn parse(space: &El, theme: &[Rgb]) -> Option<Chart> {
|
||||
@@ -39,10 +39,15 @@ impl Chart {
|
||||
cats.resize(n, String::new());
|
||||
let radar_style = ty.child("c:radarStyle").and_then(|r| r.attr("val"));
|
||||
let type_labels = show_values(ty.child("c:dLbls"));
|
||||
let type_fmt = label_format(ty.child("c:dLbls"));
|
||||
for s in &mut series {
|
||||
s.values.resize(n, None);
|
||||
s.xs.resize(n, None);
|
||||
s.labels = Some(s.labels.or(type_labels).unwrap_or(false));
|
||||
// Absent (`None`) inherits the chart type's; explicitly source-linked (`Some(None)`) does not.
|
||||
if s.label_fmt.is_none() {
|
||||
s.label_fmt.clone_from(&type_fmt);
|
||||
}
|
||||
if s.labels == Some(true) {
|
||||
s.raw = s.values.clone();
|
||||
}
|
||||
@@ -98,6 +103,13 @@ impl Chart {
|
||||
s.and_then(|s| s.color).unwrap_or_else(|| self.accent(s.and_then(|s| s.idx).unwrap_or(j)))
|
||||
}
|
||||
|
||||
/// Colour of series `j`'s markers: the marker's own fill, else the automatic colour of the
|
||||
/// series index (not the series' explicit line colour, as Word draws them).
|
||||
pub(super) fn marker_colour(&self, j: usize) -> Rgb {
|
||||
let s = self.series.get(j);
|
||||
s.and_then(|s| s.marker_color).unwrap_or_else(|| self.accent(s.and_then(|s| s.idx).unwrap_or(j)))
|
||||
}
|
||||
|
||||
/// Colour of point `i` of series `j`: its `c:dPt` override, else the varied accent (pie) or
|
||||
/// the series colour.
|
||||
pub(super) fn point_colour(&self, j: usize, i: usize) -> Rgb {
|
||||
@@ -149,7 +161,7 @@ fn to_percent(series: &mut [Series], n: usize) {
|
||||
}
|
||||
|
||||
/// The category axis (scatter: the horizontal value axis) and the value axis of a plot area.
|
||||
fn chart_axes(plot: &El, scatter: bool) -> (Axis, Axis) {
|
||||
pub(crate) fn chart_axes(plot: &El, scatter: bool) -> (Axis, Axis) {
|
||||
let vals_ax: Vec<&El> = plot.children("c:valAx").collect();
|
||||
let horiz_pos = |a: &&El| matches!(a.child("c:axPos").and_then(|p| p.attr("val")), Some("b" | "t"));
|
||||
let (val_el, x_el) = if scatter {
|
||||
@@ -180,6 +192,17 @@ fn show_values(d: Option<&El>) -> Option<bool> {
|
||||
d.child("c:showVal").map(|v| v.attr("val").is_none_or(truthy))
|
||||
}
|
||||
|
||||
/// The number format of data labels (`c:dLbls/c:numFmt`): `None` when the labels say nothing,
|
||||
/// `Some(None)` when they are linked to the source (so a chart-level format does not apply),
|
||||
/// `Some(Some(code))` for their own format code.
|
||||
fn label_format(d: Option<&El>) -> Option<Option<String>> {
|
||||
let f = d?.child("c:numFmt")?;
|
||||
if f.attr("sourceLinked").is_some_and(truthy) {
|
||||
return Some(None);
|
||||
}
|
||||
f.attr("formatCode").filter(|f| f.chars().count() <= craft_wordmodel::chart::MAX_FMT).map(|f| Some(f.to_string()))
|
||||
}
|
||||
|
||||
fn legend_pos(chart: &El) -> Option<Pos> {
|
||||
chart.child("c:legend").map(|l| match l.child("c:legendPos").and_then(|p| p.attr("val")).unwrap_or("r") {
|
||||
"l" => Pos::Left,
|
||||
@@ -206,9 +229,24 @@ impl Series {
|
||||
let sp = ser.child("c:spPr");
|
||||
let ln = sp.and_then(|p| p.child("a:ln"));
|
||||
let marker = ser.child("c:marker").and_then(|m| m.child("c:spPr")).and_then(|p| fill_colour(p, theme));
|
||||
let marker_color = marker;
|
||||
let color = sp.and_then(|p| fill_colour(p, theme)).or_else(|| ln.and_then(|l| fill_colour(l, theme))).or(marker);
|
||||
let line_w = ln.and_then(|l| l.attr("w")).and_then(|v| measure(v, 12_700.0)).map_or(LINE_W, |w| w.clamp(0.0, 20.0));
|
||||
let no_fill = sp.and_then(|p| p.child("a:noFill")).is_some();
|
||||
let line_w = ln.and_then(|l| l.attr("w")).and_then(|v| measure(v, 12_700.0)).map(|w| w.clamp(0.0, 20.0));
|
||||
let line = ln.is_none_or(|l| l.child("a:noFill").is_none());
|
||||
// `c:marker/c:symbol`: which symbol each point gets (`none` for no markers). Scatter
|
||||
// series show markers unless the file says not; line series only when it says so.
|
||||
let symbol = ser.child("c:marker").and_then(|m| m.child("c:symbol")).and_then(|s| s.attr("val")).map(str::to_string);
|
||||
let markers = symbol.as_deref().map(|s| s != "none").or_else(|| scatter.then_some(true));
|
||||
let symbol = symbol.as_deref().map_or(Symbol::Circle, Symbol::parse);
|
||||
let marker_size = ser
|
||||
.child("c:marker")
|
||||
.and_then(|m| m.child("c:size"))
|
||||
.and_then(|s| s.attr("val"))
|
||||
.and_then(|v| v.trim().parse::<f32>().ok())
|
||||
.filter(|v| v.is_finite())
|
||||
.map(|v| v.clamp(2.0, 72.0));
|
||||
let smooth = ser.child("c:smooth").and_then(|s| s.attr("val")).is_some_and(truthy);
|
||||
let (values, fmt) = numbers(ser.child(if scatter { "c:yVal" } else { "c:val" }), cap);
|
||||
let (mut xs, x_fmt) = if scatter { numbers(ser.child("c:xVal"), cap) } else { (Vec::new(), None) };
|
||||
if scatter && xs.is_empty() {
|
||||
@@ -223,7 +261,30 @@ impl Series {
|
||||
}
|
||||
}
|
||||
let labels = show_values(ser.child("c:dLbls"));
|
||||
Series { name, idx, color, line_w, line, values, xs, points, fmt, x_fmt, labels, raw: Vec::new() }
|
||||
let label_fmt = label_format(ser.child("c:dLbls"));
|
||||
let label_pos = ser.child("c:dLbls").and_then(|d| d.child("c:dLblPos")).and_then(|p| p.attr("val")).and_then(LabelPos::parse);
|
||||
Series {
|
||||
name,
|
||||
idx,
|
||||
color,
|
||||
line_w,
|
||||
line,
|
||||
no_fill,
|
||||
markers,
|
||||
symbol,
|
||||
marker_size,
|
||||
smooth,
|
||||
values,
|
||||
xs,
|
||||
points,
|
||||
fmt,
|
||||
x_fmt,
|
||||
labels,
|
||||
label_fmt,
|
||||
label_pos,
|
||||
marker_color,
|
||||
raw: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -290,7 +351,7 @@ fn pts(cache: &El, cap: usize) -> Vec<Option<String>> {
|
||||
}
|
||||
|
||||
pub(super) fn axis(a: Option<&El>) -> Axis {
|
||||
let Some(a) = a else { return Axis { deleted: true, ..Default::default() } };
|
||||
let Some(a) = a else { return Axis { deleted: true, labels: true, line: true, ..Default::default() } };
|
||||
let scaling = a.child("c:scaling");
|
||||
let num =
|
||||
|n: &str| scaling.and_then(|s| s.child(n)).and_then(|e| e.attr("val")).and_then(|v| v.trim().parse::<f64>().ok()).filter(|v| v.is_finite());
|
||||
@@ -305,8 +366,16 @@ pub(super) fn axis(a: Option<&El>) -> Axis {
|
||||
.child("c:numFmt")
|
||||
.filter(|f| f.attr("sourceLinked") != Some("1"))
|
||||
.and_then(|f| f.attr("formatCode"))
|
||||
.filter(|f| f.len() <= MAX_FORMAT_CODE)
|
||||
.filter(|f| f.chars().count() <= craft_wordmodel::chart::MAX_FMT)
|
||||
.map(str::to_string),
|
||||
title: a.child("c:title").and_then(title_text),
|
||||
title_rot: a
|
||||
.child("c:title")
|
||||
.and_then(|t| t.find("a:bodyPr"))
|
||||
.and_then(|b| b.attr("rot"))
|
||||
.and_then(|v| v.trim().parse::<i64>().ok())
|
||||
.map(|v| (v as f64 / 60_000.0).clamp(-360.0, 360.0) as f32),
|
||||
labels: a.child("c:tickLblPos").and_then(|p| p.attr("val")) != Some("none"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
use super::canvas::{Rect, fill, rect_segs, runs, seg, stroke, text, text_at, text_w, widest, wrap};
|
||||
use super::canvas::{Rect, clip_text, fill, rect_segs, runs, seg, stroke, text, text_angle, text_at, text_w, widest, wrap};
|
||||
use super::format::format_value;
|
||||
use super::model::{DataLabel, Grouping, Kind, Series};
|
||||
use super::model::{DataLabel, Grouping, Kind, LabelPos, Series, Symbol};
|
||||
use super::parse::category_text;
|
||||
use super::polar::{draw_data_labels, label_of};
|
||||
use super::{AXIS, Chart, El, GAP, GRID, GraphicItem, LINE, MAX_LABEL, MAX_TICKS, PathSeg, Rgb, ShapeKind, TextAlign, fill_colour, measure};
|
||||
use super::{AXIS, Chart, El, GAP, GRID, GraphicItem, LINE, LINE_W, MAX_LABEL, MAX_TICKS, PathSeg, Rgb, ShapeKind, TextAlign, fill_colour, measure};
|
||||
|
||||
/// Line width of the stroked marker symbols (x, plus, star), points.
|
||||
const MARKER_STROKE: f32 = 1.5;
|
||||
|
||||
pub(super) fn label(s: &str) -> String {
|
||||
s.chars().take(MAX_LABEL).collect()
|
||||
@@ -142,6 +145,8 @@ impl Axes {
|
||||
/// The room around the plot for axis labels, and the category labels as they'll be drawn.
|
||||
struct Margins {
|
||||
left: f32,
|
||||
/// Part of `left` taken by the rotated category-axis title (bars): the labels start after it.
|
||||
title: f32,
|
||||
top: f32,
|
||||
bottom: f32,
|
||||
/// Category labels in lines (none for scatter charts or a deleted category axis).
|
||||
@@ -151,22 +156,25 @@ struct Margins {
|
||||
}
|
||||
|
||||
/// The margins the axis labels need around the plot in `area`. Category labels under a column
|
||||
/// chart wrap to the band; labels that still don't fit show every k-th category.
|
||||
/// chart wrap to the band; labels that still don't fit show every k-th category. Hidden tick
|
||||
/// labels take no room; an axis title adds one line on its side.
|
||||
fn axis_margins(c: &Chart, ax: &Axes, area: Rect) -> Margins {
|
||||
let fs = c.text_size;
|
||||
let lh = fs * LINE;
|
||||
let scatter = c.kind == Kind::Scatter;
|
||||
let (cat_on, val_on) = (!c.cat.deleted, !c.val.deleted);
|
||||
let (cat_labels, val_labels) = (cat_on && c.cat.labels, val_on && c.val.labels);
|
||||
let title = title_left(c, cat_on, val_on, area.w);
|
||||
let left = if c.horizontal {
|
||||
if cat_on { widest(&c.cats, fs).min(area.w * 0.4) + GAP } else { 0.0 }
|
||||
} else if val_on {
|
||||
if cat_labels { widest(&c.cats, fs).min(area.w * 0.4) + GAP } else { 0.0 }
|
||||
} else if val_labels {
|
||||
ax.vticks.iter().map(|(_, s)| text_w(s, fs)).fold(0.0, f32::max) + GAP
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
} + title;
|
||||
let band_w = (area.w - left).max(1.0) / c.n as f32;
|
||||
let wrapped: Vec<Vec<String>> = if cat_on && !scatter {
|
||||
let width = if c.horizontal { left - GAP } else { band_w };
|
||||
let wrapped: Vec<Vec<String>> = if cat_labels && !scatter {
|
||||
let width = if c.horizontal { left - title - GAP } else { band_w };
|
||||
c.cats.iter().map(|s| wrap(s, width, fs)).collect()
|
||||
} else {
|
||||
Vec::new()
|
||||
@@ -174,15 +182,48 @@ fn axis_margins(c: &Chart, ax: &Axes, area: Rect) -> Margins {
|
||||
let widest_line = wrapped.iter().flatten().map(|s| text_w(s, fs)).fold(0.0, f32::max);
|
||||
let skip = ((widest_line / band_w).ceil() as usize).max(1);
|
||||
let label_lines = wrapped.iter().map(Vec::len).max().unwrap_or(1) as f32;
|
||||
let bottom = if cat_on && !c.horizontal {
|
||||
let bottom = (if cat_labels && !c.horizontal {
|
||||
if scatter { lh + GAP } else { label_lines * lh + GAP }
|
||||
} else if c.horizontal && val_on {
|
||||
} else if c.horizontal && val_labels {
|
||||
lh + GAP
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let top = if !c.horizontal && val_on { lh / 2.0 } else { 0.0 };
|
||||
Margins { left, top, bottom, wrapped, skip }
|
||||
}) + title_bottom(c, cat_on, val_on, lh);
|
||||
let top = if !c.horizontal && val_labels { lh / 2.0 } else { 0.0 };
|
||||
Margins { left, title, top, bottom, wrapped, skip }
|
||||
}
|
||||
|
||||
/// Room for the vertical axis title on the plot's left: one line when the axis that sits there
|
||||
/// (the value axis for columns, the category axis for bars) has a title.
|
||||
/// A horizontal title (`rot="0"`) gets a strip as wide as its wrapped lines, at most a quarter of
|
||||
/// `area_w`, so the plot keeps its room.
|
||||
fn title_left(c: &Chart, cat_on: bool, val_on: bool, area_w: f32) -> f32 {
|
||||
let side = if c.horizontal { cat_on && c.cat.title.is_some() } else { val_on && c.val.title.is_some() };
|
||||
if !side {
|
||||
return 0.0;
|
||||
}
|
||||
match flat_title_lines(c, area_w) {
|
||||
Some(lines) => widest(&lines, c.text_size).min((area_w * 0.25).max(0.0)) + GAP,
|
||||
None => c.text_size * LINE,
|
||||
}
|
||||
}
|
||||
|
||||
/// The wrapped lines of the left axis's title when the file keeps it horizontal (`rot="0"`).
|
||||
fn flat_title_lines(c: &Chart, area_w: f32) -> Option<Vec<String>> {
|
||||
let left = if c.horizontal { &c.cat } else { &c.val };
|
||||
let r = left.title_rot?.rem_euclid(360.0);
|
||||
if !(r < 1.0 || r > 359.0) {
|
||||
return None;
|
||||
}
|
||||
let width = if area_w.is_finite() { (area_w * 0.18).max(1.0) } else { 1.0 };
|
||||
left.title.as_ref().map(|t| wrap(t, width, c.text_size))
|
||||
}
|
||||
|
||||
/// Room for the horizontal axis title under the plot: one line when the axis that sits there
|
||||
/// (the category axis for columns, the value axis for bars) has a title.
|
||||
fn title_bottom(c: &Chart, cat_on: bool, val_on: bool, lh: f32) -> f32 {
|
||||
let side = if c.horizontal { val_on && c.val.title.is_some() } else { cat_on && c.cat.title.is_some() };
|
||||
if side { lh } else { 0.0 }
|
||||
}
|
||||
|
||||
/// A bar, column, line, area or scatter chart in `area`: gridlines, series, axes, then labels.
|
||||
@@ -200,6 +241,9 @@ pub(super) fn cartesian(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
|
||||
if c.val.grid {
|
||||
draw_grid(out, &g, &ax.vticks);
|
||||
}
|
||||
if c.cat.grid {
|
||||
draw_cat_grid(out, &g, &ax.xticks, c.kind);
|
||||
}
|
||||
let mut labels = Vec::new();
|
||||
match c.kind {
|
||||
Kind::Bar => bars(out, c, &g, &mut labels),
|
||||
@@ -210,12 +254,49 @@ pub(super) fn cartesian(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
|
||||
}
|
||||
draw_axes(out, c, &g);
|
||||
draw_data_labels(out, &labels, c.text_size, g.horiz && c.kind == Kind::Bar);
|
||||
if !c.val.deleted {
|
||||
draw_value_labels(out, c, &g, &ax.vticks, Rect { w: m.left, ..area });
|
||||
if !c.val.deleted && c.val.labels {
|
||||
let room = title_left(c, !c.cat.deleted, true, area.w);
|
||||
draw_value_labels(out, c, &g, &ax.vticks, Rect { x: area.x + room, w: m.left - room, ..area });
|
||||
}
|
||||
if !c.cat.deleted {
|
||||
if !c.cat.deleted && c.cat.labels {
|
||||
draw_category_labels(out, c, &g, &ax.xticks, &m, area);
|
||||
}
|
||||
draw_axis_titles(out, c, &g, area);
|
||||
}
|
||||
|
||||
/// The axis titles, one line: the title of the vertical axis (the value axis for columns, the
|
||||
/// category axis for bars) is turned 90° on the plot's left; the horizontal axis's title sits
|
||||
/// under the plot.
|
||||
fn draw_axis_titles(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, area: Rect) {
|
||||
let lh = c.text_size * LINE;
|
||||
// The axis drawn along the plot's left (the value axis for columns, the category axis for
|
||||
// bars) gets its title rotated there; the other axis's title sits under the plot.
|
||||
let (left, bottom) = if c.horizontal { (&c.cat, &c.val) } else { (&c.val, &c.cat) };
|
||||
// Word turns the vertical axis's title counter-clockwise (reading upwards) unless the file
|
||||
// turns it the other way or keeps it horizontal (`rot="0"`, wrapped in its own strip); any
|
||||
// other rotation would not fit the strip and is read as counter-clockwise.
|
||||
let up = if left.title_rot.is_some_and(|r| (r.rem_euclid(360.0) - 90.0).abs() < 1.0) { 90.0 } else { -90.0 };
|
||||
if !left.deleted
|
||||
&& let Some(lines) = flat_title_lines(c, area.w)
|
||||
{
|
||||
let strip = (g.plot.x - area.x - GAP).max(1.0);
|
||||
let top = g.plot.y + g.plot.h / 2.0 - lines.len() as f32 * lh / 2.0;
|
||||
for (k, l) in lines.iter().enumerate() {
|
||||
let l = clip_text(l, strip, c.text_size);
|
||||
text_angle(out, Rect { x: area.x, y: top + k as f32 * lh, w: strip, h: lh }, &l, c.text_size, TextAlign::Center, false, 0.0);
|
||||
}
|
||||
} else if !left.deleted
|
||||
&& let Some(t) = &left.title
|
||||
{
|
||||
let t = clip_text(t, g.plot.h, c.text_size);
|
||||
text_angle(out, Rect { x: area.x, y: g.plot.y, w: lh, h: g.plot.h }, &t, c.text_size, TextAlign::Center, false, up);
|
||||
}
|
||||
if !bottom.deleted
|
||||
&& let Some(t) = &bottom.title
|
||||
{
|
||||
let t = clip_text(t, g.plot.w, c.text_size);
|
||||
text_angle(out, Rect { x: g.plot.x, y: area.bottom() - lh, w: g.plot.w, h: lh }, &t, c.text_size, TextAlign::Center, false, 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Major gridlines at the value ticks.
|
||||
@@ -228,6 +309,31 @@ fn draw_grid(out: &mut Vec<GraphicItem>, g: &Frame, ticks: &[(f64, String)]) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Gridlines along the category axis: across the plot at each category's centre (where line and
|
||||
/// area points sit, on the tick marks when the axis crosses "midCat"), at the x ticks of a
|
||||
/// scatter chart.
|
||||
fn draw_cat_grid(out: &mut Vec<GraphicItem>, g: &Frame, xticks: &[(f64, String)], kind: Kind) {
|
||||
let plot = g.plot;
|
||||
if kind == Kind::Scatter {
|
||||
for (t, _) in xticks {
|
||||
let p = g.scat_x(*t);
|
||||
stroke(out, seg(p, plot.y, p, plot.bottom()), GRID, 0.75);
|
||||
}
|
||||
return;
|
||||
}
|
||||
for i in 0..g.n {
|
||||
let segs = if g.horiz {
|
||||
let p = g.band_start(i) + g.band() / 2.0;
|
||||
seg(plot.x, p, plot.right(), p)
|
||||
} else {
|
||||
// Bars stay in their bands; only line and area points move onto the ticks.
|
||||
let p = if matches!(kind, Kind::Line | Kind::Area) { g.cat_x(i) } else { g.band_start(i) + g.band() / 2.0 };
|
||||
seg(p, plot.y, p, plot.bottom())
|
||||
};
|
||||
stroke(out, segs, GRID, 0.75);
|
||||
}
|
||||
}
|
||||
|
||||
/// The category axis line, the plot area's outline and the value axis line, as the file shows them.
|
||||
fn draw_axes(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame) {
|
||||
let plot = g.plot;
|
||||
@@ -271,7 +377,7 @@ fn draw_category_labels(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, xticks
|
||||
for (i, lines) in m.wrapped.iter().enumerate() {
|
||||
let top = g.band_start(i) + g.band() / 2.0 - lines.len() as f32 * lh / 2.0;
|
||||
for (k, s) in lines.iter().enumerate() {
|
||||
text(out, Rect { x: area.x, y: top + k as f32 * lh, w: m.left - GAP, h: lh }, s, fs, TextAlign::Right, false);
|
||||
text(out, Rect { x: area.x + m.title, y: top + k as f32 * lh, w: m.left - m.title - GAP, h: lh }, s, fs, TextAlign::Right, false);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -361,7 +467,13 @@ fn bars(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataL
|
||||
let (p0, p1) = (g.val(v0), g.val(v1));
|
||||
let (p0, p1) = (p0.min(p1), p0.max(p1));
|
||||
let segs = if g.horiz { rect_segs(p0, a, p1, a + thick) } else { rect_segs(a, p0, a + thick, p1) };
|
||||
fill(out, segs, c.point_colour(j, i));
|
||||
if !s.no_fill {
|
||||
fill(out, segs.clone(), c.point_colour(j, i));
|
||||
}
|
||||
// An explicit outline shows without a fill too (the series' colour is the line's here).
|
||||
if let (true, true, Some(w)) = (s.line, s.no_fill, s.line_w) {
|
||||
stroke(out, segs, c.point_colour(j, i), w);
|
||||
}
|
||||
if let Some(text) = label_of(s, i) {
|
||||
let mid = a + thick / 2.0;
|
||||
let (x, y) = match (g.horiz, stacked) {
|
||||
@@ -371,7 +483,7 @@ fn bars(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataL
|
||||
(true, false) => (if v >= 0.0 { p1 } else { p0 }, mid),
|
||||
(false, false) => (mid, if v >= 0.0 { p0 } else { p1 }),
|
||||
};
|
||||
labels.push(DataLabel { x, y, text, inside: stacked });
|
||||
labels.push(DataLabel { x, y, text, inside: stacked, pos: LabelPos::Above, off: 0.0 });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -401,13 +513,109 @@ fn lines(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<Data
|
||||
.collect();
|
||||
for (i, p) in pts.iter().enumerate() {
|
||||
if let (Some((x, y)), Some(text)) = (p, label_of(s, i)) {
|
||||
labels.push(DataLabel { x: *x, y: *y, text, inside: false });
|
||||
labels.push(DataLabel {
|
||||
x: *x,
|
||||
y: *y,
|
||||
text,
|
||||
inside: false,
|
||||
pos: s.label_pos.unwrap_or(LabelPos::Right),
|
||||
off: marker_half(s, s.markers == Some(true)) + 2.0,
|
||||
});
|
||||
}
|
||||
}
|
||||
stroke(out, runs(pts.into_iter()), c.series_colour(j), s.line_w);
|
||||
if s.line {
|
||||
stroke(out, curve(pts.iter().copied(), s.smooth), c.series_colour(j), s.line_w.unwrap_or(LINE_W));
|
||||
}
|
||||
if s.markers == Some(true) {
|
||||
markers(out, pts.iter().flatten().copied(), c.marker_colour(j), s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A series' point markers in the series' symbol and size (`colour` fills or strokes them), for
|
||||
/// lines and scatter charts.
|
||||
/// Half a series' marker size, points; zero when it draws no markers.
|
||||
fn marker_half(s: &Series, drawn: bool) -> f32 {
|
||||
if drawn { s.marker_size.filter(|v| v.is_finite()).unwrap_or(6.0).clamp(2.0, 72.0) / 2.0 } else { 0.0 }
|
||||
}
|
||||
|
||||
fn markers(out: &mut Vec<GraphicItem>, pts: impl Iterator<Item = (f32, f32)>, colour: Rgb, s: &Series) {
|
||||
const MARKER: f32 = 6.0;
|
||||
let size = s.marker_size.filter(|v| v.is_finite()).unwrap_or(MARKER).clamp(2.0, 72.0);
|
||||
let h = size / 2.0;
|
||||
let shape = |out: &mut Vec<GraphicItem>, rect: [f32; 4], kind: ShapeKind| {
|
||||
out.push(GraphicItem::Shape { rect, kind, fill: Some(colour), stroke: None, stroke_width: 0.0 });
|
||||
};
|
||||
let mut strokes = Vec::new();
|
||||
for (x, y) in pts {
|
||||
match s.symbol {
|
||||
Symbol::Circle => shape(out, [x - h, y - h, size, size], ShapeKind::Ellipse),
|
||||
Symbol::Square => shape(out, [x - h, y - h, size, size], ShapeKind::Rectangle),
|
||||
Symbol::Dot => shape(out, [x - h / 2.0, y - h / 2.0, h, h], ShapeKind::Ellipse),
|
||||
Symbol::Dash => shape(out, [x - h, y - size / 6.0, size, size / 3.0], ShapeKind::Rectangle),
|
||||
Symbol::Diamond => fill(
|
||||
out,
|
||||
vec![PathSeg::Move(x, y - h), PathSeg::Line(x + h, y), PathSeg::Line(x, y + h), PathSeg::Line(x - h, y), PathSeg::Close],
|
||||
colour,
|
||||
),
|
||||
Symbol::Triangle => {
|
||||
fill(out, vec![PathSeg::Move(x, y - h), PathSeg::Line(x + h, y + h), PathSeg::Line(x - h, y + h), PathSeg::Close], colour)
|
||||
}
|
||||
Symbol::X | Symbol::Plus | Symbol::Star => {
|
||||
if s.symbol != Symbol::Plus {
|
||||
strokes.extend(seg(x - h, y - h, x + h, y + h));
|
||||
strokes.extend(seg(x - h, y + h, x + h, y - h));
|
||||
}
|
||||
if s.symbol != Symbol::X {
|
||||
strokes.extend(seg(x - h, y, x + h, y));
|
||||
strokes.extend(seg(x, y - h, x, y + h));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stroke(out, strokes, colour, MARKER_STROKE);
|
||||
}
|
||||
|
||||
/// A polyline through `pts`, a smooth curve (Catmull-Rom pieces) when `smooth` and there are at
|
||||
/// least three points in a row; gaps (`None`) end a run.
|
||||
fn curve(pts: impl Iterator<Item = Option<(f32, f32)>>, smooth: bool) -> Vec<PathSeg> {
|
||||
if !smooth {
|
||||
return runs(pts);
|
||||
}
|
||||
let pts: Vec<Option<(f32, f32)>> = pts.collect();
|
||||
let mut segs = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < pts.len() {
|
||||
let Some(first) = pts[i] else {
|
||||
i += 1;
|
||||
continue;
|
||||
};
|
||||
// Collect one run of points.
|
||||
let mut run = vec![first];
|
||||
while let Some(p) = pts.get(i + 1).copied().flatten() {
|
||||
run.push(p);
|
||||
i += 1;
|
||||
}
|
||||
i += 1;
|
||||
if run.len() < 3 {
|
||||
segs.extend(runs(run.into_iter().map(Some)));
|
||||
continue;
|
||||
}
|
||||
segs.push(PathSeg::Move(run[0].0, run[0].1));
|
||||
for k in 0..run.len() - 1 {
|
||||
// Catmull-Rom: tangent at p1 from p0 to p2 (ends mirror their own tangent).
|
||||
let p0 = run[k.max(1) - 1];
|
||||
let p1 = run[k];
|
||||
let p2 = run[k + 1];
|
||||
let p3 = run.get(k + 2).copied().unwrap_or(p2);
|
||||
let c1 = (p1.0 + (p2.0 - p0.0) / 6.0, p1.1 + (p2.1 - p0.1) / 6.0);
|
||||
let c2 = (p2.0 - (p3.0 - p1.0) / 6.0, p2.1 - (p3.1 - p1.1) / 6.0);
|
||||
segs.push(PathSeg::Cubic(c1.0, c1.1, c2.0, c2.1, p2.0, p2.1));
|
||||
}
|
||||
}
|
||||
segs
|
||||
}
|
||||
|
||||
fn areas(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataLabel>) {
|
||||
let mut base = vec![0.0f64; g.n];
|
||||
for (j, s) in c.series.iter().enumerate().filter(|(_, s)| has_values(s)) {
|
||||
@@ -422,10 +630,12 @@ fn areas(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<Data
|
||||
segs.push(PathSeg::Line(g.cat_x(i), g.val(*b)));
|
||||
}
|
||||
segs.push(PathSeg::Close);
|
||||
fill(out, segs, c.series_colour(j));
|
||||
if !s.no_fill {
|
||||
fill(out, segs, c.series_colour(j));
|
||||
}
|
||||
for (i, (t, b)) in top.iter().zip(&base).enumerate() {
|
||||
if let Some(text) = label_of(s, i) {
|
||||
labels.push(DataLabel { x: g.cat_x(i), y: g.val((t + b) / 2.0), text, inside: true });
|
||||
labels.push(DataLabel { x: g.cat_x(i), y: g.val((t + b) / 2.0), text, inside: true, pos: LabelPos::Above, off: 0.0 });
|
||||
}
|
||||
}
|
||||
if c.stacked() {
|
||||
@@ -434,9 +644,8 @@ fn areas(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<Data
|
||||
}
|
||||
}
|
||||
|
||||
/// Scatter series as lines, or as small markers when the series has no line.
|
||||
/// Scatter series as lines, and as small markers when the series asks for them.
|
||||
fn draw_scatter(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataLabel>) {
|
||||
const MARKER: f32 = 6.0;
|
||||
for (j, s) in c.series.iter().enumerate() {
|
||||
let pts: Vec<Option<(f32, f32)>> =
|
||||
s.xs.iter()
|
||||
@@ -448,16 +657,21 @@ fn draw_scatter(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut V
|
||||
.collect();
|
||||
for (i, p) in pts.iter().enumerate() {
|
||||
if let (Some((x, y)), Some(text)) = (p, label_of(s, i)) {
|
||||
labels.push(DataLabel { x: *x, y: *y - MARKER / 2.0, text, inside: false });
|
||||
labels.push(DataLabel {
|
||||
x: *x,
|
||||
y: *y,
|
||||
text,
|
||||
inside: false,
|
||||
pos: s.label_pos.unwrap_or(LabelPos::Right),
|
||||
off: marker_half(s, s.markers != Some(false)) + 2.0,
|
||||
});
|
||||
}
|
||||
}
|
||||
if s.line {
|
||||
stroke(out, runs(pts.iter().copied()), c.series_colour(j), s.line_w);
|
||||
continue;
|
||||
stroke(out, curve(pts.iter().copied(), s.smooth), c.series_colour(j), s.line_w.unwrap_or(LINE_W));
|
||||
}
|
||||
for (x, y) in pts.iter().flatten() {
|
||||
let rect = [x - MARKER / 2.0, y - MARKER / 2.0, MARKER, MARKER];
|
||||
out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(c.series_colour(j)), stroke: None, stroke_width: 0.0 });
|
||||
if s.markers != Some(false) {
|
||||
markers(out, pts.iter().flatten().copied(), c.marker_colour(j), s);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
use super::canvas::{Rect, clip_text, fill, stroke, text, text_w, widest};
|
||||
use super::format::format_value;
|
||||
use super::model::{DataLabel, Kind, Series};
|
||||
use super::model::{DataLabel, Kind, LabelPos, Series};
|
||||
use super::plot::{Axes, has_values};
|
||||
use super::{Chart, FRAC_PI_2, GAP, GRID, GraphicItem, LINE, PathSeg, Rgb, ShapeKind, TAU, TEXT, TextAlign};
|
||||
use super::{Chart, FRAC_PI_2, GAP, GRID, GraphicItem, LINE, LINE_W, PathSeg, Rgb, ShapeKind, TAU, TEXT, TextAlign};
|
||||
|
||||
pub(super) fn pie(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
|
||||
let Some(s) = c.series.first() else { return };
|
||||
@@ -38,7 +38,7 @@ pub(super) fn pie(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
|
||||
// Halfway across the ring (pie: two thirds out from the centre).
|
||||
let at = if inner > 0.0 { (r + inner) / 2.0 } else { r * 0.65 };
|
||||
let (x, y) = polar(cx, cy, at, (a + b) / 2.0);
|
||||
labels.push(DataLabel { x: x as f32, y: y as f32, text, inside: true });
|
||||
labels.push(DataLabel { x: x as f32, y: y as f32, text, inside: true, pos: LabelPos::Above, off: 0.0 });
|
||||
}
|
||||
a = b;
|
||||
}
|
||||
@@ -92,14 +92,14 @@ pub(super) fn radar(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
|
||||
if c.radar_fill {
|
||||
fill(out, segs.clone(), col);
|
||||
}
|
||||
stroke(out, segs, col, s.line_w);
|
||||
stroke(out, segs, col, s.line_w.unwrap_or(LINE_W));
|
||||
for (i, p) in ring.iter().enumerate() {
|
||||
if c.radar_marks && s.values.get(i).copied().flatten().is_some() {
|
||||
let rect = [p.0 as f32 - MARKER / 2.0, p.1 as f32 - MARKER / 2.0, MARKER, MARKER];
|
||||
out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(col), stroke: None, stroke_width: 0.0 });
|
||||
}
|
||||
if let Some(text) = label_of(s, i) {
|
||||
labels.push(DataLabel { x: p.0 as f32, y: p.1 as f32, text, inside: false });
|
||||
labels.push(DataLabel { x: p.0 as f32, y: p.1 as f32, text, inside: false, pos: LabelPos::Above, off: 0.0 });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -134,13 +134,13 @@ pub(super) fn radar(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
|
||||
}
|
||||
|
||||
/// The data label of point `i` of series `s`, when it shows one: the value as the file has it,
|
||||
/// in the series' number format.
|
||||
/// in the labels' own number format, else the series' one.
|
||||
pub(super) fn label_of(s: &Series, i: usize) -> Option<String> {
|
||||
if s.labels != Some(true) {
|
||||
return None;
|
||||
}
|
||||
let v = s.raw.get(i).copied().flatten()?;
|
||||
let fmt = s.fmt.as_deref().filter(|f| !f.eq_ignore_ascii_case("General"));
|
||||
let fmt = s.label_fmt.as_ref().map_or(s.fmt.as_deref(), |f| f.as_deref().or(s.fmt.as_deref())).filter(|f| !f.eq_ignore_ascii_case("General"));
|
||||
Some(match fmt {
|
||||
Some(f) => format_value(v, f, 1.0),
|
||||
None => general_value(v),
|
||||
@@ -165,13 +165,22 @@ pub(super) fn draw_data_labels(out: &mut Vec<GraphicItem>, labels: &[DataLabel],
|
||||
let w = text_w(&l.text, fs) + 4.0;
|
||||
let (rect, align) = if l.inside {
|
||||
(Rect { x: l.x - w / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Center)
|
||||
} else if l.off > 0.0 || l.pos != LabelPos::Above {
|
||||
let o = l.off;
|
||||
match l.pos {
|
||||
LabelPos::Right => (Rect { x: l.x + o, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Left),
|
||||
LabelPos::Left => (Rect { x: l.x - o - w, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Right),
|
||||
LabelPos::Below => (Rect { x: l.x - w / 2.0, y: l.y + o, w, h: lh }, TextAlign::Center),
|
||||
LabelPos::Center => (Rect { x: l.x - w / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Center),
|
||||
LabelPos::Above => (Rect { x: l.x - w / 2.0, y: l.y - o - lh, w, h: lh }, TextAlign::Center),
|
||||
}
|
||||
} else if beside_right {
|
||||
(Rect { x: l.x + GAP / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Left)
|
||||
} else {
|
||||
(Rect { x: l.x - w / 2.0, y: l.y - lh, w, h: lh }, TextAlign::Center)
|
||||
};
|
||||
let color = if l.inside { Rgb::WHITE } else { TEXT };
|
||||
out.push(GraphicItem::Text { rect: rect.arr(), text: l.text.clone(), size: fs, color, bold: false, align, font: None });
|
||||
out.push(GraphicItem::Text { rect: rect.arr(), text: l.text.clone(), size: fs, color, bold: false, align, font: None, angle: 0.0 });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -386,9 +386,13 @@ fn axis_format_code_length_is_capped() {
|
||||
let x = format!(r#"<c:valAx {NS}><c:numFmt formatCode="{code}" sourceLinked="0"/></c:valAx>"#);
|
||||
axis(Some(&crate::xml::parse(x.as_bytes()).unwrap())).fmt
|
||||
};
|
||||
let ok = format!("{}0", "x".repeat(254));
|
||||
use craft_wordmodel::chart::MAX_FMT;
|
||||
// The limit is the model's, counted in characters (a CJK character is one).
|
||||
let ok = format!("{}0", "x".repeat(MAX_FMT - 1));
|
||||
assert_eq!(fmt(&ok), Some(ok.clone()));
|
||||
assert_eq!(fmt(&format!("{}0", "x".repeat(300))), None);
|
||||
let cjk = format!("{}0", "\u{5143}".repeat(MAX_FMT - 1));
|
||||
assert_eq!(fmt(&cjk), Some(cjk.clone()));
|
||||
assert_eq!(fmt(&format!("{}0", "x".repeat(MAX_FMT))), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -590,3 +594,308 @@ fn series_without_a_usable_c_idx_take_their_place() {
|
||||
let big = [renumbered(&a, &format!(r#"<c:idx val="{}"/>"#, u32::MAX))];
|
||||
assert_eq!(column_colours(&big).0.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bar_category_labels_start_after_the_axis_title() {
|
||||
let long = "A category label that is much too long for its column";
|
||||
let title = r#"<c:title><c:tx><c:rich><a:p><a:r><a:t>Region</a:t></a:r></a:p></c:rich></c:tx></c:title>"#;
|
||||
let plot = format!(
|
||||
r#"<c:barChart><c:barDir val="bar"/><c:grouping val="clustered"/>{}<c:axId val="1"/><c:axId val="2"/></c:barChart><c:catAx><c:axId val="1"/>{title}</c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#,
|
||||
ser(0, "S", Some("FF0000"), &[long, "Short"], &["4", "8"]),
|
||||
);
|
||||
let items = chart_items(&space(&plot), &theme(), 300.0, 200.0);
|
||||
let rect_of = |f: &dyn Fn(&str, &TextAlign) -> bool| -> Vec<[f32; 4]> {
|
||||
items
|
||||
.iter()
|
||||
.filter_map(|it| if let GraphicItem::Text { rect, text, align, .. } = it { f(text, align).then_some(*rect) } else { None })
|
||||
.collect()
|
||||
};
|
||||
let title_rect = rect_of(&|t, _| t == "Region");
|
||||
assert_eq!(title_rect.len(), 1);
|
||||
let strip_end = title_rect[0][0] + title_rect[0][2];
|
||||
let labels = rect_of(&|t, a| matches!(a, TextAlign::Right) && t != "Region" && (long.contains(t) || t == "Short"));
|
||||
assert!(!labels.is_empty());
|
||||
assert!(labels.iter().all(|r| r[0] >= strip_end - 1e-3), "labels {labels:?} overlap the title strip ending at {strip_end}");
|
||||
assert!(labels.iter().all(|r| r[2] <= 0.4 * 300.0), "{labels:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn category_gridlines_follow_the_points_when_the_axis_crosses_mid() {
|
||||
let line = |cross: &str| {
|
||||
let plot = format!(
|
||||
r#"<c:lineChart><c:grouping val="standard"/>{}<c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/><c:majorGridlines/></c:catAx><c:valAx><c:axId val="2"/><c:crossBetween val="{cross}"/></c:valAx>"#,
|
||||
ser(0, "L", Some("FF0000"), &["Q1", "Q2", "Q3"], &["1", "5", "3"]),
|
||||
);
|
||||
let items = chart_items(&space(&plot), &theme(), 300.0, 200.0);
|
||||
let grid: Vec<f32> = items
|
||||
.iter()
|
||||
.filter_map(|it| match it {
|
||||
GraphicItem::Path { segs, stroke: Some(c), .. } if *c == GRID => match segs.first() {
|
||||
Some(PathSeg::Move(x, _)) => Some(*x),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
let pts: Vec<f32> = items
|
||||
.iter()
|
||||
.filter_map(|it| match it {
|
||||
GraphicItem::Path { segs, stroke: Some(c), .. } if *c == Rgb(255, 0, 0) => Some(
|
||||
segs.iter().filter_map(|s| if let PathSeg::Move(x, _) | PathSeg::Line(x, _) = s { Some(*x) } else { None }).collect::<Vec<f32>>(),
|
||||
),
|
||||
_ => None,
|
||||
})
|
||||
.next()
|
||||
.unwrap_or_default();
|
||||
(grid, pts)
|
||||
};
|
||||
for cross in ["midCat", "between"] {
|
||||
let (grid, pts) = line(cross);
|
||||
assert_eq!(grid.len(), 3, "{cross}");
|
||||
assert_eq!(pts.len(), 3, "{cross}");
|
||||
assert!(grid.iter().zip(&pts).all(|(g, p)| (g - p).abs() < 1e-3), "{cross}: grid {grid:?} points {pts:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A line chart with one series; `extra` goes into the series (`c:marker`, `c:dLbls`), `axes`
|
||||
/// into the value axis.
|
||||
fn line_with(extra: &str, val_axis: &str) -> Vec<GraphicItem> {
|
||||
let s = ser(0, "S", Some("FF0000"), &["Q1", "Q2", "Q3"], &["3.2", "4.1", "5"]).replace("<c:cat>", &format!("{extra}<c:cat>"));
|
||||
let plot = format!(
|
||||
r#"<c:lineChart><c:grouping val="standard"/>{s}<c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/>{val_axis}</c:valAx>"#
|
||||
);
|
||||
chart_items(&space(&plot), &theme(), 300.0, 200.0)
|
||||
}
|
||||
|
||||
fn marker_xml(symbol: &str, size: u32) -> String {
|
||||
format!(r#"<c:marker><c:symbol val="{symbol}"/><c:size val="{size}"/></c:marker>"#)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn marker_symbols_are_drawn_as_their_shape() {
|
||||
let shapes = |items: &[GraphicItem], kind: ShapeKind| {
|
||||
items
|
||||
.iter()
|
||||
.filter(|it| matches!(it, GraphicItem::Shape { kind: k, rect, .. } if *k == kind && rect[2] > 3.0 && rect[2] < 20.0 && rect[3] > 1.0))
|
||||
.count()
|
||||
};
|
||||
let paths = |items: &[GraphicItem], filled: bool| {
|
||||
items
|
||||
.iter()
|
||||
.filter(|it| matches!(it, GraphicItem::Path { fill, segs, .. } if fill.is_some() == filled && segs.len() <= 12 && segs.len() >= 3))
|
||||
.count()
|
||||
};
|
||||
// One marker per point, in the series' size.
|
||||
let circle = line_with(&marker_xml("circle", 9), "");
|
||||
assert_eq!(shapes(&circle, ShapeKind::Ellipse), 3);
|
||||
assert!(circle.iter().any(|it| matches!(it, GraphicItem::Shape { kind: ShapeKind::Ellipse, rect, .. } if (rect[2] - 9.0).abs() < 1e-3)));
|
||||
let square = line_with(&marker_xml("square", 7), "");
|
||||
assert_eq!(shapes(&square, ShapeKind::Rectangle) - shapes(&circle, ShapeKind::Rectangle), 3, "a square per point");
|
||||
assert_eq!(shapes(&square, ShapeKind::Ellipse), 0, "no circles");
|
||||
// Diamonds and triangles are filled polygons; x, plus and star are strokes.
|
||||
let base = paths(&line_with("", ""), true);
|
||||
assert_eq!(paths(&line_with(&marker_xml("diamond", 7), ""), true) - base, 3);
|
||||
assert_eq!(paths(&line_with(&marker_xml("triangle", 7), ""), true) - base, 3);
|
||||
for symbol in ["x", "plus", "star"] {
|
||||
let items = line_with(&marker_xml(symbol, 7), "");
|
||||
assert!(items.iter().any(|it| matches!(it, GraphicItem::Path { fill: None, stroke: Some(_), segs, .. } if segs.len() >= 6)), "{symbol}");
|
||||
assert_eq!(shapes(&items, ShapeKind::Ellipse), 0, "{symbol} is not a circle");
|
||||
}
|
||||
// Dash and dot are small shapes; `none` and unknown names never crash (unknown: a circle).
|
||||
assert_eq!(shapes(&line_with(&marker_xml("dash", 7), ""), ShapeKind::Rectangle) - shapes(&circle, ShapeKind::Rectangle), 3);
|
||||
assert_eq!(shapes(&line_with(&marker_xml("dot", 7), ""), ShapeKind::Ellipse), 3);
|
||||
assert_eq!(shapes(&line_with(&marker_xml("none", 7), ""), ShapeKind::Ellipse), 0);
|
||||
assert_eq!(shapes(&line_with(&marker_xml("wibble", 7), ""), ShapeKind::Ellipse), 3);
|
||||
// Hostile sizes are clamped.
|
||||
for size in ["0", "99999999999", "-4", "x"] {
|
||||
let items = line_with(&format!(r#"<c:marker><c:symbol val="square"/><c:size val="{size}"/></c:marker>"#), "");
|
||||
assert!(items.iter().all(|it| match it {
|
||||
GraphicItem::Shape { rect, .. } => rect.iter().all(|v| v.is_finite()) && rect[2] <= 300.0,
|
||||
_ => true,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_labels_use_their_own_format_code() {
|
||||
let dl = |num: &str| format!(r#"<c:dLbls>{num}<c:showVal val="1"/></c:dLbls>"#);
|
||||
let shown = |items: &[GraphicItem]| -> Vec<String> {
|
||||
items.iter().filter_map(|it| if let GraphicItem::Text { text, .. } = it { Some(text.clone()) } else { None }).collect()
|
||||
};
|
||||
let own = shown(&line_with(&dl(r#"<c:numFmt formatCode="0.00" sourceLinked="0"/>"#), ""));
|
||||
assert!(own.contains(&"3.20".to_string()) && own.contains(&"5.00".to_string()), "{own:?}");
|
||||
// Linked to the source: the series' format (General here).
|
||||
let linked = shown(&line_with(&dl(r#"<c:numFmt formatCode="0.00" sourceLinked="1"/>"#), ""));
|
||||
assert!(linked.contains(&"3.2".to_string()) && linked.contains(&"5".to_string()), "{linked:?}");
|
||||
// A format code beyond the cap is ignored, not applied.
|
||||
let long = "0".repeat(craft_wordmodel::chart::MAX_FMT + 1);
|
||||
let capped = shown(&line_with(&dl(&format!(r#"<c:numFmt formatCode="{long}" sourceLinked="0"/>"#)), ""));
|
||||
assert!(capped.contains(&"3.2".to_string()), "{capped:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn value_axis_title_turns_counter_clockwise_like_word() {
|
||||
let title = |body: &str| format!(r#"<c:title><c:tx><c:rich>{body}<a:p><a:r><a:t>Revenue</a:t></a:r></a:p></c:rich></c:tx></c:title>"#);
|
||||
let angle = |items: &[GraphicItem]| {
|
||||
items.iter().find_map(|it| if let GraphicItem::Text { text, angle, .. } = it { (text == "Revenue").then_some(*angle) } else { None })
|
||||
};
|
||||
// The file's rotation (`-5400000` is 90° counter-clockwise), or Word's default when it says none.
|
||||
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="-5400000" vert="horz"/>"#))), Some(-90.0));
|
||||
assert_eq!(angle(&line_with("", &title("<a:bodyPr/>"))), Some(-90.0));
|
||||
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="5400000" vert="horz"/>"#))), Some(90.0));
|
||||
// A horizontal title (`rot="0"`) stays horizontal: the plot makes room for it (see below).
|
||||
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="0" vert="horz"/>"#))), Some(0.0));
|
||||
// Hostile values stay finite.
|
||||
let odd = angle(&line_with("", &title(r#"<a:bodyPr rot="99999999999999999999"/>"#)));
|
||||
assert!(odd.is_some_and(f32::is_finite));
|
||||
}
|
||||
|
||||
fn first_ellipse(items: &[GraphicItem]) -> Option<([f32; 4], Rgb)> {
|
||||
items.iter().find_map(|it| match it {
|
||||
GraphicItem::Shape { kind: ShapeKind::Ellipse, rect, fill: Some(c), .. } if rect[2] > 3.0 && rect[2] < 20.0 => Some((*rect, *c)),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markers_use_the_automatic_series_colour_not_the_line_colour() {
|
||||
// The series has an explicit red line/fill but no marker fill: Word draws the markers in the
|
||||
// palette colour of the series index (accent 1 here).
|
||||
let items = line_with(&marker_xml("circle", 9), "");
|
||||
assert_eq!(first_ellipse(&items).map(|e| e.1), Some(theme()[4]));
|
||||
// A marker's own fill wins.
|
||||
let own =
|
||||
r#"<c:marker><c:symbol val="circle"/><c:size val="9"/><c:spPr><a:solidFill><a:srgbClr val="00AA00"/></a:solidFill></c:spPr></c:marker>"#;
|
||||
assert_eq!(first_ellipse(&line_with(own, "")).map(|e| e.1), Some(Rgb(0, 0xAA, 0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_data_labels_default_to_the_right_and_honour_dlblpos() {
|
||||
let dl = |pos: &str| format!(r#"{}<c:dLbls>{pos}<c:showVal val="1"/></c:dLbls>"#, marker_xml("circle", 9));
|
||||
let text_rect = |items: &[GraphicItem]| {
|
||||
items.iter().find_map(|it| if let GraphicItem::Text { text, rect, .. } = it { (text == "3.2").then_some(*rect) } else { None })
|
||||
};
|
||||
let at = |pos: &str| {
|
||||
let items = line_with(&dl(pos), "");
|
||||
let (m, _) = first_ellipse(&items).unwrap();
|
||||
let t = text_rect(&items).unwrap();
|
||||
// Centre of the marker and of the label box.
|
||||
((t[0] + t[2] / 2.0) - (m[0] + m[2] / 2.0), (t[1] + t[3] / 2.0) - (m[1] + m[3] / 2.0), t, m)
|
||||
};
|
||||
let (dx, dy, t, m) = at("");
|
||||
assert!(t[0] >= m[0] + m[2] && dy.abs() < 1.0, "default is right of the marker: {dx} {dy}");
|
||||
let (dx, dy, t, m) = at(r#"<c:dLblPos val="r"/>"#);
|
||||
assert!(t[0] >= m[0] + m[2] && dy.abs() < 1.0, "{dx} {dy}");
|
||||
let (dx, dy, _, _) = at(r#"<c:dLblPos val="t"/>"#);
|
||||
assert!(dx.abs() < 1.0 && dy < -4.0, "{dx} {dy}");
|
||||
let (dx, dy, _, _) = at(r#"<c:dLblPos val="b"/>"#);
|
||||
assert!(dx.abs() < 1.0 && dy > 4.0, "{dx} {dy}");
|
||||
let (dx, dy, t, m) = at(r#"<c:dLblPos val="l"/>"#);
|
||||
assert!(t[0] + t[2] <= m[0] && dy.abs() < 1.0, "{dx} {dy}");
|
||||
let (dx, dy, _, _) = at(r#"<c:dLblPos val="ctr"/>"#);
|
||||
assert!(dx.abs() < 1.0 && dy.abs() < 1.0, "{dx} {dy}");
|
||||
// Unknown positions fall back to the default.
|
||||
let (_, _, t, m) = at(r#"<c:dLblPos val="wibble"/>"#);
|
||||
assert!(t[0] >= m[0] + m[2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn horizontal_value_axis_title_gets_its_own_strip() {
|
||||
let title = |rot: &str, name: &str| {
|
||||
format!(r#"<c:title><c:tx><c:rich><a:bodyPr{rot}/><a:p><a:r><a:t>{name}</a:t></a:r></a:p></c:rich></c:tx></c:title>"#)
|
||||
};
|
||||
let plot_left = |items: &[GraphicItem]| {
|
||||
// The first gridline's start is the plot's left edge.
|
||||
items.iter().find_map(|it| match it {
|
||||
GraphicItem::Path { segs, stroke: Some(_), fill: None, .. } if segs.len() == 2 => match segs.first() {
|
||||
Some(PathSeg::Move(x, _)) => Some(*x),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
};
|
||||
let rect_of = |items: &[GraphicItem], name: &str| {
|
||||
items.iter().find_map(|it| if let GraphicItem::Text { text, rect, .. } = it { text.starts_with(name).then_some(*rect) } else { None })
|
||||
};
|
||||
let turned = line_with("", &title(r#" rot="-5400000" vert="horz""#, "Revenue"));
|
||||
let flat = line_with("", &title(r#" rot="0" vert="horz""#, "Revenue"));
|
||||
let (a, b) = (plot_left(&turned).unwrap(), plot_left(&flat).unwrap());
|
||||
assert!(b > a + 10.0, "a horizontal title takes more room than a turned one: {a} {b}");
|
||||
// The title box lies left of the plot and inside the chart.
|
||||
let r = rect_of(&flat, "Revenue").unwrap();
|
||||
assert!(r[0] >= 0.0 && r[0] + r[2] <= b + 0.5, "{r:?} {b}");
|
||||
// A hostile title never takes more than a quarter of the width and stays finite.
|
||||
let long = "Wibble ".repeat(500);
|
||||
let items = line_with("", &title(r#" rot="0""#, &long));
|
||||
let p = plot_left(&items).unwrap();
|
||||
assert!(p.is_finite() && p <= 300.0 * 0.3, "{p}");
|
||||
}
|
||||
|
||||
/// A one-series chart whose series carries `sp` as its `c:spPr`.
|
||||
fn chart_with_sp(kind: &str, sp: &str) -> Vec<GraphicItem> {
|
||||
let s = ser(0, "S", None, &["Q1", "Q2"], &["4", "8"]).replace("<c:cat>", &format!("{sp}<c:cat>"));
|
||||
let plot = match kind {
|
||||
"area" => format!(
|
||||
r#"<c:areaChart><c:grouping val="standard"/>{s}<c:axId val="1"/><c:axId val="2"/></c:areaChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#
|
||||
),
|
||||
dir => format!(
|
||||
r#"<c:barChart><c:barDir val="{dir}"/><c:grouping val="clustered"/>{s}<c:axId val="1"/><c:axId val="2"/></c:barChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#
|
||||
),
|
||||
};
|
||||
chart_items(&space(&plot), &theme(), 300.0, 200.0)
|
||||
}
|
||||
|
||||
/// The series' closed shapes: (filled, stroked with the given width).
|
||||
fn series_shapes(items: &[GraphicItem], width: f32) -> (usize, usize) {
|
||||
let closed = |segs: &[PathSeg]| segs.iter().any(|s| matches!(s, PathSeg::Close));
|
||||
let fills = items.iter().filter(|it| matches!(it, GraphicItem::Path { segs, fill: Some(_), .. } if closed(segs))).count();
|
||||
let strokes = items
|
||||
.iter()
|
||||
.filter(|it| matches!(it, GraphicItem::Path { segs, fill: None, stroke: Some(_), stroke_width, .. } if closed(segs) && (*stroke_width - width).abs() < 0.01))
|
||||
.count();
|
||||
(fills, strokes)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unfilled_bars_keep_their_explicit_outline() {
|
||||
let outlined = r#"<c:spPr><a:noFill/><a:ln w="25400"><a:solidFill><a:srgbClr val="FF0000"/></a:solidFill></a:ln></c:spPr>"#;
|
||||
let no_line = r#"<c:spPr><a:noFill/><a:ln><a:noFill/></a:ln></c:spPr>"#;
|
||||
for dir in ["col", "bar"] {
|
||||
// Two points, two outlines, no fill.
|
||||
assert_eq!(series_shapes(&chart_with_sp(dir, outlined), 2.0), (0, 2), "{dir}");
|
||||
// Control: no fill and no line draws nothing.
|
||||
assert_eq!(series_shapes(&chart_with_sp(dir, no_line), 2.0), (0, 0), "{dir}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_fill_area_series_is_not_painted() {
|
||||
let filled = r#"<c:spPr><a:solidFill><a:srgbClr val="FF0000"/></a:solidFill></c:spPr>"#;
|
||||
let none = r#"<c:spPr><a:noFill/><a:ln w="25400"><a:solidFill><a:srgbClr val="FF0000"/></a:solidFill></a:ln></c:spPr>"#;
|
||||
assert_eq!(series_shapes(&chart_with_sp("area", filled), 2.0).0, 1);
|
||||
assert_eq!(series_shapes(&chart_with_sp("area", none), 2.0).0, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_linked_series_labels_do_not_inherit_the_chart_format() {
|
||||
let dl = |fmt: &str| format!(r#"<c:dLbls><c:numFmt formatCode="{fmt}" sourceLinked="{{}}"/><c:showVal val="1"/></c:dLbls>"#);
|
||||
let shown = |items: &[GraphicItem]| -> Vec<String> {
|
||||
items.iter().filter_map(|it| if let GraphicItem::Text { text, .. } = it { Some(text.clone()) } else { None }).collect()
|
||||
};
|
||||
// Chart-level 0%; the series' own labels are `linked` (or have no labels element at all).
|
||||
let run = |series_dl: &str| {
|
||||
let s = ser(0, "S", Some("FF0000"), &["Q1", "Q2", "Q3"], &["3.2", "4.1", "5"]).replace("<c:cat>", &format!("{series_dl}<c:cat>"));
|
||||
let chart_dl = dl("0%").replace("{}", "0");
|
||||
let plot = format!(
|
||||
r#"<c:lineChart><c:grouping val="standard"/>{s}{chart_dl}<c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#
|
||||
);
|
||||
shown(&chart_items(&space(&plot), &theme(), 300.0, 200.0))
|
||||
};
|
||||
for linked in ["1", "true"] {
|
||||
let own = run(&dl("General").replace("{}", linked));
|
||||
assert!(own.contains(&"3.2".to_string()) && !own.contains(&"320%".to_string()), "sourceLinked={linked}: {own:?}");
|
||||
}
|
||||
// Control: a series without a label format still takes the chart's.
|
||||
let inherited = run(r#"<c:dLbls><c:showVal val="1"/></c:dLbls>"#);
|
||||
assert!(inherited.contains(&"320%".to_string()), "{inherited:?}");
|
||||
}
|
||||
@@ -463,7 +463,16 @@ fn text(sp: &El, xf: Xf, b: [f64; 4], theme: &[Rgb], filled: bool, fit: f32, out
|
||||
let mut y = y0 + (th - total) / 2.0;
|
||||
for l in lines {
|
||||
let lh = l.size * LINE;
|
||||
out.push(GraphicItem::Text { rect: [x0, y, tw, lh], text: l.text, size: l.size, color: l.color, bold: l.bold, align: l.align, font: l.font });
|
||||
out.push(GraphicItem::Text {
|
||||
rect: [x0, y, tw, lh],
|
||||
text: l.text,
|
||||
size: l.size,
|
||||
color: l.color,
|
||||
bold: l.bold,
|
||||
align: l.align,
|
||||
font: l.font,
|
||||
angle: 0.0,
|
||||
});
|
||||
y += lh;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -884,10 +884,11 @@ mod tests {
|
||||
let spec = ChartSpec {
|
||||
kind: ChartType::Column,
|
||||
categories: vec!["a".into(), "b".into()],
|
||||
series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2] }).collect(),
|
||||
series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2], ..Default::default() }).collect(),
|
||||
title: None,
|
||||
legend: None,
|
||||
data_labels: false,
|
||||
..Default::default()
|
||||
};
|
||||
let graphic = Graphic { kind: GraphicKind::Chart, w: 100.0, h: 100.0, chart: Some(Arc::new(spec)), ..Default::default() };
|
||||
InlineObject::Graphic { w: 100.0, h: 100.0, alt: String::new(), float: Default::default(), graphic: Arc::new(graphic) }
|
||||
|
||||
@@ -755,7 +755,7 @@ fn graphic_group(
|
||||
let inline = Inline::Shape { kind: *kind, w: rect[2] * sx, h: rect[3] * sy, float: None, blocks: Vec::new(), look, path: None };
|
||||
Some(Member { x: rect[0] * sx, y: rect[1] * sy, inline })
|
||||
}
|
||||
GraphicItem::Text { rect, text, size, color, bold, align, font } if finite(rect) && !text.trim().is_empty() => {
|
||||
GraphicItem::Text { rect, text, size, color, bold, align, font, .. } if finite(rect) && !text.trim().is_empty() => {
|
||||
let size = if size.is_finite() { size.clamp(1.0, 400.0) } else { 11.0 };
|
||||
// One line, centred vertically in its place as the graphic draws it.
|
||||
let line = size * 1.2;
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
[package]
|
||||
name = "craft-http-url"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license = "Apache-2.0"
|
||||
repository.workspace = true
|
||||
description = "Bounded HTTP URL parsing and redirect resolution without network access"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,25 @@
|
||||
# craft-http-url
|
||||
|
||||
HTTP/HTTPS URL parsing, dot-segment removal, relative redirects, HTTPS downgrade rejection,
|
||||
and query-redacted display. No dependencies or network operations. Parsing rejects inputs
|
||||
larger than 16 KiB and redirects are bounded before concatenation.
|
||||
|
||||
This is not an SSRF filter, DNS resolver or HTTP client. Local/private hosts are allowed.
|
||||
Applications keep destination permission, authentication, TLS, request/response limits and
|
||||
redirect-count policy. Public fields preserve the original API; manually constructed values
|
||||
must not be treated as validated URLs.
|
||||
|
||||
## Attribution
|
||||
|
||||
Extracted from LightCraft `crates/fetch/src/http.rs` at
|
||||
`2dcb258ba40738230a07484a95eb27dc84ed66fe` (Apache-2.0).
|
||||
Original parser: Daniel Voshart, `7fa7506db46d69d609e85e39b338f33786ed0293`.
|
||||
HTTPS downgrade protection: MarekSpodBiedry, `4a91a1c218bf3d0ac49cc73ebb60621a21ae55e2`.
|
||||
URL normalization, scheme, fragment and authority fixes: NetterSilver, including
|
||||
`edc369565eef92161b56b8f1e39ef53629c05489`,
|
||||
`78f0e6e650444886069f4d4ae4e6593f221b98d4`,
|
||||
`cee93496587d31c514aaeced3e6b8027cfdb41a0`,
|
||||
`dee5f7910ad8c24cf28bab528f8d007550af2b23`,
|
||||
`c3a5d9b7f1f837dc14c01942e3d0293ecbf36345` and
|
||||
`13849307a9343504339d68b086f1a32cdcbbb457`.
|
||||
Source-author commits are retained separately in this extraction.
|
||||
@@ -0,0 +1,277 @@
|
||||
//! Bounded HTTP/HTTPS URL parsing and redirect resolution, extracted from LightCraft.
|
||||
//!
|
||||
//! This is syntax handling, not a network permission or SSRF policy. It permits local/private
|
||||
//! hosts and performs no DNS, socket or TLS operations. Callers own destination allowlists,
|
||||
//! credential forwarding, redirect counts, connection deadlines and response limits.
|
||||
//! Only parsed values have validated fields; constructing public fields bypasses parsing.
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
/// A URL or redirect cannot be represented by this bounded HTTP URL syntax.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UrlError(pub String);
|
||||
|
||||
impl std::fmt::Display for UrlError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for UrlError {}
|
||||
|
||||
/// A parsed `http://` or `https://` URL.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Url {
|
||||
pub tls: bool,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
/// Path and query, starting with `/`.
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
/// Remove `.` and `..` segments from the path part of `path` (RFC 3986 section 5.2.4); the query is kept as is.
|
||||
fn remove_dot_segments(path: &str) -> String {
|
||||
let (p, query) = match path.find('?') {
|
||||
Some(i) => (path.get(..i).unwrap_or_default(), path.get(i..).unwrap_or_default()),
|
||||
None => (path, ""),
|
||||
};
|
||||
let mut out: Vec<&str> = Vec::new();
|
||||
let mut segments = p.split('/').skip(1).peekable();
|
||||
let mut trailing_slash = false;
|
||||
while let Some(seg) = segments.next() {
|
||||
let last = segments.peek().is_none();
|
||||
match seg {
|
||||
"." => trailing_slash = last,
|
||||
".." => {
|
||||
out.pop();
|
||||
trailing_slash = last;
|
||||
}
|
||||
_ => {
|
||||
out.push(seg);
|
||||
trailing_slash = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut result = format!("/{}", out.join("/"));
|
||||
if trailing_slash && !result.ends_with('/') {
|
||||
result.push('/');
|
||||
}
|
||||
result.push_str(query);
|
||||
result
|
||||
}
|
||||
|
||||
/// `s` without the leading `prefix`, matched ASCII case-insensitively (RFC 3986 §3.1).
|
||||
fn strip_scheme<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
|
||||
let head = s.get(..prefix.len())?;
|
||||
head.eq_ignore_ascii_case(prefix).then(|| s.get(prefix.len()..).unwrap_or_default())
|
||||
}
|
||||
|
||||
impl Url {
|
||||
pub fn parse(s: &str) -> Result<Url, UrlError> {
|
||||
let bad = |why: &str| UrlError(why.to_string());
|
||||
if s.len() > 16 * 1024 {
|
||||
return Err(bad("URL exceeds 16 KiB"));
|
||||
}
|
||||
let s = s.trim();
|
||||
if s.chars().any(|c| c.is_control() || c == ' ') {
|
||||
return Err(bad("contains spaces or control characters"));
|
||||
}
|
||||
let (tls, rest) = if let Some(r) = strip_scheme(s, "https://") {
|
||||
(true, r)
|
||||
} else if let Some(r) = strip_scheme(s, "http://") {
|
||||
(false, r)
|
||||
} else {
|
||||
return Err(bad("only http:// and https:// are supported"));
|
||||
};
|
||||
let rest = rest.split('#').next().unwrap_or_default();
|
||||
let (authority, path) = match rest.find(['/', '?']) {
|
||||
Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()),
|
||||
None => (rest, ""),
|
||||
};
|
||||
let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") };
|
||||
let path = remove_dot_segments(&path);
|
||||
if authority.contains('@') {
|
||||
return Err(bad("user names and passwords in URLs are not supported"));
|
||||
}
|
||||
let (host, port) = if let Some(v6) = authority.strip_prefix('[') {
|
||||
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
|
||||
let host = v6.get(..end).unwrap_or_default();
|
||||
let after = v6.get(end + 1..).unwrap_or_default();
|
||||
if !after.is_empty() && !after.starts_with(':') {
|
||||
return Err(bad("unexpected text after IPv6 address"));
|
||||
}
|
||||
(host.to_string(), after.strip_prefix(':'))
|
||||
} else {
|
||||
match authority.rsplit_once(':') {
|
||||
Some((h, p)) => (h.to_string(), Some(p)),
|
||||
None => (authority.to_string(), None),
|
||||
}
|
||||
};
|
||||
let port = match port {
|
||||
Some(p) if p.bytes().all(|b| b.is_ascii_digit()) => p.parse::<u16>().ok().filter(|p| *p != 0).ok_or_else(|| bad("bad port"))?,
|
||||
Some(_) => return Err(bad("bad port")),
|
||||
None if tls => 443,
|
||||
None => 80,
|
||||
};
|
||||
if host.is_empty() || !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | ':' | '_')) {
|
||||
return Err(bad("bad host name"));
|
||||
}
|
||||
Ok(Url { tls, host: host.to_ascii_lowercase(), port, path })
|
||||
}
|
||||
|
||||
/// `Location` of a redirect, resolved against this URL.
|
||||
pub fn join(&self, location: &str) -> Result<Url, UrlError> {
|
||||
// Reject before concatenating a hostile Location with the base URL.
|
||||
if location.len() > 16 * 1024 || self.host.len().saturating_add(self.path.len()) > 16 * 1024 {
|
||||
return Err(UrlError("URL exceeds 16 KiB".into()));
|
||||
}
|
||||
let l = location.trim();
|
||||
if strip_scheme(l, "http://").is_some() || strip_scheme(l, "https://").is_some() {
|
||||
let url = Url::parse(l)?;
|
||||
if self.tls && !url.tls {
|
||||
return Err(UrlError("redirect from https to http refused".into()));
|
||||
}
|
||||
return Ok(url);
|
||||
}
|
||||
// another scheme (`ftp:`, `file:`, `data:`…)
|
||||
if let Some((scheme, _)) = l.split_once(':')
|
||||
&& !scheme.is_empty()
|
||||
&& scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
|
||||
{
|
||||
return Err(UrlError(format!("redirect to an unsupported location `{scheme}:`")));
|
||||
}
|
||||
let scheme = if self.tls { "https" } else { "http" };
|
||||
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
|
||||
let base = format!("{scheme}://{host}:{}", self.port);
|
||||
if let Some(rest) = l.strip_prefix("//") {
|
||||
return Url::parse(&format!("{scheme}://{rest}"));
|
||||
}
|
||||
if l.starts_with('/') {
|
||||
return Url::parse(&format!("{base}{l}"));
|
||||
}
|
||||
// empty or fragment-only: the same document
|
||||
if l.is_empty() || l.starts_with('#') {
|
||||
return Ok(self.clone());
|
||||
}
|
||||
// relative to the current directory
|
||||
let dir = self.path.split('?').next().unwrap_or("/");
|
||||
// a query-only reference keeps the whole path, file name included (RFC 3986 §5.2.2)
|
||||
if l.starts_with('?') {
|
||||
return Url::parse(&format!("{base}{dir}{l}"));
|
||||
}
|
||||
let dir = dir.rsplit_once('/').map_or("", |(d, _)| d);
|
||||
Url::parse(&format!("{base}{dir}/{l}"))
|
||||
}
|
||||
|
||||
/// Authority formatted for an HTTP Host field.
|
||||
pub fn host_header(&self) -> String {
|
||||
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
|
||||
let default = if self.tls { 443 } else { 80 };
|
||||
if self.port == default { host } else { format!("{host}:{}", self.port) }
|
||||
}
|
||||
|
||||
/// The URL without its query (for messages: a query may hold a signed token).
|
||||
pub fn display(&self) -> String {
|
||||
let scheme = if self.tls { "https" } else { "http" };
|
||||
format!("{scheme}://{}{}", self.host_header(), self.path.split('?').next().unwrap_or("/"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::{Url, UrlError};
|
||||
|
||||
#[test]
|
||||
fn fragment_only_or_empty_location_keeps_file_name_and_query() {
|
||||
let base = Url::parse("https://a.example/x/y/file?q").unwrap();
|
||||
for l in ["#top", "", " "] {
|
||||
let u = base.join(l).unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/x/y/file?q"), "{l:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_oversized_and_control_character_urls() {
|
||||
let base = Url::parse("https://example.com/path").unwrap();
|
||||
assert!(base.join(&format!("#{}", "x".repeat(16 * 1024))).is_err());
|
||||
assert!(base.join(&"x".repeat(16 * 1024 + 1)).is_err());
|
||||
assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err());
|
||||
assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_query_only_keeps_file_name() {
|
||||
let u = Url::parse("https://a.example/x/y/file?q=1").unwrap();
|
||||
assert_eq!(u.join("?page=2").unwrap().path, "/x/y/file?page=2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_removes_dot_segments() {
|
||||
let base = Url::parse("https://a.example/x/y/file").unwrap();
|
||||
for (loc, want) in
|
||||
[("../z", "/x/z"), ("./z", "/x/y/z"), ("../../../z", "/z"), ("/a/./b/../c", "/a/c"), ("../z?q=1", "/x/z?q=1"), ("..", "/x/")]
|
||||
{
|
||||
assert_eq!(base.join(loc).unwrap().path, want, "{loc}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_scheme_is_matched_case_insensitively() {
|
||||
let u = Url::parse("HTTPS://A.example/z").unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "a.example", 443, "/z"));
|
||||
assert!(Url::parse("Http://a.example/").is_ok());
|
||||
let j = Url::parse("https://a.example/start").unwrap().join("HTTPS://cdn.example/file").unwrap();
|
||||
assert_eq!((j.tls, j.host.as_str(), j.path.as_str()), (true, "cdn.example", "/file"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_urls() {
|
||||
let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap();
|
||||
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "cdn.example.com", 443, "/models/sam3/model.safetensors?sig=1"));
|
||||
assert_eq!(u.display(), "https://cdn.example.com/models/sam3/model.safetensors");
|
||||
let u = Url::parse("http://127.0.0.1:8080").unwrap();
|
||||
assert_eq!((u.tls, u.port, u.path.as_str()), (false, 8080, "/"));
|
||||
let u = Url::parse("http://[::1]:9/a").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 9));
|
||||
for bad in ["ftp://x/y", "https://", "https://a b/", "https://u:p@h/", "http://h:0/", "http://h:99999/", "https://h/\r\nX: y", "file:///etc"]
|
||||
{
|
||||
assert!(Url::parse(bad).is_err(), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_signed_ports() {
|
||||
for bad in ["http://example.com:+80/x", "http://example.com:-80/x", "http://[::1]:+80/x"] {
|
||||
assert!(Url::parse(bad).is_err(), "{bad}");
|
||||
}
|
||||
assert!(Url::parse("http://a.test/").unwrap().join("http://example.com:+80/x").is_err());
|
||||
assert_eq!(Url::parse("http://example.com:80/x").unwrap().port, 80);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_text_after_a_bracketed_ipv6_host() {
|
||||
for bad in ["http://[::1]junk/x", "https://[::1]junk/x", "http://[::1]8080/x", "http://[::1]8080"] {
|
||||
assert!(matches!(Url::parse(bad), Err(UrlError(_))), "{bad}");
|
||||
}
|
||||
let u = Url::parse("http://[::1]:8080/x").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 8080));
|
||||
let u = Url::parse("http://[::1]/x").unwrap();
|
||||
assert_eq!((u.host.as_str(), u.port), ("::1", 80));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_redirects() {
|
||||
let u = Url::parse("https://a.example/x/y/file?q").unwrap();
|
||||
assert_eq!(u.join("https://b.example/z").unwrap().host, "b.example");
|
||||
assert_eq!(u.join("/root").unwrap().path, "/root");
|
||||
assert_eq!(u.join("other").unwrap().path, "/x/y/other");
|
||||
assert_eq!(u.join("//c.example/p").unwrap().host, "c.example");
|
||||
assert!(u.join("ftp://nope").is_err());
|
||||
// no downgrade from https to http, however the redirect is written
|
||||
assert!(u.join("http://a.example/z").is_err());
|
||||
let plain = Url::parse("http://127.0.0.1:8080/a").unwrap();
|
||||
assert!(!plain.join("http://127.0.0.1:9/b").unwrap().tls);
|
||||
assert!(plain.join("https://b.example/z").unwrap().tls);
|
||||
}
|
||||
}
|
||||
@@ -842,6 +842,10 @@ fn read_metadata(c: &Container) -> (Option<Vec<u8>>, Option<String>) {
|
||||
/// Decodes the primary image. `keep_orientation` leaves the container's `clap`, `irot` and
|
||||
/// `imir` unapplied (the stored pixels), as for HEIF.
|
||||
pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<Image> {
|
||||
decode_with_depth(bytes, limits, keep_orientation).map(|(image, _)| image)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_with_depth(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> Result<(Image, u8)> {
|
||||
let c = parse(bytes)?;
|
||||
let id = c.primary;
|
||||
let (w, h) = item_size(&c, id)?;
|
||||
@@ -890,9 +894,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits, keep_orientation: bool) -> R
|
||||
// PQ or HLG: linear light in 32-bit float, as for HEIF.
|
||||
if cicp.is_hdr() {
|
||||
limits.check(w, h, layout, SampleType::F32)?;
|
||||
return crate::cicp::linearize(img, cicp);
|
||||
return crate::cicp::linearize(img, cicp).map(|image| (image, bits));
|
||||
}
|
||||
Ok(img)
|
||||
Ok((img, bits))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1301,6 +1305,9 @@ mod tests {
|
||||
let src = Image::from_u8(w as u32, h as u32, ChannelLayout::Rgb, card(w, h, 3)).unwrap();
|
||||
let file = encode_avif_with_speed(&src, 95, 10).unwrap();
|
||||
let img = crate::decode(&file).unwrap();
|
||||
let (with_depth, depth) = crate::decode_avif_with_depth(&file, &crate::DecodeOptions::default()).unwrap();
|
||||
assert_eq!(depth, 10);
|
||||
assert_eq!(with_depth, img);
|
||||
assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (48, 32, ChannelLayout::Rgb, SampleType::U16));
|
||||
let diff = mean_abs_diff(&rgb8(&img), &card(w, h, 3));
|
||||
assert!(diff < 2.5, "mean difference {diff}");
|
||||
|
||||
@@ -121,6 +121,14 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
|
||||
finish_format(format, bytes, img, opts)
|
||||
}
|
||||
|
||||
/// Decode AVIF pixels and report their original AV1 bit depth (8, 10 or 12).
|
||||
/// The returned image may use a wider sample type, particularly for linear HDR.
|
||||
#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))]
|
||||
pub fn decode_avif_with_depth(bytes: &[u8], opts: &DecodeOptions) -> Result<(Image, u8), CodecError> {
|
||||
let (image, depth) = codecs::avif_decode::decode_with_depth(bytes, &opts.limits, opts.keep_orientation)?;
|
||||
Ok((finish_format(Format::Avif, bytes, image, opts)?, depth))
|
||||
}
|
||||
|
||||
/// What [`decode_as_with`] does with a PNG decoded in steps by [`PngDecodeRows`] (`img` is its
|
||||
/// output): the resolution from the metadata, the upright orientation and the final limit check,
|
||||
/// so the result equals [`decode_as_with`]'s on the same file.
|
||||
|
||||
@@ -229,7 +229,7 @@ impl Reader {
|
||||
};
|
||||
let name = s.label.as_deref().and_then(|a| cells_of(a).first().copied()).map(|at| label(self.cell(at))).unwrap_or_default();
|
||||
let name = if name.is_empty() { format!("Series {}", i + 1) } else { name };
|
||||
ChartSeries { name, values }
|
||||
ChartSeries { name, values, ..Default::default() }
|
||||
})
|
||||
.collect();
|
||||
// A scatter chart's x values: its first series' domain.
|
||||
@@ -248,7 +248,10 @@ impl Reader {
|
||||
_ => LegendPos::Right,
|
||||
});
|
||||
let title = self.title.as_deref().map(str::trim).filter(|t| !t.is_empty()).map(str::to_string);
|
||||
Some(ChartSpec { kind, categories, series, title, legend, data_labels }.sanitized())
|
||||
// ODF axis formatting is not read; the value axis keeps the look every chart had before
|
||||
// axes were formattable (gridlines, no axis line), a round chart has none.
|
||||
let val_axis = if kind.is_round() { Default::default() } else { craft_wordmodel::chart::value_axis() };
|
||||
Some(ChartSpec { kind, categories, series, title, legend, data_labels, val_axis, ..Default::default() }.sanitized())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,8 +57,8 @@ fn spec(chart: &str) -> Option<ChartSpec> {
|
||||
|
||||
fn north_south() -> Vec<ChartSeries> {
|
||||
vec![
|
||||
ChartSeries { name: "North".into(), values: vec![Some(10.0), Some(14.0), Some(8.0), Some(16.0)] },
|
||||
ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)] },
|
||||
ChartSeries { name: "North".into(), values: vec![Some(10.0), Some(14.0), Some(8.0), Some(16.0)], ..Default::default() },
|
||||
ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)], ..Default::default() },
|
||||
]
|
||||
}
|
||||
|
||||
@@ -78,6 +78,8 @@ fn a_column_chart_reads_its_data_title_and_legend() {
|
||||
title: Some("Sales by quarter".into()),
|
||||
legend: Some(LegendPos::Right),
|
||||
data_labels: false,
|
||||
val_axis: craft_wordmodel::chart::value_axis(),
|
||||
..Default::default()
|
||||
}
|
||||
);
|
||||
// The frame's size, in points.
|
||||
@@ -135,7 +137,7 @@ fn line_area_pie_doughnut_and_scatter_classes() {
|
||||
let s = spec(&chart_xml("chart:scatter", "", "", "", scatter)).unwrap();
|
||||
assert_eq!(s.kind, ChartType::Scatter);
|
||||
assert_eq!(s.categories, vec!["10", "14", "8", "16"]);
|
||||
assert_eq!(s.series, vec![ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)] }]);
|
||||
assert_eq!(s.series, vec![ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)], ..Default::default() }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -249,7 +251,15 @@ fn every_chart_type_is_saved_as_a_chart_object_and_reads_back() {
|
||||
let bytes = crate::text::export(&doc_with(spec.clone(), false)).unwrap();
|
||||
let back = reread(&bytes);
|
||||
assert_eq!(back.len(), 1, "{kind:?}");
|
||||
assert_eq!(*back[0].spec, spec.clone().sanitized(), "{kind:?}");
|
||||
// ODF keeps no series or axis formatting here: only the data, the title and the legend
|
||||
// come back; the value axis has the gridlines every chart had before axes were formattable.
|
||||
let mut expected = spec.clone().sanitized();
|
||||
for s in &mut expected.series {
|
||||
*s = ChartSeries { name: std::mem::take(&mut s.name), values: std::mem::take(&mut s.values), ..Default::default() };
|
||||
}
|
||||
expected.cat_axis = Default::default();
|
||||
expected.val_axis = if kind.is_round() { Default::default() } else { craft_wordmodel::chart::value_axis() };
|
||||
assert_eq!(*back[0].spec, expected, "{kind:?}");
|
||||
assert_eq!((back[0].w, back[0].h, back[0].alt.as_str()), (300.0, 180.0, "Sales chart"), "{kind:?}");
|
||||
// The package lists the chart object.
|
||||
let z = crate::Package::open(&bytes, &crate::Limits::DEFAULT).unwrap();
|
||||
|
||||
@@ -130,6 +130,7 @@ fn a_diagram_is_saved_as_a_group_of_its_drawing() {
|
||||
bold: false,
|
||||
align: TextAlign::Center,
|
||||
font: None,
|
||||
angle: 0.0,
|
||||
},
|
||||
GraphicItem::Shape { rect: [90.0, 9.0, 18.0, 18.0], kind: ShapeKind::Arrow, fill: Some(blue), stroke: None, stroke_width: 0.0 },
|
||||
],
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
//! Callers map their page and color models to these neutral records. This module owns PDF object
|
||||
//! layout, image streams, ICC/Separation resources, page labels, marks, xref and trailer syntax.
|
||||
|
||||
/// Multi-image 8/16-bit print pages with per-image ICC resources and explicit byte budgets.
|
||||
#[path = "raster_print.rs"]
|
||||
pub mod print;
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Write as _;
|
||||
use std::io::Write as _;
|
||||
|
||||
@@ -0,0 +1,658 @@
|
||||
//! A small PDF writer for print output: pages of a given size in points, each with raster images
|
||||
//! placed on it, tagged with their ICC profile (`ICCBased`) so the numbers mean the same colour in
|
||||
//! every reader and RIP, Flate-compressed (lossless) at 8 or 16 bits per component, or carried as
|
||||
//! JPEG (`DCTDecode`) where small files matter more than exact samples. Optionally an output intent:
|
||||
//! the printing condition (the printer / paper profile) the file was made for, as PDF/X readers
|
||||
//! expect it in the catalog. Profiles shared by several images are written once.
|
||||
//!
|
||||
//! The file is PDF 1.6 (16-bit images need 1.5): classic cross-reference table, no object streams,
|
||||
//! deterministic (no dates or IDs), so the same pages always give the same bytes.
|
||||
//!
|
||||
//! Encoded JPEG and Flate payloads are copied, not decoded or verified against declared
|
||||
//! dimensions. Callers own their encoding validity and ICC/profile-channel compatibility.
|
||||
//! An output intent describes a printing condition; this API does not certify PDF/X.
|
||||
//! Byte limits bound accounted input and emitted output, not peak process memory or
|
||||
//! caller-owned image construction. This is not the streaming slideshow writer.
|
||||
//!
|
||||
//! Sources: ISO 32000-1:2008 (§7.3 objects, §7.5 file structure, §8.6.5.5 ICCBased colour spaces,
|
||||
//! §8.9 images, §14.11.5 output intents). Own design.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Write as _;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Pages per document: a print job beyond this is a mistake, not a book.
|
||||
pub const MAX_PAGES: usize = 1000;
|
||||
/// Images per page.
|
||||
pub const MAX_IMAGES_PER_PAGE: usize = 256;
|
||||
/// Largest page side in points (200 inches): more than any printer's roll.
|
||||
pub const MAX_PAGE_POINTS: f64 = 14_400.0;
|
||||
/// Largest image, in pixels per side.
|
||||
pub const MAX_IMAGE_SIDE: u32 = 65_535;
|
||||
|
||||
/// Serialization budgets; input records already belong to the caller. These are not a
|
||||
/// process-wide peak-memory guarantee. Repeated references count toward the input budget.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Limits {
|
||||
pub input_bytes: usize,
|
||||
pub output_bytes: usize,
|
||||
}
|
||||
impl Default for Limits {
|
||||
fn default() -> Self {
|
||||
Self { input_bytes: 512 * 1024 * 1024, output_bytes: 512 * 1024 * 1024 }
|
||||
}
|
||||
}
|
||||
|
||||
/// How an image's samples are stored.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ImageData {
|
||||
/// Interleaved samples, `bits` (8 or 16) per component, 16-bit values big-endian: compressed
|
||||
/// losslessly in the file.
|
||||
Raw { bits: u8, samples: Vec<u8> },
|
||||
/// A baseline JPEG stream, copied into the file as it is.
|
||||
Jpeg(Vec<u8>),
|
||||
/// Samples already compressed with zlib (see [`PdfImage::deflated`]), `bits` per component.
|
||||
Flate { bits: u8, data: Vec<u8> },
|
||||
}
|
||||
|
||||
/// A raster image and the colour space its samples are in.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PdfImage {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Components per pixel: 1 (grey), 3 (RGB) or 4 (CMYK).
|
||||
pub channels: u8,
|
||||
pub data: ImageData,
|
||||
/// The ICC profile the samples are in; without one the image is device colour.
|
||||
pub icc: Option<Arc<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl PdfImage {
|
||||
/// An image from interleaved 16-bit samples.
|
||||
pub fn from_u16(width: u32, height: u32, channels: u8, samples: &[u16], icc: Option<Arc<Vec<u8>>>) -> PdfImage {
|
||||
let bytes = samples.iter().flat_map(|v| v.to_be_bytes()).collect();
|
||||
PdfImage { width, height, channels, data: ImageData::Raw { bits: 16, samples: bytes }, icc }
|
||||
}
|
||||
|
||||
/// The same image with its samples compressed now (checked first), so a long print doesn't
|
||||
/// hold every page's uncompressed samples until the file is written.
|
||||
pub fn deflated(self) -> Result<PdfImage, String> {
|
||||
self.check()?;
|
||||
if matches!(&self.data, ImageData::Raw { samples, .. } if samples.len() > Limits::default().input_bytes) {
|
||||
return Err("PDF input byte limit exceeded".into());
|
||||
}
|
||||
let data = match self.data {
|
||||
ImageData::Raw { bits, samples } => ImageData::Flate { bits, data: deflate(&samples, Limits::default().output_bytes)? },
|
||||
other => other,
|
||||
};
|
||||
Ok(PdfImage { data, ..self })
|
||||
}
|
||||
|
||||
fn check(&self) -> Result<(), String> {
|
||||
if self.width == 0 || self.height == 0 || self.width > MAX_IMAGE_SIDE || self.height > MAX_IMAGE_SIDE {
|
||||
return Err(format!("an image of {} × {} pixels can't be written", self.width, self.height));
|
||||
}
|
||||
if !matches!(self.channels, 1 | 3 | 4) {
|
||||
return Err(format!("images have 1, 3 or 4 components, not {}", self.channels));
|
||||
}
|
||||
match &self.data {
|
||||
ImageData::Raw { bits, samples } => {
|
||||
if !matches!(bits, 8 | 16) {
|
||||
return Err(format!("images have 8 or 16 bits per component, not {bits}"));
|
||||
}
|
||||
let want = u64::from(self.width) * u64::from(self.height) * u64::from(self.channels) * u64::from(bits / 8);
|
||||
if samples.len() as u64 != want {
|
||||
return Err(format!("the image needs {want} bytes of samples, not {}", samples.len()));
|
||||
}
|
||||
}
|
||||
ImageData::Jpeg(data) => {
|
||||
if !data.starts_with(&[0xFF, 0xD8]) {
|
||||
return Err("the image isn't a JPEG stream".into());
|
||||
}
|
||||
}
|
||||
ImageData::Flate { bits, data } => {
|
||||
if !matches!(bits, 8 | 16) || data.is_empty() {
|
||||
return Err("a compressed image has 8 or 16 bits per component and some data".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Where an image goes on its page, in points from the page's bottom-left corner.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Placement {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
}
|
||||
|
||||
/// One page: its size in points and the images on it, drawn in order (later ones on top).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PdfPage {
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
pub images: Vec<(PdfImage, Placement)>,
|
||||
}
|
||||
|
||||
/// The printing condition the document is made for (PDF/X-style output intent).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct OutputIntent {
|
||||
/// The printer / paper profile.
|
||||
pub icc: Arc<Vec<u8>>,
|
||||
/// Its components: 1, 3 or 4.
|
||||
pub channels: u8,
|
||||
/// A name for the condition (the profile's description).
|
||||
pub condition: String,
|
||||
}
|
||||
|
||||
/// Document-level settings.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct PdfInfo {
|
||||
pub title: Option<String>,
|
||||
pub output_intent: Option<OutputIntent>,
|
||||
/// Application name, not a PDF/X conformance claim.
|
||||
pub producer: String,
|
||||
}
|
||||
|
||||
fn finite(v: f64) -> bool {
|
||||
v.is_finite()
|
||||
}
|
||||
|
||||
/// A number as PDF writes it: up to four decimals, no exponent, no trailing zeros.
|
||||
fn num(v: f64) -> String {
|
||||
let s = format!("{:.4}", if finite(v) { v } else { 0.0 });
|
||||
let s = s.trim_end_matches('0').trim_end_matches('.');
|
||||
if s == "-0" || s.is_empty() { "0".into() } else { s.into() }
|
||||
}
|
||||
|
||||
/// A text string: plain ASCII as a literal string, anything else as UTF-16BE with a byte order mark.
|
||||
fn text(s: &str) -> String {
|
||||
if s.bytes().all(|b| (0x20..0x7f).contains(&b)) {
|
||||
let mut out = String::with_capacity(s.len() + 2);
|
||||
out.push('(');
|
||||
for c in s.chars() {
|
||||
if matches!(c, '(' | ')' | '\\') {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out.push(')');
|
||||
out
|
||||
} else {
|
||||
super::pdf_unicode_string(s)
|
||||
}
|
||||
}
|
||||
|
||||
struct Writer {
|
||||
out: Vec<u8>,
|
||||
offsets: Vec<usize>,
|
||||
limit: usize,
|
||||
}
|
||||
impl Writer {
|
||||
fn put(&mut self, bytes: &[u8]) -> Result<(), String> {
|
||||
let end = self.out.len().checked_add(bytes.len()).ok_or("PDF output size overflow")?;
|
||||
if end > self.limit {
|
||||
return Err("PDF output byte limit exceeded".into());
|
||||
}
|
||||
self.out.try_reserve(bytes.len()).map_err(|_| "PDF output allocation failed")?;
|
||||
self.out.extend_from_slice(bytes);
|
||||
Ok(())
|
||||
}
|
||||
fn reserve(&mut self) -> Result<usize, String> {
|
||||
self.offsets.try_reserve(1).map_err(|_| "PDF object allocation failed")?;
|
||||
self.offsets.push(0);
|
||||
Ok(self.offsets.len())
|
||||
}
|
||||
fn object(&mut self, id: usize, dict: &str, stream: Option<&[u8]>) -> Result<(), String> {
|
||||
let pos = self.out.len();
|
||||
*self.offsets.get_mut(id.checked_sub(1).ok_or("invalid PDF object")?).ok_or("invalid PDF object")? = pos;
|
||||
self.put(format!("{id} 0 obj\n{dict}").as_bytes())?;
|
||||
if let Some(data) = stream {
|
||||
self.put(b"\nstream\n")?;
|
||||
self.put(data)?;
|
||||
self.put(b"\nendstream")?;
|
||||
}
|
||||
self.put(b"\nendobj\n")
|
||||
}
|
||||
}
|
||||
|
||||
fn deflate(data: &[u8], max: usize) -> Result<Vec<u8>, String> {
|
||||
struct Sink {
|
||||
bytes: Vec<u8>,
|
||||
max: usize,
|
||||
}
|
||||
impl std::io::Write for Sink {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
if bytes.len() > self.max.saturating_sub(self.bytes.len()) {
|
||||
return Err(std::io::Error::other("PDF compressed byte limit exceeded"));
|
||||
}
|
||||
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut encoder = flate2::write::ZlibEncoder::new(Sink { bytes: Vec::new(), max }, flate2::Compression::new(6));
|
||||
encoder.write_all(data).map_err(|e| e.to_string())?;
|
||||
Ok(encoder.finish().map_err(|e| e.to_string())?.bytes)
|
||||
}
|
||||
|
||||
/// Write a document. Errors name the first page or image that can't be written.
|
||||
pub fn write(pages: &[PdfPage], info: &PdfInfo) -> Result<Vec<u8>, String> {
|
||||
write_with_limits(pages, info, Limits::default())
|
||||
}
|
||||
|
||||
/// Write with explicit total input and serialized output budgets. ICC bytes are opaque:
|
||||
/// callers validate their profile semantics; resources are keyed by bytes AND channels.
|
||||
pub fn write_with_limits(pages: &[PdfPage], info: &PdfInfo, limits: Limits) -> Result<Vec<u8>, String> {
|
||||
let mut input = 0usize;
|
||||
let mut account = |n: usize| -> Result<(), String> {
|
||||
input = input.checked_add(n).ok_or("PDF input size overflow")?;
|
||||
if input > limits.input_bytes {
|
||||
return Err("PDF input byte limit exceeded".into());
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
account(info.producer.len())?;
|
||||
account(info.title.as_ref().map_or(0, String::len))?;
|
||||
if let Some(intent) = &info.output_intent {
|
||||
account(intent.icc.len())?;
|
||||
account(intent.condition.len())?;
|
||||
}
|
||||
|
||||
if pages.is_empty() {
|
||||
return Err("there are no pages to write".into());
|
||||
}
|
||||
if pages.len() > MAX_PAGES {
|
||||
return Err(format!("a document holds at most {MAX_PAGES} pages"));
|
||||
}
|
||||
for (n, page) in pages.iter().enumerate() {
|
||||
let n = n + 1;
|
||||
if !(finite(page.width) && finite(page.height))
|
||||
|| page.width <= 0.0
|
||||
|| page.height <= 0.0
|
||||
|| page.width > MAX_PAGE_POINTS
|
||||
|| page.height > MAX_PAGE_POINTS
|
||||
{
|
||||
return Err(format!("page {n} has no usable size"));
|
||||
}
|
||||
if page.images.len() > MAX_IMAGES_PER_PAGE {
|
||||
return Err(format!("page {n} has more than {MAX_IMAGES_PER_PAGE} images"));
|
||||
}
|
||||
for (image, at) in &page.images {
|
||||
account(match &image.data {
|
||||
ImageData::Raw { samples, .. } => samples.len(),
|
||||
ImageData::Jpeg(data) | ImageData::Flate { data, .. } => data.len(),
|
||||
})?;
|
||||
if let Some(icc) = &image.icc {
|
||||
account(icc.len())?;
|
||||
}
|
||||
image.check().map_err(|e| format!("page {n}: {e}"))?;
|
||||
if ![at.x, at.y, at.width, at.height].into_iter().all(finite) || at.width <= 0.0 || at.height <= 0.0 {
|
||||
return Err(format!("page {n}: an image has no usable placement"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(intent) = &info.output_intent
|
||||
&& !matches!(intent.channels, 1 | 3 | 4)
|
||||
{
|
||||
return Err("the output intent's profile has 1, 3 or 4 components".into());
|
||||
}
|
||||
|
||||
let mut w = Writer { out: Vec::new(), offsets: Vec::new(), limit: limits.output_bytes };
|
||||
w.put(b"%PDF-1.6\n%\xE2\xE3\xCF\xD3\n")?;
|
||||
let catalog = w.reserve()?;
|
||||
let pages_id = w.reserve()?;
|
||||
let info_id = w.reserve()?;
|
||||
|
||||
// ICC profiles, each written once (keyed by content) with its component count
|
||||
let mut profiles: HashMap<(Arc<Vec<u8>>, u8), usize> = HashMap::new();
|
||||
let mut profile = |w: &mut Writer, icc: &Arc<Vec<u8>>, n: u8| -> Result<usize, String> {
|
||||
if let Some(&id) = profiles.get(&(Arc::clone(icc), n)) {
|
||||
return Ok(id);
|
||||
}
|
||||
let id = w.reserve()?;
|
||||
let data = deflate(icc, limits.output_bytes)?;
|
||||
let alternate = match n {
|
||||
1 => "/DeviceGray",
|
||||
4 => "/DeviceCMYK",
|
||||
_ => "/DeviceRGB",
|
||||
};
|
||||
w.object(id, &format!("<< /N {n} /Alternate {alternate} /Filter /FlateDecode /Length {} >>", data.len()), Some(&data))?;
|
||||
profiles.try_reserve(1).map_err(|_| "PDF profile allocation failed")?;
|
||||
profiles.insert((Arc::clone(icc), n), id);
|
||||
Ok(id)
|
||||
};
|
||||
|
||||
let intent_id = info.output_intent.as_ref().map(|intent| profile(&mut w, &intent.icc, intent.channels)).transpose()?;
|
||||
|
||||
let mut kids = Vec::with_capacity(pages.len());
|
||||
for page in pages {
|
||||
let page_id = w.reserve()?;
|
||||
let mut xobjects = String::new();
|
||||
let mut content = String::new();
|
||||
for (k, (image, at)) in page.images.iter().enumerate() {
|
||||
let space = match &image.icc {
|
||||
Some(icc) => format!("[/ICCBased {} 0 R]", profile(&mut w, icc, image.channels)?),
|
||||
None => match image.channels {
|
||||
1 => "/DeviceGray".into(),
|
||||
4 => "/DeviceCMYK".into(),
|
||||
_ => "/DeviceRGB".into(),
|
||||
},
|
||||
};
|
||||
let id = w.reserve()?;
|
||||
let (bits, filter, data): (u8, &str, std::borrow::Cow<[u8]>) = match &image.data {
|
||||
ImageData::Raw { bits, samples } => (*bits, "/FlateDecode", deflate(samples, limits.output_bytes)?.into()),
|
||||
ImageData::Flate { bits, data } => (*bits, "/FlateDecode", data.as_slice().into()),
|
||||
ImageData::Jpeg(jpeg) => (8, "/DCTDecode", jpeg.as_slice().into()),
|
||||
};
|
||||
w.object(
|
||||
id,
|
||||
&format!(
|
||||
"<< /Type /XObject /Subtype /Image /Width {} /Height {} /ColorSpace {space} /BitsPerComponent {bits} /Filter {filter} /Length {} >>",
|
||||
image.width,
|
||||
image.height,
|
||||
data.len()
|
||||
),
|
||||
Some(&data),
|
||||
)?;
|
||||
xobjects.push_str(&format!("/Im{k} {id} 0 R "));
|
||||
content.push_str(&format!("q {} 0 0 {} {} {} cm /Im{k} Do Q\n", num(at.width), num(at.height), num(at.x), num(at.y)));
|
||||
}
|
||||
let content_id = w.reserve()?;
|
||||
let stream = deflate(content.as_bytes(), limits.output_bytes)?;
|
||||
w.object(content_id, &format!("<< /Filter /FlateDecode /Length {} >>", stream.len()), Some(&stream))?;
|
||||
w.object(
|
||||
page_id,
|
||||
&format!(
|
||||
"<< /Type /Page /Parent {pages_id} 0 R /MediaBox [0 0 {} {}] /Resources << /XObject << {xobjects}>> >> /Contents {content_id} 0 R >>",
|
||||
num(page.width),
|
||||
num(page.height)
|
||||
),
|
||||
None,
|
||||
)?;
|
||||
kids.push(format!("{page_id} 0 R"));
|
||||
}
|
||||
w.object(pages_id, &format!("<< /Type /Pages /Count {} /Kids [{}] >>", kids.len(), kids.join(" ")), None)?;
|
||||
|
||||
let intents = match (&info.output_intent, intent_id) {
|
||||
(Some(intent), Some(id)) => format!(
|
||||
" /OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX /OutputConditionIdentifier (Custom) /Info {} /DestOutputProfile {id} 0 R >>]",
|
||||
text(&intent.condition)
|
||||
),
|
||||
_ => String::new(),
|
||||
};
|
||||
w.object(catalog, &format!("<< /Type /Catalog /Pages {pages_id} 0 R{intents} >>"), None)?;
|
||||
let title = info.title.as_deref().map(|t| format!(" /Title {}", text(t))).unwrap_or_default();
|
||||
w.object(info_id, &format!("<< /Producer {}{title} >>", text(&info.producer)), None)?;
|
||||
|
||||
let xref = w.out.len();
|
||||
let mut table = format!("xref\n0 {}\n0000000000 65535 f \n", w.offsets.len() + 1);
|
||||
for offset in &w.offsets {
|
||||
table.push_str(&format!("{offset:010} 00000 n \n"));
|
||||
}
|
||||
table.push_str(&format!("trailer\n<< /Size {} /Root {catalog} 0 R /Info {info_id} 0 R >>\nstartxref\n{xref}\n%%EOF\n", w.offsets.len() + 1));
|
||||
w.put(table.as_bytes())?;
|
||||
Ok(w.out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The body of object `id` (dictionary and raw stream bytes), found through the xref table.
|
||||
fn object(pdf: &[u8], id: usize) -> (String, Vec<u8>) {
|
||||
// byte offsets: the streams are binary, so search the bytes, not a lossy string
|
||||
let start = pdf.windows(10).rposition(|w| w == b"startxref\n").unwrap() + 10;
|
||||
let tail = std::str::from_utf8(&pdf[start..]).unwrap();
|
||||
let xref: usize = tail.lines().next().unwrap().trim().parse().unwrap();
|
||||
let table = std::str::from_utf8(&pdf[xref..]).unwrap();
|
||||
let line = table.lines().nth(2 + id).unwrap();
|
||||
let offset: usize = line[..10].parse().unwrap();
|
||||
let head = format!("{id} 0 obj\n");
|
||||
assert_eq!(&pdf[offset..offset + head.len()], head.as_bytes(), "xref entry for {id}");
|
||||
let rest = &pdf[offset + head.len()..];
|
||||
let end = rest.windows(7).position(|w| w == b"\nendobj").unwrap();
|
||||
let body = &rest[..end];
|
||||
match body.windows(8).position(|w| w == b"\nstream\n") {
|
||||
Some(p) => {
|
||||
let data = &body[p + 8..body.len() - "\nendstream".len()];
|
||||
(String::from_utf8_lossy(&body[..p]).into_owned(), data.to_vec())
|
||||
}
|
||||
None => (String::from_utf8_lossy(body).into_owned(), vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
fn inflate(data: &[u8]) -> Vec<u8> {
|
||||
{
|
||||
use std::io::Read;
|
||||
let mut out = Vec::new();
|
||||
flate2::read::ZlibDecoder::new(data).read_to_end(&mut out).unwrap();
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Object ids of every image XObject, in file order.
|
||||
fn images(pdf: &[u8]) -> Vec<usize> {
|
||||
let s = String::from_utf8_lossy(pdf);
|
||||
s.match_indices(" 0 obj\n<< /Type /XObject /Subtype /Image").map(|(i, _)| s[..i].rsplit('\n').next().unwrap().parse().unwrap()).collect()
|
||||
}
|
||||
|
||||
fn reference(dict: &str, key: &str) -> usize {
|
||||
let at = dict.find(key).unwrap() + key.len();
|
||||
dict[at..].trim_start().split(' ').next().unwrap().parse().unwrap()
|
||||
}
|
||||
|
||||
fn srgb() -> Arc<Vec<u8>> {
|
||||
Arc::new(b"opaque RGB profile fixture".to_vec())
|
||||
}
|
||||
|
||||
fn page(images: Vec<(PdfImage, Placement)>) -> PdfPage {
|
||||
PdfPage { width: 612.0, height: 792.0, images }
|
||||
}
|
||||
|
||||
fn full(w: f64, h: f64) -> Placement {
|
||||
Placement { x: 0.0, y: 0.0, width: w, height: h }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_16_bit_icc_image_round_trips() {
|
||||
let samples: Vec<u16> = (0..2 * 3 * 3).map(|i| (i * 3851) as u16).collect();
|
||||
let img = PdfImage::from_u16(3, 2, 3, &samples, Some(srgb()));
|
||||
let pdf = write(&[page(vec![(img, full(612.0, 792.0))])], &PdfInfo::default()).unwrap();
|
||||
assert!(pdf.starts_with(b"%PDF-1.6\n"));
|
||||
assert!(pdf.ends_with(b"%%EOF\n"));
|
||||
let ids = images(&pdf);
|
||||
assert_eq!(ids.len(), 1);
|
||||
let (dict, data) = object(&pdf, ids[0]);
|
||||
assert!(dict.contains("/Width 3 /Height 2") && dict.contains("/BitsPerComponent 16") && dict.contains("/FlateDecode"), "{dict}");
|
||||
let back: Vec<u16> = inflate(&data).chunks(2).map(|b| u16::from_be_bytes([b[0], b[1]])).collect();
|
||||
assert_eq!(back, samples);
|
||||
let (profile, icc) = object(&pdf, reference(&dict, "/ICCBased"));
|
||||
assert!(profile.contains("/N 3") && profile.contains("/Alternate /DeviceRGB"), "{profile}");
|
||||
assert_eq!(inflate(&icc), *srgb());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pages_sizes_placements_and_a_shared_profile() {
|
||||
let px = |v: u16| PdfImage::from_u16(1, 1, 3, &[v, v, v], Some(srgb()));
|
||||
let at = Placement { x: 36.0, y: 72.5, width: 288.0, height: 216.0 };
|
||||
let pages = [page(vec![(px(1), at), (px(2), full(612.0, 792.0))]), PdfPage { width: 841.89, height: 595.28, images: vec![(px(3), at)] }];
|
||||
let pdf = write(&pages, &PdfInfo::default()).unwrap();
|
||||
let s = String::from_utf8_lossy(&pdf);
|
||||
assert!(s.contains("/Type /Pages /Count 2"), "two pages");
|
||||
assert!(s.contains("/MediaBox [0 0 612 792]") && s.contains("/MediaBox [0 0 841.89 595.28]"));
|
||||
// the sRGB profile is written once for all three images
|
||||
assert_eq!(s.matches("/N 3 /Alternate").count(), 1);
|
||||
let first: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
|
||||
assert!(first.windows(2).all(|w| w[0] == w[1]), "{first:?}");
|
||||
// placements reach the content streams
|
||||
let contents: Vec<String> = s
|
||||
.match_indices("/Contents ")
|
||||
.map(|(i, _)| s[i + 10..].split(' ').next().unwrap().parse::<usize>().unwrap())
|
||||
.map(|id| String::from_utf8(inflate(&object(&pdf, id).1)).unwrap())
|
||||
.collect();
|
||||
assert_eq!(contents[0], "q 288 0 0 216 36 72.5 cm /Im0 Do Q\nq 612 0 0 792 0 0 cm /Im1 Do Q\n");
|
||||
assert_eq!(write(&pages, &PdfInfo::default()).unwrap(), pdf, "deterministic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_intent_and_title() {
|
||||
let cmyk_like = srgb();
|
||||
let info = PdfInfo {
|
||||
title: Some("Garden – prints (2)".into()),
|
||||
producer: "Fixture".into(),
|
||||
output_intent: Some(OutputIntent { icc: cmyk_like.clone(), channels: 3, condition: "Glossy (Photo) \\ test".into() }),
|
||||
};
|
||||
let img = PdfImage::from_u16(1, 1, 3, &[0, 0, 0], Some(cmyk_like));
|
||||
let pdf = write(&[page(vec![(img, full(10.0, 10.0))])], &info).unwrap();
|
||||
let (catalog, _) = object(&pdf, 1);
|
||||
assert!(catalog.contains("/OutputIntents [<< /Type /OutputIntent /S /GTS_PDFX"), "{catalog}");
|
||||
assert!(catalog.contains("/Info (Glossy \\(Photo\\) \\\\ test)"), "{catalog}");
|
||||
// the intent and the image share one profile object
|
||||
let dest = reference(&catalog, "/DestOutputProfile");
|
||||
assert_eq!(reference(&object(&pdf, images(&pdf)[0]).0, "/ICCBased"), dest);
|
||||
let (info_dict, _) = object(&pdf, 3);
|
||||
assert!(info_dict.contains("/Title <FEFF"), "non-ASCII titles are UTF-16: {info_dict}");
|
||||
assert!(info_dict.contains("2013"), "the en dash: {info_dict}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmyk_grey_jpeg_and_device_images() {
|
||||
let jpeg = vec![0xFF, 0xD8, 0xFF, 0xD9];
|
||||
let imgs = vec![
|
||||
(
|
||||
PdfImage { width: 1, height: 1, channels: 4, data: ImageData::Raw { bits: 8, samples: vec![0, 10, 20, 30] }, icc: None },
|
||||
full(1.0, 1.0),
|
||||
),
|
||||
(PdfImage { width: 2, height: 1, channels: 1, data: ImageData::Raw { bits: 8, samples: vec![0, 255] }, icc: None }, full(1.0, 1.0)),
|
||||
(PdfImage { width: 4, height: 4, channels: 3, data: ImageData::Jpeg(jpeg.clone()), icc: None }, full(1.0, 1.0)),
|
||||
];
|
||||
let pdf = write(&[page(imgs)], &PdfInfo::default()).unwrap();
|
||||
let ids = images(&pdf);
|
||||
let dicts: Vec<_> = ids.iter().map(|&id| object(&pdf, id)).collect();
|
||||
assert!(dicts[0].0.contains("/ColorSpace /DeviceCMYK") && dicts[0].0.contains("/BitsPerComponent 8"));
|
||||
assert!(dicts[1].0.contains("/ColorSpace /DeviceGray"));
|
||||
assert!(dicts[2].0.contains("/DCTDecode") && dicts[2].0.contains("/DeviceRGB"));
|
||||
assert_eq!(dicts[2].1, jpeg, "JPEG streams are copied as they are");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressing_early_writes_the_same_file() {
|
||||
let samples: Vec<u16> = (0..4 * 3 * 3).map(|i| (i * 977) as u16).collect();
|
||||
let raw = PdfImage::from_u16(4, 3, 3, &samples, Some(srgb()));
|
||||
let early = raw.clone().deflated().unwrap();
|
||||
assert!(matches!(early.data, ImageData::Flate { bits: 16, .. }));
|
||||
let at = full(100.0, 75.0);
|
||||
let a = write(&[page(vec![(raw, at)])], &PdfInfo::default()).unwrap();
|
||||
let b = write(&[page(vec![(early, at)])], &PdfInfo::default()).unwrap();
|
||||
assert_eq!(a, b);
|
||||
let mut short = PdfImage::from_u16(4, 3, 3, &samples, None);
|
||||
short.width = 5;
|
||||
assert!(short.deflated().is_err(), "checked before compressing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_documents_are_errors() {
|
||||
let ok = || PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
|
||||
let at = full(10.0, 10.0);
|
||||
let bad = |pages: Vec<PdfPage>| write(&pages, &PdfInfo::default()).is_err();
|
||||
assert!(bad(vec![]));
|
||||
assert!(bad(vec![PdfPage { width: f64::NAN, height: 10.0, images: vec![] }]));
|
||||
assert!(bad(vec![PdfPage { width: -1.0, height: 10.0, images: vec![] }]));
|
||||
assert!(bad(vec![PdfPage { width: 1e9, height: 10.0, images: vec![] }]));
|
||||
assert!(bad(vec![page(vec![(ok(), Placement { x: f64::INFINITY, ..at })])]));
|
||||
assert!(bad(vec![page(vec![(ok(), Placement { width: 0.0, ..at })])]));
|
||||
let mut short = ok();
|
||||
short.data = ImageData::Raw { bits: 16, samples: vec![0; 5] };
|
||||
assert!(bad(vec![page(vec![(short, at)])]));
|
||||
let mut empty = ok();
|
||||
empty.width = 0;
|
||||
assert!(bad(vec![page(vec![(empty, at)])]));
|
||||
let mut odd = ok();
|
||||
odd.channels = 2;
|
||||
assert!(bad(vec![page(vec![(odd, at)])]));
|
||||
let mut not_jpeg = ok();
|
||||
not_jpeg.data = ImageData::Jpeg(vec![1, 2, 3]);
|
||||
assert!(bad(vec![page(vec![(not_jpeg, at)])]));
|
||||
let too_many = (0..MAX_PAGES + 1).map(|_| page(vec![])).collect();
|
||||
assert!(bad(too_many));
|
||||
let info = PdfInfo {
|
||||
producer: String::new(),
|
||||
title: None,
|
||||
output_intent: Some(OutputIntent { icc: srgb(), channels: 2, condition: String::new() }),
|
||||
};
|
||||
assert!(write(&[page(vec![])], &info).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbers_and_strings() {
|
||||
assert_eq!(num(612.0), "612");
|
||||
assert_eq!(num(72.5), "72.5");
|
||||
assert_eq!(num(-0.00001), "0");
|
||||
assert_eq!(num(f64::NAN), "0");
|
||||
assert_eq!(num(1.0 / 3.0), "0.3333");
|
||||
assert_eq!(text("a(b)c\\"), "(a\\(b\\)c\\\\)");
|
||||
assert_eq!(text("é"), "<FEFF00E9>");
|
||||
}
|
||||
#[test]
|
||||
fn distinct_profiles_and_component_counts_are_not_conflated() {
|
||||
let common = Arc::new(b"opaque profile A".to_vec());
|
||||
let different = Arc::new(b"opaque profile B".to_vec());
|
||||
let image = |channels, profile| PdfImage {
|
||||
width: 1,
|
||||
height: 1,
|
||||
channels,
|
||||
data: ImageData::Raw { bits: 8, samples: vec![7; channels as usize] },
|
||||
icc: Some(profile),
|
||||
};
|
||||
let pages = [page(vec![
|
||||
(image(3, common.clone()), full(1.0, 1.0)),
|
||||
(image(4, common.clone()), full(1.0, 1.0)),
|
||||
(image(3, different.clone()), full(1.0, 1.0)),
|
||||
(image(3, common.clone()), full(1.0, 1.0)),
|
||||
])];
|
||||
let pdf = write(&pages, &PdfInfo::default()).unwrap();
|
||||
let refs: Vec<_> = images(&pdf).iter().map(|&id| reference(&object(&pdf, id).0, "/ICCBased")).collect();
|
||||
assert_ne!(refs[0], refs[1]);
|
||||
assert_ne!(refs[0], refs[2]);
|
||||
assert_eq!(refs[0], refs[3]);
|
||||
for (i, channels, bytes) in [(0, 3, common.as_slice()), (1, 4, common.as_slice()), (2, 3, different.as_slice())] {
|
||||
let (dict, data) = object(&pdf, refs[i]);
|
||||
assert!(dict.contains(&format!("/N {channels}")));
|
||||
assert_eq!(inflate(&data), bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gray_rgb_cmyk_raw_depths_preserve_samples() {
|
||||
for channels in [1, 3, 4] {
|
||||
for bits in [8, 16] {
|
||||
let bytes: Vec<u8> = (0..channels * (bits / 8)).collect();
|
||||
let image = PdfImage { width: 1, height: 1, channels, data: ImageData::Raw { bits, samples: bytes.clone() }, icc: None };
|
||||
let pdf = write(&[page(vec![(image, full(1.0, 1.0))])], &PdfInfo::default()).unwrap();
|
||||
let (dict, data) = object(&pdf, images(&pdf)[0]);
|
||||
assert!(dict.contains(&format!("/BitsPerComponent {bits}")));
|
||||
assert_eq!(inflate(&data), bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_and_output_budgets_fail_without_partial_success() {
|
||||
let image = PdfImage::from_u16(1, 1, 3, &[1, 2, 3], None);
|
||||
let pages = [page(vec![(image, full(1.0, 1.0))])];
|
||||
let info = PdfInfo::default();
|
||||
let expected = write(&pages, &info).unwrap();
|
||||
assert_eq!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() }).unwrap(), expected);
|
||||
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 5, output_bytes: expected.len() }).is_err());
|
||||
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: expected.len() - 1 }).is_err());
|
||||
assert!(write_with_limits(&pages, &info, Limits { input_bytes: 6, output_bytes: 0 }).is_err());
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,7 @@ The format layer ends where arithmetic on the picture begins.
|
||||
|---|---|
|
||||
| Recognising containers (`probe`), walking TIFF/IFD structure on `craft-tiff` | Black subtraction, white scaling, cropping to the active area |
|
||||
| Decompressing the samples: unpacking, lossless JPEG, lossy JPEG, Deflate, JPEG XL | Demosaicing, binned previews, hot-pixel and highlight handling |
|
||||
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
|
||||
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values; left unapplied when `OpcodeList1` is present, so the develop pipeline can run that list on the stored codes first) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
|
||||
| Black and white levels, CFA layout, active area, default crop, orientation, pixel aspect | The DNG colour model's arithmetic (illuminant interpolation, camera to working space) |
|
||||
| Colour tags as data: matrices, calibration illuminants, as-shot neutral / white | White balance, Kelvin scales, spectral or fitted camera colour |
|
||||
| Opcode lists parsed and written (`opcodes`), profile tables and gain table maps as data | Applying opcodes (gain maps, warps, bad pixels), profile look tables, tone curves |
|
||||
|
||||
@@ -852,6 +852,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
// makes a neutral surface magenta. WB edits remain relative to this as-shot RGB.
|
||||
wb_multipliers: if linear_rgb { Some([1.0; 3]) } else { wb },
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
// only validated components become opcodes (see `arw_lens`, and `arw_lens_rm6` for the ILCE-7RM6)
|
||||
opcodes: OpcodeLists { list3: warp.into_iter().chain(rm6).collect(), ..Default::default() },
|
||||
|
||||
@@ -311,6 +311,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData { kelvin_presets, ..Default::default() },
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -388,6 +388,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode, limits: &
|
||||
// the camera's white balance is already applied, as in Sony's linear YCbCr
|
||||
wb_multipliers: Some([1.0; 3]),
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -269,6 +269,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: levels.wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: ifd0.map(Camera::from_ifd0).unwrap_or_default(),
|
||||
@@ -824,6 +825,7 @@ mod tests {
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Camera::default(),
|
||||
|
||||
@@ -59,6 +59,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, caller: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: h.wb_multipliers,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera,
|
||||
|
||||
+30
-14
@@ -299,17 +299,11 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
deinterleave(&mut data, w, h, cpp, factor(t::ROW_INTERLEAVE_FACTOR), factor(t::COLUMN_INTERLEAVE_FACTOR))?;
|
||||
}
|
||||
|
||||
// linearization table (integer data only)
|
||||
let mut linearized = false;
|
||||
// linearization table (integer data only) and opcode lists. OpcodeList1 runs on the stored
|
||||
// codes first (DNG 1.7 chapter 7); see [`linearization_and_opcodes`].
|
||||
let (linearized, linearization_table, opcodes) = linearization_and_opcodes(pixels, raw, &mut data);
|
||||
// the largest value the table can give a sample: the white level when the file states none
|
||||
let mut table_max = None;
|
||||
if let (RawData::U16(v), Some(table)) = (&mut data, pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty())) {
|
||||
let table: Vec<u16> = table.iter().map(|&x| x.min(65535) as u16).collect();
|
||||
let last = table.len() - 1;
|
||||
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
|
||||
linearized = true;
|
||||
table_max = table.iter().copied().max().filter(|&m| m > 0);
|
||||
}
|
||||
let table_max = linearization_table.as_deref().and_then(|t| t.iter().copied().max().filter(|&m| m > 0));
|
||||
|
||||
// a reduced copy's own levels, where it states them
|
||||
let levels = |tag| if pixels.contains(tag) { pixels } else { raw };
|
||||
@@ -442,10 +436,6 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
None
|
||||
};
|
||||
|
||||
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
|
||||
let opcodes =
|
||||
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
|
||||
|
||||
let color = color_data(ifd0, raw, tiff.order);
|
||||
let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2))
|
||||
&& color.profile.gain_table_map.is_none();
|
||||
@@ -465,6 +455,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
color,
|
||||
wb_multipliers: None,
|
||||
linearized,
|
||||
linearization_table,
|
||||
pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()),
|
||||
opcodes,
|
||||
camera: Camera::from_ifd0(ifd0),
|
||||
@@ -477,6 +468,31 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
/// Parse opcode lists and, for integer data, the LinearizationTable.
|
||||
///
|
||||
/// OpcodeList1 runs on the stored codes, then the table maps them to sensor values (DNG 1.7
|
||||
/// chapter 7). Applying opcodes is the develop pipeline's work, so a non-empty list 1 leaves the
|
||||
/// table unapplied (`linearized` is false) and available on the image.
|
||||
fn linearization_and_opcodes(pixels: &Ifd, raw: &Ifd, data: &mut RawData) -> (bool, Option<Vec<u16>>, crate::OpcodeLists) {
|
||||
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
|
||||
let opcodes =
|
||||
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
|
||||
let linearization_table = if matches!(data, RawData::U16(_)) {
|
||||
pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty()).map(|table| table.iter().map(|&x| x.min(65535) as u16).collect::<Vec<u16>>())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut linearized = false;
|
||||
if opcodes.list1.is_empty()
|
||||
&& let (RawData::U16(v), Some(table)) = (data, linearization_table.as_deref())
|
||||
{
|
||||
let last = table.len() - 1;
|
||||
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
|
||||
linearized = true;
|
||||
}
|
||||
(linearized, linearization_table, opcodes)
|
||||
}
|
||||
|
||||
/// CinemaDNG `TimeCodes` (CinemaDNG Image Data Format Specification 1.0, tag 51043).
|
||||
const TIME_CODES: u16 = 51043;
|
||||
/// CinemaDNG `FrameRate` (tag 51044).
|
||||
|
||||
@@ -165,6 +165,14 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result<Vec<u8>> {
|
||||
} else {
|
||||
ifd.set(t::WHITE_LEVEL, Value::Long(white));
|
||||
}
|
||||
// A table still to be applied (OpcodeList1 has to run on the stored codes first) is written
|
||||
// with the samples. Once the samples have been mapped, writing the tag would apply it twice
|
||||
// on the next read of a file without list 1.
|
||||
if !raw.linearized
|
||||
&& let Some(table) = raw.linearization_table.as_deref().filter(|t| !t.is_empty())
|
||||
{
|
||||
ifd.set(t::LINEARIZATION_TABLE, Value::Short(table.to_vec()));
|
||||
}
|
||||
let a = raw.active_area;
|
||||
ifd.set(t::ACTIVE_AREA, Value::Long(vec![a.y as u32, a.x as u32, (a.y + a.height) as u32, (a.x + a.width) as u32]));
|
||||
ifd.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![raw.crop.x as u32, raw.crop.y as u32]));
|
||||
@@ -457,6 +465,7 @@ mod tests {
|
||||
color: crate::ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Default::default(),
|
||||
@@ -514,6 +523,7 @@ mod tests {
|
||||
color: crate::ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Default::default(),
|
||||
|
||||
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
|
||||
+126
-13
@@ -4,16 +4,19 @@
|
||||
|
||||
use crate::{RawError, Result};
|
||||
|
||||
/// A canonical-by-table Huffman decoder with a 12-bit lookup.
|
||||
/// A canonical-by-table Huffman decoder with a 12-bit lookup that also holds the difference when code and
|
||||
/// additional bits fit in 12 bits.
|
||||
pub(crate) struct Huffman {
|
||||
/// Indexed by the next 12 bits: (symbol, length); length 0 = invalid.
|
||||
lut: Vec<(u8, u8)>,
|
||||
/// Indexed by the next 12 bits. Per entry: bits 0-7 the symbol, bits 8-11 the code length (0 = invalid), bits
|
||||
/// 12-15 the length of code plus additional bits when both fit in 12 bits (0 = they do not), bits 16-31 the
|
||||
/// signed difference in that case.
|
||||
lut: Box<[u32; 4096]>,
|
||||
}
|
||||
|
||||
impl Huffman {
|
||||
/// From `(left-aligned 12-bit code, length)` per symbol.
|
||||
pub fn new(codes: &[(u16, u8)]) -> Result<Huffman> {
|
||||
let mut lut = vec![(0u8, 0u8); 4096];
|
||||
let mut lut = Box::new([0u32; 4096]);
|
||||
for (sym, &(code, len)) in codes.iter().enumerate() {
|
||||
if len == 0 {
|
||||
continue;
|
||||
@@ -21,10 +24,19 @@ impl Huffman {
|
||||
if len > 12 || sym > 16 {
|
||||
return Err(RawError::Unsupported(format!("PEF Huffman code of length {len}")));
|
||||
}
|
||||
let (len, s) = (len as u32, sym as u32);
|
||||
let first = (code >> (12 - len) << (12 - len)) as usize;
|
||||
for e in &mut lut[first..first + (1 << (12 - len))] {
|
||||
if e.1 == 0 {
|
||||
*e = (sym as u8, len);
|
||||
for (i, e) in lut[first..first + (1 << (12 - len))].iter_mut().enumerate() {
|
||||
if (*e >> 8) & 15 == 0 {
|
||||
let mut entry = (len << 8) | s;
|
||||
if s == 0 {
|
||||
entry |= len << 12;
|
||||
} else if len + s <= 12 {
|
||||
let v = (((first + i) >> (12 - len - s)) & ((1 << s) - 1)) as i32;
|
||||
let d = if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v };
|
||||
entry |= ((len + s) << 12) | ((d as i16 as u16 as u32) << 16);
|
||||
}
|
||||
*e = entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -85,21 +97,122 @@ impl<'a> Bits<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode one difference: Huffman category then T.81 additional bits.
|
||||
/// Decode one difference: Huffman category then T.81 additional bits, in one lookup when both fit in 12 bits.
|
||||
#[inline]
|
||||
pub(crate) fn diff(bits: &mut Bits, h: &Huffman) -> Option<i32> {
|
||||
let (sym, len) = h.lut[bits.peek(12) as usize];
|
||||
let e = h.lut[(bits.peek(12) & 0xfff) as usize];
|
||||
let len = (e >> 8) & 15;
|
||||
if len == 0 {
|
||||
return None;
|
||||
}
|
||||
bits.skip(len as u32);
|
||||
let s = sym as u32;
|
||||
if s == 0 {
|
||||
return Some(0);
|
||||
let total = (e >> 12) & 15;
|
||||
if total > 0 {
|
||||
bits.skip(total);
|
||||
return Some((e as i32) >> 16);
|
||||
}
|
||||
bits.skip(len);
|
||||
let s = e & 0xff;
|
||||
if s > 16 {
|
||||
return None;
|
||||
}
|
||||
let v = bits.get(s) as i32;
|
||||
Some(if v < 1 << (s - 1) { v - (1 << s) + 1 } else { v })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Canonical left-aligned 12-bit codes for symbols 0..lens.len() (shorter codes first, ties by symbol).
|
||||
fn canonical(lens: &[u8]) -> Vec<(u16, u8)> {
|
||||
let mut order: Vec<usize> = (0..lens.len()).collect();
|
||||
order.sort_by_key(|&s| (lens[s], s));
|
||||
let mut codes = vec![(0u16, 0u8); lens.len()];
|
||||
let (mut code, mut prev) = (0u32, 0u8);
|
||||
for s in order {
|
||||
code <<= lens[s] - prev;
|
||||
prev = lens[s];
|
||||
codes[s] = ((code << (12 - lens[s])) as u16, lens[s]);
|
||||
code += 1;
|
||||
}
|
||||
codes
|
||||
}
|
||||
|
||||
fn bit(src: &[u8], i: usize) -> u32 {
|
||||
src.get(i / 8).map_or(0, |b| ((b >> (7 - i % 8)) & 1) as u32)
|
||||
}
|
||||
|
||||
/// Reference: match the code bit by bit, then read the category's additional bits (T.81 F.1.2.1).
|
||||
fn reference(src: &[u8], pos: &mut usize, codes: &[(u16, u8)]) -> Option<i32> {
|
||||
let window: u32 = (0..12).fold(0, |a, k| (a << 1) | bit(src, *pos + k));
|
||||
let (sym, len) = codes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(s, &(c, l))| (l > 0 && (c as u32 >> (12 - l)) == window >> (12 - l)).then_some((s as u32, l as usize)))?;
|
||||
*pos += len;
|
||||
if sym == 0 {
|
||||
return Some(0);
|
||||
}
|
||||
let v = (0..sym as usize).fold(0u32, |a, k| (a << 1) | bit(src, *pos + k)) as i32;
|
||||
*pos += sym as usize;
|
||||
Some(if v < 1 << (sym - 1) { v - (1 << sym) + 1 } else { v })
|
||||
}
|
||||
|
||||
/// `diff` against the reference on random streams (invalid codes are stepped over) and on every (code, extra
|
||||
/// bits) pair of every symbol.
|
||||
fn check_table(lens: &[u8]) {
|
||||
let codes = canonical(lens);
|
||||
let h = Huffman::new(&codes).unwrap();
|
||||
for (sym, &(code, len)) in codes.iter().enumerate() {
|
||||
for v in 0..1u64 << sym {
|
||||
let (len, n) = (len as u32, sym as u32);
|
||||
let word = (((code >> (12 - len)) as u64) << n | v) << (64 - len - n);
|
||||
let src = word.to_be_bytes();
|
||||
let mut pos = 0;
|
||||
let want = reference(&src, &mut pos, &codes);
|
||||
let mut bits = Bits::new(&src);
|
||||
assert_eq!(diff(&mut bits, &h), want, "symbol {sym} bits {v:#b}");
|
||||
assert_eq!(bits.consumed_bits(), pos);
|
||||
}
|
||||
}
|
||||
let mut s = 0x1234_5678_9abc_def0u64;
|
||||
let (mut invalid, mut long) = (0, 0);
|
||||
for _ in 0..40 {
|
||||
let src: Vec<u8> = (0..400)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
(s >> 40) as u8
|
||||
})
|
||||
.collect();
|
||||
let mut bits = Bits::new(&src);
|
||||
let mut pos = 0;
|
||||
while pos < src.len() * 8 - 64 {
|
||||
let before = pos;
|
||||
let want = reference(&src, &mut pos, &codes);
|
||||
assert_eq!(diff(&mut bits, &h), want);
|
||||
if want.is_none() {
|
||||
// step over the invalid code
|
||||
invalid += 1;
|
||||
pos += 1;
|
||||
bits.skip(1);
|
||||
} else {
|
||||
long += (pos - before > 12) as u32;
|
||||
}
|
||||
assert_eq!(bits.consumed_bits(), pos);
|
||||
}
|
||||
}
|
||||
assert!(long > 0, "{invalid} invalid codes, no symbol longer than the lookup");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_matches_bitwise_reference_for_every_category() {
|
||||
// categories 0..=16, code lengths up to 12 so that short codes combine with their bits and long ones do not
|
||||
check_table(&[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_matches_reference_when_code_and_bits_fill_the_lookup_exactly() {
|
||||
// category 8 with a 4-bit code and category 7 with a 5-bit code: len + category == 12
|
||||
check_table(&[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12]);
|
||||
}
|
||||
}
|
||||
@@ -115,6 +115,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
+32
-5
@@ -10,11 +10,13 @@
|
||||
//!
|
||||
//! # Where this crate stops
|
||||
//!
|
||||
//! The samples come out as the file stores them (after a DNG `LinearizationTable`): black is not subtracted,
|
||||
//! nothing is scaled, cropped, white balanced or demosaiced, opcode lists and profile tables are parsed but not
|
||||
//! applied, and no colour transform is computed from the matrices. Normalisation, demosaicing, highlight
|
||||
//! handling, the DNG colour model's arithmetic, lens corrections, tone and everything after belong to the
|
||||
//! develop pipeline, which reads the public fields of the types here.
|
||||
//! The samples come out as the file stores them (after a DNG `LinearizationTable`, except when the file
|
||||
//! also has an `OpcodeList1`: that list runs on the stored codes first, and applying it is the develop
|
||||
//! pipeline's work, so the table is left unapplied and carried on [`RawImage::linearization_table`]).
|
||||
//! Black is not subtracted, nothing is scaled, cropped, white balanced or demosaiced, opcode lists and
|
||||
//! profile tables are parsed but not applied, and no colour transform is computed from the matrices.
|
||||
//! Normalisation, demosaicing, highlight handling, the DNG colour model's arithmetic, lens corrections,
|
||||
//! tone and everything after belong to the develop pipeline, which reads the public fields of the types here.
|
||||
//!
|
||||
//! # Formats
|
||||
//!
|
||||
@@ -645,6 +647,8 @@ pub fn orientation(bytes: &[u8]) -> Option<u16> {
|
||||
}
|
||||
// A damaged/unsupported pixel IFD must not hide usable orientation metadata from preview
|
||||
// fallback. CR3 keeps IFD0 in its CMT1 box; TIFF-based raws keep it at the file start.
|
||||
// The `cr3` module exists only with the `cr3` feature: this gate is required so
|
||||
// `cargo check -p craft-raw --no-default-features --features dng` builds (E0433 otherwise).
|
||||
#[cfg(feature = "cr3")]
|
||||
if probe(bytes) == Some(RawFormat::Cr3) {
|
||||
return cr3::orientation(bytes);
|
||||
@@ -1041,6 +1045,12 @@ pub struct RawImage {
|
||||
pub wb_multipliers: Option<[f32; 3]>,
|
||||
/// Whether a `LinearizationTable` was applied to `data`.
|
||||
pub linearized: bool,
|
||||
/// The file's `LinearizationTable`, when present. `data` has already been mapped through it when
|
||||
/// [`Self::linearized`] is true. When `linearized` is false and this is `Some`, the table was left
|
||||
/// unapplied so `OpcodeList1` can run on the stored samples first (DNG chapter 7); call
|
||||
/// [`Self::apply_linearization`] after that list.
|
||||
#[serde(default)]
|
||||
pub linearization_table: Option<Vec<u16>>,
|
||||
/// Height of a sensor pixel over its width (1.0 for square pixels). The D1X and D1H write 2.0: their frame is
|
||||
/// 4028 x 1324 and the picture is 3:2 only once the rows are doubled. A developer stretches the rows by
|
||||
/// this factor.
|
||||
@@ -1119,6 +1129,23 @@ impl RawImage {
|
||||
self.check(true)
|
||||
}
|
||||
|
||||
/// Map integer `data` through [`Self::linearization_table`] when the table has not yet been applied.
|
||||
/// No-op when [`Self::linearized`] is already true, the table is missing, or the samples are not
|
||||
/// integers. Call this after `OpcodeList1`, which DNG applies to the stored codes (chapter 7).
|
||||
pub fn apply_linearization(&mut self) {
|
||||
if self.linearized {
|
||||
return;
|
||||
}
|
||||
let Some(table) = self.linearization_table.as_deref().filter(|t| !t.is_empty()) else {
|
||||
return;
|
||||
};
|
||||
if let RawData::U16(v) = &mut self.data {
|
||||
let last = table.len() - 1;
|
||||
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
|
||||
self.linearized = true;
|
||||
}
|
||||
}
|
||||
|
||||
fn check(&self, samples: bool) -> Result<()> {
|
||||
if self.width == 0 || self.height == 0 || !(1..=4).contains(&self.cpp) {
|
||||
return Err(RawError::Corrupt("bad dimensions".into()));
|
||||
|
||||
+129
-39
@@ -32,15 +32,17 @@ fn err(s: &str) -> RawError {
|
||||
/// Huffman decoding table (T.81 Annex C "BITS"/"HUFFVAL").
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Huffman {
|
||||
/// For peeks of `LOOKUP` bits: (code length, symbol); length 0 = slow path.
|
||||
fast: Vec<(u8, u8)>,
|
||||
/// Indexed by the next `LOOKUP` bits; length 0 = slow path. Per entry: bits 0-7 the symbol, bits 8-11 the code
|
||||
/// length, bits 12-15 the length of code plus extra bits when both fit in the lookup (0 = they do not), and
|
||||
/// bits 16-31 the signed difference in that case.
|
||||
fast: Vec<u32>,
|
||||
maxcode: [i32; 18],
|
||||
valptr: [i32; 17],
|
||||
mincode: [i32; 17],
|
||||
values: Vec<u8>,
|
||||
}
|
||||
|
||||
const LOOKUP: u32 = 10;
|
||||
const LOOKUP: u32 = 12;
|
||||
|
||||
impl Huffman {
|
||||
/// Build from the 16 code-length counts and the symbol list.
|
||||
@@ -52,7 +54,7 @@ impl Huffman {
|
||||
let mut maxcode = [-1i32; 18];
|
||||
let mut valptr = [0i32; 17];
|
||||
let mut mincode = [0i32; 17];
|
||||
let mut fast = vec![(0u8, 0u8); 1 << LOOKUP];
|
||||
let mut fast = vec![0u32; 1 << LOOKUP];
|
||||
let mut code = 0i32;
|
||||
let mut k = 0usize;
|
||||
for len in 1..=16usize {
|
||||
@@ -63,9 +65,10 @@ impl Huffman {
|
||||
if len as u32 <= LOOKUP {
|
||||
let shift = LOOKUP - len as u32;
|
||||
let base = (code as usize) << shift;
|
||||
let ssss = values[k] as u32;
|
||||
for i in 0..(1usize << shift) {
|
||||
if let Some(slot) = fast.get_mut(base + i) {
|
||||
*slot = (len as u8, values[k]);
|
||||
*slot = fast_entry(len as u32, ssss, base + i);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -84,12 +87,41 @@ impl Huffman {
|
||||
|
||||
#[inline]
|
||||
pub fn decode<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
|
||||
let peek = br.peek(LOOKUP);
|
||||
let (len, sym) = self.fast[peek as usize];
|
||||
let e = self.fast[br.peek(LOOKUP) as usize];
|
||||
let len = (e >> 8) & 15;
|
||||
if len > 0 {
|
||||
br.consume(len as u32);
|
||||
return Ok(sym);
|
||||
br.consume(len);
|
||||
return Ok(e as u8);
|
||||
}
|
||||
self.decode_slow(br)
|
||||
}
|
||||
|
||||
/// One symbol and its T.81 difference in a single lookup when code and extra bits fit in `LOOKUP` bits; the same
|
||||
/// as [`Huffman::decode`] followed by [`diff_value`]. `strict`: a category above 16 is an error (subsampled
|
||||
/// scans) instead of reading 16 bits.
|
||||
#[inline]
|
||||
pub fn decode_diff<B: Bits>(&self, br: &mut B, strict: bool) -> Result<i32, RawError> {
|
||||
let e = self.fast[br.peek(LOOKUP) as usize];
|
||||
let len = (e >> 8) & 15;
|
||||
let total = (e >> 12) & 15;
|
||||
if total > 0 {
|
||||
br.consume(total);
|
||||
return Ok((e as i32) >> 16);
|
||||
}
|
||||
let ssss = if len > 0 {
|
||||
br.consume(len);
|
||||
e as u8
|
||||
} else {
|
||||
self.decode_slow(br)?
|
||||
};
|
||||
if strict && ssss > 16 {
|
||||
return Err(err("invalid difference category"));
|
||||
}
|
||||
Ok(diff_value(br, ssss))
|
||||
}
|
||||
|
||||
#[cold]
|
||||
fn decode_slow<B: Bits>(&self, br: &mut B) -> Result<u8, RawError> {
|
||||
let mut code = br.peek(LOOKUP + 1) as i32;
|
||||
let mut len = LOOKUP as usize + 1;
|
||||
while len <= 16 && code > self.maxcode[len] {
|
||||
@@ -105,6 +137,21 @@ impl Huffman {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lookup entry for a code of `len` bits (`len <= LOOKUP`) naming category `ssss`, at window position `index`
|
||||
/// (the code is the top `len` bits of the `LOOKUP`-bit window). See [`Huffman::fast`].
|
||||
fn fast_entry(len: u32, ssss: u32, index: usize) -> u32 {
|
||||
let base = (len << 8) | ssss;
|
||||
if ssss == 0 {
|
||||
return base | (len << 12);
|
||||
}
|
||||
if ssss > 11 || len + ssss > LOOKUP {
|
||||
return base;
|
||||
}
|
||||
let v = ((index >> (LOOKUP - len - ssss)) & ((1 << ssss) - 1)) as i32;
|
||||
let d = if v < (1 << (ssss - 1)) { v - (1 << ssss) + 1 } else { v };
|
||||
base | ((len + ssss) << 12) | ((d as i16 as u16 as u32) << 16)
|
||||
}
|
||||
|
||||
/// A source of entropy-coded bits, first bit most significant.
|
||||
pub trait Bits {
|
||||
/// The next `n` (1..=32) bits without consuming them; zeros past the end.
|
||||
@@ -614,11 +661,7 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
|
||||
Prediction::Sequential if mx == 0 && dx == 0 && dy == 0 => row_first[c],
|
||||
Prediction::Sequential => last[c],
|
||||
};
|
||||
let ssss = tables[c].decode(&mut br)?;
|
||||
if ssss > 16 {
|
||||
return Err(err("invalid difference category"));
|
||||
}
|
||||
let value = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16;
|
||||
let value = ((pred + tables[c].decode_diff(&mut br, true)?) & 0xffff) as u16;
|
||||
plane[i] = value;
|
||||
last[c] = i32::from(value);
|
||||
if mx == 0 && dx == 0 && dy == 0 {
|
||||
@@ -642,6 +685,44 @@ pub fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -
|
||||
Ok(FrameSubsampled { width: h.width, height: h.height, vertical_subsampling: vertical, planes })
|
||||
}
|
||||
|
||||
/// T.81 Table H.1 predictor `pred` (1..=7; 0 = none) from the left, above and above-left neighbours.
|
||||
#[inline(always)]
|
||||
fn predict(pred: u8, ra: i32, rb: i32, rc: i32) -> i32 {
|
||||
match pred {
|
||||
0 => 0,
|
||||
1 => ra,
|
||||
2 => rb,
|
||||
3 => rc,
|
||||
4 => ra + rb - rc,
|
||||
5 => ra + ((rb - rc) >> 1),
|
||||
6 => rb + ((ra - rc) >> 1),
|
||||
_ => (ra + rb) >> 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode the pixels `x0..x1` (`x0 >= 1`) of one row with the fixed predictor `P`. `prev` is empty on the first
|
||||
/// line, which only predicts from the left (`P = 1`).
|
||||
#[inline(always)]
|
||||
fn decode_run<const P: u8>(
|
||||
tables: &[&Huffman],
|
||||
br: &mut BitReader<'_>,
|
||||
cur: &mut [u16],
|
||||
prev: &[u16],
|
||||
nc: usize,
|
||||
x0: usize,
|
||||
x1: usize,
|
||||
) -> Result<(), RawError> {
|
||||
for x in x0..x1 {
|
||||
for (c, table) in tables.iter().enumerate() {
|
||||
let i = x * nc + c;
|
||||
let ra = cur[i - nc] as i32;
|
||||
let p = if P <= 1 { predict(P, ra, 0, 0) } else { predict(P, ra, prev[i] as i32, prev[i - nc] as i32) };
|
||||
cur[i] = ((p + table.decode_diff(br, false)?) & 0xffff) as u16;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decode a lossless JPEG stream. `max_samples` bounds the allocation (hostile headers).
|
||||
pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
|
||||
let h = parse_header(d)?;
|
||||
@@ -680,18 +761,23 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
|
||||
let (prev_rows, cur_rows) = out.split_at_mut(y * row_len);
|
||||
let cur = &mut cur_rows[..row_len];
|
||||
let prev = if y > 0 { &prev_rows[(y - 1) * row_len..] } else { &[][..] };
|
||||
for x in 0..w {
|
||||
if h.restart > 0 {
|
||||
if restart_left == 0 {
|
||||
if !br.restart() {
|
||||
return Err(err("missing restart marker"));
|
||||
}
|
||||
restart_left = h.restart;
|
||||
first_line = true;
|
||||
reset_pending = true;
|
||||
let mut x = 0;
|
||||
while x < w {
|
||||
if h.restart > 0 && restart_left == 0 {
|
||||
if !br.restart() {
|
||||
return Err(err("missing restart marker"));
|
||||
}
|
||||
restart_left -= 1;
|
||||
restart_left = h.restart;
|
||||
first_line = true;
|
||||
reset_pending = true;
|
||||
}
|
||||
// pixels up to the next restart marker or the end of the row
|
||||
let mut end = w;
|
||||
if h.restart > 0 {
|
||||
end = w.min(x + restart_left);
|
||||
restart_left -= end - x;
|
||||
}
|
||||
// the first pixel of the run has the special predictions (start of the interval, first column)
|
||||
#[allow(clippy::needless_range_loop)] // `c` indexes the tables and the sample position alike
|
||||
for c in 0..nc {
|
||||
let i = x * nc + c;
|
||||
@@ -702,24 +788,28 @@ pub fn decode(d: &[u8], max_samples: usize) -> Result<Frame, RawError> {
|
||||
} else if x == 0 || pred == 0 {
|
||||
if pred == 0 { 0 } else { prev[i] as i32 }
|
||||
} else {
|
||||
let ra = cur[i - nc] as i32;
|
||||
let rb = prev[i] as i32;
|
||||
let rc = prev[i - nc] as i32;
|
||||
match pred {
|
||||
1 => ra,
|
||||
2 => rb,
|
||||
3 => rc,
|
||||
4 => ra + rb - rc,
|
||||
5 => ra + ((rb - rc) >> 1),
|
||||
6 => rb + ((ra - rc) >> 1),
|
||||
_ => (ra + rb) >> 1,
|
||||
}
|
||||
predict(pred, cur[i - nc] as i32, prev[i] as i32, prev[i - nc] as i32)
|
||||
};
|
||||
let ssss = tables[c].decode(&mut br)?;
|
||||
let diff = diff_value(&mut br, ssss);
|
||||
cur[i] = ((p + diff) & mask) as u16;
|
||||
cur[i] = ((p + tables[c].decode_diff(&mut br, false)?) & mask) as u16;
|
||||
}
|
||||
reset_pending = false;
|
||||
// the rest of the run: the predictor is fixed, so it is chosen once here, not per sample
|
||||
if first_line {
|
||||
// the first line of an interval predicts from the left neighbour
|
||||
decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?;
|
||||
} else {
|
||||
match pred {
|
||||
0 => decode_run::<0>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
1 => decode_run::<1>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
2 => decode_run::<2>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
3 => decode_run::<3>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
4 => decode_run::<4>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
5 => decode_run::<5>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
6 => decode_run::<6>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
_ => decode_run::<7>(&tables, &mut br, cur, prev, nc, x + 1, end)?,
|
||||
}
|
||||
}
|
||||
x = end;
|
||||
}
|
||||
first_line = false;
|
||||
if br.overrun > 64 * 8 {
|
||||
|
||||
@@ -125,6 +125,121 @@ fn roundtrip_extremes_and_restarts() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-label an encoded stream as `precision` bits with point transform `pt`: the encoder codes the samples at
|
||||
/// `precision - pt` bits, which is what a transformed scan holds.
|
||||
fn with_point_transform(mut enc: Vec<u8>, precision: u8, pt: u8) -> Vec<u8> {
|
||||
enc[6] = precision; // SOF3 precision follows SOI, marker and length
|
||||
let sos = enc.windows(2).position(|w| w == [0xff, 0xda]).unwrap();
|
||||
let seg_len = u16::from_be_bytes([enc[sos + 2], enc[sos + 3]]) as usize;
|
||||
enc[sos + 2 + seg_len - 1] = pt;
|
||||
enc
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_point_transform_all_predictors() {
|
||||
for pred in 1..=7u8 {
|
||||
for comps in [1usize, 2, 3] {
|
||||
for (bits, pt) in [(12u8, 1u8), (14, 2), (16, 4), (8, 3)] {
|
||||
let (w, h) = (11, 6);
|
||||
let data = noise(w * h * comps, (bits - pt) as u32, pred as u64 * 17 + comps as u64);
|
||||
let enc = with_point_transform(encode(&data, w, h, comps, bits - pt, pred, 0), bits, pt);
|
||||
let f = decode(&enc, usize::MAX).unwrap();
|
||||
assert_eq!((f.precision, f.point_transform, f.predictor), (bits, pt, pred));
|
||||
let want: Vec<u16> = data.iter().map(|&v| v << pt).collect();
|
||||
assert_eq!(f.data, want, "pred {pred} comps {comps} bits {bits} pt {pt}");
|
||||
}
|
||||
}
|
||||
}
|
||||
// with restart intervals that end mid-row
|
||||
let data = noise(20 * 9 * 2, 10, 9);
|
||||
for rst in [3usize, 7, 20] {
|
||||
let enc = with_point_transform(encode(&data, 20, 9, 2, 10, 4, rst), 12, 2);
|
||||
let want: Vec<u16> = data.iter().map(|&v| v << 2).collect();
|
||||
assert_eq!(decode(&enc, usize::MAX).unwrap().data, want, "restart {rst}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_skewed_categories_with_long_codes() {
|
||||
// category k with probability 2^-(k+1): the table gets codes longer than the lookup, rare 16-bit differences
|
||||
let mut s = 0x9e37_79b9_7f4a_7c15u64;
|
||||
let mut last = 0u32;
|
||||
let data: Vec<u16> = (0..256 * 256)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
let r = (s >> 32) as u32;
|
||||
let cat = r.trailing_zeros().min(16);
|
||||
let mag = (((s >> 8) as u32) & ((1u32 << cat) - 1)) | (1 << cat) >> 1;
|
||||
let d = if s & 1 == 0 { mag } else { 0u32.wrapping_sub(mag) };
|
||||
last = last.wrapping_add(d) & 0xffff;
|
||||
last as u16
|
||||
})
|
||||
.collect();
|
||||
for pred in [1u8, 4, 7] {
|
||||
let enc = encode(&data, 256, 256, 1, 16, pred, 0);
|
||||
assert_eq!(decode(&enc, usize::MAX).unwrap().data, data, "pred {pred}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical decoding from the code lengths alone, bit by bit (T.81 Annex F, Figure F.16): the reference for the
|
||||
/// lookup tables.
|
||||
fn canonical_symbol(br: &mut BitReader, lens: &[u8]) -> Option<u8> {
|
||||
let window = br.peek(16);
|
||||
let (mut first, mut index) = (0u32, 0usize);
|
||||
for len in 1..=16u8 {
|
||||
let code = window >> (16 - len);
|
||||
let n = lens.iter().filter(|&&l| l == len).count() as u32;
|
||||
if code < first + n {
|
||||
br.consume(len as u32);
|
||||
return Some((index + (code - first) as usize) as u8);
|
||||
}
|
||||
index += n as usize;
|
||||
first = (first + n) << 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// `decode_diff` against canonical decoding and `diff_value`, for tables with codes inside and beyond the lookup
|
||||
/// and with `len + category` equal to the lookup width.
|
||||
#[test]
|
||||
fn decode_diff_matches_canonical_decoding() {
|
||||
let tables: [[u8; 17]; 3] = [
|
||||
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 12, 12],
|
||||
[2, 2, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 16],
|
||||
[2, 3, 3, 5, 5, 5, 6, 5, 4, 7, 8, 9, 10, 11, 12, 12, 12],
|
||||
];
|
||||
for lens in tables {
|
||||
let mut counts = [0u8; 16];
|
||||
for &l in &lens {
|
||||
counts[l as usize - 1] += 1;
|
||||
}
|
||||
// symbols in code order: by length, ties by symbol
|
||||
let mut values: Vec<u8> = (0..17).collect();
|
||||
values.sort_by_key(|&v| (lens[v as usize], v));
|
||||
let sorted: Vec<u8> = values.iter().map(|&v| lens[v as usize]).collect();
|
||||
let t = Huffman::new(&counts, &values).unwrap();
|
||||
let mut s = 77u64;
|
||||
let src: Vec<u8> = (0..6000)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
((s >> 40) as u8).min(0xfe)
|
||||
})
|
||||
.collect();
|
||||
let (mut a, mut b) = (BitReader::new(&src), BitReader::new(&src));
|
||||
for n in 0..8000 {
|
||||
let want = canonical_symbol(&mut a, &sorted).map(|i| diff_value(&mut a, values[i as usize]));
|
||||
let got = t.decode_diff(&mut b, false);
|
||||
assert_eq!(got.ok(), want, "symbol {n}");
|
||||
assert_eq!(a.peek(32), b.peek(32), "symbol {n}");
|
||||
if want.is_none() {
|
||||
// step past the invalid code so the walk goes on
|
||||
a.consume(1);
|
||||
b.consume(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constant_image_single_symbol_table() {
|
||||
let data = vec![1234u16; 16 * 4];
|
||||
|
||||
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
|
||||
@@ -165,6 +165,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.as_ref().and_then(|t| t.ifds.first()).map(Camera::from_ifd0).unwrap_or_default(),
|
||||
cinema: Default::default(),
|
||||
|
||||
@@ -329,6 +329,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect,
|
||||
// Nikon's in-file distortion correction, about the optical centre (which follows the sensor shift)
|
||||
opcodes: OpcodeLists {
|
||||
@@ -671,6 +672,7 @@ fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::ima
|
||||
// the stored colour is already white balanced (see above)
|
||||
wb_multipliers: Some([1.0; 3]),
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -357,6 +357,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -238,6 +238,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -298,6 +298,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData { kelvin_presets, baseline_exposure: baseline_exposure(&h), ..Default::default() },
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists { list3: crate::raf_lens::corrections(&camera_name(bytes), &ifd, active), ..Default::default() },
|
||||
camera: ifd0
|
||||
|
||||
@@ -672,6 +672,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists { list3: distortion(ifd0, active).into_iter().collect(), ..Default::default() },
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -186,6 +186,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
|
||||
|
||||
@@ -44,6 +44,7 @@ fn an_active_area_that_wraps_past_the_image_is_corrupt() {
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Camera::default(),
|
||||
cinema: Default::default(),
|
||||
|
||||
+154
-47
@@ -2,7 +2,7 @@
|
||||
|
||||
#[cfg(feature = "dng")]
|
||||
use crate::unpack::inflate;
|
||||
use crate::unpack::{f16_to_f32, f24_to_f32, read_u16s, undo_diff_u16, undo_float_predictor, unpack_lsb, unpack_msb, unpack_words32_msb};
|
||||
use crate::unpack::{f16_to_f32, f24_to_f32, read_u16s, undo_diff_u16, undo_float_predictor, unpack_lsb, unpack_msb, unpack_words32_msb_with};
|
||||
use crate::{Limits, RawData, RawError, Result, ljpeg};
|
||||
use craft_tiff::image::{Chunk, ImageInfo, chunk_bytes};
|
||||
use craft_tiff::{ByteOrder, tags::compression as comp};
|
||||
@@ -18,8 +18,8 @@ pub enum Packing {
|
||||
Lsb,
|
||||
/// Each sample stored in 16 bits in file byte order, regardless of `bits`.
|
||||
Word16,
|
||||
/// MSB-first bit stream inside 32-bit words stored in file byte order ([`unpack_words32_msb`]); rows start on a
|
||||
/// word boundary.
|
||||
/// MSB-first bit stream inside 32-bit words stored in file byte order ([`unpack_words32_msb`](crate::unpack::unpack_words32_msb));
|
||||
/// rows start on a word boundary.
|
||||
Words32Msb,
|
||||
}
|
||||
|
||||
@@ -184,46 +184,72 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin
|
||||
.par_iter()
|
||||
.map(|c| decode_chunk(data, info, c, order, packing, ccpp, bits, float, limits).map(|p| (*c, p)))
|
||||
.collect();
|
||||
for r in decoded {
|
||||
// a chunk that cannot be decoded is not a partial image: the file stays preview-only
|
||||
let (c, px) = r?;
|
||||
let (cw, ch) = (c.width as usize, c.height as usize);
|
||||
let (x0, y0) = (c.x as usize, c.y as usize);
|
||||
if x0 >= w || y0 >= h {
|
||||
continue;
|
||||
}
|
||||
let copy_w = cw.min(w - x0);
|
||||
let copy_h = ch.min(h - y0);
|
||||
for y in 0..copy_h {
|
||||
for x in 0..copy_w {
|
||||
for s in 0..ccpp {
|
||||
let src = (y * cw + x) * ccpp + s;
|
||||
let dst_s = if planar { c.plane as usize } else { s };
|
||||
if dst_s >= cpp {
|
||||
continue;
|
||||
}
|
||||
let dst = ((y0 + y) * w + x0 + x) * cpp + dst_s;
|
||||
match &px {
|
||||
ChunkPx::U16(v) => {
|
||||
if let (Some(o), Some(&s)) = (out_u16.get_mut(dst), v.get(src)) {
|
||||
*o = s;
|
||||
}
|
||||
}
|
||||
ChunkPx::F32(v) => {
|
||||
if let (Some(o), Some(&s)) = (out_f32.get_mut(dst), v.get(src)) {
|
||||
*o = s;
|
||||
}
|
||||
}
|
||||
}
|
||||
// a chunk that cannot be decoded is not a partial image: the file stays preview-only
|
||||
let decoded = decoded.into_iter().collect::<Result<Vec<_>>>()?;
|
||||
// The copy runs in parallel over bands of output rows: each band takes its rows of every chunk of the
|
||||
// group that reaches it, in chunk order (so overlapping chunks resolve as a serial copy would).
|
||||
let row_samples = w * cpp;
|
||||
let threads = rayon::current_num_threads().max(1);
|
||||
let band_rows = h.div_ceil(threads * 4).max(1);
|
||||
if float {
|
||||
out_f32.par_chunks_mut(band_rows * row_samples).enumerate().for_each(|(bi, band)| {
|
||||
for (c, px) in &decoded {
|
||||
if let ChunkPx::F32(v) = px {
|
||||
place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
} else {
|
||||
out_u16.par_chunks_mut(band_rows * row_samples).enumerate().for_each(|(bi, band)| {
|
||||
for (c, px) in &decoded {
|
||||
if let ChunkPx::U16(v) = px {
|
||||
place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
decode_from = to;
|
||||
}
|
||||
Ok(if float { RawData::F32(out_f32) } else { RawData::U16(out_u16) })
|
||||
}
|
||||
|
||||
/// Copy the rows of chunk `c` (samples `src`, `c.width` per row, `ccpp` samples per pixel) that fall into the output
|
||||
/// band `band`, which holds the image rows from `band_y0` on (`w` pixels of `cpp` samples each, `h` rows in the whole
|
||||
/// image). Pixels past the image edge and samples `src` does not hold are skipped; a planar chunk (`planar`) writes
|
||||
/// its one sample per pixel into the channel `c.plane`, a chunk of a plane the image does not have is skipped.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn place_chunk<T: Copy>(band: &mut [T], band_y0: usize, c: &Chunk, src: &[T], w: usize, h: usize, cpp: usize, planar: bool) {
|
||||
let (cw, ch) = (c.width as usize, c.height as usize);
|
||||
let (x0, y0) = (c.x as usize, c.y as usize);
|
||||
if x0 >= w || y0 >= h || cpp == 0 {
|
||||
return;
|
||||
}
|
||||
let plane = c.plane as usize;
|
||||
if planar && plane >= cpp {
|
||||
return;
|
||||
}
|
||||
let ccpp = if planar { 1 } else { cpp };
|
||||
let copy_w = cw.min(w - x0);
|
||||
let copy_h = ch.min(h - y0);
|
||||
let band_rows = band.len() / (w * cpp);
|
||||
let first = y0.max(band_y0);
|
||||
let last = (y0 + copy_h).min(band_y0 + band_rows);
|
||||
for y in first..last {
|
||||
let s0 = (y - y0) * cw * ccpp;
|
||||
let Some(avail) = src.get(s0..) else { break };
|
||||
let d0 = ((y - band_y0) * w + x0) * cpp;
|
||||
if planar {
|
||||
let row = &mut band[d0..d0 + copy_w * cpp];
|
||||
for (o, &v) in row[plane..].iter_mut().step_by(cpp).zip(&avail[..copy_w.min(avail.len())]) {
|
||||
*o = v;
|
||||
}
|
||||
} else {
|
||||
let n = (copy_w * cpp).min(avail.len());
|
||||
band[d0..d0 + n].copy_from_slice(&avail[..n]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Framing precedes allocation even in header-only probes: missing source rows may not
|
||||
/// be reported as a decodable image and later silently filled with zero samples.
|
||||
fn check_uncompressed_chunks(data: &[u8], info: &ImageInfo, chunks: &[Chunk], packing: Packing) -> Result<()> {
|
||||
@@ -512,13 +538,14 @@ fn unpack_chunk(
|
||||
let row_bytes = row_n * bp;
|
||||
let mut out = vec![0f32; row_n * ch];
|
||||
let mut rowbuf = vec![0u8; row_bytes];
|
||||
let mut scratch = Vec::new();
|
||||
for y in 0..ch {
|
||||
let s = src.get(y * row_bytes..).unwrap_or(&[]);
|
||||
let len = s.len().min(row_bytes);
|
||||
rowbuf[..len].copy_from_slice(&s[..len]);
|
||||
rowbuf[len..].fill(0);
|
||||
let big = if fp {
|
||||
undo_float_predictor(&mut rowbuf, row_n, bp, cpp * factor);
|
||||
undo_float_predictor(&mut rowbuf, row_n, bp, cpp * factor, &mut scratch);
|
||||
true
|
||||
} else {
|
||||
if factor > 0 {
|
||||
@@ -526,16 +553,15 @@ fn unpack_chunk(
|
||||
}
|
||||
order == ByteOrder::Big
|
||||
};
|
||||
for (i, o) in out[y * row_n..(y + 1) * row_n].iter_mut().enumerate() {
|
||||
let b = &rowbuf[i * bp..(i + 1) * bp];
|
||||
*o = match (bp, big) {
|
||||
(2, true) => f16_to_f32(u16::from_be_bytes([b[0], b[1]])),
|
||||
(2, false) => f16_to_f32(u16::from_le_bytes([b[0], b[1]])),
|
||||
(3, true) => f24_to_f32(u32::from_be_bytes([0, b[0], b[1], b[2]])),
|
||||
(3, false) => f24_to_f32(u32::from_le_bytes([b[0], b[1], b[2], 0])),
|
||||
(4, true) => f32::from_be_bytes([b[0], b[1], b[2], b[3]]),
|
||||
_ => f32::from_le_bytes([b[0], b[1], b[2], b[3]]),
|
||||
};
|
||||
let dst = &mut out[y * row_n..(y + 1) * row_n];
|
||||
let samples = rowbuf.chunks_exact(bp);
|
||||
match (bp, big) {
|
||||
(2, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f16_to_f32(u16::from_be_bytes([b[0], b[1]]))),
|
||||
(2, false) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f16_to_f32(u16::from_le_bytes([b[0], b[1]]))),
|
||||
(3, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f24_to_f32(u32::from_be_bytes([0, b[0], b[1], b[2]]))),
|
||||
(3, false) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f24_to_f32(u32::from_le_bytes([b[0], b[1], b[2], 0]))),
|
||||
(4, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f32::from_be_bytes([b[0], b[1], b[2], b[3]])),
|
||||
_ => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f32::from_le_bytes([b[0], b[1], b[2], b[3]])),
|
||||
}
|
||||
}
|
||||
return Ok(ChunkPx::F32(out));
|
||||
@@ -544,6 +570,7 @@ fn unpack_chunk(
|
||||
return Err(RawError::Unsupported("floating-point predictor on integer data".into()));
|
||||
}
|
||||
let mut out = vec![0u16; row_n * ch];
|
||||
let mut words = Vec::new();
|
||||
let (row_bytes, kind) = match (bits, packing) {
|
||||
(8, _) => (row_n, 0),
|
||||
(16, _) | (_, Packing::Word16) => (row_n * 2, 1),
|
||||
@@ -559,7 +586,7 @@ fn unpack_chunk(
|
||||
0 => row.iter_mut().zip(s).for_each(|(o, &b)| *o = b as u16),
|
||||
1 => read_u16s(s, order, row),
|
||||
2 => unpack_msb(s, bits, row),
|
||||
4 => unpack_words32_msb(s, order, bits, row),
|
||||
4 => unpack_words32_msb_with(s, order, bits, row, &mut words),
|
||||
_ => unpack_lsb(s, bits, row),
|
||||
}
|
||||
if factor > 0 {
|
||||
@@ -581,6 +608,86 @@ fn unpack_chunk(
|
||||
Ok(ChunkPx::U16(out))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod place_tests {
|
||||
use super::*;
|
||||
|
||||
/// The serial per-sample copy `place_chunk` replaced: the oracle.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn reference(out: &mut [u16], c: &Chunk, v: &[u16], w: usize, h: usize, cpp: usize, planar: bool) {
|
||||
let ccpp = if planar { 1 } else { cpp };
|
||||
let (cw, ch) = (c.width as usize, c.height as usize);
|
||||
let (x0, y0) = (c.x as usize, c.y as usize);
|
||||
if x0 >= w || y0 >= h {
|
||||
return;
|
||||
}
|
||||
for y in 0..ch.min(h - y0) {
|
||||
for x in 0..cw.min(w - x0) {
|
||||
for s in 0..ccpp {
|
||||
let src = (y * cw + x) * ccpp + s;
|
||||
let dst_s = if planar { c.plane as usize } else { s };
|
||||
if dst_s >= cpp {
|
||||
continue;
|
||||
}
|
||||
let dst = ((y0 + y) * w + x0 + x) * cpp + dst_s;
|
||||
if let (Some(o), Some(&s)) = (out.get_mut(dst), v.get(src)) {
|
||||
*o = s;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn chunk(x: u32, y: u32, width: u32, height: u32, plane: u16) -> Chunk {
|
||||
Chunk { index: 0, x, y, width, height, plane, offset: 0, len: 0 }
|
||||
}
|
||||
|
||||
/// Chunks that overlap (the later one wins), run past the image edge, hold fewer samples than their size, or name a
|
||||
/// plane the image does not have, placed through bands of several heights, equal the serial copy.
|
||||
#[test]
|
||||
fn place_chunk_matches_the_serial_copy_for_odd_chunks() {
|
||||
let (w, h) = (11, 9);
|
||||
for cpp in [1usize, 3] {
|
||||
for planar in [false, true] {
|
||||
let ccpp = if planar { 1 } else { cpp };
|
||||
let specs = [
|
||||
(0, 0, 8, 6, 0, 1u16),
|
||||
(4, 3, 8, 8, 0, 2), // overlaps the first and runs past both edges
|
||||
(4, 3, 8, 8, 0, 3), // the same place again, later wins
|
||||
(0, 5, 5, 4, 1, 4),
|
||||
(2, 2, 6, 6, 2, 5),
|
||||
(0, 0, 4, 4, 3, 6), // plane 3: not an image plane for cpp <= 3
|
||||
(9, 0, 6, 12, 0, 7), // only two columns inside
|
||||
(20, 0, 4, 4, 0, 8), // right of the image
|
||||
(0, 0, 11, 9, 0, 9), // whole image, short source below
|
||||
];
|
||||
let chunks: Vec<(Chunk, Vec<u16>)> = specs
|
||||
.iter()
|
||||
.map(|&(x, y, cw, ch, plane, val)| {
|
||||
let n = (cw * ch) as usize * ccpp;
|
||||
// the last chunk holds only 40 samples
|
||||
let n = if val == 9 { 40 } else { n };
|
||||
(chunk(x, y, cw, ch, plane), (0..n).map(|i| val * 100 + i as u16).collect())
|
||||
})
|
||||
.collect();
|
||||
let mut want = vec![0u16; w * h * cpp];
|
||||
for (c, v) in &chunks {
|
||||
reference(&mut want, c, v, w, h, cpp, planar);
|
||||
}
|
||||
for band_rows in [1usize, 2, 4, 9, 20] {
|
||||
let mut got = vec![0u16; w * h * cpp];
|
||||
for (bi, band) in got.chunks_mut(band_rows * w * cpp).enumerate() {
|
||||
for (c, v) in &chunks {
|
||||
place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar);
|
||||
}
|
||||
}
|
||||
assert_eq!(got, want, "cpp {cpp} planar {planar} band rows {band_rows}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(feature = "dng")]
|
||||
mod lossy_tests {
|
||||
|
||||
@@ -46,12 +46,18 @@ pub fn unpack_lsb(src: &[u8], bits: u32, out: &mut [u16]) {
|
||||
/// data, which is not the byte-wise MSB stream of [`unpack_msb`] unless `order` is big-endian. A short last word
|
||||
/// is padded with zeros.
|
||||
pub fn unpack_words32_msb(src: &[u8], order: ByteOrder, bits: u32, out: &mut [u16]) {
|
||||
let words = src.chunks(4).flat_map(|c| {
|
||||
unpack_words32_msb_with(src, order, bits, out, &mut Vec::new());
|
||||
}
|
||||
|
||||
/// [`unpack_words32_msb`] with the byte-swapped copy of `src` kept in `scratch` (reused across rows).
|
||||
pub fn unpack_words32_msb_with(src: &[u8], order: ByteOrder, bits: u32, out: &mut [u16], scratch: &mut Vec<u8>) {
|
||||
scratch.clear();
|
||||
scratch.extend(src.chunks(4).flat_map(|c| {
|
||||
let mut w = [0u8; 4];
|
||||
w[..c.len()].copy_from_slice(c);
|
||||
order.u32(w).to_be_bytes()
|
||||
});
|
||||
unpack_msb(&words.collect::<Vec<u8>>(), bits, out);
|
||||
}));
|
||||
unpack_msb(scratch, bits, out);
|
||||
}
|
||||
|
||||
pub fn read_u16s(src: &[u8], order: ByteOrder, out: &mut [u16]) {
|
||||
@@ -126,10 +132,12 @@ pub fn undo_diff_u8(row: &mut [u8], stride: usize) {
|
||||
|
||||
/// Undo the floating-point predictor on one row: `row` holds `n` samples of `bytes_per` bytes, byte-differenced
|
||||
/// with stride `stride` samples, then stored as byte planes (most significant first). Returns the samples as
|
||||
/// big-endian byte groups in place.
|
||||
pub fn undo_float_predictor(row: &mut [u8], n: usize, bytes_per: usize, stride: usize) {
|
||||
/// big-endian byte groups in place. `scratch` holds the byte planes (reused across rows).
|
||||
pub fn undo_float_predictor(row: &mut [u8], n: usize, bytes_per: usize, stride: usize, scratch: &mut Vec<u8>) {
|
||||
undo_diff_u8(row, stride);
|
||||
let tmp = row.to_vec();
|
||||
scratch.clear();
|
||||
scratch.extend_from_slice(row);
|
||||
let tmp = &scratch[..];
|
||||
for i in 0..n {
|
||||
for b in 0..bytes_per {
|
||||
if let (Some(dst), Some(&s)) = (row.get_mut(i * bytes_per + b), tmp.get(b * n + i)) {
|
||||
@@ -167,6 +175,22 @@ mod tests {
|
||||
assert_eq!(o, [0x000, 0x012]); // the word 0x00001234, top bits first
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn words32_scratch_is_reusable() {
|
||||
let long: Vec<u8> = (0u8..24).map(|i| i.wrapping_mul(37).wrapping_add(5)).collect();
|
||||
let short = &long[3..10];
|
||||
let mut scratch = Vec::new();
|
||||
for order in [ByteOrder::Little, ByteOrder::Big] {
|
||||
for (src, n) in [(&long[..], 16usize), (short, 4)] {
|
||||
let mut want = vec![0u16; n];
|
||||
unpack_words32_msb(src, order, 12, &mut want);
|
||||
let mut got = vec![0u16; n];
|
||||
unpack_words32_msb_with(src, order, 12, &mut got, &mut scratch);
|
||||
assert_eq!(got, want);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packing() {
|
||||
// 12-bit MSB: 0xABC, 0xDEF -> AB CD EF
|
||||
@@ -210,7 +234,7 @@ mod tests {
|
||||
for i in (1..row.len()).rev() {
|
||||
row[i] = row[i].wrapping_sub(row[i - 1]);
|
||||
}
|
||||
undo_float_predictor(&mut row, 2, 4, 1);
|
||||
undo_float_predictor(&mut row, 2, 4, 1, &mut Vec::new());
|
||||
assert_eq!(f32::from_be_bytes([row[0], row[1], row[2], row[3]]), 1.0);
|
||||
assert_eq!(f32::from_be_bytes([row[4], row[5], row[6], row[7]]), 2.0);
|
||||
}
|
||||
|
||||
+233
-1
@@ -1,7 +1,7 @@
|
||||
//! Synthetic DNG round trips: every storage variant we decode is written with our own TIFF writer (and LJ92
|
||||
//! encoder) and must decode bit-exactly.
|
||||
|
||||
use craft_raw::opcodes::{Area, Opcode};
|
||||
use craft_raw::opcodes::{self, Area, Opcode};
|
||||
use craft_raw::{
|
||||
BlackLevel, Cfa, ColorData, DngCompression, DngWriteOptions, OpcodeLists, Orientation, RawData, RawFormat, RawImage, Rect, decode,
|
||||
embedded_preview, probe, write_dng,
|
||||
@@ -58,6 +58,7 @@ fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
|
||||
color,
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera,
|
||||
@@ -403,6 +404,172 @@ fn floating_point_dngs() {
|
||||
assert_eq!(decode(&dng(raw, ByteOrder::Big)).unwrap().data, RawData::F32(vals));
|
||||
}
|
||||
|
||||
/// The chunks (tiles of `tw` x `th`, or strips when `tw == w`) of an image of `cpp` interleaved samples per pixel, each
|
||||
/// nominal-sized (strips: clipped to the image height) with zero padding past the image edge: one chunk per tile, and for `planar` one set of chunks per
|
||||
/// plane (one sample per pixel), plane after plane.
|
||||
fn chunk_samples<T: Copy + Default>(px: &[T], w: usize, h: usize, cpp: usize, tw: usize, th: usize, planar: bool) -> Vec<Vec<T>> {
|
||||
let planes = if planar { cpp } else { 1 };
|
||||
let ccpp = if planar { 1 } else { cpp };
|
||||
let mut out = Vec::new();
|
||||
for plane in 0..planes {
|
||||
for ty in 0..h.div_ceil(th) {
|
||||
for tx in 0..w.div_ceil(tw) {
|
||||
// a strip holds only the rows inside the image, a tile its nominal size
|
||||
let rows = if tw == w { th.min(h - ty * th) } else { th };
|
||||
let mut c = vec![T::default(); tw * rows * ccpp];
|
||||
for y in 0..th.min(h - ty * th) {
|
||||
for x in 0..tw.min(w - tx * tw) {
|
||||
for s in 0..ccpp {
|
||||
let src = ((ty * th + y) * w + tx * tw + x) * cpp + if planar { plane } else { s };
|
||||
c[(y * tw + x) * ccpp + s] = px[src];
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn layout_image(raw: &mut IfdBuilder, chunks: Vec<Vec<u8>>, w: usize, tile: (usize, usize)) {
|
||||
raw.set_image(if tile.0 == w {
|
||||
ImageData::Strips { rows_per_strip: tile.1 as u32, strips: chunks }
|
||||
} else {
|
||||
ImageData::Tiles { tile_width: tile.0 as u32, tile_height: tile.1 as u32, tiles: chunks }
|
||||
});
|
||||
}
|
||||
|
||||
/// Uncompressed integer DNGs in every chunk layout (chunky and planar, tiles and strips with edge chunks, several chunks
|
||||
/// per image) at several bit depths and byte orders decode to the samples they were built from.
|
||||
#[test]
|
||||
fn uncompressed_chunk_layouts_place_every_sample() {
|
||||
let (w, h) = (37, 23);
|
||||
for cpp in [1usize, 3] {
|
||||
for bits in [8u32, 12, 16] {
|
||||
let px = pattern(w, h, cpp, bits);
|
||||
// tiles overhanging both edges, strips with a shorter last strip
|
||||
for tile in [(16, 16), (w, 5), (w, h), (16, 32)] {
|
||||
for planar in [false, true] {
|
||||
if planar && cpp == 1 {
|
||||
continue;
|
||||
}
|
||||
for order in [ByteOrder::Little, ByteOrder::Big] {
|
||||
let ccpp = if planar { 1 } else { cpp };
|
||||
let chunks: Vec<Vec<u8>> = chunk_samples(&px, w, h, cpp, tile.0, tile.1, planar)
|
||||
.iter()
|
||||
.map(|c| match bits {
|
||||
8 => c.iter().map(|&v| v as u8).collect(),
|
||||
16 => u16_bytes(c, order),
|
||||
_ => pack_msb(c, bits, tile.0 * ccpp),
|
||||
})
|
||||
.collect();
|
||||
let mut raw = base_ifd(w, h, bits as u16, cpp, cpp == 1);
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![1]));
|
||||
if planar {
|
||||
raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
|
||||
}
|
||||
layout_image(&mut raw, chunks, w, tile);
|
||||
let img = decode(&dng(raw, order)).unwrap();
|
||||
assert_eq!(img.cpp, cpp);
|
||||
assert_eq!(img.data, RawData::U16(px.clone()), "cpp {cpp} bits {bits} tile {tile:?} planar {planar} {order:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Deflate with the integer and floating-point predictors, chunky and planar tiles and strips.
|
||||
#[test]
|
||||
fn deflate_chunk_layouts_place_every_sample() {
|
||||
let (w, h) = (37, 23);
|
||||
for planar in [false, true] {
|
||||
let cpp = 3;
|
||||
let ccpp = if planar { 1 } else { cpp };
|
||||
let px = pattern(w, h, cpp, 16);
|
||||
for tile in [(16, 16), (w, 5)] {
|
||||
// integer predictor 2 on 16-bit samples
|
||||
let chunks: Vec<Vec<u8>> = chunk_samples(&px, w, h, cpp, tile.0, tile.1, planar)
|
||||
.into_iter()
|
||||
.map(|mut c| {
|
||||
for row in c.chunks_mut(tile.0 * ccpp) {
|
||||
for i in (ccpp..row.len()).rev() {
|
||||
row[i] = row[i].wrapping_sub(row[i - ccpp]);
|
||||
}
|
||||
}
|
||||
miniz_oxide::deflate::compress_to_vec_zlib(&u16_bytes(&c, ByteOrder::Little), 6)
|
||||
})
|
||||
.collect();
|
||||
let mut raw = base_ifd(w, h, 16, cpp, false);
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE]));
|
||||
raw.set(t::PREDICTOR, Value::Short(vec![2]));
|
||||
if planar {
|
||||
raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
|
||||
}
|
||||
layout_image(&mut raw, chunks, w, tile);
|
||||
let img = decode(&dng(raw, ByteOrder::Little)).unwrap();
|
||||
assert_eq!(img.data, RawData::U16(px.clone()), "integer predictor, tile {tile:?} planar {planar}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Floating-point DNGs (uncompressed and with the floating-point predictor) in tiled and strip, chunky and planar
|
||||
/// layouts with several chunks and edge chunks.
|
||||
#[test]
|
||||
fn float_chunk_layouts_place_every_sample() {
|
||||
let (w, h) = (37, 23);
|
||||
for cpp in [1usize, 3] {
|
||||
let vals: Vec<f32> = (0..w * h * cpp).map(|i| ((i * 37) % 200) as f32 / 64.0).collect();
|
||||
for tile in [(16, 16), (w, 5)] {
|
||||
for planar in [false, true] {
|
||||
if planar && cpp == 1 {
|
||||
continue;
|
||||
}
|
||||
let ccpp = if planar { 1 } else { cpp };
|
||||
let parts = chunk_samples(&vals, w, h, cpp, tile.0, tile.1, planar);
|
||||
for bits in [16u16, 24, 32] {
|
||||
let bp = bits as usize / 8;
|
||||
for (pred, order) in [(1u16, ByteOrder::Little), (1, ByteOrder::Big), (3, ByteOrder::Little)] {
|
||||
if pred == 1 && bits != 16 && bits != 32 {
|
||||
continue; // uncompressed 24-bit floats are not written by this test
|
||||
}
|
||||
let chunks: Vec<Vec<u8>> = parts
|
||||
.iter()
|
||||
.map(|c| {
|
||||
if pred == 3 {
|
||||
let enc = fp_encode(c, bp, ccpp, tile.0 * ccpp);
|
||||
miniz_oxide::deflate::compress_to_vec_zlib(&enc, 6)
|
||||
} else if bits == 16 {
|
||||
c.iter()
|
||||
.flat_map(
|
||||
|&v| if order == ByteOrder::Big { f32_to_f16(v).to_be_bytes() } else { f32_to_f16(v).to_le_bytes() },
|
||||
)
|
||||
.collect()
|
||||
} else {
|
||||
c.iter().flat_map(|&v| if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() }).collect()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let mut raw = base_ifd(w, h, bits, cpp, cpp == 1);
|
||||
raw.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![if pred == 3 { compression::DEFLATE } else { 1 }]));
|
||||
if pred == 3 {
|
||||
raw.set(t::PREDICTOR, Value::Short(vec![3]));
|
||||
}
|
||||
if planar {
|
||||
raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
|
||||
}
|
||||
layout_image(&mut raw, chunks, w, tile);
|
||||
let img = decode(&dng(raw, order)).unwrap();
|
||||
assert_eq!(img.data, RawData::F32(vals.clone()), "cpp {cpp} {bits}-bit pred {pred} tile {tile:?} planar {planar} {order:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_finite_float_samples_decode_as_levels() {
|
||||
// NaN samples in a float CFA DNG spread through any arithmetic downstream: the decoder hands on numbers only
|
||||
@@ -1177,12 +1344,77 @@ fn white_level_defaults_to_the_linearization_tables_maximum() {
|
||||
};
|
||||
let img = build(None);
|
||||
assert!(img.linearized);
|
||||
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
|
||||
assert_eq!(img.data, RawData::U16(codes.iter().map(|&c| table[c as usize]).collect()));
|
||||
assert_eq!(img.white, vec![51000.0], "255 × 200, the table's top");
|
||||
// a stated level still wins
|
||||
assert_eq!(build(Some(40000)).white, vec![40000.0]);
|
||||
}
|
||||
|
||||
/// DNG chapter 7: OpcodeList1 runs on the stored codes, then the LinearizationTable maps them to
|
||||
/// sensor values. When list 1 is present, decode leaves the table unapplied so the develop pipeline
|
||||
/// can keep that order. Applying the table first would give a different picture.
|
||||
#[test]
|
||||
fn opcode_list1_runs_on_stored_samples_before_linearization() {
|
||||
let (w, h) = (4, 2);
|
||||
let stored = vec![2u16; w * h];
|
||||
// table: 2 → 20, 7 → 30, 20 → 99 (so the two orders disagree)
|
||||
let mut table = vec![0u16; 32];
|
||||
table[2] = 20;
|
||||
table[7] = 30;
|
||||
table[20] = 99;
|
||||
// MapTable: 2 → 7, 20 → 8
|
||||
let mut map: Vec<u16> = (0..32).collect();
|
||||
map[2] = 7;
|
||||
map[20] = 8;
|
||||
let area = Area { top: 0, left: 0, bottom: h as u32, right: w as u32, plane: 0, planes: 1, row_pitch: 1, col_pitch: 1 };
|
||||
let list1 = opcodes::write_list(&[Opcode::MapTable { area, table: map.clone() }]);
|
||||
|
||||
let mut raw = base_ifd(w, h, 16, 1, true);
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![compression::NONE]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![u16_bytes(&stored, ByteOrder::Little)] });
|
||||
raw.set(t::LINEARIZATION_TABLE, Value::Short(table.clone()));
|
||||
raw.set(t::OPCODE_LIST_1, Value::Undefined(list1));
|
||||
raw.set(t::WHITE_LEVEL, Value::Long(vec![100]));
|
||||
let img = decode(&dng(raw, ByteOrder::Little)).unwrap();
|
||||
assert!(!img.linearized, "list 1 is present, so the table must wait");
|
||||
assert_eq!(img.data, RawData::U16(stored.clone()));
|
||||
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
|
||||
assert_eq!(img.opcodes.list1.len(), 1);
|
||||
|
||||
let apply_map = |samples: &mut [u16]| {
|
||||
for s in samples {
|
||||
let i = (*s as usize).min(map.len() - 1);
|
||||
*s = map[i];
|
||||
}
|
||||
};
|
||||
|
||||
let mut correct = img.clone();
|
||||
if let RawData::U16(v) = &mut correct.data {
|
||||
apply_map(v);
|
||||
}
|
||||
correct.apply_linearization();
|
||||
assert!(correct.linearized);
|
||||
// 2 → 7 (MapTable) → 30 (linearization)
|
||||
assert_eq!(correct.data, RawData::U16(vec![30; w * h]));
|
||||
|
||||
let mut wrong = img.clone();
|
||||
wrong.apply_linearization();
|
||||
if let RawData::U16(v) = &mut wrong.data {
|
||||
apply_map(v);
|
||||
}
|
||||
// 2 → 20 (linearization) → 8 (MapTable)
|
||||
assert_eq!(wrong.data, RawData::U16(vec![8; w * h]));
|
||||
assert_ne!(correct.data, wrong.data);
|
||||
|
||||
// the unapplied table and list 1 survive a write / decode round trip
|
||||
let back = decode(&write_dng(&img, &DngWriteOptions { compression: DngCompression::Uncompressed, ..Default::default() }).unwrap()).unwrap();
|
||||
assert!(!back.linearized);
|
||||
assert_eq!(back.data, RawData::U16(stored));
|
||||
assert_eq!(back.linearization_table.as_deref(), Some(table.as_slice()));
|
||||
assert_eq!(back.opcodes.list1.len(), 1);
|
||||
}
|
||||
|
||||
/// PhotoCraft: `AsShotWhiteXY` is a chromaticity, so both coordinates lie inside (0, 1).
|
||||
#[test]
|
||||
fn an_as_shot_white_outside_the_chromaticity_range_is_ignored() {
|
||||
|
||||
@@ -22,6 +22,7 @@ fn sample(comp: DngCompression) -> Vec<u8> {
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
linearization_table: None,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
camera: Default::default(),
|
||||
|
||||
@@ -2,6 +2,6 @@
|
||||
|
||||
Bounded wire primitives for USD Crate, independent of application scene and material models.
|
||||
|
||||
Provides bounded integer-delta array decoding and Crate-framed LZ4 decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and export remain outside this extraction. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation.
|
||||
Provides bounded integer-delta array decoding and Crate-framed LZ4 compression/decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and full scene export remain outside this extraction. Compression uses a caller cap on worst-case reserved output and fallible allocation. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation.
|
||||
|
||||
Adapted from TextureCraft `crates/formats/src/mesh_import/usd/codec.rs`, introduced by bflatastic in `bdbb52c9b66b221a55b370105b42f485a76e6fc7`. The LZ4 backend is MIT-licensed `lz4_flex`.
|
||||
@@ -0,0 +1,76 @@
|
||||
//! Bounded single-block USD Crate LZ4 writing.
|
||||
|
||||
/// Compression admission, allocation or backend failure.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
/// Input or worst-case output exceeds the supported/caller budget.
|
||||
#[error("USD Crate LZ4 compression exceeds budget")]
|
||||
Limit,
|
||||
/// The bounded output reservation failed.
|
||||
#[error("cannot allocate USD Crate LZ4 output")]
|
||||
Allocation,
|
||||
/// The backend could not encode into the preflighted output buffer.
|
||||
#[error("USD Crate LZ4 compression: {0}")]
|
||||
Encode(#[from] lz4_flex::block::CompressError),
|
||||
}
|
||||
|
||||
fn capacity(input: usize, max_output_bytes: usize) -> Result<usize, Error> {
|
||||
// Keep inputs below the LZ4 single-block limit and preflight the backend's
|
||||
// 20 + input*110/100 sizing arithmetic before calling it, including the count byte.
|
||||
if input > 0x7e00_0000 {
|
||||
return Err(Error::Limit);
|
||||
}
|
||||
input
|
||||
.checked_mul(110)
|
||||
.map(|n| n / 100)
|
||||
.and_then(|n| n.checked_add(21))
|
||||
.filter(|n| *n <= max_output_bytes && *n <= isize::MAX as usize)
|
||||
.ok_or(Error::Limit)
|
||||
}
|
||||
|
||||
/// Write one Crate-framed LZ4 block, beginning with a zero chunk-count byte.
|
||||
///
|
||||
/// `max_output_bytes` bounds the reserved output including framing. Admission uses
|
||||
/// the backend's worst-case size, so even highly compressible input must fit that bound.
|
||||
/// Allocation is fallible; backend scratch memory is not included in the output cap.
|
||||
/// Uses TextureCraft's original LZ4 backend/configuration and produces the same block bytes.
|
||||
pub fn compress(input: &[u8], max_output_bytes: usize) -> Result<Vec<u8>, Error> {
|
||||
let capacity = capacity(input.len(), max_output_bytes)?;
|
||||
let mut output = Vec::new();
|
||||
output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?;
|
||||
output.resize(capacity, 0);
|
||||
let target = output.get_mut(1..).ok_or(Error::Limit)?;
|
||||
let written = lz4_flex::block::compress_into(input, target)?;
|
||||
output.truncate(written.checked_add(1).ok_or(Error::Limit)?);
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn original_wire_bytes_and_roundtrip() {
|
||||
for size in [0, 1, 15, 16, 255, 4096, 65536] {
|
||||
for repeated in [false, true] {
|
||||
let input: Vec<u8> = (0..size).map(|i| if repeated { 17 } else { (i % 251) as u8 }).collect();
|
||||
let mut old = vec![0];
|
||||
old.extend(lz4_flex::block::compress(&input));
|
||||
let encoded = compress(&input, 1 << 20).unwrap();
|
||||
assert_eq!(encoded, old);
|
||||
assert_eq!(crate::decompress(&encoded, size).unwrap(), input);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checks_worst_case_before_allocation() {
|
||||
let input = [0; 100];
|
||||
assert!(matches!(compress(&input, 130), Err(Error::Limit)));
|
||||
assert!(compress(&input, 131).is_ok());
|
||||
assert!(capacity(usize::MAX, usize::MAX).is_err());
|
||||
assert!(capacity(0x7e00_0001, usize::MAX).is_err());
|
||||
assert_eq!(capacity(0, 21).unwrap(), 21);
|
||||
}
|
||||
}
|
||||
@@ -80,6 +80,72 @@ pub fn decode(data: &[u8], count: usize, width: Width, max_output_bytes: usize)
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// Encode integers with their original signed wrapping and largest-delta tie-break policy.
|
||||
///
|
||||
/// `max_bytes` bounds the combined worst-case output reservation and sorted delta scratch.
|
||||
/// Both reservations are fallible. Sorting uses no extra heap storage; input storage and
|
||||
/// allocator bookkeeping are outside this budget. Empty input emits no bytes.
|
||||
pub fn encode(values: &[i64], width: Width, max_bytes: usize) -> Result<Vec<u8>, Error> {
|
||||
if values.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let size = width.bytes();
|
||||
let codes_len = values.len().div_ceil(4);
|
||||
let capacity = values.len().checked_mul(size).and_then(|n| n.checked_add(size)).and_then(|n| n.checked_add(codes_len)).ok_or(Error::Limit)?;
|
||||
let scratch = values.len().checked_mul(8).ok_or(Error::Limit)?;
|
||||
if capacity > isize::MAX as usize || scratch > isize::MAX as usize || capacity.checked_add(scratch).is_none_or(|n| n > max_bytes) {
|
||||
return Err(Error::Limit);
|
||||
}
|
||||
let wrap = |v: i64| match width {
|
||||
Width::I32 => i64::from(v as i32),
|
||||
Width::I64 => v,
|
||||
};
|
||||
let mut deltas = Vec::new();
|
||||
deltas.try_reserve_exact(values.len()).map_err(|_| Error::Allocation)?;
|
||||
let mut previous = 0i64;
|
||||
for &value in values {
|
||||
deltas.push(wrap(value.wrapping_sub(previous)));
|
||||
previous = value;
|
||||
}
|
||||
deltas.sort_unstable();
|
||||
let mut best = (0usize, 0i64);
|
||||
let mut at = 0;
|
||||
while at < deltas.len() {
|
||||
let value = deltas[at];
|
||||
let start = at;
|
||||
while at < deltas.len() && deltas[at] == value {
|
||||
at += 1;
|
||||
}
|
||||
best = best.max((at - start, value));
|
||||
}
|
||||
let common = best.1;
|
||||
let mut output = Vec::new();
|
||||
output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?;
|
||||
output.extend_from_slice(&common.to_le_bytes()[..size]);
|
||||
output.resize(size + codes_len, 0);
|
||||
previous = 0;
|
||||
for (i, &value) in values.iter().enumerate() {
|
||||
let delta = wrap(value.wrapping_sub(previous));
|
||||
previous = value;
|
||||
let small = |bits: u32| delta >= -(1i64 << (bits - 1)) && delta < (1i64 << (bits - 1));
|
||||
let code: u8 = if delta == common {
|
||||
0
|
||||
} else if small(size as u32 * 2) {
|
||||
1
|
||||
} else if small(size as u32 * 4) {
|
||||
2
|
||||
} else {
|
||||
3
|
||||
};
|
||||
if code != 0 {
|
||||
let bytes = (size >> (3 - usize::from(code))).max(1);
|
||||
output.extend_from_slice(&delta.to_le_bytes()[..bytes]);
|
||||
}
|
||||
output[size + i / 4] |= code << (2 * (i % 4));
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
@@ -125,3 +191,81 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod encode_tests {
|
||||
use super::*;
|
||||
fn oracle(values: &[i64], width: usize) -> Vec<u8> {
|
||||
if values.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let wrap = |v: i64| if width == 4 { i64::from(v as i32) } else { v };
|
||||
let mut prev = 0i64;
|
||||
let deltas: Vec<i64> = values
|
||||
.iter()
|
||||
.map(|&v| {
|
||||
let d = wrap(v.wrapping_sub(prev));
|
||||
prev = v;
|
||||
d
|
||||
})
|
||||
.collect();
|
||||
let mut counts = std::collections::HashMap::new();
|
||||
for d in &deltas {
|
||||
*counts.entry(*d).or_insert(0usize) += 1;
|
||||
}
|
||||
let common = counts.into_iter().max_by_key(|&(v, c)| (c, v)).map_or(0, |(v, _)| v);
|
||||
let mut out = Vec::with_capacity(width + values.len().div_ceil(4) + values.len() * width);
|
||||
out.extend_from_slice(&common.to_le_bytes()[..width]);
|
||||
let codes_at = out.len();
|
||||
out.resize(codes_at + values.len().div_ceil(4), 0);
|
||||
for (i, &d) in deltas.iter().enumerate() {
|
||||
let small = |bits: u32| d >= -(1i64 << (bits - 1)) && d < (1i64 << (bits - 1));
|
||||
let code: u8 = if d == common {
|
||||
0
|
||||
} else if small(width as u32 * 2) {
|
||||
1
|
||||
} else if small(width as u32 * 4) {
|
||||
2
|
||||
} else {
|
||||
3
|
||||
};
|
||||
if code > 0 {
|
||||
let w = (width >> (3 - code as usize)).max(1);
|
||||
out.extend_from_slice(&d.to_le_bytes()[..w]);
|
||||
}
|
||||
if let Some(b) = out.get_mut(codes_at + i / 4) {
|
||||
*b |= code << (2 * (i % 4));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_original_bytes_and_wrapping() {
|
||||
for width in [Width::I32, Width::I64] {
|
||||
for n in [0, 1, 2, 3, 4, 5, 127, 1024] {
|
||||
let mut seed = 17u64;
|
||||
let values: Vec<i64> = (0..n)
|
||||
.map(|_| {
|
||||
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
|
||||
seed as i64
|
||||
})
|
||||
.collect();
|
||||
let bytes = encode(&values, width, 32768).unwrap();
|
||||
assert_eq!(bytes, oracle(&values, width.bytes()));
|
||||
let expected: Vec<i64> = values.iter().map(|&v| if width == Width::I32 { i64::from(v as i32) } else { v }).collect();
|
||||
assert_eq!(decode(&bytes, n, width, 32768).unwrap(), expected);
|
||||
}
|
||||
for values in [vec![1, 3, 4, 6], vec![i64::MAX, i64::MIN, 0], vec![7; 32]] {
|
||||
assert_eq!(encode(&values, width, 4096).unwrap(), oracle(&values, width.bytes()));
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn combined_reservation_budget_is_enforced() {
|
||||
assert_eq!(encode(&[], Width::I64, 0).unwrap(), Vec::<u8>::new());
|
||||
assert!(matches!(encode(&[1], Width::I32, 16), Err(Error::Limit)));
|
||||
assert!(encode(&[1], Width::I32, 17).is_ok());
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,9 @@
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
/// Bounded single-block Crate LZ4 compression.
|
||||
pub mod compress;
|
||||
|
||||
/// Bounded Crate integer delta arrays.
|
||||
pub mod integers;
|
||||
|
||||
|
||||
@@ -73,11 +73,17 @@ with libdav1d: **bit-exact** on all 6 fixtures.
|
||||
|
||||
`Decoder::new()` uses all cores (`Decoder::with_threads(1)` decodes on the calling thread; on
|
||||
wasm32, or without the default `threads` feature, everything runs on the calling thread). Headers are parsed in order; each frame is
|
||||
then decoded by a frame worker once the reference frames it reads are finished, so independent
|
||||
frames (hierarchical GOP layers) decode concurrently; tiles of a frame decode in parallel into
|
||||
private buffers that are merged afterwards; the in-loop filters (deblocking, CDEF, loop
|
||||
restoration) each run in parallel bands of 64 rows (`postfilter.rs` explains why the bands are
|
||||
independent). Results never depend on scheduling (the oracle tests compare 1 thread, many threads
|
||||
then decoded by a frame worker. With worker threads it first reads the symbols of all its tiles,
|
||||
which needs only the parse results of its references (CDFs, segment ids, saved motion vectors),
|
||||
publishes its own, and then reconstructs from the recorded blocks. Frames with several tiles
|
||||
read the symbols of their tiles in parallel (one after another with intra block copy). A frame reconstructs superblock row by superblock
|
||||
row (that row of every tile), each row waiting only for the reference rows its motion vectors
|
||||
reach (row wavefront), and the in-loop filters (deblocking, CDEF, loop restoration) follow a few
|
||||
64-row bands behind and publish each final band for the frames predicting from it
|
||||
(`decoder.rs` and `postfilter.rs` give the dependency rules). Super-resolution frames wait for
|
||||
their whole references and filter the whole frame. There is one frame worker per thread (at most
|
||||
16), and the frames in flight are bounded by their size. Independent frames (hierarchical GOP
|
||||
layers) decode concurrently. Results never depend on scheduling (the oracle tests compare 1 thread, many threads
|
||||
and libdav1d). With frame threads, pictures may be returned by a
|
||||
later `decode_pts` call or by `flush`; each carries the `pts` of the temporal unit that showed it.
|
||||
|
||||
@@ -130,6 +136,44 @@ motion vectors nor CDFs reach another frame — skip deblocking, CDEF and loop r
|
||||
(super-resolution still runs, it sets the output size) and are flagged `Picture::draft`. In
|
||||
SVT-AV1's hierarchical GOPs that is the top layer, half of the frames.
|
||||
|
||||
Frame threading on a Ryzen 7800X3D (8 cores / 16 threads, release `examples/av1_av1dec`, median of 7
|
||||
alternating runs against Filmcraft `6909e461`, 2026-11-08), wall time for the whole file:
|
||||
|
||||
| Fixture | Threads | Before | After | Speed-up |
|
||||
|---|---|---|---|---|
|
||||
| `perf_1080p_gop` (60 frames) | 1 | 69.2 fps | 69.1 fps | 1.00x |
|
||||
| `perf_1080p_gop` | 16 | 159.6 fps | 227.3 fps | 1.42x |
|
||||
| `perf_2160p_gop` (30 frames) | 1 | 17.9 fps | 17.8 fps | 1.00x |
|
||||
| `perf_2160p_gop` | 16 | 40.2 fps | 57.7 fps | 1.43x |
|
||||
|
||||
Peak RSS at 16 threads goes from 278 to 365 MB (1080p) and from 1031 to 1217 MB (2160p, +18%):
|
||||
every frame in flight holds its parse record and its final samples are kept in 64-row bands.
|
||||
|
||||
With one frame worker per thread, the row wavefront for frames with several tiles and parse
|
||||
waits limited to what parsing reads (same machine, median of 5 alternating runs, 2026-11-08):
|
||||
|
||||
| Fixture | Threads | Before (`6909e461`) | After | Speed-up | Peak RSS before / after |
|
||||
|---|---|---|---|---|---|
|
||||
| `perf_1080p_gop` | 4 | 95.7 fps | 185.2 fps | 1.94x | 148 / 236 MB |
|
||||
| `perf_1080p_gop` | 16 | 155.0 fps | 259.7 fps | 1.68x | 274 / 560 MB |
|
||||
| `perf_2160p_gop` | 4 | 23.6 fps | 47.1 fps | 2.00x | 512 / 798 MB |
|
||||
| `perf_2160p_gop` | 16 | 39.9 fps | 65.8 fps | 1.65x | 994 / 1353 MB |
|
||||
| `perf_1080p_gop_tiles` (4x2 tiles) | 16 | 186.3 fps | 237.2 fps | 1.27x | 328 / 617 MB |
|
||||
| SVT-AV1 2160p, 4x2 tiles (30 frames) | 16 | 57.1 fps | 70.8 fps | 1.24x | 1197 / 1502 MB |
|
||||
|
||||
Known limitations of the frame threading (the output is the same in every case, only the
|
||||
overlap is smaller):
|
||||
|
||||
- A frame whose parse record would pass 32 bytes per luma sample of a tile (pathological streams:
|
||||
tiny blocks, large palettes, every coefficient coded), or that cannot reserve its record from
|
||||
the decoder-wide budget of 1 GiB shared by the frames in flight, is decoded again in one pass.
|
||||
- A superblock row using local warped motion, global motion beyond translation or a scaled
|
||||
reference waits for the whole reference instead of the rows its motion vectors reach.
|
||||
- Super-resolution frames do not use the row wavefront: they wait for their whole references
|
||||
and filter the whole frame.
|
||||
- The tile columns of a superblock row are reconstructed one after another on the frame's worker
|
||||
(their symbols are still read in parallel).
|
||||
|
||||
## Limits
|
||||
|
||||
Frames beyond the AV1 level 6.x limits (Annex A: at most 16384 wide, 8704 high and 35 651 584 luma samples
|
||||
|
||||
@@ -10,19 +10,33 @@ use crate::av1::decode::state::FrameState;
|
||||
/// equals CurrFrame). Bands of 64 luma rows (whole 64x64 filter blocks) are filtered in
|
||||
/// parallel: every block reads only CurrFrame and writes only its own samples.
|
||||
pub(crate) fn apply(fs: &FrameState, pool: &Pool) -> Option<FrameBuf> {
|
||||
if !active(fs) {
|
||||
return None;
|
||||
}
|
||||
let mut out = fs.cur.clone();
|
||||
let ranges = bands(pool, fs.fh.mi_rows as usize * 4);
|
||||
filter_bands(fs, pool, out.row_bands(&ranges), &ranges);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Whether CDEF filters any block of the frame (it reads only parse results).
|
||||
pub(crate) fn active(fs: &FrameState) -> bool {
|
||||
let fh = &fs.fh;
|
||||
if fh.coded_lossless || fh.allow_intrabc || !fs.seq.enable_cdef {
|
||||
return None;
|
||||
return false;
|
||||
}
|
||||
let (mi_rows, mi_cols) = (fh.mi_rows as usize, fh.mi_cols as usize);
|
||||
let fb_rows = mi_rows.div_ceil(16);
|
||||
let fb_cols = mi_cols.div_ceil(16);
|
||||
if !(0..fb_rows).any(|fr| (0..fb_cols).any(|fc| fb_skip_mask(fs, fr * 16, fc * 16).is_some())) {
|
||||
return None;
|
||||
}
|
||||
let mut out = fs.cur.clone();
|
||||
let ranges = bands(pool, mi_rows * 4);
|
||||
let mut planes = out.row_bands(&ranges);
|
||||
(0..fb_rows).any(|fr| (0..fb_cols).any(|fc| fb_skip_mask(fs, fr * 16, fc * 16).is_some()))
|
||||
}
|
||||
|
||||
/// Filter the bands `planes` covering luma rows `ranges` (whole 64x64 filter block rows), which
|
||||
/// hold CurrFrame; reads CurrFrame up to 2 rows beyond each band.
|
||||
pub(crate) fn filter_bands(fs: &FrameState, pool: &Pool, mut planes: Vec<[RowBand; 3]>, ranges: &[(usize, usize)]) {
|
||||
let (mi_rows, mi_cols) = (fs.fh.mi_rows as usize, fs.fh.mi_cols as usize);
|
||||
let fb_rows = mi_rows.div_ceil(16);
|
||||
let fb_cols = mi_cols.div_ceil(16);
|
||||
pool.map_mut(&mut planes, |b, band| {
|
||||
let (start, end) = ranges[b];
|
||||
let mut buf = [0i16; BUF_LEN];
|
||||
@@ -34,8 +48,6 @@ pub(crate) fn apply(fs: &FrameState, pool: &Pool) -> Option<FrameBuf> {
|
||||
}
|
||||
}
|
||||
});
|
||||
drop(planes);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// The filtered 8x8 blocks of the 64x64 filter block at (r, c): bit `(i << 3) | j` is set for
|
||||
|
||||
+1044
-110
File diff suppressed because it is too large.
Load diff
@@ -197,6 +197,50 @@ impl FrameBuf {
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy luma rows [y0, y1) (chroma rows scaled down; clipped to the stored rows) of `src`, a
|
||||
/// frame of the same geometry.
|
||||
pub fn copy_rows_from(&mut self, src: &FrameBuf, y0: usize, y1: usize) {
|
||||
for p in 0..self.num_planes {
|
||||
let sy = if p == 0 { 0 } else { self.subsampling_y };
|
||||
let (d, s) = (&mut self.planes[p], &src.planes[p]);
|
||||
if d.stride != s.stride {
|
||||
continue;
|
||||
}
|
||||
let a = (y0 >> sy).min(d.rows).min(s.rows) * d.stride;
|
||||
let b = (y1 >> sy).min(d.rows).min(s.rows) * d.stride;
|
||||
if let (Some(dst), Some(src)) = (d.data.get_mut(a..b), s.data.get(a..b)) {
|
||||
dst.copy_from_slice(src);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies of luma rows [y0, y1) (chroma rows scaled down; clipped to the stored rows), one
|
||||
/// plane region each.
|
||||
pub fn rows_copy(&self, y0: usize, y1: usize) -> [Plane; 3] {
|
||||
std::array::from_fn(|p| {
|
||||
if p >= self.num_planes {
|
||||
return Plane::default();
|
||||
}
|
||||
let sy = if p == 0 { 0 } else { self.subsampling_y };
|
||||
let s = &self.planes[p];
|
||||
let (a, b) = ((y0 >> sy).min(s.rows), (y1 >> sy).min(s.rows));
|
||||
let mut out = Plane::region(0, a, s.stride, b - a);
|
||||
if let Some(src) = s.data.get(a * s.stride..b * s.stride) {
|
||||
out.data.copy_from_slice(src);
|
||||
}
|
||||
out
|
||||
})
|
||||
}
|
||||
|
||||
/// A frame of this geometry (samples not copied: zero).
|
||||
pub fn same_geometry(&self) -> FrameBuf {
|
||||
let planes = std::array::from_fn(|p| {
|
||||
let pl = &self.planes[p];
|
||||
Plane { data: zeroed(pl.data.len()), stride: pl.stride, rows: pl.rows, ox: pl.ox, oy: pl.oy }
|
||||
});
|
||||
FrameBuf { planes, ..*self }
|
||||
}
|
||||
|
||||
/// [`Plane::row_bands`] of all three planes for ranges of luma rows (chroma rows scaled
|
||||
/// down): element `i` holds the bands of range `i`.
|
||||
pub fn row_bands(&mut self, luma: &[(usize, usize)]) -> Vec<[RowBand<'_>; 3]> {
|
||||
@@ -208,6 +252,128 @@ impl FrameBuf {
|
||||
}
|
||||
}
|
||||
|
||||
/// Luma rows per published band of a [`BandFrame`] (log2).
|
||||
pub const BAND_LOG2: u32 = 6;
|
||||
|
||||
/// One plane of a frame's final samples, published band by band while the rest of the frame
|
||||
/// is still being decoded (see [`BandFrame`]). A band is written once and then only read.
|
||||
pub struct BandPlane {
|
||||
bands: Vec<std::sync::OnceLock<Plane>>,
|
||||
stride: usize,
|
||||
rows: usize,
|
||||
log2: u32,
|
||||
/// Returned for rows of a band that is not published: zeros instead of a panic. Readers wait
|
||||
/// for the row progress first and references of another chroma format are refused, so
|
||||
/// this only happens through a scheduling bug.
|
||||
missing: Box<[u16]>,
|
||||
}
|
||||
|
||||
impl BandPlane {
|
||||
fn new(stride: usize, rows: usize, log2: u32) -> BandPlane {
|
||||
let n = rows.div_ceil(1 << log2);
|
||||
BandPlane { bands: (0..n).map(|_| std::sync::OnceLock::new()).collect(), stride, rows, log2, missing: vec![0; stride].into_boxed_slice() }
|
||||
}
|
||||
|
||||
/// Row `y` (`stride` samples) of a published band.
|
||||
#[inline(always)]
|
||||
pub fn row(&self, y: usize) -> &[u16] {
|
||||
let o = (y & ((1 << self.log2) - 1)) * self.stride;
|
||||
match self.bands.get(y >> self.log2).and_then(|b| b.get()).and_then(|b| b.data.get(o..o + self.stride)) {
|
||||
Some(r) => r,
|
||||
None => {
|
||||
debug_assert!(false, "row {y} read before its band was published");
|
||||
&self.missing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish band `band` copied from the rows of the whole plane `src` (same geometry); a
|
||||
/// band is published once.
|
||||
fn publish(&self, band: usize, src: &Plane) {
|
||||
let y0 = band << self.log2;
|
||||
let y1 = ((band + 1) << self.log2).min(self.rows);
|
||||
let mut p = Plane::region(0, y0, self.stride, y1.saturating_sub(y0));
|
||||
let w = self.stride.min(src.stride);
|
||||
for (y, out) in (y0..y1).zip(p.data.chunks_exact_mut(self.stride.max(1))) {
|
||||
if let Some(r) = src.data.get(y * src.stride..y * src.stride + w) {
|
||||
out[..w].copy_from_slice(r);
|
||||
}
|
||||
}
|
||||
self.publish_owned(band, p);
|
||||
}
|
||||
|
||||
/// Publish band `band` as `p`, whose rows start at the band's first row (stride `stride`;
|
||||
/// rows past the band are ignored).
|
||||
fn publish_owned(&self, band: usize, p: Plane) {
|
||||
if let Some(slot) = self.bands.get(band)
|
||||
&& p.stride == self.stride
|
||||
{
|
||||
let _ = slot.set(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A frame's final samples (every in-loop filter applied) in bands of 64 luma rows, so later
|
||||
/// frames can predict from the top of a reference while its bottom is still being decoded.
|
||||
/// Each band is published once, before the frame's row progress says it is final.
|
||||
pub struct BandFrame {
|
||||
pub planes: [BandPlane; 3],
|
||||
pub num_planes: usize,
|
||||
pub subsampling_x: usize,
|
||||
pub subsampling_y: usize,
|
||||
}
|
||||
|
||||
impl BandFrame {
|
||||
/// Bands for the visible rows of a `width` x `height` frame stored like [`FrameBuf::new`].
|
||||
pub fn new(width: usize, height: usize, num_planes: usize, subsampling_x: usize, subsampling_y: usize) -> BandFrame {
|
||||
let aw = width.div_ceil(128) * 128 + 160;
|
||||
let planes = std::array::from_fn(|p| {
|
||||
if p >= num_planes {
|
||||
return BandPlane::new(0, 0, BAND_LOG2);
|
||||
}
|
||||
let (sx, sy) = if p == 0 { (0, 0) } else { (subsampling_x, subsampling_y) };
|
||||
BandPlane::new(aw >> sx, (height + sy) >> sy, BAND_LOG2 - sy as u32)
|
||||
});
|
||||
BandFrame { planes, num_planes, subsampling_x, subsampling_y }
|
||||
}
|
||||
|
||||
/// Number of bands (of 64 luma rows).
|
||||
pub fn bands(&self) -> usize {
|
||||
self.planes[0].bands.len()
|
||||
}
|
||||
|
||||
/// Publish luma band `band` (and the chroma rows it covers) copied from `src`.
|
||||
pub fn publish(&self, band: usize, src: &FrameBuf) {
|
||||
for p in 0..self.num_planes {
|
||||
self.planes[p].publish(band, &src.planes[p]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish luma band `band` from per-plane buffers starting at the band's first row.
|
||||
pub fn publish_owned(&self, band: usize, planes: [Plane; 3]) {
|
||||
for (p, pl) in planes.into_iter().enumerate().take(self.num_planes) {
|
||||
self.planes[p].publish_owned(band, pl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A plane to predict from: a whole stored plane or the published bands of a reference.
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum PlaneRef<'a> {
|
||||
Whole(&'a Plane),
|
||||
Bands(&'a BandPlane),
|
||||
}
|
||||
|
||||
impl<'a> PlaneRef<'a> {
|
||||
#[inline(always)]
|
||||
pub fn row(&self, y: usize) -> &'a [u16] {
|
||||
match *self {
|
||||
PlaneRef::Whole(p) => p.row(y),
|
||||
PlaneRef::Bands(b) => b.row(y),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A motion vector (row, col) in 1/8 sample units.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Mv {
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use crate::av1::decode::frame::Plane;
|
||||
use crate::av1::decode::frame::{Plane, PlaneRef};
|
||||
use crate::av1::decode::mvpred::round2_signed64;
|
||||
use crate::av1::decode::spec_tables::{
|
||||
BLOCK_8X8, BLOCK_SIZES, COMPOUND_AVERAGE, COMPOUND_DIFFWTD, COMPOUND_DISTANCE, COMPOUND_INTRA, COMPOUND_WEDGE, DIV_LUT, DIV_LUT_BITS,
|
||||
@@ -227,12 +227,9 @@ impl TileDecoder<'_, '_> {
|
||||
(start_x as i32, start_y as i32, step_x as i32, step_y as i32)
|
||||
}
|
||||
|
||||
fn ref_plane(&self, ref_idx: i32, plane: usize) -> &Plane {
|
||||
fn ref_plane(&self, ref_idx: i32, plane: usize) -> PlaneRef<'_> {
|
||||
// A missing reference (rejected earlier for valid streams) predicts from the current frame.
|
||||
match usize::try_from(ref_idx).ok().and_then(|i| self.fs.refs.get(i)).and_then(|r| r.as_ref()) {
|
||||
Some(r) => &r.buf.planes[plane],
|
||||
None => &self.t.cur.planes[plane],
|
||||
}
|
||||
self.fs.ref_samples(ref_idx, plane).unwrap_or(PlaneRef::Whole(&self.t.cur.planes[plane]))
|
||||
}
|
||||
|
||||
/// Block inter prediction process (7.11.3.4) into `pred` (stride PRED_STRIDE); `tmp` holds
|
||||
@@ -512,12 +509,12 @@ impl TileDecoder<'_, '_> {
|
||||
return false;
|
||||
}
|
||||
let fs = self.fs;
|
||||
let Some(r) = usize::try_from(ref_idx).ok().and_then(|i| fs.refs.get(i)).and_then(|r| r.as_ref()) else {
|
||||
let Some(refp) = fs.ref_samples(ref_idx, plane) else {
|
||||
return false;
|
||||
};
|
||||
let dims = self.ref_plane_dims(ref_idx, plane);
|
||||
let max = ((1u32 << fs.bit_depth) - 1) as u16;
|
||||
copy_fullpel(&r.buf.planes[plane], dims, McBlock { x: sx, y: sy, w, h }, &mut self.t.cur.planes[plane], (x, y), max);
|
||||
copy_fullpel(refp, dims, McBlock { x: sx, y: sy, w, h }, &mut self.t.cur.planes[plane], (x, y), max);
|
||||
true
|
||||
}
|
||||
|
||||
@@ -763,7 +760,7 @@ impl TileDecoder<'_, '_> {
|
||||
/// Unscaled single-reference prediction at a whole-sample position: both filter phases are the
|
||||
/// identity and Round2(128 * Round2(128 * v, InterRound0), InterRound1) == v for every bit
|
||||
/// depth, so the prediction is the (clamped) reference sample.
|
||||
fn copy_fullpel(refp: &Plane, (last_x, last_y): (i32, i32), src_blk: McBlock, dst: &mut Plane, (dx, dy): (usize, usize), max: u16) {
|
||||
fn copy_fullpel(refp: PlaneRef, (last_x, last_y): (i32, i32), src_blk: McBlock, dst: &mut Plane, (dx, dy): (usize, usize), max: u16) {
|
||||
let McBlock { x, y, w, h } = src_blk;
|
||||
let x0 = x >> 10;
|
||||
let y0 = y >> 10;
|
||||
@@ -831,7 +828,7 @@ fn vpass<const W: usize>(rows: &[i32], vf: &[i16; 8], out: &mut [i32], half: i32
|
||||
/// Block inter prediction without reference scaling: every column / row uses the same filter
|
||||
/// phase, so the filters run over whole rows (bit-identical to the per-sample process).
|
||||
fn mc_unscaled(
|
||||
refp: &Plane,
|
||||
refp: PlaneRef,
|
||||
(last_x, last_y): (i32, i32),
|
||||
blk: McBlock,
|
||||
(hf, vf): (&[i16; 8], &[i16; 8]),
|
||||
@@ -982,9 +979,17 @@ mod tests {
|
||||
[(0, 0), (-5, -3), (7, 9), (last_x - w as i32 / 2, last_y - h as i32 / 2), (last_x + 3, last_y + 2), (-(w as i32) - 4, 3)]
|
||||
{
|
||||
let (x, y) = ((px << 10) | 32, (py << 10) | 32);
|
||||
mc_unscaled(&refp, (last_x, last_y), McBlock { x, y, w, h }, (&IDENTITY_TAPS, &IDENTITY_TAPS), rnd, &mut pred, &mut tmp);
|
||||
mc_unscaled(
|
||||
PlaneRef::Whole(&refp),
|
||||
(last_x, last_y),
|
||||
McBlock { x, y, w, h },
|
||||
(&IDENTITY_TAPS, &IDENTITY_TAPS),
|
||||
rnd,
|
||||
&mut pred,
|
||||
&mut tmp,
|
||||
);
|
||||
let mut dst = Plane::new(w + 3, h);
|
||||
copy_fullpel(&refp, (last_x, last_y), McBlock { x, y, w, h }, &mut dst, (3, 0), max);
|
||||
copy_fullpel(PlaneRef::Whole(&refp), (last_x, last_y), McBlock { x, y, w, h }, &mut dst, (3, 0), max);
|
||||
for i in 0..h {
|
||||
for j in 0..w {
|
||||
let want = spec_phase0(&refp, (last_x, last_y), (px, py), (j, i), (w * 3 + h) % SUBPEL_FILTERS.len(), rnd)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! In-loop post filters run by decode_frame_wrapup: loop filter (7.14), CDEF (7.15),
|
||||
//! super-resolution upscaling (7.16) and loop restoration (7.17).
|
||||
|
||||
use crate::av1::decode::frame::RowBand;
|
||||
use crate::av1::decode::frame::{FrameBuf, Plane, RowBand};
|
||||
use crate::av1::decode::spec_tables::{
|
||||
GLOBAL_GLOBALMV, GLOBALMV, INTRA_FRAME, MAX_LOOP_FILTER, NEARESTMV, NUM_4X4_BLOCKS_HIGH, NUM_4X4_BLOCKS_WIDE, SEG_LVL_ALT_LF_Y_V, TX_HEIGHT,
|
||||
TX_WIDTH,
|
||||
@@ -48,6 +48,121 @@ pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, pool: &crate::
|
||||
stats.add(Stage::Restoration, t.lap());
|
||||
}
|
||||
|
||||
/// The in-loop filters run band by band behind reconstruction (row wavefront): each stage of a
|
||||
/// band starts once the stage before has finished the band below it, which bounds what the
|
||||
/// stage reads beyond the band (the loop filter changes up to 7 rows above a band and intra
|
||||
/// prediction of the next superblock row reads the unfiltered row above it; CDEF reads 2 rows
|
||||
/// and loop restoration 3 rows beyond a band). Bands are 64 luma rows. The result is the same
|
||||
/// as running each filter over the whole frame. Super-resolution frames use [`apply`].
|
||||
///
|
||||
/// The last active stage writes each band into its own buffers, which are handed out as the
|
||||
/// band's final samples without a copy.
|
||||
pub(crate) struct Pipeline {
|
||||
bands: Vec<(usize, usize)>,
|
||||
/// Luma rows of the coded frame (mi_rows * 4).
|
||||
rows: usize,
|
||||
lf_on: bool,
|
||||
cdef_on: bool,
|
||||
lr_on: bool,
|
||||
/// Bands finished by each stage.
|
||||
lf: usize,
|
||||
cdef: usize,
|
||||
lr: usize,
|
||||
pending: [Vec<u16>; 3],
|
||||
/// CdefFrame when loop restoration reads it.
|
||||
cdef_out: Option<FrameBuf>,
|
||||
}
|
||||
|
||||
/// Final bands from [`Pipeline::advance`]: (band, its samples starting at the band's first row).
|
||||
pub(crate) type FinalBands = Vec<(usize, [Plane; 3])>;
|
||||
|
||||
impl Pipeline {
|
||||
pub fn new(fs: &FrameState, draft: bool) -> Pipeline {
|
||||
let rows = fs.fh.mi_rows as usize * 4;
|
||||
let n = rows.div_ceil(BAND_ROWS).max(1);
|
||||
let bands = (0..n).map(|b| (b * BAND_ROWS, if b + 1 == n { usize::MAX } else { (b + 1) * BAND_ROWS })).collect();
|
||||
let lvl = fs.fh.lf.level;
|
||||
let cdef_on = !draft && crate::av1::decode::cdef::active(fs);
|
||||
let lr_on = fs.fh.lr.uses_lr && !draft;
|
||||
let cdef_out = (cdef_on && lr_on).then(|| fs.cur.same_geometry());
|
||||
Pipeline {
|
||||
bands,
|
||||
rows,
|
||||
lf_on: (lvl[0] != 0 || lvl[1] != 0) && !draft,
|
||||
cdef_on,
|
||||
lr_on,
|
||||
lf: 0,
|
||||
cdef: 0,
|
||||
lr: 0,
|
||||
pending: Default::default(),
|
||||
cdef_out,
|
||||
}
|
||||
}
|
||||
|
||||
/// Filter what luma rows [0, `recon_rows`) of reconstructed samples allow (everything once
|
||||
/// they cover the frame); returns the bands whose samples became final.
|
||||
pub fn advance(&mut self, fs: &mut FrameState, stats: &mut DecodeStats, pool: &crate::av1::decode::par::Pool, recon_rows: usize) -> FinalBands {
|
||||
let n = self.bands.len();
|
||||
let all = recon_rows >= self.rows;
|
||||
let mut out = FinalBands::new();
|
||||
let mut t = Timer::start();
|
||||
// The loop filter of a band waits for the band below it to be reconstructed.
|
||||
let lf_to = if all { n } else { (recon_rows / BAND_ROWS).saturating_sub(1) };
|
||||
if lf_to > self.lf {
|
||||
if self.lf_on {
|
||||
loop_filter_bands(fs, pool, &self.bands, self.lf..lf_to, &mut self.pending);
|
||||
}
|
||||
self.lf = lf_to;
|
||||
}
|
||||
stats.add(Stage::LoopFilter, t.lap());
|
||||
let cdef_to = if all { n } else { self.lf.saturating_sub(1) };
|
||||
if cdef_to > self.cdef {
|
||||
let r = &self.bands[self.cdef..cdef_to];
|
||||
if self.cdef_on {
|
||||
if let Some(cdef) = &mut self.cdef_out {
|
||||
cdef.copy_rows_from(&fs.cur, r[0].0, r[r.len() - 1].1);
|
||||
crate::av1::decode::cdef::filter_bands(fs, pool, cdef.row_bands(r), r);
|
||||
} else {
|
||||
let mut own: Vec<[Plane; 3]> = r.iter().map(|&(a, b)| fs.cur.rows_copy(a, b)).collect();
|
||||
crate::av1::decode::cdef::filter_bands(fs, pool, own_bands(&mut own), r);
|
||||
out.extend((self.cdef..cdef_to).zip(own));
|
||||
}
|
||||
} else if !self.lr_on {
|
||||
// The loop filter is the last stage: the band is final once the band below it
|
||||
// is filtered.
|
||||
out.extend((self.cdef..cdef_to).zip(r.iter().map(|&(a, b)| fs.cur.rows_copy(a, b))));
|
||||
}
|
||||
self.cdef = cdef_to;
|
||||
}
|
||||
stats.add(Stage::Cdef, t.lap());
|
||||
let lr_to = if all { n } else { self.cdef.saturating_sub(1) };
|
||||
if self.lr_on && lr_to > self.lr {
|
||||
let r = &self.bands[self.lr..lr_to];
|
||||
let cdef = self.cdef_out.as_ref().unwrap_or(&fs.cur);
|
||||
let mut own: Vec<[Plane; 3]> = r.iter().map(|&(a, b)| cdef.rows_copy(a, b)).collect();
|
||||
crate::av1::decode::restoration::restore_bands(fs, &fs.cur, cdef, own_bands(&mut own), pool, r);
|
||||
out.extend((self.lr..lr_to).zip(own));
|
||||
self.lr = lr_to;
|
||||
}
|
||||
stats.add(Stage::Restoration, t.lap());
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Row bands over per-band buffers made by [`FrameBuf::rows_copy`].
|
||||
fn own_bands(own: &mut [[Plane; 3]]) -> Vec<[RowBand<'_>; 3]> {
|
||||
fn band(p: &mut Plane) -> RowBand<'_> {
|
||||
let (stride, y0) = (p.stride.max(1), p.oy);
|
||||
RowBand { data: p.data.as_mut_slice(), stride, y0 }
|
||||
}
|
||||
own.iter_mut()
|
||||
.map(|planes| {
|
||||
let [a, b, c] = planes;
|
||||
[band(a), band(b), band(c)]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Loop filter (7.14)
|
||||
|
||||
@@ -69,23 +184,40 @@ const LF_MARGIN: usize = 8;
|
||||
/// written back; the band above never touches them.
|
||||
fn loop_filter(fs: &mut FrameState, pool: &crate::av1::decode::par::Pool) {
|
||||
let luma = bands(pool, fs.fh.mi_rows as usize * 4);
|
||||
loop_filter_bands(fs, pool, &luma, 0..luma.len(), &mut Default::default());
|
||||
}
|
||||
|
||||
/// The loop filter on bands `range` of `all` (the frame's bands). The bands above the range
|
||||
/// are filtered already: `pending` holds the unfiltered rows above the first band of the range
|
||||
/// (empty for band 0), saved before the band above was filtered, and is set to those of the
|
||||
/// band after the range.
|
||||
fn loop_filter_bands(
|
||||
fs: &mut FrameState,
|
||||
pool: &crate::av1::decode::par::Pool,
|
||||
all: &[(usize, usize)],
|
||||
range: std::ops::Range<usize>,
|
||||
pending: &mut [Vec<u16>; 3],
|
||||
) {
|
||||
let Some(luma) = all.get(range.clone()) else { return };
|
||||
let planes: Vec<usize> = (0..fs.num_planes).filter(|&p| p == 0 || fs.fh.lf.level[1 + p] != 0).collect();
|
||||
let mut cur = std::mem::take(&mut fs.cur);
|
||||
let f: &FrameState = fs;
|
||||
// Unfiltered copies of the rows above each band but the first.
|
||||
let margins: Vec<[Vec<u16>; 3]> = luma
|
||||
.iter()
|
||||
.map(|&(start, _)| {
|
||||
std::array::from_fn(|p| {
|
||||
let sub_y = if p == 0 { 0 } else { f.ssy };
|
||||
let pl = &cur.planes[p];
|
||||
let (y0, y1) = ((start.saturating_sub(LF_MARGIN)) >> sub_y, start >> sub_y);
|
||||
let rows = pl.data.get(y0 * pl.stride..y1 * pl.stride);
|
||||
if start == 0 || !planes.contains(&p) { Vec::new() } else { rows.map(<[u16]>::to_vec).unwrap_or_default() }
|
||||
})
|
||||
// Unfiltered copies of the rows above a band (none above the first).
|
||||
let margin = |cur: &crate::av1::decode::frame::FrameBuf, start: usize| -> [Vec<u16>; 3] {
|
||||
std::array::from_fn(|p| {
|
||||
let sub_y = if p == 0 { 0 } else { f.ssy };
|
||||
let pl = &cur.planes[p];
|
||||
let (y0, y1) = ((start.saturating_sub(LF_MARGIN)) >> sub_y, start >> sub_y);
|
||||
let rows = pl.data.get(y0 * pl.stride..y1 * pl.stride);
|
||||
if start == 0 || !planes.contains(&p) { Vec::new() } else { rows.map(<[u16]>::to_vec).unwrap_or_default() }
|
||||
})
|
||||
.collect();
|
||||
let mut bands: Vec<([RowBand; 3], [Vec<u16>; 3])> = cur.row_bands(&luma).into_iter().zip(margins).collect();
|
||||
};
|
||||
let margins: Vec<[Vec<u16>; 3]> =
|
||||
luma.iter().enumerate().map(|(i, &(start, _))| if i == 0 { std::mem::take(pending) } else { margin(&cur, start) }).collect();
|
||||
if let Some(&(next, _)) = all.get(range.end) {
|
||||
*pending = margin(&cur, next);
|
||||
}
|
||||
let mut bands: Vec<([RowBand; 3], [Vec<u16>; 3])> = cur.row_bands(luma).into_iter().zip(margins).collect();
|
||||
let changed = pool.map_mut(&mut bands, |b, (band, margin)| {
|
||||
let mut changed: [Vec<(usize, u16)>; 3] = Default::default();
|
||||
for &p in &planes {
|
||||
|
||||
@@ -70,10 +70,16 @@ struct LrPlane<'a> {
|
||||
/// Bands of 64 luma rows are restored in parallel: every unit reads only the two input frames
|
||||
/// and writes only its own samples.
|
||||
pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf, pool: &Pool) -> FrameBuf {
|
||||
let fh = &fs.fh;
|
||||
let mut lr = cdef.clone();
|
||||
let ranges = bands(pool, fh.frame_height as usize);
|
||||
let mut planes = lr.row_bands(&ranges);
|
||||
let ranges = bands(pool, fs.fh.frame_height as usize);
|
||||
restore_bands(fs, cur, cdef, lr.row_bands(&ranges), pool, &ranges);
|
||||
lr
|
||||
}
|
||||
|
||||
/// Restore the bands `planes` covering luma rows `ranges`, which hold UpscaledCdefFrame; reads
|
||||
/// both inputs up to 3 rows beyond each band.
|
||||
pub(crate) fn restore_bands(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf, mut planes: Vec<[RowBand; 3]>, pool: &Pool, ranges: &[(usize, usize)]) {
|
||||
let fh = &fs.fh;
|
||||
pool.map_mut(&mut planes, |b, band| {
|
||||
let (start, end) = ranges[b];
|
||||
let mut scratch = Scratch::default();
|
||||
@@ -100,8 +106,6 @@ pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf,
|
||||
p.restore(out, start >> sub_y, end >> sub_y, &mut scratch);
|
||||
}
|
||||
});
|
||||
drop(planes);
|
||||
lr
|
||||
}
|
||||
|
||||
/// Row buffers reused across stripes.
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
//! with several, each tile decodes into private buffers covering its area (tiles never read
|
||||
//! outside themselves), which are then copied into the frame.
|
||||
|
||||
use crate::av1::decode::decoder::RefFrame;
|
||||
use crate::av1::decode::frame::{FrameBuf, MiInfo};
|
||||
use crate::av1::decode::decoder::{Parsed, RefFrame};
|
||||
use crate::av1::decode::frame::{BandFrame, FrameBuf, MiInfo, PlaneRef};
|
||||
use crate::av1::decode::header::{FrameHeader, RefInfo, SequenceHeader};
|
||||
use crate::av1::decode::spec_tables::{DEFAULT_INTRA_FRAME_Y_MODE_CDF, NUM_REF_FRAMES, RESTORE_NONE, SUBSAMPLED_SIZE};
|
||||
use std::ops::{Deref, DerefMut};
|
||||
@@ -26,6 +26,11 @@ pub(crate) struct FrameShared {
|
||||
pub cdef_cols: usize,
|
||||
/// Reference slots (FrameStore and saved state) used by this frame.
|
||||
pub refs: [Option<Arc<RefFrame>>; NUM_REF_FRAMES],
|
||||
/// Parse results (CDFs, segment ids, saved motion vectors) of the same slots.
|
||||
pub parsed: [Option<Arc<Parsed>>; NUM_REF_FRAMES],
|
||||
/// The published bands of references that are still being decoded (row wavefront): read
|
||||
/// only below the row progress that was waited for.
|
||||
pub ref_rows: [Option<Arc<BandFrame>>; NUM_REF_FRAMES],
|
||||
pub ref_info: [RefInfo; NUM_REF_FRAMES],
|
||||
/// MotionFieldMvs[ ref ][ y8 * w8 + x8 ] (7.9); empty when unused.
|
||||
pub motion_field: [Vec<[i32; 2]>; 8],
|
||||
@@ -34,6 +39,16 @@ pub(crate) struct FrameShared {
|
||||
}
|
||||
|
||||
impl FrameShared {
|
||||
/// The samples of reference slot `ref_idx` to predict from (None for a missing reference).
|
||||
#[inline]
|
||||
pub fn ref_samples(&self, ref_idx: i32, plane: usize) -> Option<PlaneRef<'_>> {
|
||||
let i = usize::try_from(ref_idx).ok()?;
|
||||
if let Some(b) = self.ref_rows.get(i)?.as_ref() {
|
||||
return b.planes.get(plane).map(PlaneRef::Bands);
|
||||
}
|
||||
self.refs.get(i)?.as_ref().map(|r| r.plane(plane))
|
||||
}
|
||||
|
||||
pub fn plane_residual_size(&self, subsize: usize, plane: usize) -> usize {
|
||||
let sx = if plane > 0 { self.ssx } else { 0 };
|
||||
let sy = if plane > 0 { self.ssy } else { 0 };
|
||||
@@ -210,6 +225,8 @@ impl FrameState {
|
||||
lr_unit_cols,
|
||||
cdef_cols,
|
||||
refs: Default::default(),
|
||||
parsed: Default::default(),
|
||||
ref_rows: Default::default(),
|
||||
ref_info: Default::default(),
|
||||
motion_field: Default::default(),
|
||||
prev_segment_ids: vec![0; mi_cols * mi_rows],
|
||||
@@ -283,6 +300,14 @@ impl FrameState {
|
||||
pub fn merge_tile_state(&mut self, t: &TileState) {
|
||||
for p in 0..self.num_planes {
|
||||
t.cur.planes[p].copy_into(&mut self.cur.planes[p]);
|
||||
}
|
||||
self.merge_tile_modes(t);
|
||||
}
|
||||
|
||||
/// Copy a tile's private buffers except the samples (after parsing, before reconstruction)
|
||||
/// into the frame.
|
||||
pub fn merge_tile_modes(&mut self, t: &TileState) {
|
||||
for p in 0..self.num_planes {
|
||||
t.lf_tx_size[p].copy_into(&mut self.lf_tx_size[p]);
|
||||
for &u in &t.lr_written[p] {
|
||||
self.lr.lr_type[p][u] = t.lr.lr_type[p][u];
|
||||
|
||||
@@ -94,6 +94,191 @@ pub(crate) struct Block {
|
||||
pub local_warp_params: [i32; 6],
|
||||
}
|
||||
|
||||
/// What a tile decoder does with the blocks it visits.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub(crate) enum Pass {
|
||||
/// Read the symbols and reconstruct each block right away (single-threaded decoding).
|
||||
Full,
|
||||
/// Read the symbols only, recording what reconstruction needs in a [`TileRecord`].
|
||||
Parse,
|
||||
/// Reconstruct from a [`TileRecord`], reading no symbols.
|
||||
Reconstruct,
|
||||
}
|
||||
|
||||
/// The per-block values reconstruction needs beyond the mode info (which is final after
|
||||
/// parsing): the block variables as parsed, its quantizer index and the warp samples / palette
|
||||
/// color maps when used.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct BlockRecord {
|
||||
b: Block,
|
||||
current_q_index: i32,
|
||||
warp_samples: Option<Box<(usize, [[i32; 4]; crate::av1::decode::spec_tables::LEAST_SQUARES_SAMPLES_MAX])>>,
|
||||
color_maps: Option<Box<(Vec<u8>, Vec<u8>)>>,
|
||||
}
|
||||
|
||||
/// A parsed tile: its blocks in decoding order, how many of them each superblock holds and the
|
||||
/// coefficients of every coded transform block as a stream of
|
||||
/// `eob, tx_type, count, (position, value) * count` (only non-zero values).
|
||||
#[derive(Default)]
|
||||
pub(crate) struct TileRecord {
|
||||
blocks: Vec<BlockRecord>,
|
||||
sb_blocks: Vec<u32>,
|
||||
coeffs: Vec<i32>,
|
||||
/// Read positions while reconstructing.
|
||||
next_block: usize,
|
||||
next_coeff: usize,
|
||||
next_sb: usize,
|
||||
next_sb_row: usize,
|
||||
/// Bytes recorded and the most allowed.
|
||||
pub bytes: usize,
|
||||
pub limit: usize,
|
||||
/// The decoder-wide budget the recorded bytes are reserved from, and how much is reserved.
|
||||
pub budget: Option<std::sync::Arc<RecordBudget>>,
|
||||
reserved: usize,
|
||||
}
|
||||
|
||||
impl Drop for TileRecord {
|
||||
fn drop(&mut self) {
|
||||
if let Some(b) = &self.budget {
|
||||
b.release(self.reserved);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The parse record bytes all frames of a decoder may hold at once (frames in flight decode in
|
||||
/// one pass once it is used up).
|
||||
pub(crate) struct RecordBudget {
|
||||
used: std::sync::atomic::AtomicUsize,
|
||||
max: usize,
|
||||
/// Most bytes ever reserved at once (tests).
|
||||
#[cfg(test)]
|
||||
pub peak: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
/// Bytes reserved from a [`RecordBudget`] at a time.
|
||||
const RESERVE_STEP: usize = 1 << 20;
|
||||
|
||||
impl RecordBudget {
|
||||
pub fn new(max: usize) -> RecordBudget {
|
||||
RecordBudget {
|
||||
used: Default::default(),
|
||||
max,
|
||||
#[cfg(test)]
|
||||
peak: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserve `n` bytes; false (nothing reserved) if that would go over the budget.
|
||||
fn reserve(&self, n: usize) -> bool {
|
||||
use std::sync::atomic::Ordering;
|
||||
let ok = self.used.try_update(Ordering::AcqRel, Ordering::Acquire, |u| u.checked_add(n).filter(|&v| v <= self.max)).is_ok();
|
||||
#[cfg(test)]
|
||||
if ok {
|
||||
self.peak.fetch_max(self.used.load(Ordering::Acquire), Ordering::AcqRel);
|
||||
}
|
||||
ok
|
||||
}
|
||||
|
||||
fn release(&self, n: usize) {
|
||||
self.used.fetch_sub(n, std::sync::atomic::Ordering::AcqRel);
|
||||
}
|
||||
|
||||
/// Bytes reserved now (tests).
|
||||
#[cfg(test)]
|
||||
pub fn used(&self) -> usize {
|
||||
self.used.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
/// The motion of a recorded block (what a reference row wait depends on).
|
||||
pub(crate) struct BlockMotion {
|
||||
pub mi_row: usize,
|
||||
pub bh4: usize,
|
||||
/// Inter prediction from reference frames (not intra block copy).
|
||||
pub inter: bool,
|
||||
pub ref_frame: [i8; 2],
|
||||
/// Mv[ list ] (row, col) in 1/8 samples.
|
||||
pub mv: [[i32; 2]; 2],
|
||||
pub motion_mode: u8,
|
||||
pub y_mode: usize,
|
||||
}
|
||||
|
||||
impl TileRecord {
|
||||
/// Whether recording stopped at [`TileRecord::limit`] (the record is unusable).
|
||||
pub fn over_limit(&self) -> bool {
|
||||
self.bytes > self.limit
|
||||
}
|
||||
|
||||
/// Account for `n` more recorded bytes; false once over the limit (nothing more is kept).
|
||||
fn grow(&mut self, n: usize) -> bool {
|
||||
self.bytes = self.bytes.saturating_add(n);
|
||||
if self.bytes > self.limit {
|
||||
return false;
|
||||
}
|
||||
if self.bytes > self.reserved
|
||||
&& let Some(b) = &self.budget
|
||||
{
|
||||
let step = (self.bytes - self.reserved).max(RESERVE_STEP);
|
||||
if !b.reserve(step) {
|
||||
// The decoder-wide budget is used up: this frame is decoded in one pass.
|
||||
self.limit = 0;
|
||||
return false;
|
||||
}
|
||||
self.reserved += step;
|
||||
}
|
||||
true
|
||||
}
|
||||
/// The recorded blocks of superblock rows [first, end), for a tile `sbs_per_row` superblocks wide.
|
||||
pub fn row_blocks(&self, sbs_per_row: usize, first: usize, end: usize) -> impl Iterator<Item = BlockMotion> + '_ {
|
||||
let start: usize = self.sb_blocks.iter().take(first * sbs_per_row).map(|&n| n as usize).sum();
|
||||
let count: usize = self.sb_blocks.iter().skip(first * sbs_per_row).take((end - first) * sbs_per_row).map(|&n| n as usize).sum();
|
||||
self.blocks.iter().skip(start).take(count).map(|r| BlockMotion {
|
||||
mi_row: r.b.mi_row,
|
||||
bh4: r.b.bh4,
|
||||
inter: r.b.is_inter && !r.b.use_intrabc,
|
||||
ref_frame: r.b.ref_frame,
|
||||
mv: r.b.mv_i,
|
||||
motion_mode: r.b.motion_mode,
|
||||
y_mode: r.b.y_mode,
|
||||
})
|
||||
}
|
||||
|
||||
fn next_coeff(&mut self) -> Result<i32> {
|
||||
let v = *self.coeffs.get(self.next_coeff).ok_or(crate::av1::decode::Error::Invalid("coefficient record exhausted"))?;
|
||||
self.next_coeff += 1;
|
||||
Ok(v)
|
||||
}
|
||||
}
|
||||
|
||||
/// The work buffers of a [`TileDecoder`] (`quant` all zero between transform blocks).
|
||||
pub(crate) struct Buffers {
|
||||
quant: Vec<i32>,
|
||||
dequant: Vec<i32>,
|
||||
residual: Vec<i32>,
|
||||
color_map_y: Vec<u8>,
|
||||
color_map_uv: Vec<u8>,
|
||||
pred_buf: [Vec<i32>; 2],
|
||||
mask: Vec<i32>,
|
||||
mc_tmp: Vec<i32>,
|
||||
levels: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Buffers {
|
||||
pub fn new() -> Buffers {
|
||||
Buffers {
|
||||
quant: vec![0; 1024],
|
||||
dequant: vec![0; 64 * 64],
|
||||
residual: vec![0; 64 * 64],
|
||||
color_map_y: vec![0; 64 * 64],
|
||||
color_map_uv: vec![0; 64 * 64],
|
||||
pred_buf: [vec![0; 128 * 128], vec![0; 128 * 128]],
|
||||
mask: vec![0; 128 * 128],
|
||||
mc_tmp: vec![0; (2 * 128 + 8) * 128],
|
||||
levels: vec![0; (32 + TX_PAD) * (32 + TX_PAD)],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decoder for one tile.
|
||||
pub(crate) struct TileDecoder<'a, 'f> {
|
||||
pub fs: &'f FrameShared,
|
||||
@@ -128,6 +313,9 @@ pub(crate) struct TileDecoder<'a, 'f> {
|
||||
pub mc_tmp: Vec<i32>,
|
||||
/// Coefficient levels of the current transform block, padded (see TX_PAD).
|
||||
levels: Vec<u8>,
|
||||
pub pass: Pass,
|
||||
/// Written by [`Pass::Parse`], read by [`Pass::Reconstruct`].
|
||||
pub record: TileRecord,
|
||||
}
|
||||
|
||||
const SB_MAX: usize = 32; // superblock size in 4x4 units (128 / 4)
|
||||
@@ -141,6 +329,21 @@ fn block_height(bs: usize) -> usize {
|
||||
|
||||
impl<'a, 'f> TileDecoder<'a, 'f> {
|
||||
pub fn new(fs: &'f FrameShared, t: &'f mut TileState, data: &'a [u8], cdf: Box<Cdfs>, tile_row: usize, tile_col: usize) -> Self {
|
||||
Self::with_buffers(fs, t, data, cdf, tile_row, tile_col, Buffers::new())
|
||||
}
|
||||
|
||||
/// [`TileDecoder::new`] reusing the work buffers of an earlier decoder of the same frame
|
||||
/// (see [`TileDecoder::into_buffers`]).
|
||||
pub fn with_buffers(
|
||||
fs: &'f FrameShared,
|
||||
t: &'f mut TileState,
|
||||
data: &'a [u8],
|
||||
cdf: Box<Cdfs>,
|
||||
tile_row: usize,
|
||||
tile_col: usize,
|
||||
bufs: Buffers,
|
||||
) -> Self {
|
||||
let Buffers { quant, dequant, residual, color_map_y, color_map_uv, pred_buf, mask, mc_tmp, levels } = bufs;
|
||||
let ti = &fs.fh.tile_info;
|
||||
let (rs, re) = (ti.mi_row_starts[tile_row] as usize, ti.mi_row_starts[tile_row + 1] as usize);
|
||||
let (cs, ce) = (ti.mi_col_starts[tile_col] as usize, ti.mi_col_starts[tile_col + 1] as usize);
|
||||
@@ -164,19 +367,27 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
|
||||
b: Block::default(),
|
||||
mvs: Default::default(),
|
||||
nb: Default::default(),
|
||||
quant: vec![0; 1024],
|
||||
dequant: vec![0; 64 * 64],
|
||||
residual: vec![0; 64 * 64],
|
||||
color_map_y: vec![0; 64 * 64],
|
||||
color_map_uv: vec![0; 64 * 64],
|
||||
quant,
|
||||
dequant,
|
||||
residual,
|
||||
color_map_y,
|
||||
color_map_uv,
|
||||
plane_tx_type: 0,
|
||||
pred_buf: [vec![0; 128 * 128], vec![0; 128 * 128]],
|
||||
mask: vec![0; 128 * 128],
|
||||
mc_tmp: vec![0; (2 * 128 + 8) * 128],
|
||||
levels: vec![0; (32 + TX_PAD) * (32 + TX_PAD)],
|
||||
pred_buf,
|
||||
mask,
|
||||
mc_tmp,
|
||||
levels,
|
||||
pass: Pass::Full,
|
||||
record: TileRecord::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The work buffers and CDFs, for [`TileDecoder::with_buffers`].
|
||||
pub fn into_buffers(self) -> (Buffers, Box<Cdfs>) {
|
||||
let TileDecoder { quant, dequant, residual, color_map_y, color_map_uv, pred_buf, mask, mc_tmp, levels, cdf, .. } = self;
|
||||
(Buffers { quant, dequant, residual, color_map_y, color_map_uv, pred_buf, mask, mc_tmp, levels }, cdf)
|
||||
}
|
||||
|
||||
fn seq(&self) -> &SequenceHeader {
|
||||
&self.fs.seq
|
||||
}
|
||||
@@ -189,8 +400,11 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
|
||||
c >= self.mi_col_start as isize && c < self.mi_col_end as isize && r >= self.mi_row_start as isize && r < self.mi_row_end as isize
|
||||
}
|
||||
|
||||
/// decode_tile( )
|
||||
/// decode_tile( ), or its reconstruction from [`TileDecoder::record`] in [`Pass::Reconstruct`].
|
||||
pub fn decode_tile(&mut self) -> Result<()> {
|
||||
if self.pass == Pass::Reconstruct {
|
||||
return self.reconstruct_tile();
|
||||
}
|
||||
// clear_above_context( )
|
||||
let t = &mut *self.t;
|
||||
for p in 0..3 {
|
||||
@@ -223,7 +437,11 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
|
||||
self.clear_cdef(r, c);
|
||||
self.clear_block_decoded_flags(r, c, sb4);
|
||||
self.read_lr(r, c, sb_size);
|
||||
let before = self.record.blocks.len();
|
||||
self.decode_partition(r, c, sb_size)?;
|
||||
if self.pass == Pass::Parse {
|
||||
self.record.sb_blocks.push((self.record.blocks.len() - before) as u32);
|
||||
}
|
||||
c += sb4;
|
||||
}
|
||||
r += sb4;
|
||||
@@ -231,6 +449,76 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The reconstruction half of decode_tile( ): the superblocks in the same order, each block
|
||||
/// predicted and reconstructed from its record.
|
||||
fn reconstruct_tile(&mut self) -> Result<()> {
|
||||
self.reconstruct_sb_rows(usize::MAX)
|
||||
}
|
||||
|
||||
/// Reconstruct the next `rows` superblock rows of the tile (the record keeps the position).
|
||||
pub fn reconstruct_sb_rows(&mut self, rows: usize) -> Result<()> {
|
||||
let sb4 = if self.seq().use_128x128_superblock { 32 } else { 16 };
|
||||
let mut r = self.mi_row_start + self.record.next_sb_row * sb4;
|
||||
let mut done = 0;
|
||||
while r < self.mi_row_end && done < rows {
|
||||
let mut c = self.mi_col_start;
|
||||
while c < self.mi_col_end {
|
||||
self.clear_block_decoded_flags(r, c, sb4);
|
||||
let n = *self.record.sb_blocks.get(self.record.next_sb).ok_or(crate::av1::decode::Error::Invalid("block record exhausted"))?;
|
||||
self.record.next_sb += 1;
|
||||
for _ in 0..n {
|
||||
self.reconstruct_block()?;
|
||||
}
|
||||
c += sb4;
|
||||
}
|
||||
r += sb4;
|
||||
done += 1;
|
||||
self.record.next_sb_row += 1;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reconstruct_block(&mut self) -> Result<()> {
|
||||
let rec = self.record.blocks.get(self.record.next_block).ok_or(crate::av1::decode::Error::Invalid("block record exhausted"))?;
|
||||
self.record.next_block += 1;
|
||||
self.b = rec.b.clone();
|
||||
self.current_q_index = rec.current_q_index;
|
||||
if let Some(w) = &rec.warp_samples {
|
||||
self.mvs.num_samples = w.0;
|
||||
self.mvs.cand_list = w.1;
|
||||
}
|
||||
if let Some(maps) = &rec.color_maps {
|
||||
if let Some(d) = self.color_map_y.get_mut(..maps.0.len()) {
|
||||
d.copy_from_slice(&maps.0);
|
||||
}
|
||||
if let Some(d) = self.color_map_uv.get_mut(..maps.1.len()) {
|
||||
d.copy_from_slice(&maps.1);
|
||||
}
|
||||
}
|
||||
self.compute_prediction()?;
|
||||
self.residual()
|
||||
}
|
||||
|
||||
/// Records the block for [`Pass::Reconstruct`] (its variables as parsed).
|
||||
fn record_block(&mut self) {
|
||||
let b = &self.b;
|
||||
// The rows of the color maps (stride 64) the block's palette prediction reads.
|
||||
let map_rows = |n: usize, h: usize| if n > 0 { (h + 2).min(64) * 64 } else { 0 };
|
||||
let rows_y = map_rows(b.palette_size_y, block_height(b.mi_size));
|
||||
let rows_uv = map_rows(b.palette_size_uv, block_height(b.mi_size) >> self.fs.ssy);
|
||||
let size = std::mem::size_of::<BlockRecord>()
|
||||
+ rows_y
|
||||
+ rows_uv
|
||||
+ if b.motion_mode as usize == crate::av1::decode::spec_tables::LOCALWARP { 160 } else { 0 };
|
||||
if !self.record.grow(size) {
|
||||
return;
|
||||
}
|
||||
let warp_samples =
|
||||
(b.motion_mode as usize == crate::av1::decode::spec_tables::LOCALWARP).then(|| Box::new((self.mvs.num_samples, self.mvs.cand_list)));
|
||||
let color_maps = (rows_y + rows_uv > 0).then(|| Box::new((self.color_map_y[..rows_y].to_vec(), self.color_map_uv[..rows_uv].to_vec())));
|
||||
self.record.blocks.push(BlockRecord { b: b.clone(), current_q_index: self.current_q_index, warp_samples, color_maps });
|
||||
}
|
||||
|
||||
fn clear_cdef(&mut self, r: usize, c: usize) {
|
||||
let t = &mut *self.t;
|
||||
t.set_cdef_idx(r, c, -1);
|
||||
@@ -461,7 +749,11 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
|
||||
}
|
||||
}
|
||||
}
|
||||
self.compute_prediction()?;
|
||||
if self.pass == Pass::Parse {
|
||||
self.record_block();
|
||||
} else {
|
||||
self.compute_prediction()?;
|
||||
}
|
||||
self.residual()?;
|
||||
let b = &self.b;
|
||||
let delta_lf = self.delta_lf;
|
||||
|
||||
@@ -459,6 +459,58 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Moves the coefficients `coeffs` left in `quant` into the record (leaving `quant` zero, as
|
||||
/// reconstruction does).
|
||||
pub(super) fn record_coeffs(&mut self, eob: usize, tx_sz: usize) {
|
||||
let scan = self.get_scan(tx_sz);
|
||||
let scan = scan.get(..eob).unwrap_or(scan);
|
||||
if !self.record.grow(4 * (3 + 2 * scan.len())) {
|
||||
// Over the limit: the record is dropped (the frame is decoded again in one pass);
|
||||
// `quant` must still be left zero.
|
||||
for &pos in scan {
|
||||
if let Some(q) = self.quant.get_mut(pos as usize) {
|
||||
*q = 0;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
let rec = &mut self.record.coeffs;
|
||||
rec.push(eob as i32);
|
||||
if eob == 0 {
|
||||
return;
|
||||
}
|
||||
rec.push(self.plane_tx_type as i32);
|
||||
let count_at = rec.len();
|
||||
rec.push(0);
|
||||
let mut count = 0;
|
||||
for &pos in scan {
|
||||
let v = std::mem::take(&mut self.quant[pos as usize]);
|
||||
if v != 0 {
|
||||
rec.push(pos as i32);
|
||||
rec.push(v);
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
rec[count_at] = count;
|
||||
}
|
||||
|
||||
/// The next transform block's coefficients from the record into `quant` (and its type);
|
||||
/// returns its eob.
|
||||
pub(super) fn replay_coeffs(&mut self) -> crate::av1::decode::Result<usize> {
|
||||
let eob = self.record.next_coeff()?;
|
||||
if eob <= 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
self.plane_tx_type = usize::try_from(self.record.next_coeff()?).map_err(|_| crate::av1::decode::Error::Invalid("coefficient record"))?;
|
||||
let count = self.record.next_coeff()?;
|
||||
for _ in 0..count {
|
||||
let pos = self.record.next_coeff()? as usize;
|
||||
let v = self.record.next_coeff()?;
|
||||
*self.quant.get_mut(pos).ok_or(crate::av1::decode::Error::Invalid("coefficient record"))? = v;
|
||||
}
|
||||
Ok(eob as usize)
|
||||
}
|
||||
|
||||
/// Reconstruct process (7.12.3).
|
||||
pub(super) fn reconstruct(&mut self, plane: usize, x: usize, y: usize, tx_sz: usize, dc_only: bool) {
|
||||
// dqDenom (7.12.3) is 1, 2 or 4 and the dividend is non-negative.
|
||||
|
||||
@@ -213,7 +213,8 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
|
||||
if start_x >= max_x || start_y >= max_y {
|
||||
return Ok(());
|
||||
}
|
||||
if !self.b.is_inter {
|
||||
let parse_only = self.pass == super::Pass::Parse;
|
||||
if !self.b.is_inter && !parse_only {
|
||||
let palette = if plane == 0 { self.b.palette_size_y } else { self.b.palette_size_uv };
|
||||
if palette > 0 {
|
||||
self.predict_palette(plane, start_x, start_y, x, y, tx_sz);
|
||||
@@ -266,9 +267,23 @@ impl<'a, 'f> TileDecoder<'a, 'f> {
|
||||
}
|
||||
}
|
||||
if !self.b.skip {
|
||||
let eob = self.coeffs(plane, start_x, start_y, tx_sz);
|
||||
if eob > 0 {
|
||||
self.reconstruct(plane, start_x, start_y, tx_sz, eob == 1);
|
||||
match self.pass {
|
||||
super::Pass::Full => {
|
||||
let eob = self.coeffs(plane, start_x, start_y, tx_sz);
|
||||
if eob > 0 {
|
||||
self.reconstruct(plane, start_x, start_y, tx_sz, eob == 1);
|
||||
}
|
||||
}
|
||||
super::Pass::Parse => {
|
||||
let eob = self.coeffs(plane, start_x, start_y, tx_sz);
|
||||
self.record_coeffs(eob, tx_sz);
|
||||
}
|
||||
super::Pass::Reconstruct => {
|
||||
let eob = self.replay_coeffs()?;
|
||||
if eob > 0 {
|
||||
self.reconstruct(plane, start_x, start_y, tx_sz, eob == 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for i in 0..step_y {
|
||||
|
||||
@@ -152,38 +152,35 @@ impl<'a, S: crate::hevc::decode::picture::Sample> SliceDecoder<'a, S> {
|
||||
col.ref_idx[1] = 0;
|
||||
col_avail = true;
|
||||
}
|
||||
let mut cands: Vec<MvField> = list[..n].to_vec();
|
||||
if col_avail {
|
||||
cands.push(col);
|
||||
list[n] = col;
|
||||
n += 1;
|
||||
}
|
||||
if merge_idx < cands.len() {
|
||||
return self.finish_merge(cands[merge_idx], w0, h0);
|
||||
if merge_idx < n {
|
||||
return self.finish_merge(list[merge_idx], w0, h0);
|
||||
}
|
||||
// combined bi-predictive candidates (8.5.3.2.4)
|
||||
let num_orig = cands.len();
|
||||
let num_orig = n;
|
||||
if self.sh.is_b() && num_orig > 1 && num_orig < max {
|
||||
let mut comb = 0;
|
||||
while comb < num_orig * (num_orig - 1) && cands.len() < max {
|
||||
while comb < num_orig * (num_orig - 1) && n < max {
|
||||
let (i0, i1) = COMB[comb];
|
||||
let (c0, c1) = (cands[i0], cands[i1]);
|
||||
let (c0, c1) = (list[i0], list[i1]);
|
||||
if c0.pred(0) && c1.pred(1) && (self.ref_pic(0, c0.ref_idx[0]).poc != self.ref_pic(1, c1.ref_idx[1]).poc || c0.mv[0] != c1.mv[1]) {
|
||||
cands.push(MvField { mv: [c0.mv[0], c1.mv[1]], ref_idx: [c0.ref_idx[0], c1.ref_idx[1]] });
|
||||
list[n] = MvField { mv: [c0.mv[0], c1.mv[1]], ref_idx: [c0.ref_idx[0], c1.ref_idx[1]] };
|
||||
n += 1;
|
||||
}
|
||||
comb += 1;
|
||||
}
|
||||
}
|
||||
if merge_idx < cands.len() {
|
||||
return self.finish_merge(cands[merge_idx], w0, h0);
|
||||
if merge_idx < n {
|
||||
return self.finish_merge(list[merge_idx], w0, h0);
|
||||
}
|
||||
// zero candidates (8.5.3.2.5)
|
||||
// zero candidates (8.5.3.2.5): candidate merge_idx is the (merge_idx - n)-th one
|
||||
let num_ref = if self.sh.is_b() { self.sh.num_ref_idx[0].min(self.sh.num_ref_idx[1]) } else { self.sh.num_ref_idx[0] } as usize;
|
||||
let mut zero_idx = 0;
|
||||
while cands.len() <= merge_idx {
|
||||
let r = if zero_idx < num_ref { zero_idx as i8 } else { 0 };
|
||||
cands.push(MvField { mv: [[0; 2]; 2], ref_idx: [r, if self.sh.is_b() { r } else { -1 }] });
|
||||
zero_idx += 1;
|
||||
}
|
||||
self.finish_merge(cands[merge_idx], w0, h0)
|
||||
let zero_idx = merge_idx - n;
|
||||
let r = if zero_idx < num_ref { zero_idx as i8 } else { 0 };
|
||||
self.finish_merge(MvField { mv: [[0; 2]; 2], ref_idx: [r, if self.sh.is_b() { r } else { -1 }] }, w0, h0)
|
||||
}
|
||||
|
||||
fn finish_merge(&self, mut f: MvField, w: i32, h: i32) -> MvField {
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::hevc::decode::spec_tables::{
|
||||
INTRA_CHROMA, LAST_X_PREFIX, LAST_Y_PREFIX, MERGE_FLAG, MERGE_IDX, MVP_FLAG, PART_MODE, PRED_MODE, PREV_INTRA_LUMA, REF_IDX, RQT_ROOT_CBF,
|
||||
SAO_MERGE, SAO_TYPE, SIG_COEFF, SPLIT_CU, SPLIT_TRANSFORM, TRANSFORM_SKIP,
|
||||
};
|
||||
use crate::hevc::decode::tables::{CTX_IDX_MAP, INTRA_CHROMA_MODE_422, LEVEL_SCALE, SCAN_4, qpc, scan_order};
|
||||
use crate::hevc::decode::tables::{INTRA_CHROMA_MODE_422, LEVEL_SCALE, SCAN_4, SIG_CTX, qpc, scan_order, scan_pos};
|
||||
use crate::hevc::decode::transform;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -340,6 +340,8 @@ pub struct SliceDecoder<'a, S> {
|
||||
pub cu_part: PartMode,
|
||||
// scratch
|
||||
pub coeffs: Vec<i32>,
|
||||
/// Scaled non-zero coefficients of the current transform block: (index in `coeffs`, value).
|
||||
pub nz: Vec<(u16, i32)>,
|
||||
pub pred0: Vec<i16>,
|
||||
pub pred1: Vec<i16>,
|
||||
pub mc: crate::hevc::decode::inter::McScratch,
|
||||
@@ -461,6 +463,7 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
|
||||
intra_split: false,
|
||||
cu_part: PartMode::P2Nx2N,
|
||||
coeffs: vec![0; 32 * 32],
|
||||
nz: vec![(0, 0); 32 * 32],
|
||||
pred0: vec![0; 64 * 64],
|
||||
pred1: vec![0; 64 * 64],
|
||||
mc: Default::default(),
|
||||
@@ -632,3 +635,30 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::hevc::decode::transform;
|
||||
|
||||
/// `reconstruct` clears only the coefficient bounding box, so the inverse transform must not
|
||||
/// read the stale values left outside it by the previous block.
|
||||
#[test]
|
||||
fn inverse_transform_reads_only_the_coefficient_box() {
|
||||
for &(n, dst) in &[(4usize, true), (4, false), (8, false), (16, false), (32, false)] {
|
||||
for (max_x, max_y) in [(0, 1), (1, 0), (2, 3), (n / 2, 1), (n - 1, 0), (n - 1, n - 1)] {
|
||||
let (mut clean, mut stale) = (vec![0i32; n * n], vec![0i32; n * n]);
|
||||
for (i, (c, s)) in clean.iter_mut().zip(stale.iter_mut()).enumerate() {
|
||||
let (x, y) = (i % n, i / n);
|
||||
*s = 12345 - i as i32;
|
||||
if x <= max_x && y <= max_y {
|
||||
*c = (i as i32 * 37) % 201 - 100;
|
||||
*s = *c;
|
||||
}
|
||||
}
|
||||
transform::inverse_transform(&mut clean, n, dst, max_x, max_y, 12);
|
||||
transform::inverse_transform(&mut stale, n, dst, max_x, max_y, 12);
|
||||
assert_eq!(clean, stale, "n {n} dst {dst} box ({max_x}, {max_y})");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::{
|
||||
CBF_CHROMA, CBF_LUMA, CODED_SUB_BLOCK, CTX_IDX_MAP, CU_QP_DELTA_ABS, ChromaFormat, E_TU_H, E_TU_V, F_CBF, F_INTRA, GT1, GT2, LAST_X_PREFIX,
|
||||
LAST_Y_PREFIX, LEVEL_SCALE, PartMode, Result, SCAN_4, SIG_COEFF, SPLIT_TRANSFORM, Sample, SliceDecoder, TRANSFORM_SKIP, ensure, for_runs, intra,
|
||||
scan_order, transform,
|
||||
CBF_CHROMA, CBF_LUMA, CODED_SUB_BLOCK, CU_QP_DELTA_ABS, ChromaFormat, E_TU_H, E_TU_V, F_CBF, F_INTRA, GT1, GT2, LAST_X_PREFIX, LAST_Y_PREFIX,
|
||||
LEVEL_SCALE, PartMode, Result, SCAN_4, SIG_COEFF, SIG_CTX, SPLIT_TRANSFORM, Sample, SliceDecoder, TRANSFORM_SKIP, ensure, for_runs, intra,
|
||||
scan_order, scan_pos, transform,
|
||||
};
|
||||
|
||||
/// Significant scan positions of one sub-block, in decoding (descending) order.
|
||||
@@ -44,54 +44,26 @@ struct GreaterFlags {
|
||||
g1ctx: u32,
|
||||
}
|
||||
|
||||
/// sigCtx increment of sig_coeff_flag (9.3.4.2.5) for the position `(xp, yp)` inside the sub-block.
|
||||
fn sig_ctx_inc(p: &SigMapParams, (xp, yp): (usize, usize)) -> usize {
|
||||
let (xc, yc) = (p.xs * 4 + xp, p.ys * 4 + yp);
|
||||
/// sigCtx increments (9.3.4.2.5) of the 16 scan positions of one sub-block.
|
||||
fn sig_ctx_table(p: &SigMapParams) -> [u8; 16] {
|
||||
if p.log2 == 2 {
|
||||
return CTX_IDX_MAP[(yc << 2) + xc] as usize;
|
||||
return SIG_CTX[p.scan_idx][4];
|
||||
}
|
||||
if xc + yc == 0 {
|
||||
return 0;
|
||||
}
|
||||
let mut s = match p.prev_csbf {
|
||||
0 => {
|
||||
if xp + yp == 0 {
|
||||
2
|
||||
} else if xp + yp < 3 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
1 => {
|
||||
if yp == 0 {
|
||||
2
|
||||
} else if yp == 1 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
2 => {
|
||||
if xp == 0 {
|
||||
2
|
||||
} else if xp == 1 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
_ => 2,
|
||||
};
|
||||
if p.c == 0 {
|
||||
if p.xs + p.ys > 0 {
|
||||
s += 3;
|
||||
}
|
||||
s += if p.log2 == 3 { if p.scan_idx == 0 { 9 } else { 15 } } else { 21 };
|
||||
let off = if p.c == 0 {
|
||||
(if p.xs + p.ys > 0 { 3 } else { 0 }) + if p.log2 == 3 { if p.scan_idx == 0 { 9 } else { 15 } } else { 21 }
|
||||
} else if p.log2 == 3 {
|
||||
9
|
||||
} else {
|
||||
s += if p.log2 == 3 { 9 } else { 12 };
|
||||
12
|
||||
};
|
||||
let mut t = SIG_CTX[p.scan_idx][p.prev_csbf as usize];
|
||||
for v in t.iter_mut() {
|
||||
*v += off;
|
||||
}
|
||||
s
|
||||
if p.xs + p.ys == 0 {
|
||||
t[0] = 0;
|
||||
}
|
||||
t
|
||||
}
|
||||
|
||||
/// A node of the transform tree: the block at (`x0`, `y0`) of size 2^`log2`, its parent's origin
|
||||
@@ -333,14 +305,13 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
|
||||
/// sig_coeff_flag decoding of one coded sub-block, appending the significant scan positions
|
||||
/// (descending) to `list`.
|
||||
fn read_sig_map(&mut self, p: &SigMapParams, list: &mut SigList) {
|
||||
let pos_scan = &SCAN_4[p.scan_idx];
|
||||
let c_off_sig = if p.c == 0 { 0 } else { 27 };
|
||||
let ctx_tab = sig_ctx_table(p);
|
||||
let mut infer_dc = p.infer_dc;
|
||||
let mut np = p.start;
|
||||
while np >= 0 {
|
||||
let (xp, yp) = (pos_scan[np as usize].0 as usize, pos_scan[np as usize].1 as usize);
|
||||
if np > 0 || !infer_dc {
|
||||
let sig_ctx = sig_ctx_inc(p, (xp, yp));
|
||||
let sig_ctx = ctx_tab[np as usize] as usize;
|
||||
if self.c.decode_decision(SIG_COEFF + c_off_sig + sig_ctx) == 1 {
|
||||
list.push(np as u8);
|
||||
infer_dc = false;
|
||||
@@ -438,9 +409,23 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
|
||||
let sb_scan = scan_order(log2sb, scan_idx);
|
||||
let pos_scan = &SCAN_4[scan_idx];
|
||||
let sb_w = 1usize << log2sb;
|
||||
let last_sb = sb_scan.iter().position(|&(x, y)| x as u32 == lx >> 2 && y as u32 == ly >> 2).unwrap_or(0);
|
||||
let last_pos = pos_scan.iter().position(|&(x, y)| x as u32 == lx & 3 && y as u32 == ly & 3).unwrap_or(0);
|
||||
self.coeffs[..n * n].fill(0);
|
||||
let last_sb = scan_pos(log2sb, scan_idx, lx >> 2, ly >> 2);
|
||||
let last_pos = scan_pos(2, scan_idx, lx & 3, ly & 3);
|
||||
// scaling (8.6.3), applied as each coefficient is decoded
|
||||
let bd = if c == 0 { self.pic.bd_y } else { self.pic.bd_c };
|
||||
let bd_shift = bd + log2 - 5;
|
||||
let round = 1i64 << (bd_shift - 1);
|
||||
let scale = (LEVEL_SCALE[(qp % 6) as usize] as i64) << (qp / 6);
|
||||
let matrix = match &self.pic.scaling {
|
||||
Some(s) if !(bypass || (ts && n > 4)) => {
|
||||
let size_id = (log2 - 2) as usize;
|
||||
let matrix_id = if self.cu_intra { c } else { 3 + c };
|
||||
Some((s.clone(), size_id * 6 + matrix_id))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let m: Option<&[u8]> = matrix.as_ref().map(|(s, i)| &s[*i][..]);
|
||||
let mut nnz = 0;
|
||||
let mut csbf = [0u8; 64];
|
||||
let mut prev_g1: Option<u32> = None;
|
||||
let sdh = self.pps.sign_data_hiding && !bypass;
|
||||
@@ -485,12 +470,18 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
|
||||
for k in 0..list.n {
|
||||
let np = list.pos[k] as usize;
|
||||
let (xc, yc) = (xs * 4 + pos_scan[np].0 as usize, ys * 4 + pos_scan[np].1 as usize);
|
||||
self.coeffs[yc * n + xc] = values[k];
|
||||
let mut v = values[k];
|
||||
if !bypass {
|
||||
let f = m.map_or(16, |m| m[yc * n + xc] as i64);
|
||||
v = ((v as i64 * f * scale + round) >> bd_shift).clamp(-32768, 32767) as i32;
|
||||
}
|
||||
self.nz[nnz] = ((yc * n + xc) as u16, v);
|
||||
nnz += 1;
|
||||
max_x = max_x.max(xc);
|
||||
max_y = max_y.max(yc);
|
||||
}
|
||||
}
|
||||
self.reconstruct((x0, y0), log2, c, qp, ts, (max_x, max_y))
|
||||
self.reconstruct((x0, y0), log2, c, ts, nnz, (max_x, max_y))
|
||||
}
|
||||
|
||||
fn coeff_abs_level_remaining(&mut self, rice: u32) -> Result<i32> {
|
||||
@@ -508,55 +499,70 @@ impl<'a, S: Sample> SliceDecoder<'a, S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Scaling, transform and addition to the prediction.
|
||||
fn reconstruct(&mut self, (x0, y0): (i32, i32), log2: u32, c: usize, qp: i32, ts: bool, (max_x, max_y): (usize, usize)) -> Result<()> {
|
||||
/// Transform of the scaled coefficients `nz[..nnz]` and addition to the prediction.
|
||||
fn reconstruct(&mut self, (x0, y0): (i32, i32), log2: u32, c: usize, ts: bool, nnz: usize, (max_x, max_y): (usize, usize)) -> Result<()> {
|
||||
let n = 1usize << log2;
|
||||
let bd = if c == 0 { self.pic.bd_y } else { self.pic.bd_c };
|
||||
let coeffs = &mut self.coeffs[..n * n];
|
||||
let mut dc_only = false;
|
||||
if !self.cu_transquant_bypass {
|
||||
// scaling (8.6.3)
|
||||
let bd_shift = bd + log2 - 5;
|
||||
let round = 1i64 << (bd_shift - 1);
|
||||
let scale = (LEVEL_SCALE[(qp % 6) as usize] as i64) << (qp / 6);
|
||||
let m: Option<&[u8]> = match &self.pic.scaling {
|
||||
Some(s) if !(ts && n > 4) => {
|
||||
let size_id = (log2 - 2) as usize;
|
||||
let matrix_id = if self.cu_intra { c } else { 3 + c };
|
||||
Some(&s[size_id * 6 + matrix_id])
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
for y in 0..=max_y {
|
||||
for x in 0..=max_x {
|
||||
let v = coeffs[y * n + x];
|
||||
if v != 0 {
|
||||
let f = m.map_or(16, |m| m[y * n + x] as i64);
|
||||
coeffs[y * n + x] = ((v as i64 * f * scale + round) >> bd_shift).clamp(-32768, 32767) as i32;
|
||||
}
|
||||
}
|
||||
let bypass = self.cu_transquant_bypass;
|
||||
let dst = self.cu_intra && c == 0 && n == 4;
|
||||
let (stride, max) = (if c == 0 { self.pic.width } else { self.pic.cwidth }, (1i32 << bd) - 1);
|
||||
let off = y0 as usize * stride + x0 as usize;
|
||||
let plane = &mut self.pic.planes[c];
|
||||
let bd_shift = 20 - bd;
|
||||
if !bypass && !ts && max_x == 0 && max_y == 0 && !dst {
|
||||
let v = transform::inverse_dc(self.nz[0].1, bd_shift);
|
||||
match n {
|
||||
4 => add_dc::<S, 4>(plane, off, stride, v, max),
|
||||
8 => add_dc::<S, 8>(plane, off, stride, v, max),
|
||||
16 => add_dc::<S, 16>(plane, off, stride, v, max),
|
||||
_ => add_dc::<S, 32>(plane, off, stride, v, max),
|
||||
}
|
||||
let bd_shift = 20 - bd;
|
||||
return Ok(());
|
||||
}
|
||||
// the inverse transform reads only the bounding box of the coefficients
|
||||
let coeffs = &mut self.coeffs[..n * n];
|
||||
if bypass || ts {
|
||||
coeffs.fill(0);
|
||||
} else {
|
||||
for row in coeffs.chunks_exact_mut(n).take(max_y + 1) {
|
||||
row[..=max_x].fill(0);
|
||||
}
|
||||
}
|
||||
for &(p, v) in &self.nz[..nnz] {
|
||||
coeffs[p as usize] = v;
|
||||
}
|
||||
if !bypass {
|
||||
if ts {
|
||||
transform::transform_skip(coeffs, n, bd_shift);
|
||||
} else if max_x == 0 && max_y == 0 && !(self.cu_intra && c == 0 && n == 4) {
|
||||
let v = transform::inverse_dc(coeffs[0], bd_shift);
|
||||
coeffs.fill(v);
|
||||
dc_only = true;
|
||||
} else {
|
||||
let dst = self.cu_intra && c == 0 && n == 4;
|
||||
transform::inverse_transform(coeffs, n, dst, max_x, max_y, bd_shift);
|
||||
}
|
||||
}
|
||||
let _ = dc_only;
|
||||
let (stride, max) = (if c == 0 { self.pic.width } else { self.pic.cwidth }, (1i32 << bd) - 1);
|
||||
let plane = &mut self.pic.planes[c];
|
||||
for y in 0..n {
|
||||
let row = &mut plane[(y0 as usize + y) * stride + x0 as usize..][..n];
|
||||
for (x, s) in row.iter_mut().enumerate() {
|
||||
*s = S::from_i32((s.to_i32() + coeffs[y * n + x]).clamp(0, max));
|
||||
}
|
||||
match n {
|
||||
4 => add_residual::<S, 4>(plane, off, stride, coeffs, max),
|
||||
8 => add_residual::<S, 8>(plane, off, stride, coeffs, max),
|
||||
16 => add_residual::<S, 16>(plane, off, stride, coeffs, max),
|
||||
_ => add_residual::<S, 32>(plane, off, stride, coeffs, max),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn add_residual<S: Sample, const N: usize>(plane: &mut [S], off: usize, stride: usize, res: &[i32], max: i32) {
|
||||
for (y, r) in res.as_chunks::<N>().0.iter().take(N).enumerate() {
|
||||
let row = &mut plane[off + y * stride..][..N];
|
||||
for (s, &d) in row.iter_mut().zip(r) {
|
||||
*s = S::from_i32((s.to_i32() + d).max(0).min(max));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn add_dc<S: Sample, const N: usize>(plane: &mut [S], off: usize, stride: usize, v: i32, max: i32) {
|
||||
for y in 0..N {
|
||||
for s in &mut plane[off + y * stride..][..N] {
|
||||
*s = S::from_i32((s.to_i32() + v).max(0).min(max));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -51,6 +51,72 @@ pub fn scan_order(log2: u32, scan_idx: usize) -> &'static [(u8, u8)] {
|
||||
}
|
||||
}
|
||||
|
||||
/// Inverse of a scan of an `N x N` block: scan position at `[y * N + x]`.
|
||||
const fn inverse<const L: usize>(scan: [(u8, u8); L], n: usize) -> [u8; L] {
|
||||
let mut out = [0u8; L];
|
||||
let mut i = 0;
|
||||
while i < L {
|
||||
out[scan[i].1 as usize * n + scan[i].0 as usize] = i as u8;
|
||||
i += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
static INV_SCAN_1: [[u8; 1]; 3] = [[0], [0], [0]];
|
||||
static INV_SCAN_2: [[u8; 4]; 3] = [inverse(diag::<4>(2), 2), inverse(hv::<4>(2, false), 2), inverse(hv::<4>(2, true), 2)];
|
||||
static INV_SCAN_4: [[u8; 16]; 3] = [inverse(diag::<16>(4), 4), inverse(hv::<16>(4, false), 4), inverse(hv::<16>(4, true), 4)];
|
||||
static INV_SCAN_8: [[u8; 64]; 3] = [inverse(diag::<64>(8), 8), inverse(hv::<64>(8, false), 8), inverse(hv::<64>(8, true), 8)];
|
||||
|
||||
/// Position of (x, y) in ScanOrder[log2BlockSize][scanIdx] (log2BlockSize 0..=3).
|
||||
#[inline]
|
||||
pub fn scan_pos(log2: u32, scan_idx: usize, x: u32, y: u32) -> usize {
|
||||
let i = ((y << log2) + x) as usize;
|
||||
(match log2 {
|
||||
0 => INV_SCAN_1[scan_idx][i],
|
||||
1 => INV_SCAN_2[scan_idx][i],
|
||||
2 => INV_SCAN_4[scan_idx][i],
|
||||
_ => INV_SCAN_8[scan_idx][i],
|
||||
}) as usize
|
||||
}
|
||||
|
||||
/// sig_coeff_flag ctxInc (9.3.4.2.5) by scan position n inside a 4x4 sub-block, `[scanIdx][k][n]`:
|
||||
/// k = prevCsbf (0..=3) gives sigCtx of blocks larger than 4x4 before the position and size
|
||||
/// offsets; k = 4 gives ctxIdxMap of 4x4 blocks.
|
||||
pub static SIG_CTX: [[[u8; 16]; 5]; 3] = [sig_ctx(diag::<16>(4)), sig_ctx(hv::<16>(4, false)), sig_ctx(hv::<16>(4, true))];
|
||||
|
||||
const fn sig_ctx(scan: [(u8, u8); 16]) -> [[u8; 16]; 5] {
|
||||
let mut out = [[0u8; 16]; 5];
|
||||
let mut n = 0;
|
||||
while n < 16 {
|
||||
let (x, y) = (scan[n].0, scan[n].1);
|
||||
out[0][n] = if x + y == 0 {
|
||||
2
|
||||
} else if x + y < 3 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
out[1][n] = if y == 0 {
|
||||
2
|
||||
} else if y == 1 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
out[2][n] = if x == 0 {
|
||||
2
|
||||
} else if x == 1 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
out[3][n] = 2;
|
||||
out[4][n] = CTX_IDX_MAP[(y as usize) * 4 + x as usize];
|
||||
n += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Up-right diagonal scan for log2 size 2 or 3.
|
||||
pub fn scan_diag(log2: u32) -> &'static [(u8, u8)] {
|
||||
scan_order(log2, 0)
|
||||
@@ -81,7 +147,7 @@ pub fn qpc(qpi: i32, chroma_array_type_1: bool) -> i32 {
|
||||
}
|
||||
|
||||
/// Table 9-50 ctxIdxMap for 4x4 sig_coeff_flag contexts.
|
||||
pub static CTX_IDX_MAP: [u8; 16] = [0, 1, 4, 5, 2, 3, 4, 5, 6, 6, 8, 8, 7, 7, 8, 8];
|
||||
pub const CTX_IDX_MAP: [u8; 16] = [0, 1, 4, 5, 2, 3, 4, 5, 6, 6, 8, 8, 7, 7, 8, 8];
|
||||
|
||||
/// levelScale (8-309).
|
||||
pub static LEVEL_SCALE: [i32; 6] = [40, 45, 51, 57, 64, 72];
|
||||
@@ -137,6 +203,17 @@ mod tests {
|
||||
assert_eq!(SCAN_4[1][1], (1, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_pos_inverts_scan_order() {
|
||||
for log2 in 0..4 {
|
||||
for scan_idx in 0..3 {
|
||||
for (i, &(x, y)) in scan_order(log2, scan_idx).iter().enumerate() {
|
||||
assert_eq!(scan_pos(log2, scan_idx, x as u32, y as u32), i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dct_matrix_properties() {
|
||||
// rows are nearly orthogonal with norm ~ 64^2 * 32
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
//! Hostile-input robustness: truncation, header-region sweeps and random mutation of small
|
||||
//! ffmpeg-generated AV1 streams (frame by frame) must never panic or run away. Skips without ffmpeg.
|
||||
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
@@ -20,8 +19,15 @@ impl Rng {
|
||||
}
|
||||
}
|
||||
|
||||
/// (name, encoder args, pix_fmt)
|
||||
const SEEDS: &[(&str, &str, &str)] = &[("av1_fz_default", "-c:v libsvtav1 -g 4", "yuv420p"), ("av1_fz_10bit", "-c:v libsvtav1 -g 6", "yuv420p10le")];
|
||||
// Seeds are at least 64x64: SVT-AV1 before 3.0 (the Ubuntu runners have 2.3) refuses a source under 64 lines.
|
||||
/// (name, encoder args, pix_fmt, size, decoder thread counts)
|
||||
const SEEDS: &[(&str, &str, &str, &str, &[usize])] = &[
|
||||
("av1_fz_default", "-c:v libsvtav1 -g 4", "yuv420p", "64x64", &[1, 3]),
|
||||
("av1_fz_10bit", "-c:v libsvtav1 -g 6", "yuv420p10le", "64x64", &[1, 3]),
|
||||
// Several superblock rows, so corrupt references meet frames already predicting from their
|
||||
// top rows (row wavefront with frame threads).
|
||||
("av1_fz_rows", "-c:v libsvtav1 -g 6", "yuv420p", "64x136", &[1, 8]),
|
||||
];
|
||||
|
||||
fn ivf_frames(data: &[u8]) -> Vec<Vec<u8>> {
|
||||
let mut out = Vec::new();
|
||||
@@ -37,14 +43,13 @@ fn ivf_frames(data: &[u8]) -> Vec<Vec<u8>> {
|
||||
out
|
||||
}
|
||||
|
||||
// Seeds are 64x64: SVT-AV1 before 3.0 (the Ubuntu runners have 2.3) refuses a source under 64 lines.
|
||||
fn make(ff: &std::path::Path, name: &str, args: &str, pix: &str) -> Option<Vec<Vec<u8>>> {
|
||||
fn make(ff: &std::path::Path, name: &str, args: &str, pix: &str, size: &str) -> Option<Vec<Vec<u8>>> {
|
||||
let out = craft_media_testkit::fixtures_dir("av1").join(format!("{name}.ivf"));
|
||||
let p = craft_media_testkit::fixtures::generate(&out, |tmp| {
|
||||
let a: Vec<&str> = args.split_whitespace().collect();
|
||||
matches!(
|
||||
Command::new(ff)
|
||||
.args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=64x64:rate=25", "-frames:v", "6", "-pix_fmt", pix])
|
||||
.args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", &format!("testsrc2=size={size}:rate=25"), "-frames:v", "6", "-pix_fmt", pix])
|
||||
.args(a)
|
||||
.args(["-f", "ivf"])
|
||||
.arg(tmp)
|
||||
@@ -69,20 +74,29 @@ fn count_panics() {
|
||||
});
|
||||
}
|
||||
|
||||
fn run_all(frames: &[Vec<u8>], label: &str) {
|
||||
fn run_all(frames: &[Vec<u8>], threads: &[usize], label: &str) {
|
||||
craft_media_testkit::set_progress(label);
|
||||
count_panics();
|
||||
let before = PANICS.load(Ordering::SeqCst);
|
||||
let res = catch_unwind(AssertUnwindSafe(|| {
|
||||
for threads in [1, 3] {
|
||||
let mut dec = Decoder::with_threads(threads);
|
||||
for (i, f) in frames.iter().enumerate() {
|
||||
let _ = dec.decode_pts(f, i as i64);
|
||||
}
|
||||
let _ = dec.flush_result();
|
||||
}
|
||||
}));
|
||||
assert!(res.is_ok() && PANICS.load(Ordering::SeqCst) == before, "panic: {label}");
|
||||
// Every thread count decodes the stream at the same time, each on a thread of its own.
|
||||
let ok = std::thread::scope(|s| {
|
||||
let runs: Vec<_> = threads
|
||||
.iter()
|
||||
.map(|&threads| {
|
||||
s.spawn(move || {
|
||||
let mut dec = Decoder::with_threads(threads);
|
||||
for (i, f) in frames.iter().enumerate() {
|
||||
let _ = dec.decode_pts(f, i as i64);
|
||||
}
|
||||
let _ = dec.flush_result();
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
// Join every run (no short cut), then check them all.
|
||||
let joined: Vec<bool> = runs.into_iter().map(|r| r.join().is_ok()).collect();
|
||||
joined.into_iter().all(|ok| ok)
|
||||
});
|
||||
assert!(ok && PANICS.load(Ordering::SeqCst) == before, "panic: {label}");
|
||||
}
|
||||
|
||||
/// Random-mutation rounds per seed (raise with FILMCRAFT_FUZZ_ITERS for a long hunt).
|
||||
@@ -95,17 +109,23 @@ const DEADLINE_SECS: u64 = 600;
|
||||
|
||||
#[test]
|
||||
fn hostile_streams_never_panic() {
|
||||
craft_media_testkit::with_deadline(DEADLINE_SECS, sweep);
|
||||
craft_media_testkit::with_deadline(DEADLINE_SECS, || sweep(&SEEDS[..2]));
|
||||
}
|
||||
|
||||
fn sweep() {
|
||||
/// The seed with several superblock rows, in a test of its own so it runs beside the others.
|
||||
#[test]
|
||||
fn hostile_streams_with_superblock_rows_never_panic() {
|
||||
craft_media_testkit::with_deadline(DEADLINE_SECS, || sweep(&SEEDS[2..]));
|
||||
}
|
||||
|
||||
fn sweep(seeds: &[(&str, &str, &str, &str, &[usize])]) {
|
||||
let Some(ff) = craft_media_testkit::ffmpeg_or_skip("AV1 fuzz") else { return };
|
||||
for &(name, args, pix) in SEEDS {
|
||||
for &(name, args, pix, size, threads) in seeds {
|
||||
if !craft_media_testkit::ffmpeg_has_encoder(&ff, "libsvtav1") {
|
||||
craft_media_testkit::skip_test("AV1 fuzz", &format!("this ffmpeg has no libsvtav1 encoder (seed {name})"));
|
||||
continue;
|
||||
}
|
||||
let Some(seed) = make(&ff, name, args, pix) else {
|
||||
let Some(seed) = make(&ff, name, args, pix, size) else {
|
||||
panic!("ffmpeg has libsvtav1 but could not encode the {name} seed");
|
||||
};
|
||||
if seed.is_empty() {
|
||||
@@ -118,7 +138,7 @@ fn sweep() {
|
||||
for n in (0..len).step_by((len / 200).max(1)) {
|
||||
let mut d = seed.clone();
|
||||
d[fi].truncate(n);
|
||||
run_all(&d, &format!("{name} frame {fi} truncate {n}"));
|
||||
run_all(&d, threads, &format!("{name} frame {fi} truncate {n}"));
|
||||
}
|
||||
}
|
||||
// header region of every frame: every byte to a few values, every bit
|
||||
@@ -127,12 +147,12 @@ fn sweep() {
|
||||
for v in [0u8, 0xff, 0x80, 0x01, 0x7f] {
|
||||
let mut d = seed.clone();
|
||||
d[fi][p] = v;
|
||||
run_all(&d, &format!("{name} frame {fi} byte {p}={v:#x}"));
|
||||
run_all(&d, threads, &format!("{name} frame {fi} byte {p}={v:#x}"));
|
||||
}
|
||||
for b in 0..8 {
|
||||
let mut d = seed.clone();
|
||||
d[fi][p] ^= 1 << b;
|
||||
run_all(&d, &format!("{name} frame {fi} flip {p}.{b}"));
|
||||
run_all(&d, threads, &format!("{name} frame {fi} flip {p}.{b}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -151,7 +171,7 @@ fn sweep() {
|
||||
d[fi][p] = rng.next() as u8;
|
||||
}
|
||||
}
|
||||
run_all(&d, &format!("{name} random {i}"));
|
||||
run_all(&d, threads, &format!("{name} random {i}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,3 +9,4 @@ mod hostile_dims;
|
||||
mod oracle_inter;
|
||||
mod oracle_intra;
|
||||
mod perf;
|
||||
mod wavefront;
|
||||
@@ -0,0 +1,93 @@
|
||||
//! Stress tests of the frame threading (parse / reconstruct split and the row wavefront across
|
||||
//! frames): every thread count gives the single-thread pictures, run after run, and dropping
|
||||
//! or flushing a busy decoder never hangs. A hang fails the test through the deadline.
|
||||
|
||||
use crate::common;
|
||||
use common::*;
|
||||
use craft_video::av1::decode::{Decoder, Picture};
|
||||
|
||||
/// A GOP with many superblock rows (the wavefront has rows to overlap), hierarchical
|
||||
/// references and motion reaching across rows.
|
||||
fn spec() -> Spec {
|
||||
Spec::new("wavefront_640x480", "testsrc2=s=640x480:r=25,rotate=a=t*0.5:c=black", 24, "yuv420p", &["-preset", "8", "-crf", "38"])
|
||||
}
|
||||
|
||||
/// Seconds a test may take before it counts as a hang (override: FILMCRAFT_HOSTILE_TIMEOUT_SECS).
|
||||
const DEADLINE_SECS: u64 = 300;
|
||||
|
||||
fn decode(tus: &[Vec<u8>], threads: usize) -> Vec<Picture> {
|
||||
let mut dec = Decoder::with_threads(threads);
|
||||
let mut pics = Vec::new();
|
||||
for (i, tu) in tus.iter().enumerate() {
|
||||
pics.extend(dec.decode_pts(tu, i as i64).unwrap_or_else(|e| panic!("threads {threads}, unit {i}: {e}")));
|
||||
}
|
||||
pics.extend(dec.flush_result().unwrap_or_else(|e| panic!("threads {threads}, flush: {e}")));
|
||||
pics
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_thread_count_gives_the_single_thread_pictures_repeatedly() {
|
||||
let Some(ff) = ffmpeg() else { return };
|
||||
let tus = ivf_frames(&make(&ff, &spec()));
|
||||
craft_media_testkit::with_deadline(DEADLINE_SECS, move || {
|
||||
let want = decode(&tus, 1);
|
||||
assert_eq!(want.len(), 24);
|
||||
for threads in [2, 3, 4, 5, 8, 16] {
|
||||
for round in 0..4 {
|
||||
assert!(decode(&tus, threads) == want, "threads {threads}, round {round}: pictures differ");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// 128x128 superblocks in two tile columns (libaom), with motion field estimation: the
|
||||
/// wavefront over the superblock rows of every tile column.
|
||||
fn spec_sb128_tiles() -> Spec {
|
||||
let enc = ["-cpu-used", "8", "-crf", "40", "-tile-columns", "1", "-lag-in-frames", "16", "-aom-params", "sb-size=128:enable-ref-frame-mvs=1"];
|
||||
Spec::new("wavefront_sb128_tiles", "testsrc2=s=640x400:r=25,rotate=a=t*0.5:c=black", 16, "yuv420p", &enc).encoder("libaom-av1")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn superblocks_of_128_in_tile_columns_give_the_single_thread_pictures() {
|
||||
let Some(ff) = ffmpeg_libaom() else { return };
|
||||
let tus = ivf_frames(&make(&ff, &spec_sb128_tiles()));
|
||||
craft_media_testkit::with_deadline(DEADLINE_SECS, move || {
|
||||
let want = decode(&tus, 1);
|
||||
assert_eq!(want.len(), 16);
|
||||
for threads in [2, 3, 4, 8, 16] {
|
||||
for round in 0..2 {
|
||||
assert!(decode(&tus, threads) == want, "threads {threads}, round {round}: pictures differ");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_or_flushing_a_busy_decoder_never_hangs() {
|
||||
let Some(ff) = ffmpeg() else { return };
|
||||
let tus = ivf_frames(&make(&ff, &spec()));
|
||||
craft_media_testkit::with_deadline(DEADLINE_SECS, move || {
|
||||
let want = decode(&tus, 1);
|
||||
for threads in [2, 3, 8, 16] {
|
||||
for cut in [1, 2, 7, 12, 23] {
|
||||
// Dropped with frames in flight (no flush).
|
||||
let mut dec = Decoder::with_threads(threads);
|
||||
for (i, tu) in tus.iter().take(cut).enumerate() {
|
||||
dec.decode_pts(tu, i as i64).expect("decode");
|
||||
}
|
||||
drop(dec);
|
||||
// Flushed mid-stream (as before a seek), then continued: the same pictures.
|
||||
let mut dec = Decoder::with_threads(threads);
|
||||
let mut pics = Vec::new();
|
||||
for (i, tu) in tus.iter().enumerate() {
|
||||
if i == cut {
|
||||
pics.extend(dec.flush_result().expect("flush"));
|
||||
}
|
||||
pics.extend(dec.decode_pts(tu, i as i64).expect("decode"));
|
||||
}
|
||||
pics.extend(dec.flush_result().expect("flush"));
|
||||
assert!(pics == want, "threads {threads}, flush before unit {cut}: pictures differ");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -11,6 +11,8 @@
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::props::Rgb;
|
||||
|
||||
/// Most series in a chart.
|
||||
pub const MAX_SERIES: usize = 255;
|
||||
/// Most categories (points per series) in a chart.
|
||||
@@ -123,17 +125,111 @@ impl LegendPos {
|
||||
}
|
||||
}
|
||||
|
||||
/// One series: its name and a value per category (`None` = no value, a gap).
|
||||
/// One series: its name, a value per category (`None` = no value, a gap) and how it is drawn
|
||||
/// (Chart Design ▸ Format Series): fill, line, markers, smooth lines, its own data labels and
|
||||
/// number format. Unset fields follow the chart's defaults.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ChartSeries {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub values: Vec<Option<f64>>,
|
||||
/// Fill colour; `None`: the chart's accent colour for the series.
|
||||
#[serde(default)]
|
||||
pub fill: Option<Rgb>,
|
||||
/// "None" fill: the series draws no fill.
|
||||
#[serde(default)]
|
||||
pub no_fill: bool,
|
||||
/// Line width in points; `None`: the chart's default.
|
||||
#[serde(default)]
|
||||
pub line_w: Option<f32>,
|
||||
/// "None" line: the series draws no line.
|
||||
#[serde(default)]
|
||||
pub no_line: bool,
|
||||
/// Markers on points; `None`: unset (scatter charts draw them; a saved scatter series then has no `c:marker`, so Word picks its own symbols).
|
||||
#[serde(default)]
|
||||
pub markers: Option<bool>,
|
||||
/// Draw the line as a smooth curve.
|
||||
#[serde(default)]
|
||||
pub smooth: bool,
|
||||
/// Data labels for this series; `None`: follow the chart's setting.
|
||||
#[serde(default)]
|
||||
pub labels: Option<bool>,
|
||||
/// Number format of this series's labels and values; `None`: linked to the source.
|
||||
#[serde(default)]
|
||||
pub fmt: Option<String>,
|
||||
}
|
||||
|
||||
/// One chart axis as Word formats it (Format Axis): title, scale bounds, tick-label number
|
||||
/// format, gridlines, tick labels and the axis line.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ChartAxis {
|
||||
/// `None`: no axis title; text cut to [`MAX_TEXT`] characters.
|
||||
#[serde(default)]
|
||||
pub title: Option<String>,
|
||||
/// Fixed lower bound; `None`: automatic.
|
||||
#[serde(default)]
|
||||
pub min: Option<f64>,
|
||||
/// Fixed upper bound; `None`: automatic.
|
||||
#[serde(default)]
|
||||
pub max: Option<f64>,
|
||||
/// Tick-label number format (`#,##0`, `0.00`, `0%`…); `None`: linked to the source data.
|
||||
#[serde(default)]
|
||||
pub fmt: Option<String>,
|
||||
/// Major gridlines drawn along this axis.
|
||||
#[serde(default)]
|
||||
pub gridlines: bool,
|
||||
/// Tick labels; `false` hides the axis the way Word's Axis Labels ▸ None does.
|
||||
#[serde(default = "the_true")]
|
||||
pub labels: bool,
|
||||
/// The axis line.
|
||||
#[serde(default = "the_true")]
|
||||
pub line: bool,
|
||||
}
|
||||
|
||||
fn the_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
impl Default for ChartAxis {
|
||||
fn default() -> Self {
|
||||
ChartAxis { title: None, min: None, max: None, fmt: None, gridlines: false, labels: true, line: true }
|
||||
}
|
||||
}
|
||||
|
||||
/// The value axis as a new chart has it: gridlines, labels, no axis line.
|
||||
pub fn value_axis() -> ChartAxis {
|
||||
ChartAxis { gridlines: true, line: false, ..ChartAxis::default() }
|
||||
}
|
||||
|
||||
impl ChartAxis {
|
||||
/// Make the axis safe to store: texts cleaned, bounds finite, `min` under `max` (an
|
||||
/// impossible pair drops both), the format code cut to [`MAX_FMT`] characters.
|
||||
pub fn sanitize(&mut self) {
|
||||
self.title = self.title.as_deref().map(clean_text).filter(|t| !t.is_empty());
|
||||
self.min = self.min.filter(|v| v.is_finite());
|
||||
self.max = self.max.filter(|v| v.is_finite());
|
||||
if matches!((self.min, self.max), (Some(lo), Some(hi)) if lo >= hi) {
|
||||
self.min = None;
|
||||
self.max = None;
|
||||
}
|
||||
self.fmt = self.fmt.as_deref().map(clean_fmt).filter(|f| !f.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// Longest axis number format kept, characters.
|
||||
pub const MAX_FMT: usize = 128;
|
||||
|
||||
/// A number format code: control characters as spaces, ends trimmed, at most [`MAX_FMT`]
|
||||
/// characters.
|
||||
pub fn clean_fmt(s: &str) -> String {
|
||||
let s: String = s.chars().take(MAX_FMT * 4).map(|c| if c.is_control() { ' ' } else { c }).collect();
|
||||
s.trim().chars().take(MAX_FMT).collect::<String>().trim_end().to_string()
|
||||
}
|
||||
|
||||
/// A chart the app can edit.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ChartSpec {
|
||||
#[serde(rename = "type")]
|
||||
@@ -151,6 +247,29 @@ pub struct ChartSpec {
|
||||
/// Show each point's value next to it.
|
||||
#[serde(default)]
|
||||
pub data_labels: bool,
|
||||
/// The category axis (scatter charts: the x axis).
|
||||
#[serde(default)]
|
||||
pub cat_axis: ChartAxis,
|
||||
/// The value axis (scatter charts: the y axis); gridlines on, no axis line, as a new chart has them.
|
||||
#[serde(default = "value_axis")]
|
||||
pub val_axis: ChartAxis,
|
||||
}
|
||||
|
||||
impl Default for ChartSpec {
|
||||
/// An empty chart whose value axis is the one a new chart (and a missing JSON field) has:
|
||||
/// gridlines on, no axis line.
|
||||
fn default() -> Self {
|
||||
ChartSpec {
|
||||
kind: ChartType::default(),
|
||||
categories: Vec::new(),
|
||||
series: Vec::new(),
|
||||
title: None,
|
||||
legend: None,
|
||||
data_labels: false,
|
||||
cat_axis: ChartAxis::default(),
|
||||
val_axis: value_axis(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChartSpec {
|
||||
@@ -168,10 +287,23 @@ impl ChartSpec {
|
||||
ChartSpec {
|
||||
kind,
|
||||
categories: categories.into_iter().map(str::to_string).collect(),
|
||||
series: series.into_iter().map(|(n, v)| ChartSeries { name: n.to_string(), values: v.into_iter().map(Some).collect() }).collect(),
|
||||
series: series
|
||||
.into_iter()
|
||||
.map(|(n, v)| ChartSeries {
|
||||
name: n.to_string(),
|
||||
values: v.into_iter().map(Some).collect(),
|
||||
// Scatter series are drawn as markers without a line.
|
||||
no_line: kind == ChartType::Scatter,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
title: None,
|
||||
legend: Some(LegendPos::Bottom),
|
||||
data_labels: false,
|
||||
cat_axis: ChartAxis::default(),
|
||||
// Pie and doughnut charts have no axes: their axes stay plain, so the chart reads back
|
||||
// as the same model.
|
||||
val_axis: if kind.is_round() { ChartAxis::default() } else { value_axis() },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,8 +335,12 @@ impl ChartSpec {
|
||||
*v = v.filter(|x| x.is_finite());
|
||||
}
|
||||
s.values.resize(n, None);
|
||||
s.fmt = s.fmt.as_deref().map(clean_fmt).filter(|f| !f.is_empty());
|
||||
s.line_w = s.line_w.filter(|w| w.is_finite()).map(|w| w.clamp(0.0, 20.0));
|
||||
}
|
||||
self.title = self.title.as_deref().map(clean_text).filter(|t| !t.is_empty());
|
||||
self.cat_axis.sanitize();
|
||||
self.val_axis.sanitize();
|
||||
}
|
||||
|
||||
/// The chart after `sanitize`.
|
||||
@@ -229,7 +365,11 @@ mod tests {
|
||||
let mut c = ChartSpec {
|
||||
categories: vec!["a".into(); MAX_POINTS + 10],
|
||||
series: (0..MAX_SERIES + 5)
|
||||
.map(|i| ChartSeries { name: format!("{}\n{}", "x".repeat(400), i), values: vec![Some(f64::NAN), Some(f64::INFINITY), Some(1.0)] })
|
||||
.map(|i| ChartSeries {
|
||||
name: format!("{}\n{}", "x".repeat(400), i),
|
||||
values: vec![Some(f64::NAN), Some(f64::INFINITY), Some(1.0)],
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
title: Some(" ".into()),
|
||||
..Default::default()
|
||||
@@ -250,7 +390,7 @@ mod tests {
|
||||
fn short_categories_are_padded_to_the_longest_series() {
|
||||
let c = ChartSpec {
|
||||
categories: vec!["a".into()],
|
||||
series: vec![ChartSeries { name: "s".into(), values: vec![Some(1.0), Some(2.0)] }],
|
||||
series: vec![ChartSeries { name: "s".into(), values: vec![Some(1.0), Some(2.0)], ..Default::default() }],
|
||||
..Default::default()
|
||||
}
|
||||
.sanitized();
|
||||
@@ -258,4 +398,53 @@ mod tests {
|
||||
assert_eq!(ChartType::from_id("BARSTACKED"), Some(ChartType::BarStacked));
|
||||
assert_eq!(LegendPos::from_id("top"), Some(LegendPos::Top));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_and_missing_json_axis_fields_agree() {
|
||||
let from_json: ChartSpec = serde_json::from_str(r#"{"type":"column"}"#).unwrap();
|
||||
let d = ChartSpec::default();
|
||||
assert_eq!((d.cat_axis.clone(), d.val_axis.clone()), (from_json.cat_axis, from_json.val_axis));
|
||||
assert!(d.val_axis.gridlines && !d.val_axis.line);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn axis_bounds_and_formats_are_cleaned() {
|
||||
let mut a = ChartAxis {
|
||||
title: Some(" Sales\n".into()),
|
||||
min: Some(f64::NAN),
|
||||
max: Some(50.0),
|
||||
fmt: Some("x".repeat(MAX_FMT + 40)),
|
||||
gridlines: true,
|
||||
labels: false,
|
||||
line: true,
|
||||
};
|
||||
a.sanitize();
|
||||
assert_eq!(a.title.as_deref(), Some("Sales"));
|
||||
assert_eq!(a.min, None);
|
||||
assert_eq!(a.max, Some(50.0));
|
||||
assert_eq!(a.fmt.as_deref().map(|f| f.chars().count()), Some(MAX_FMT));
|
||||
|
||||
// min not under max drops the pair; a format of only spaces is no format.
|
||||
let mut b = ChartAxis { min: Some(50.0), max: Some(50.0), fmt: Some(" ".into()), ..Default::default() };
|
||||
b.sanitize();
|
||||
assert_eq!((b.min, b.max), (None, None));
|
||||
assert_eq!(b.fmt, None);
|
||||
|
||||
let mut spec = ChartSpec {
|
||||
series: vec![ChartSeries { name: "s".into(), values: vec![Some(1.0)], ..Default::default() }],
|
||||
categories: vec!["a".into()],
|
||||
..Default::default()
|
||||
};
|
||||
spec.series[0].line_w = Some(f32::NAN);
|
||||
spec.series[0].fmt = Some("0.00".into());
|
||||
spec.sanitize();
|
||||
assert_eq!(spec.series[0].line_w, None);
|
||||
assert_eq!(spec.series[0].fmt.as_deref(), Some("0.00"));
|
||||
let mut spec = ChartSpec {
|
||||
series: vec![ChartSeries { name: "s".into(), values: vec![], line_w: Some(100.0), ..Default::default() }],
|
||||
..Default::default()
|
||||
};
|
||||
spec.sanitize();
|
||||
assert_eq!(spec.series[0].line_w, Some(20.0), "a wild width clamps to 20 pt");
|
||||
}
|
||||
}
|
||||
@@ -115,7 +115,9 @@ pub enum GraphicItem {
|
||||
#[serde(default)]
|
||||
stroke_width: f32,
|
||||
},
|
||||
/// One line of text in `rect`, aligned horizontally by `align`, centred vertically.
|
||||
/// One line of text in `rect`, aligned horizontally by `align`, centred vertically; `angle`
|
||||
/// degrees clockwise turns the text about `rect`'s centre as it is drawn (chart axis titles
|
||||
/// turn the vertical axis's title -90°, reading upwards, as Word does).
|
||||
Text {
|
||||
rect: [f32; 4],
|
||||
text: String,
|
||||
@@ -127,6 +129,8 @@ pub enum GraphicItem {
|
||||
align: TextAlign,
|
||||
#[serde(default)]
|
||||
font: Option<String>,
|
||||
#[serde(default)]
|
||||
angle: f32,
|
||||
},
|
||||
/// A picture (`Document::media` key) stretched into `rect`.
|
||||
Image { rect: [f32; 4], media: String },
|
||||
|
||||
+16
-11
@@ -83,17 +83,22 @@ for new pictures (`vm = i + 1`) and the `XLDAPR` dynamic-array metadata (`cm = 1
|
||||
The parts this crate writes use the transitional namespaces; a strict file's kept parts stay as
|
||||
they were. A binary workbook is not written.
|
||||
|
||||
A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not
|
||||
read (each cell keeps its format index). Its cell formulas are decoded from their tokens into the
|
||||
text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four
|
||||
ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in
|
||||
the formula, numbers as Excel spells them), with
|
||||
`FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell
|
||||
metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the
|
||||
file keeps the tokens once. A formula whose decoding needs something the reader does not read
|
||||
keeps only its saved result, with a warning saying why: user defined names, table references,
|
||||
other workbooks, Python code stored outside the cell, data tables. The XLSB function table
|
||||
(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s.
|
||||
A binary workbook gives the same `Workbook`, strings, rows and merged cells, and the same `Styles`:
|
||||
number formats, fonts, fills, borders, cell and cell style formats, cell styles, differential
|
||||
formats and the indexed colours, as Excel writes them in `styles.xml` (default fill colours and
|
||||
empty settings left out, a cell style format's `apply*="0"`, a number format with a newer locale tag
|
||||
in its Excel 2007 spelling, as `formatCode` has it). Custom table styles and recent colours are not
|
||||
read; a damaged styles part is a warning and no styles, while one past a cap (65,536 entries in a
|
||||
table, 1,024 gradient stops) or past `binary_text` is too large. Its cell formulas are decoded from
|
||||
their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from
|
||||
the four ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks
|
||||
typed in the formula, numbers as Excel spells them), with `FormulaKind::Shared` or
|
||||
`FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a
|
||||
shared formula gets its own text, decoded at the cell, since the file keeps the tokens once. A
|
||||
formula whose decoding needs something the reader does not read keeps only its saved result, with a
|
||||
warning saying why: user defined names, table references, other workbooks, Python code stored
|
||||
outside the cell, data tables. The XLSB function table (`xlsb/ftab.rs`) and the expression printer
|
||||
are this crate's own, apart from `craft-xls`'s.
|
||||
|
||||
## Hostile input
|
||||
|
||||
|
||||
@@ -209,8 +209,8 @@ pub struct Document {
|
||||
}
|
||||
|
||||
impl Document {
|
||||
/// Reads every part. A binary workbook is refused: its formulas and styles are not read, so
|
||||
/// it cannot be written back as SpreadsheetML without loss.
|
||||
/// Reads every part. A binary workbook is refused: only some of its parts are read, so it
|
||||
/// cannot be written back as SpreadsheetML without loss.
|
||||
pub fn read(bytes: &[u8], limits: &Limits) -> Result<Document, Error> {
|
||||
let mut p = Package::open(bytes, limits)?;
|
||||
if p.is_binary() {
|
||||
|
||||
@@ -220,16 +220,28 @@ impl<'a> Package<'a> {
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// The styles part, `None` when the workbook has none (or is binary: its styles are not read).
|
||||
/// The styles part, `None` when the workbook has none. A damaged one is a warning and `None`;
|
||||
/// only a limit is an error.
|
||||
pub fn styles(&mut self) -> Result<Option<Styles>, Error> {
|
||||
if self.binary.is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
let rels = self.workbook_rels()?;
|
||||
let Some(rel) = rels.first("styles").filter(|r| !r.external) else {
|
||||
self.warn("no styles part");
|
||||
return Ok(None);
|
||||
};
|
||||
if let Some(types) = &self.binary {
|
||||
let read = xlsb::check_type(types, &rel.target, xlsb::STYLES).and_then(|()| {
|
||||
let bytes = self.opc.read(&rel.target)?.ok_or_else(|| xlsb::invalid(format!("missing part {}", rel.target)))?;
|
||||
xlsb::styles(&bytes, &self.limits, &self.budget, &mut self.warnings)
|
||||
});
|
||||
return match read {
|
||||
Ok(styles) => Ok(Some(styles)),
|
||||
Err(Error::TooLarge(m)) => Err(Error::TooLarge(m)),
|
||||
Err(e) => {
|
||||
self.warn(format!("skipped {}: {e}", rel.target));
|
||||
Ok(None)
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(self.optional_xml(&rel.target.clone())?.map(Styles::read))
|
||||
}
|
||||
|
||||
|
||||
+27
-8
@@ -1,11 +1,13 @@
|
||||
//! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheet cells, from
|
||||
//! their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an
|
||||
//! `.xlsx` file holds; one that can't be keeps only its saved result, with a warning saying why.
|
||||
//! Styles are not read. A record that does not parse fails the read.
|
||||
//! The styles part is read into the same [`crate::styles::Styles`] ([`styles`]). A record that
|
||||
//! does not parse fails the read.
|
||||
|
||||
mod expr;
|
||||
mod formula;
|
||||
mod ftab;
|
||||
mod styles;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
@@ -19,11 +21,13 @@ use crate::workbook::{SheetEntry, SheetKind, Workbook};
|
||||
use crate::{Error, Limits, Warnings};
|
||||
pub(crate) use formula::Links;
|
||||
use formula::{Book, Mode, Name, Xti};
|
||||
pub(crate) use styles::styles;
|
||||
|
||||
pub(crate) const MAIN: &str = "application/vnd.ms-excel.main";
|
||||
const BINARY_MAIN: &str = "application/vnd.ms-excel.sheet.binary.macroEnabled.main";
|
||||
pub(crate) const WORKSHEET: &str = "application/vnd.ms-excel.worksheet";
|
||||
pub(crate) const STRINGS: &str = "application/vnd.ms-excel.sharedStrings";
|
||||
pub(crate) const STYLES: &str = "application/vnd.ms-excel.styles";
|
||||
/// Most sheets in the workbook.
|
||||
const MAX_SHEETS: usize = 1024;
|
||||
/// Supporting links (this workbook, add-ins, other workbooks) read at most; past it, formulas
|
||||
@@ -174,12 +178,16 @@ struct Records<'a, 'l> {
|
||||
bytes: Bytes<'a>,
|
||||
limits: &'l Limits,
|
||||
budget: &'l Budget,
|
||||
blocks: Vec<u16>,
|
||||
/// Each open block's end record, and whether its records are read.
|
||||
blocks: Vec<(u16, bool)>,
|
||||
/// Read the alternate content meant for Excel 2007 (version 12), as an `.xlsx` reader sees
|
||||
/// the main attribute where Excel keeps a newer spelling in an extension.
|
||||
excel_2007: bool,
|
||||
}
|
||||
|
||||
impl<'a, 'l> Records<'a, 'l> {
|
||||
fn new(bytes: &'a [u8], limits: &'l Limits, budget: &'l Budget) -> Self {
|
||||
Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![] }
|
||||
Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![], excel_2007: false }
|
||||
}
|
||||
|
||||
fn next(&mut self, warnings: &mut Warnings) -> Result<Option<(u16, &'a [u8])>, Error> {
|
||||
@@ -208,6 +216,7 @@ impl<'a, 'l> Records<'a, 'l> {
|
||||
// BrtFRTBegin, BrtACBegin.
|
||||
35 | 37 => {
|
||||
let mut b = Bytes::new(payload);
|
||||
let mut read = false;
|
||||
if id == 35 {
|
||||
b.take(4)?;
|
||||
} else {
|
||||
@@ -215,22 +224,32 @@ impl<'a, 'l> Records<'a, 'l> {
|
||||
if count == 0 {
|
||||
return Err(invalid("empty alternate-content version list"));
|
||||
}
|
||||
b.take(usize::from(count) * 4)?;
|
||||
// ACProductVersion: fileVersion, then fileProduct (15 bits) and
|
||||
// fileExtension (the version and later).
|
||||
for _ in 0..count {
|
||||
let (version, product) = (b.u16()?, b.u16()?);
|
||||
let later = product & 0x8000 != 0;
|
||||
read |= self.excel_2007 && product & 0x7FFF == 0 && (version == 12 || later && version <= 12);
|
||||
}
|
||||
}
|
||||
b.finish()?;
|
||||
if self.blocks.len() >= 64 {
|
||||
return Err(invalid("extension blocks nested too deeply"));
|
||||
}
|
||||
self.blocks.push(id + 1);
|
||||
warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped");
|
||||
// Inside a skipped block, everything is skipped.
|
||||
let read = read && self.blocks.iter().all(|(_, r)| *r);
|
||||
self.blocks.push((id + 1, read));
|
||||
if !read {
|
||||
warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped");
|
||||
}
|
||||
}
|
||||
// BrtFRTEnd, BrtACEnd.
|
||||
36 | 38 => {
|
||||
if self.blocks.pop() != Some(id) || !payload.is_empty() {
|
||||
if self.blocks.pop().map(|(end, _)| end) != Some(id) || !payload.is_empty() {
|
||||
return Err(invalid("mismatched extension block end"));
|
||||
}
|
||||
}
|
||||
_ if !self.blocks.is_empty() => {}
|
||||
_ if self.blocks.iter().any(|(_, read)| !read) => {}
|
||||
_ => return Ok(Some((id, payload))),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,484 @@
|
||||
//! The styles part of a binary workbook (`styles.bin`, [MS-XLSB] 2.1.7.50), read into the same
|
||||
//! [`Styles`] the XML part gives: what Excel writes in `styles.xml` for the same workbook. Table
|
||||
//! styles, recent colours and future records are not read.
|
||||
|
||||
use craft_sheet::{BorderStyle, HAlign, Pattern, Script, Underline, VAlign};
|
||||
use craft_xml::El;
|
||||
|
||||
use super::{Budget, Bytes, Records, invalid};
|
||||
use crate::styles::{
|
||||
Alignment, Border, BorderEdge, CellStyle, Color, ColorSource, Dxf, Fill, Font, FontScheme, GradientFill, NumFmt, PatternFill, Protection, Styles,
|
||||
Xf,
|
||||
};
|
||||
use crate::{Error, Limits, Warnings};
|
||||
|
||||
/// Characters of a font name or number format kept, as the XML side keeps of a font name.
|
||||
const MAX_NAME_CHARS: usize = 255;
|
||||
/// Longest string read from the part (Excel's own limits are 31 and 255 characters).
|
||||
const MAX_STRING_UNITS: u32 = 32_767;
|
||||
/// Most stops in one gradient fill (Excel writes a handful).
|
||||
const MAX_STOPS: u32 = 1024;
|
||||
/// Most entries in one table of the part (Excel allows about 64,000 cell formats). Each entry is
|
||||
/// an allocation, and the record allowance alone would let a small part hold millions.
|
||||
const MAX_ENTRIES: usize = 65_536;
|
||||
|
||||
/// `BrtColor` and `XFPropColor`: the same eight bytes. `None` for "not set" (`XFPropColor` type 4).
|
||||
fn color(b: &mut Bytes<'_>) -> Result<Option<Color>, Error> {
|
||||
let kind = b.u8()? >> 1;
|
||||
let index = b.u8()?;
|
||||
let tint = i16::from_le_bytes(<[u8; 2]>::try_from(b.take(2)?).map_err(|_| invalid("truncated colour"))?);
|
||||
let [r, g, bl, a] = <[u8; 4]>::try_from(b.take(4)?).map_err(|_| invalid("truncated colour"))?;
|
||||
let source = match kind {
|
||||
0 => ColorSource::Auto,
|
||||
1 => ColorSource::Indexed(u32::from(index)),
|
||||
2 => ColorSource::Rgb(u32::from_be_bytes([a, r, g, bl])),
|
||||
3 => ColorSource::Theme(u32::from(index)),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
Ok(Some(Color { source, tint: (f64::from(tint) / 32767.0).clamp(-1.0, 1.0) }))
|
||||
}
|
||||
|
||||
fn pattern(fls: u32) -> Result<Pattern, Error> {
|
||||
const PATTERNS: [Pattern; 19] = [
|
||||
Pattern::None,
|
||||
Pattern::Solid,
|
||||
Pattern::Gray50,
|
||||
Pattern::Gray75,
|
||||
Pattern::Gray25,
|
||||
Pattern::DarkHorizontal,
|
||||
Pattern::DarkVertical,
|
||||
Pattern::DarkDown,
|
||||
Pattern::DarkUp,
|
||||
Pattern::DarkGrid,
|
||||
Pattern::DarkTrellis,
|
||||
Pattern::LightHorizontal,
|
||||
Pattern::LightVertical,
|
||||
Pattern::LightDown,
|
||||
Pattern::LightUp,
|
||||
Pattern::LightGrid,
|
||||
Pattern::LightTrellis,
|
||||
Pattern::Gray125,
|
||||
Pattern::Gray0625,
|
||||
];
|
||||
PATTERNS.get(fls as usize).copied().ok_or_else(|| invalid("unknown fill pattern"))
|
||||
}
|
||||
|
||||
fn border_style(dg: u16) -> Result<BorderStyle, Error> {
|
||||
const STYLES: [BorderStyle; 14] = [
|
||||
BorderStyle::None,
|
||||
BorderStyle::Thin,
|
||||
BorderStyle::Medium,
|
||||
BorderStyle::Dashed,
|
||||
BorderStyle::Dotted,
|
||||
BorderStyle::Thick,
|
||||
BorderStyle::Double,
|
||||
BorderStyle::Hair,
|
||||
BorderStyle::MediumDashed,
|
||||
BorderStyle::DashDot,
|
||||
BorderStyle::MediumDashDot,
|
||||
BorderStyle::DashDotDot,
|
||||
BorderStyle::MediumDashDotDot,
|
||||
BorderStyle::SlantDashDot,
|
||||
];
|
||||
STYLES.get(usize::from(dg)).copied().ok_or_else(|| invalid("unknown border style"))
|
||||
}
|
||||
|
||||
fn halign(alc: u32) -> Option<HAlign> {
|
||||
[HAlign::General, HAlign::Left, HAlign::Center, HAlign::Right, HAlign::Fill, HAlign::Justify, HAlign::CenterAcross, HAlign::Distributed]
|
||||
.get(alc as usize)
|
||||
.copied()
|
||||
}
|
||||
|
||||
fn valign(alcv: u32) -> Option<VAlign> {
|
||||
[VAlign::Top, VAlign::Center, VAlign::Bottom, VAlign::Justify, VAlign::Distributed].get(alcv as usize).copied()
|
||||
}
|
||||
|
||||
fn underline(uls: u32) -> Underline {
|
||||
match uls {
|
||||
0 => Underline::None,
|
||||
2 => Underline::Double,
|
||||
0x21 => Underline::SingleAccounting,
|
||||
0x22 => Underline::DoubleAccounting,
|
||||
_ => Underline::Single,
|
||||
}
|
||||
}
|
||||
|
||||
fn script(sss: u32) -> Script {
|
||||
match sss {
|
||||
1 => Script::Superscript,
|
||||
2 => Script::Subscript,
|
||||
_ => Script::Baseline,
|
||||
}
|
||||
}
|
||||
|
||||
fn scheme(s: u32) -> Option<FontScheme> {
|
||||
match s {
|
||||
0 => Some(FontScheme::None),
|
||||
1 => Some(FontScheme::Major),
|
||||
2 => Some(FontScheme::Minor),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn name(text: String) -> String {
|
||||
text.chars().take(MAX_NAME_CHARS).collect()
|
||||
}
|
||||
|
||||
/// `BrtFont`: only what Excel's `.xlsx` states for the same font (no `<b val="0"/>`, no scheme
|
||||
/// `none`, no family or character set 0).
|
||||
fn font(b: &mut Bytes<'_>) -> Result<Font, Error> {
|
||||
let height = b.u16()?;
|
||||
let flags = b.u16()?;
|
||||
let weight = b.u16()?;
|
||||
let sss = b.u16()?;
|
||||
let uls = b.u8()?;
|
||||
let family = b.u8()?;
|
||||
let charset = b.u8()?;
|
||||
b.u8()?;
|
||||
let color = color(b)?;
|
||||
let font_scheme = b.u8()?;
|
||||
let font_name = name(b.text(MAX_STRING_UNITS)?);
|
||||
let on = |bit: u16| (flags & bit != 0).then_some(true);
|
||||
Ok(Font {
|
||||
name: Some(font_name),
|
||||
size: Some(f64::from(height) / 20.0),
|
||||
bold: (weight >= 0x2BC).then_some(true),
|
||||
italic: on(0x02),
|
||||
strike: on(0x08),
|
||||
outline: on(0x10),
|
||||
shadow: on(0x20),
|
||||
condense: on(0x40),
|
||||
extend: on(0x80),
|
||||
underline: (uls != 0).then(|| underline(u32::from(uls))),
|
||||
vert_align: (sss != 0).then(|| script(u32::from(sss))),
|
||||
family: (family != 0).then_some(u32::from(family)),
|
||||
charset: (charset != 0).then_some(u32::from(charset)),
|
||||
scheme: scheme(u32::from(font_scheme)).filter(|s| *s != FontScheme::None),
|
||||
// An automatic colour is a font with no `<color>` in `.xlsx`.
|
||||
color: color.filter(|c| c.source != ColorSource::Auto),
|
||||
})
|
||||
}
|
||||
|
||||
/// A gradient's angle and inner rectangle, as `BrtFill` and `XFPropGradient` hold them.
|
||||
fn gradient(b: &mut Bytes<'_>, path: bool) -> Result<GradientFill, Error> {
|
||||
Ok(GradientFill { path, degree: b.f64()?, left: b.f64()?, right: b.f64()?, top: b.f64()?, bottom: b.f64()?, stops: vec![] })
|
||||
}
|
||||
|
||||
/// `BrtFill`. Excel's `.xlsx` leaves out the default colours: system foreground (indexed 64) as
|
||||
/// the pattern colour and system background (indexed 65) as the background, unless the pattern
|
||||
/// colour has a tint (as the built-in "20% - Accent1" styles do), when it states the background.
|
||||
fn fill(b: &mut Bytes<'_>) -> Result<Fill, Error> {
|
||||
let fls = b.u32()?;
|
||||
let fg = color(b)?;
|
||||
let bg = color(b)?;
|
||||
let kind = b.u32()?;
|
||||
let mut g = gradient(b, kind == 1)?;
|
||||
let count = b.u32()?;
|
||||
if fls == 0x28 {
|
||||
if count > MAX_STOPS {
|
||||
return Err(Error::TooLarge(format!("more than {MAX_STOPS} gradient stops")));
|
||||
}
|
||||
for _ in 0..count {
|
||||
let c = color(b)?.unwrap_or_default();
|
||||
g.stops.push((b.f64()?, c));
|
||||
}
|
||||
return Ok(Fill::Gradient(g));
|
||||
}
|
||||
let system = |c: Color, i: u32| c.source == ColorSource::Indexed(i) && c.tint == 0.0;
|
||||
let tinted = fg.is_some_and(|c| c.tint != 0.0);
|
||||
Ok(Fill::Pattern(PatternFill { pattern: Some(pattern(fls)?), fg: fg.filter(|c| !system(*c, 64)), bg: bg.filter(|c| tinted || !system(*c, 65)) }))
|
||||
}
|
||||
|
||||
/// `BrtBorder`: the five edges a cell border has, each stated (`<left/>` when it has no line).
|
||||
fn border(b: &mut Bytes<'_>) -> Result<Border, Error> {
|
||||
let flags = b.u8()?;
|
||||
let mut edge = || -> Result<BorderEdge, Error> {
|
||||
let style = border_style(u16::from(b.u8()?))?;
|
||||
b.u8()?;
|
||||
let c = color(b)?;
|
||||
Ok(BorderEdge { style, color: c.filter(|_| style != BorderStyle::None) })
|
||||
};
|
||||
let (top, bottom, left, right, diagonal) = (edge()?, edge()?, edge()?, edge()?, edge()?);
|
||||
Ok(Border {
|
||||
left: Some(left),
|
||||
right: Some(right),
|
||||
top: Some(top),
|
||||
bottom: Some(bottom),
|
||||
diagonal: Some(diagonal),
|
||||
vertical: None,
|
||||
horizontal: None,
|
||||
diagonal_up: flags & 2 != 0,
|
||||
diagonal_down: flags & 1 != 0,
|
||||
outline: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// `BrtXF`. A cell style format (`style`) marks what it leaves out (`apply*="0"`); a cell format
|
||||
/// marks what it sets apart from its style (`apply*="1"`).
|
||||
fn xf(b: &mut Bytes<'_>, style: bool) -> Result<Xf, Error> {
|
||||
const APPLY: [(u8, &str); 6] = [
|
||||
(0x01, "applyNumberFormat"),
|
||||
(0x02, "applyFont"),
|
||||
(0x10, "applyFill"),
|
||||
(0x08, "applyBorder"),
|
||||
(0x04, "applyAlignment"),
|
||||
(0x20, "applyProtection"),
|
||||
];
|
||||
let parent = b.u16()?;
|
||||
let num_fmt_id = u32::from(b.u16()?);
|
||||
let font_id = u32::from(b.u16()?);
|
||||
let fill_id = u32::from(b.u16()?);
|
||||
let border_id = u32::from(b.u16()?);
|
||||
let rotation = u32::from(b.u8()?);
|
||||
let indent = u32::from(b.u8()?);
|
||||
let flags = u32::from(b.u16()?);
|
||||
let apply = b.u8()?;
|
||||
b.u8()?;
|
||||
let alignment = Alignment {
|
||||
horizontal: halign(flags & 7).filter(|h| *h != HAlign::General),
|
||||
vertical: valign((flags >> 3) & 7).filter(|v| *v != VAlign::Bottom),
|
||||
text_rotation: (rotation != 0).then_some(rotation),
|
||||
wrap_text: flags & 0x40 != 0,
|
||||
justify_last_line: flags & 0x80 != 0,
|
||||
shrink_to_fit: flags & 0x100 != 0,
|
||||
indent: (indent != 0).then_some(indent),
|
||||
reading_order: Some((flags >> 10) & 3).filter(|r| *r != 0),
|
||||
};
|
||||
let protection = Protection { locked: (flags & 0x1000 == 0).then_some(false), hidden: (flags & 0x2000 != 0).then_some(true) };
|
||||
let mut el = El::new("xf");
|
||||
for (bit, attr) in APPLY {
|
||||
if apply & bit != 0 {
|
||||
el = el.with_attr(attr, if style { "0" } else { "1" });
|
||||
}
|
||||
}
|
||||
Ok(Xf {
|
||||
num_fmt_id,
|
||||
font_id,
|
||||
fill_id,
|
||||
border_id,
|
||||
xf_id: (!style).then_some(u32::from(parent)),
|
||||
alignment: (alignment != Alignment::default()).then_some(alignment),
|
||||
protection: (protection != Protection::default()).then_some(protection),
|
||||
quote_prefix: flags & 0x8000 != 0,
|
||||
el,
|
||||
})
|
||||
}
|
||||
|
||||
/// `BrtStyle`.
|
||||
fn cell_style(b: &mut Bytes<'_>) -> Result<CellStyle, Error> {
|
||||
let xf_id = b.u32()?;
|
||||
let flags = b.u16()?;
|
||||
let builtin = b.u8()?;
|
||||
let level = b.u8()?;
|
||||
let style_name = b.string(MAX_STRING_UNITS, true)?.map(name).unwrap_or_default();
|
||||
let is_builtin = flags & 1 != 0;
|
||||
Ok(CellStyle {
|
||||
name: style_name,
|
||||
xf_id,
|
||||
builtin_id: is_builtin.then_some(u32::from(builtin)),
|
||||
level: (is_builtin && matches!(builtin, 1 | 2)).then_some(u32::from(level)),
|
||||
hidden: flags & 2 != 0,
|
||||
custom_builtin: flags & 4 != 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// An unsigned integer of the size a property has (1, 2 or 4 bytes).
|
||||
fn uint(data: &[u8]) -> Result<u32, Error> {
|
||||
match *data {
|
||||
[a] => Ok(u32::from(a)),
|
||||
[a, b] => Ok(u32::from(u16::from_le_bytes([a, b]))),
|
||||
[a, b, c, d] => Ok(u32::from_le_bytes([a, b, c, d])),
|
||||
_ => Err(invalid("unexpected formatting property size")),
|
||||
}
|
||||
}
|
||||
|
||||
/// A string with a 2-byte length (`LPWideString`, and the number format of `XFProp` 0x26).
|
||||
fn short_string(b: &mut Bytes<'_>) -> Result<String, Error> {
|
||||
let count = usize::from(b.u16()?);
|
||||
let mut text = String::new();
|
||||
for ch in char::decode_utf16(b.take(count * 2)?.as_chunks::<2>().0.iter().copied().map(u16::from_le_bytes)) {
|
||||
text.push(ch.map_err(|_| invalid("invalid UTF-16 string"))?);
|
||||
}
|
||||
Ok(name(text))
|
||||
}
|
||||
|
||||
fn dxf_font(d: &mut Dxf) -> &mut Font {
|
||||
d.font.get_or_insert_with(Font::default)
|
||||
}
|
||||
|
||||
/// `BrtDXF`: its `XFProps`, each stating one property.
|
||||
fn dxf(b: &mut Bytes<'_>) -> Result<Dxf, Error> {
|
||||
b.u16()?;
|
||||
b.u16()?;
|
||||
let count = b.u16()?;
|
||||
let mut d = Dxf::default();
|
||||
let mut gradient_fill: Option<GradientFill> = None;
|
||||
let mut pattern_fill: Option<PatternFill> = None;
|
||||
for _ in 0..count {
|
||||
let kind = b.u16()?;
|
||||
let size = usize::from(b.u16()?).checked_sub(4).ok_or_else(|| invalid("formatting property too short"))?;
|
||||
let data = b.take(size)?;
|
||||
let mut p = Bytes::new(data);
|
||||
match kind {
|
||||
0x00 => pattern_fill.get_or_insert_with(PatternFill::default).pattern = Some(pattern(uint(data)?)?),
|
||||
0x01 => pattern_fill.get_or_insert_with(PatternFill::default).fg = color(&mut p)?,
|
||||
0x02 => pattern_fill.get_or_insert_with(PatternFill::default).bg = color(&mut p)?,
|
||||
0x03 => {
|
||||
let path = p.u32()? == 1;
|
||||
gradient_fill = Some(gradient(&mut p, path)?);
|
||||
}
|
||||
0x04 => {
|
||||
let g = gradient_fill.as_mut().ok_or_else(|| invalid("gradient stop without a gradient"))?;
|
||||
if g.stops.len() >= MAX_STOPS as usize {
|
||||
return Err(Error::TooLarge(format!("more than {MAX_STOPS} gradient stops")));
|
||||
}
|
||||
p.u16()?;
|
||||
let position = p.f64()?;
|
||||
g.stops.push((position, color(&mut p)?.unwrap_or_default()));
|
||||
}
|
||||
0x05 => dxf_font(&mut d).color = color(&mut p)?,
|
||||
0x06..=0x0C => {
|
||||
let c = color(&mut p)?;
|
||||
let edge = Some(BorderEdge { style: border_style(p.u16()?)?, color: c });
|
||||
let border = d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() });
|
||||
match kind {
|
||||
0x06 => border.top = edge,
|
||||
0x07 => border.bottom = edge,
|
||||
0x08 => border.left = edge,
|
||||
0x09 => border.right = edge,
|
||||
0x0A => border.diagonal = edge,
|
||||
0x0B => border.vertical = edge,
|
||||
_ => border.horizontal = edge,
|
||||
}
|
||||
}
|
||||
0x0D => d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }).diagonal_up = uint(data)? != 0,
|
||||
0x0E => d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }).diagonal_down = uint(data)? != 0,
|
||||
0x0F..=0x16 => {
|
||||
let v = uint(data)?;
|
||||
let a = d.alignment.get_or_insert_with(Alignment::default);
|
||||
match kind {
|
||||
0x0F => a.horizontal = halign(v),
|
||||
0x10 => a.vertical = valign(v),
|
||||
0x11 => a.text_rotation = Some(v),
|
||||
0x12 => a.indent = Some(v),
|
||||
0x13 => a.reading_order = Some(v),
|
||||
0x14 => a.wrap_text = v != 0,
|
||||
0x15 => a.justify_last_line = v != 0,
|
||||
_ => a.shrink_to_fit = v != 0,
|
||||
}
|
||||
}
|
||||
0x18 => dxf_font(&mut d).name = Some(short_string(&mut p)?),
|
||||
0x19 => dxf_font(&mut d).bold = Some(uint(data)? >= 0x2BC),
|
||||
0x1A => dxf_font(&mut d).underline = Some(underline(uint(data)?)),
|
||||
0x1B => dxf_font(&mut d).vert_align = Some(script(uint(data)?)),
|
||||
0x1C => dxf_font(&mut d).italic = Some(uint(data)? != 0),
|
||||
0x1D => dxf_font(&mut d).strike = Some(uint(data)? != 0),
|
||||
0x1E => dxf_font(&mut d).outline = Some(uint(data)? != 0),
|
||||
0x1F => dxf_font(&mut d).shadow = Some(uint(data)? != 0),
|
||||
0x20 => dxf_font(&mut d).condense = Some(uint(data)? != 0),
|
||||
0x21 => dxf_font(&mut d).extend = Some(uint(data)? != 0),
|
||||
0x22 => dxf_font(&mut d).charset = Some(uint(data)?),
|
||||
0x23 => dxf_font(&mut d).family = Some(uint(data)?),
|
||||
0x24 => dxf_font(&mut d).size = Some(f64::from(uint(data)?) / 20.0),
|
||||
0x25 => dxf_font(&mut d).scheme = scheme(uint(data)?),
|
||||
0x26 => d.num_fmt.get_or_insert_with(NumFmt::default).code = Some(short_string(&mut p)?),
|
||||
0x29 => d.num_fmt.get_or_insert_with(NumFmt::default).id = Some(uint(data)?),
|
||||
0x2B => d.protection.get_or_insert_with(Protection::default).locked = Some(uint(data)? != 0),
|
||||
0x2C => d.protection.get_or_insert_with(Protection::default).hidden = Some(uint(data)? != 0),
|
||||
// Merged cells (0x17) and relative indents (0x2A) have no place in `<dxf>`.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
d.fill = gradient_fill.map(Fill::Gradient).or(pattern_fill.map(Fill::Pattern));
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
/// The styles part: `BrtBeginStyleSheet` to `BrtEndStyleSheet`.
|
||||
pub(crate) fn styles(bytes: &[u8], limits: &Limits, budget: &Budget, warnings: &mut Warnings) -> Result<Styles, Error> {
|
||||
let mut records = Records::new(bytes, limits, budget);
|
||||
// Excel keeps a number format with a newer locale tag in two alternate-content blocks: the
|
||||
// newer spelling for itself, and for Excel 2007 the one `.xlsx` has as `formatCode`.
|
||||
records.excel_2007 = true;
|
||||
records.first(278, warnings)?;
|
||||
let mut s = Styles::default();
|
||||
// Inside BrtBeginCellStyleXFs (true) or BrtBeginCellXFs (false).
|
||||
let mut xfs: Option<bool> = None;
|
||||
let mut palette = false;
|
||||
while let Some((id, payload)) = records.next(warnings)? {
|
||||
let mut b = Bytes::new(payload);
|
||||
let entries = match id {
|
||||
44 => s.num_fmts.len(),
|
||||
43 => s.fonts.len(),
|
||||
45 => s.fills.len(),
|
||||
46 => s.borders.len(),
|
||||
47 => s.cell_style_xfs.len().max(s.cell_xfs.len()),
|
||||
48 => s.cell_styles.len(),
|
||||
507 => s.dxfs.len(),
|
||||
475 => s.indexed_colors.len(),
|
||||
_ => 0,
|
||||
};
|
||||
if entries >= MAX_ENTRIES {
|
||||
return Err(Error::TooLarge(format!("more than {MAX_ENTRIES} entries in a styles table")));
|
||||
}
|
||||
match id {
|
||||
279 => {
|
||||
records.end(payload, warnings)?;
|
||||
return Ok(s);
|
||||
}
|
||||
44 => {
|
||||
let id = u32::from(b.u16()?);
|
||||
let code = name(b.text(MAX_STRING_UNITS)?);
|
||||
budget.text(code.len(), limits)?;
|
||||
s.num_fmts.push((id, code));
|
||||
}
|
||||
43 => {
|
||||
let f = font(&mut b)?;
|
||||
budget.text(f.name.as_ref().map_or(0, String::len), limits)?;
|
||||
s.fonts.push(f);
|
||||
}
|
||||
45 => s.fills.push(fill(&mut b)?),
|
||||
46 => s.borders.push(border(&mut b)?),
|
||||
626 => xfs = Some(true),
|
||||
617 => xfs = Some(false),
|
||||
627 | 618 => xfs = None,
|
||||
47 => match xfs {
|
||||
Some(true) => s.cell_style_xfs.push(xf(&mut b, true)?),
|
||||
Some(false) => s.cell_xfs.push(xf(&mut b, false)?),
|
||||
None => return Err(invalid("cell format outside its list")),
|
||||
},
|
||||
48 => {
|
||||
let c = cell_style(&mut b)?;
|
||||
budget.text(c.name.len(), limits)?;
|
||||
s.cell_styles.push(c);
|
||||
}
|
||||
507 => {
|
||||
let d = dxf(&mut b)?;
|
||||
// What it keeps of its strings, after a later property has replaced an earlier.
|
||||
let font = d.font.as_ref().and_then(|f| f.name.as_ref()).map_or(0, String::len);
|
||||
let format = d.num_fmt.as_ref().and_then(|n| n.code.as_ref()).map_or(0, String::len);
|
||||
budget.text(font + format, limits)?;
|
||||
s.dxfs.push(d);
|
||||
}
|
||||
// BrtBeginIndexedColors, BrtEndIndexedColors.
|
||||
565 => palette = true,
|
||||
566 => palette = false,
|
||||
475 if palette => {
|
||||
let [r, g, bl, _] = <[u8; 4]>::try_from(b.take(4)?).map_err(|_| invalid("truncated colour"))?;
|
||||
s.indexed_colors.push(u32::from_be_bytes([0, r, g, bl]));
|
||||
}
|
||||
// BrtBeginTableStyles: the count of custom table styles, which are not read.
|
||||
508 => {
|
||||
if b.u32()? > 0 {
|
||||
warnings.warn("custom table styles in a binary workbook were not read");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
// The records read above are read whole; the rest are skipped.
|
||||
if matches!(id, 44 | 43 | 45 | 46 | 47 | 48 | 507 | 475) {
|
||||
b.finish()?;
|
||||
}
|
||||
}
|
||||
Err(invalid("missing styles end record"))
|
||||
}
|
||||
+288
-1
@@ -5,7 +5,10 @@ mod common;
|
||||
|
||||
use common::{read, read_with};
|
||||
use craft_xlsx::testing::zip;
|
||||
use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Range, SheetKind, Value, Visibility, is_xlsb};
|
||||
use craft_xlsx::{
|
||||
CellError, CellRef, Color, ColorSource, Error, Fill, FormulaKind, Limits, Package, Pattern, PatternFill, Protection, Range, SheetKind, Styles,
|
||||
Value, Visibility, is_xlsb,
|
||||
};
|
||||
|
||||
const ROOT: &str = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="book" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="records/book.bin"/></Relationships>"#;
|
||||
const RELS: &str = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="one" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="../data/one.bin"/><Relationship Id="two" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="../data/two.bin"/><Relationship Id="strings" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" Target="../text/strings.bin"/></Relationships>"#;
|
||||
@@ -931,6 +934,280 @@ fn a_dynamic_array_kept_as_its_value_has_no_cell_metadata() {
|
||||
assert!(warned(&wb, "missing from the file"), "{:?}", wb.warnings);
|
||||
}
|
||||
|
||||
// Styles ([MS-XLSB] 2.1.7.50): a styles part assembled from records, and the `styles.xml` Excel
|
||||
// writes for the same formats when it saves the workbook as `.xlsx`.
|
||||
|
||||
const STYLES_RELS: &str = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="one" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="../data/one.bin"/><Relationship Id="two" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="../data/two.bin"/><Relationship Id="styles" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="../look/styles.bin"/></Relationships>"#;
|
||||
const STYLES_TYPES: &str = r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Override PartName="/records/book.bin" ContentType="application/vnd.ms-excel.sheet.binary.macroEnabled.main"/><Override PartName="/data/one.bin" ContentType="application/vnd.ms-excel.worksheet"/><Override PartName="/data/two.bin" ContentType="application/vnd.ms-excel.worksheet"/><Override PartName="/look/styles.bin" ContentType="application/vnd.ms-excel.styles"/></Types>"#;
|
||||
|
||||
fn styled_file(styles: &[u8], types: &str) -> Vec<u8> {
|
||||
let sheet = worksheet(&[row(0), cell(5, 0, &1.0f64.to_le_bytes())].concat(), &[]);
|
||||
zip(&[
|
||||
("[Content_Types].xml", types.as_bytes()),
|
||||
("_rels/.rels", ROOT.as_bytes()),
|
||||
("records/book.bin", &book(false)),
|
||||
("records/_rels/book.bin.rels", STYLES_RELS.as_bytes()),
|
||||
("data/one.bin", &sheet),
|
||||
("data/two.bin", &worksheet(&[], &[])),
|
||||
("look/styles.bin", styles),
|
||||
])
|
||||
}
|
||||
|
||||
/// The styles part of a package, and the warnings reading it gave.
|
||||
fn styles_of(bytes: &[u8]) -> (Option<Styles>, Vec<String>) {
|
||||
let mut p = Package::open(bytes, &Limits::default()).unwrap();
|
||||
let styles = p.styles().unwrap();
|
||||
(styles, p.warnings().to_vec())
|
||||
}
|
||||
|
||||
/// `BrtColor`: auto 0, indexed 1, RGB 2, theme 3.
|
||||
fn color(kind: u8, index: u8, tint: i16, rgba: [u8; 4]) -> Vec<u8> {
|
||||
[&[kind << 1 | 1, index][..], &tint.to_le_bytes(), &rgba].concat()
|
||||
}
|
||||
fn indexed(i: u8) -> Vec<u8> {
|
||||
color(1, i, 0, [0, 0, 0, 255])
|
||||
}
|
||||
fn theme(i: u8, tint: i16) -> Vec<u8> {
|
||||
color(3, i, tint, [0, 0, 0, 255])
|
||||
}
|
||||
fn rgb(r: u8, g: u8, b: u8) -> Vec<u8> {
|
||||
color(2, 255, 0, [r, g, b, 255])
|
||||
}
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn font(twips: u16, flags: u16, weight: u16, script: u16, underline: u8, family: u8, charset: u8, c: &[u8], scheme: u8, name: &str) -> Vec<u8> {
|
||||
let head =
|
||||
[&twips.to_le_bytes()[..], &flags.to_le_bytes(), &weight.to_le_bytes(), &script.to_le_bytes(), &[underline, family, charset, 0]].concat();
|
||||
record(43, &[&head[..], c, &[scheme], &wide(name)].concat())
|
||||
}
|
||||
fn pattern_fill(fls: u32, fg: &[u8], bg: &[u8]) -> Vec<u8> {
|
||||
record(45, &[&fls.to_le_bytes()[..], fg, bg, &[0; 48]].concat())
|
||||
}
|
||||
fn edge(dg: u8, c: &[u8]) -> Vec<u8> {
|
||||
[&[dg, 0][..], c].concat()
|
||||
}
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn brt_xf(parent: u16, fmt: u16, font: u16, fill: u16, border: u16, rotation: u8, indent: u8, flags: u16, apply: u8) -> Vec<u8> {
|
||||
let ids = [parent, fmt, font, fill, border].map(u16::to_le_bytes).concat();
|
||||
record(47, &[&ids[..], &[rotation, indent], &flags.to_le_bytes(), &[apply, 0]].concat())
|
||||
}
|
||||
fn style(xf: u32, flags: u16, builtin: u8, level: u8, name: &str) -> Vec<u8> {
|
||||
record(48, &[&xf.to_le_bytes()[..], &flags.to_le_bytes(), &[builtin, level], &wide(name)].concat())
|
||||
}
|
||||
fn prop(kind: u16, data: &[u8]) -> Vec<u8> {
|
||||
[&kind.to_le_bytes()[..], &(data.len() as u16 + 4).to_le_bytes(), data].concat()
|
||||
}
|
||||
fn short(text: &str) -> Vec<u8> {
|
||||
let units: Vec<u16> = text.encode_utf16().collect();
|
||||
[(units.len() as u16).to_le_bytes().to_vec(), units.iter().flat_map(|u| u.to_le_bytes()).collect()].concat()
|
||||
}
|
||||
fn brt_dxf(props: &[Vec<u8>]) -> Vec<u8> {
|
||||
record(507, &[&[0, 0x80, 0, 0][..], &(props.len() as u16).to_le_bytes(), &props.concat()].concat())
|
||||
}
|
||||
fn begin(id: u16, count: u32) -> Vec<u8> {
|
||||
record(id, &count.to_le_bytes())
|
||||
}
|
||||
|
||||
/// Every kind of record the styles part has, as Excel writes them.
|
||||
fn styles_bin() -> Vec<u8> {
|
||||
let normal = font(220, 0, 400, 0, 0, 2, 0, &theme(1, 0), 2, "Aptos Narrow");
|
||||
// Bold, italic, struck out, superscript, double accounting underline; family and
|
||||
// character set 0 and scheme "none" are not written in `.xlsx`.
|
||||
let fancy = font(290, 0x0A, 700, 1, 0x22, 0, 2, &rgb(0, 128, 255), 0, "Wingdings");
|
||||
let heading = font(220, 0, 400, 0, 0, 2, 0, &theme(4, 13106), 1, "Aptos Display");
|
||||
// An automatic colour: no `<color>` in `.xlsx`.
|
||||
let auto = font(220, 0x02, 400, 0, 0, 2, 0, &color(0, 64, 0, [0, 0, 0, 255]), 2, "Aptos Narrow");
|
||||
let gradient = [
|
||||
&0x28u32.to_le_bytes()[..],
|
||||
&indexed(64),
|
||||
&indexed(65),
|
||||
&1u32.to_le_bytes(),
|
||||
&[0.0f64, 0.5, 0.5, 0.5, 0.5].map(f64::to_le_bytes).concat(),
|
||||
&2u32.to_le_bytes(),
|
||||
&theme(0, 0),
|
||||
&0.0f64.to_le_bytes(),
|
||||
&theme(4, 0),
|
||||
&1.0f64.to_le_bytes(),
|
||||
]
|
||||
.concat();
|
||||
let none = edge(0, &color(0, 0, 0, [0, 0, 0, 0]));
|
||||
let parts: Vec<Vec<u8>> = vec![
|
||||
record(278, &[]),
|
||||
begin(615, 2),
|
||||
record(44, &[&164u16.to_le_bytes()[..], &wide(r"yyyy\-mm\-dd")].concat()),
|
||||
// A format in two alternate-content blocks: the newer spelling for version 13 and later
|
||||
// (skipped), and the one for versions 12 and 13, which `.xlsx` has as `formatCode`.
|
||||
record(37, &[1, 0, 13, 0, 0, 0x80]),
|
||||
record(44, &[&165u16.to_le_bytes()[..], &wide("[$-ja-JP-x-gannen]ge")].concat()),
|
||||
record(38, &[]),
|
||||
record(37, &[2, 0, 12, 0, 0, 0, 13, 0, 0, 0]),
|
||||
record(44, &[&165u16.to_le_bytes()[..], &wide("[$]ge")].concat()),
|
||||
record(38, &[]),
|
||||
record(616, &[]),
|
||||
begin(611, 4),
|
||||
normal,
|
||||
fancy,
|
||||
heading,
|
||||
auto,
|
||||
record(612, &[]),
|
||||
begin(603, 6),
|
||||
pattern_fill(0, &indexed(64), &indexed(65)),
|
||||
pattern_fill(17, &indexed(64), &indexed(65)),
|
||||
pattern_fill(1, &rgb(255, 255, 0), &indexed(64)),
|
||||
pattern_fill(9, &theme(5, -8191), &indexed(65)),
|
||||
record(45, &gradient),
|
||||
// A tinted pattern colour: `.xlsx` states the system background, as for darkGrid above.
|
||||
pattern_fill(1, &theme(4, 26213), &indexed(65)),
|
||||
record(604, &[]),
|
||||
begin(613, 2),
|
||||
record(46, &[&[0][..], &none, &none, &none, &none, &none].concat()),
|
||||
record(
|
||||
46,
|
||||
&[&[3][..], &edge(2, &rgb(200, 0, 0)), &none, &edge(7, &indexed(64)), &edge(13, &color(0, 0, 0, [0, 0, 0, 0])), &edge(1, &theme(4, 0))]
|
||||
.concat(),
|
||||
),
|
||||
record(614, &[]),
|
||||
begin(626, 3),
|
||||
brt_xf(0xFFFF, 0, 0, 0, 0, 0, 0, 0x1010, 0),
|
||||
brt_xf(0xFFFF, 9, 0, 0, 0, 0, 0, 0x1010, 0x3E),
|
||||
brt_xf(0xFFFF, 0, 2, 0, 0, 0, 0, 0x1010, 0),
|
||||
record(627, &[]),
|
||||
begin(617, 4),
|
||||
brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0),
|
||||
brt_xf(0, 164, 0, 0, 0, 0, 0, 0x1010, 0x01),
|
||||
// Left, top, wrapped, shrunk, right to left, unlocked, hidden, quote prefix.
|
||||
brt_xf(0, 0, 0, 0, 0, 180, 2, 0x01 | 0x40 | 0x100 | 0x800 | 0x2000 | 0x8000, 0x24),
|
||||
brt_xf(1, 0, 1, 4, 1, 0, 0, 0x1010, 0x1A),
|
||||
record(618, &[]),
|
||||
begin(619, 3),
|
||||
style(0, 1, 0, 0, "Normal"),
|
||||
// Excel wraps each style in alternate content, which is skipped.
|
||||
record(37, &[1, 0, 2, 0x0e, 0, 0x80]),
|
||||
record(3072, &[0; 16]),
|
||||
record(38, &[]),
|
||||
style(1, 1 | 4, 1, 1, "RowLevel_2"),
|
||||
style(2, 2, 0, 0, "Probe Style"),
|
||||
record(620, &[]),
|
||||
begin(505, 2),
|
||||
brt_dxf(&[
|
||||
prop(0x05, &rgb(0x9C, 0, 6)),
|
||||
prop(0x19, &700u16.to_le_bytes()),
|
||||
prop(0x1C, &[0]),
|
||||
prop(0x1A, &0x21u16.to_le_bytes()),
|
||||
prop(0x24, &240u32.to_le_bytes()),
|
||||
prop(0x18, &short("Arial")),
|
||||
prop(0x29, &170u16.to_le_bytes()),
|
||||
prop(0x26, &short("0.0")),
|
||||
prop(0x02, &rgb(0xFF, 0xC7, 0xCE)),
|
||||
prop(0x0F, &[2]),
|
||||
prop(0x14, &[1]),
|
||||
prop(0x2B, &[0]),
|
||||
prop(0x07, &[&color(0, 0, 0, [0, 0, 0, 0])[..], &1u16.to_le_bytes()].concat()),
|
||||
]),
|
||||
brt_dxf(&[
|
||||
prop(0x03, &[&0u32.to_le_bytes()[..], &[90.0f64, 0.0, 0.0, 0.0, 0.0].map(f64::to_le_bytes).concat()].concat()),
|
||||
prop(0x04, &[&[0, 0][..], &0.0f64.to_le_bytes(), &rgb(255, 255, 255)].concat()),
|
||||
prop(0x04, &[&[0, 0][..], &1.0f64.to_le_bytes(), &theme(4, 0)].concat()),
|
||||
]),
|
||||
record(506, &[]),
|
||||
record(508, &[&0u32.to_le_bytes()[..], &wide("TableStyleMedium2"), &wide("PivotStyleLight16")].concat()),
|
||||
record(509, &[]),
|
||||
record(473, &[]),
|
||||
record(565, &[]),
|
||||
record(475, &[0x11, 0x22, 0x33, 0]),
|
||||
record(475, &[0xFF, 0xFF, 0xFF, 0]),
|
||||
record(566, &[]),
|
||||
record(474, &[]),
|
||||
record(279, &[]),
|
||||
];
|
||||
parts.concat()
|
||||
}
|
||||
|
||||
const STYLES_XML: &str = r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><numFmts count="2"><numFmt numFmtId="164" formatCode="yyyy\-mm\-dd"/><numFmt numFmtId="165" formatCode="[$]ge"/></numFmts><fonts count="3"><font><sz val="11"/><color theme="1"/><name val="Aptos Narrow"/><family val="2"/><scheme val="minor"/></font><font><b/><i/><strike/><u val="doubleAccounting"/><vertAlign val="superscript"/><sz val="14.5"/><color rgb="FF0080FF"/><name val="Wingdings"/><charset val="2"/></font><font><sz val="11"/><color theme="4" tint="0.39997558519241921"/><name val="Aptos Display"/><family val="2"/><scheme val="major"/></font><font><i/><sz val="11"/><name val="Aptos Narrow"/><family val="2"/><scheme val="minor"/></font></fonts><fills count="5"><fill><patternFill patternType="none"/></fill><fill><patternFill patternType="gray125"/></fill><fill><patternFill patternType="solid"><fgColor rgb="FFFFFF00"/><bgColor indexed="64"/></patternFill></fill><fill><patternFill patternType="darkGrid"><fgColor theme="5" tint="-0.249977111117893"/><bgColor indexed="65"/></patternFill></fill><fill><gradientFill type="path" left="0.5" right="0.5" top="0.5" bottom="0.5"><stop position="0"><color theme="0"/></stop><stop position="1"><color theme="4"/></stop></gradientFill></fill><fill><patternFill patternType="solid"><fgColor theme="4" tint="0.79998168889431442"/><bgColor indexed="65"/></patternFill></fill></fills><borders count="2"><border><left/><right/><top/><bottom/><diagonal/></border><border diagonalUp="1" diagonalDown="1"><left style="hair"><color indexed="64"/></left><right style="slantDashDot"><color auto="1"/></right><top style="medium"><color rgb="FFC80000"/></top><bottom/><diagonal style="thin"><color theme="4"/></diagonal></border></borders><cellStyleXfs count="3"><xf numFmtId="0" fontId="0" fillId="0" borderId="0"/><xf numFmtId="9" fontId="0" fillId="0" borderId="0" applyFont="0" applyFill="0" applyBorder="0" applyAlignment="0" applyProtection="0"/><xf numFmtId="0" fontId="2" fillId="0" borderId="0"/></cellStyleXfs><cellXfs count="4"><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0"/><xf numFmtId="164" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0" quotePrefix="1" applyAlignment="1" applyProtection="1"><alignment horizontal="left" vertical="top" textRotation="180" wrapText="1" indent="2" shrinkToFit="1" readingOrder="2"/><protection locked="0" hidden="1"/></xf><xf numFmtId="0" fontId="1" fillId="4" borderId="1" xfId="1" applyFont="1" applyFill="1" applyBorder="1"/></cellXfs><cellStyles count="3"><cellStyle name="Normal" xfId="0" builtinId="0"/><cellStyle name="RowLevel_2" xfId="1" builtinId="1" iLevel="1" customBuiltin="1"/><cellStyle name="Probe Style" xfId="2" hidden="1"/></cellStyles><dxfs count="2"><dxf><font><b/><i val="0"/><u val="singleAccounting"/><sz val="12"/><color rgb="FF9C0006"/><name val="Arial"/></font><numFmt numFmtId="170" formatCode="0.0"/><fill><patternFill><bgColor rgb="FFFFC7CE"/></patternFill></fill><alignment horizontal="center" wrapText="1"/><protection locked="0"/><border><bottom style="thin"><color auto="1"/></bottom></border></dxf><dxf><fill><gradientFill degree="90"><stop position="0"><color rgb="FFFFFFFF"/></stop><stop position="1"><color theme="4"/></stop></gradientFill></fill></dxf></dxfs><colors><indexedColors><rgbColor rgb="00112233"/><rgbColor rgb="00FFFFFF"/></indexedColors></colors></styleSheet>"#;
|
||||
|
||||
/// The same styles from a SpreadsheetML package.
|
||||
fn xml_styles(xml: &str) -> Styles {
|
||||
let types = r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/></Types>"#;
|
||||
let root = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="r" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#;
|
||||
let rels = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="s" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/></Relationships>"#;
|
||||
let bytes = zip(&[
|
||||
("[Content_Types].xml", types.as_bytes()),
|
||||
("_rels/.rels", root.as_bytes()),
|
||||
("xl/workbook.xml", br#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"/>"#),
|
||||
("xl/_rels/workbook.xml.rels", rels.as_bytes()),
|
||||
("xl/styles.xml", xml.as_bytes()),
|
||||
]);
|
||||
styles_of(&bytes).0.unwrap()
|
||||
}
|
||||
|
||||
/// What both readers lift out: an `<xf>`'s element keeps only its `apply*` flags, and the XML
|
||||
/// part's own element is dropped.
|
||||
fn lifted(mut s: Styles) -> Styles {
|
||||
for xf in s.cell_style_xfs.iter_mut().chain(&mut s.cell_xfs) {
|
||||
let attrs = xf.el.attrs.iter().filter(|(k, _)| k.starts_with("apply")).cloned().collect();
|
||||
xf.el = craft_xml::El::new("xf");
|
||||
xf.el.attrs = attrs;
|
||||
}
|
||||
s.root = craft_xml::El::default();
|
||||
s
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn styles_are_read_as_their_xlsx_spelling() {
|
||||
let (binary, warnings) = styles_of(&styled_file(&styles_bin(), STYLES_TYPES));
|
||||
let binary = binary.expect("styles are read");
|
||||
assert_eq!(lifted(binary.clone()), lifted(xml_styles(STYLES_XML)));
|
||||
// A few of the conversions, spelled out.
|
||||
assert_eq!(binary.fonts[1].size, Some(14.5));
|
||||
assert_eq!(binary.fonts[1].family, None);
|
||||
let solid = Fill::Pattern(PatternFill {
|
||||
pattern: Some(Pattern::Solid),
|
||||
fg: Some(Color { source: ColorSource::Rgb(0xFFFF_FF00), tint: 0.0 }),
|
||||
bg: Some(Color { source: ColorSource::Indexed(64), tint: 0.0 }),
|
||||
});
|
||||
assert_eq!(binary.fills[2], solid);
|
||||
assert_eq!(binary.cell_xfs[2].protection, Some(Protection { locked: Some(false), hidden: Some(true) }));
|
||||
assert_eq!(binary.palette()[0], 0x11_2233);
|
||||
// Written back as SpreadsheetML, a cell style format says what it leaves out, as Excel's does.
|
||||
let written = String::from_utf8(binary.to_xml()).unwrap();
|
||||
assert!(written.contains(r#"applyFont="0" applyFill="0" applyBorder="0" applyAlignment="0" applyProtection="0""#), "{written}");
|
||||
assert!(warnings.iter().any(|w| w.contains("alternate-content")), "{warnings:?}");
|
||||
assert!(!warnings.iter().any(|w| w.contains("table styles")), "{warnings:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn damaged_styles_are_a_warning_and_the_workbook_opens() {
|
||||
let good = styles_bin();
|
||||
let end = record(279, &[]);
|
||||
let body = &good[..good.len() - end.len()];
|
||||
let with = |extra: &[u8]| [body, extra, &end].concat();
|
||||
let cases: Vec<(&str, Vec<u8>, &str)> = vec![
|
||||
("truncated", good[..good.len() - 1].to_vec(), STYLES_TYPES),
|
||||
("no end record", body.to_vec(), STYLES_TYPES),
|
||||
("unknown fill pattern", with(&pattern_fill(99, &indexed(64), &indexed(65))), STYLES_TYPES),
|
||||
("unknown border style", with(&record(46, &[&[0][..], &edge(14, &indexed(64)), &[0; 40]].concat())), STYLES_TYPES),
|
||||
("cell format outside its list", with(&brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0)), STYLES_TYPES),
|
||||
("short property", with(&brt_dxf(&[vec![0x19, 0, 2, 0]])), STYLES_TYPES),
|
||||
("wrong content type", good.clone(), TYPES),
|
||||
];
|
||||
for (why, part, types) in cases {
|
||||
let bytes = styled_file(&part, types);
|
||||
let (styles, warnings) = styles_of(&bytes);
|
||||
assert!(styles.is_none(), "{why}");
|
||||
assert!(warnings.iter().any(|w| w.contains("look/styles.bin")), "{why}: {warnings:?}");
|
||||
assert_eq!(read(&bytes).unwrap().sheets[0].value(CellRef::new(0, 0)), Value::Number(1.0), "{why}");
|
||||
}
|
||||
// A table past 65,536 entries is past a fixed cap: refused before it is held, as too large.
|
||||
let xfs = brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0).repeat(65_537);
|
||||
let bytes = styled_file(&with(&[&begin(617, 65_537)[..], &xfs, &record(618, &[])].concat()), STYLES_TYPES);
|
||||
let mut p = Package::open(&bytes, &Limits::default()).unwrap();
|
||||
assert!(matches!(p.styles(), Err(Error::TooLarge(m)) if m.contains("65536 entries")));
|
||||
// Custom table styles are not read, and say so.
|
||||
let table_styles = [body, &record(508, &[&1u32.to_le_bytes()[..], &wide("Probe Table"), &wide("")].concat()), &end].concat();
|
||||
let (styles, warnings) = styles_of(&styled_file(&table_styles, STYLES_TYPES));
|
||||
assert!(styles.is_some());
|
||||
assert!(warnings.iter().any(|w| w.contains("table styles")), "{warnings:?}");
|
||||
}
|
||||
|
||||
/// `PtgAttrSpace` ([MS-XLSB] 2.5.98.31): its type (0 spaces before the next token, 1 line breaks
|
||||
/// before it, 2 spaces before a `(`, 4 spaces before a `)`) and count.
|
||||
fn space(kind: u8, n: u8) -> Vec<u8> {
|
||||
@@ -1063,6 +1340,16 @@ fn a_spent_formula_allowance_skips_decoding() {
|
||||
assert!(!warned(&wb, "unknown formula token"), "{:?}", wb.warnings);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn differential_format_strings_are_charged_to_the_text_allowance() {
|
||||
// A DXF's font name and number format are text the styles part keeps, charged like a font's.
|
||||
let dxf = |name: &str| [record(278, &[]), begin(505, 1), brt_dxf(&[prop(0x18, &short(name))]), record(506, &[]), record(279, &[])].concat();
|
||||
let limits = Limits { binary_text: 100, ..Limits::default() };
|
||||
let open = |part: &[u8]| Package::open(&styled_file(part, STYLES_TYPES), &limits).unwrap().styles();
|
||||
assert!(open(&dxf("Arial")).is_ok_and(|s| s.is_some_and(|s| s.dxfs.len() == 1)));
|
||||
assert!(matches!(open(&dxf(&"x".repeat(200))), Err(Error::TooLarge(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_sheets_and_supporting_links_are_bounded() {
|
||||
// 1,025 very hidden module sheets (no relationship) before the worksheets: each keeps a place
|
||||
|
||||
Reference in new issue
Block a user