Merge pull request #1861 from storytold/render-parity-harness

craft-render-parity: compare two renders against a stated tolerance (#587)
This commit is contained in:
FredApps authored and GitHub committed 2026-11-16 15:02:00 +01:00
commit e60702ab1c
7 files changed
+368

No files matched your search

Generated
+7
View File
@@ -2658,6 +2658,13 @@ dependencies = [
"zune-jpegxl",
]
[[package]]
name = "craft-render-parity"
version = "0.1.0"
dependencies = [
"craft-cms",
]
[[package]]
name = "craft-rtf"
version = "0.1.0"
+1
View File
@@ -17,6 +17,7 @@ craft-usd = { path = "crates/usd" }
craft-smime = { path = "crates/smime" }
craft-ber = { path = "crates/ber" }
craft-proto = { path = "crates/proto" }
craft-render-parity = { path = "crates/render-parity" }
acadrust = "=0.6.6"
log = "0.4"
# internal: add each crate here as `craft-<name> = { path = "crates/<name>" }`
+3
View File
@@ -200,6 +200,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
Hasselblad 3FR/FFF, Minolta MRW, Epson ERF, Mamiya MEF, Kodak DCS, Pentax PEF, Samsung SRW, Canon CR2, Nikon NEF/NRW, Panasonic RW2, Fujifilm RAF, Olympus ORF (uncompressed), Sony ARW, Canon CR3 and Canon CRW (through
`craft-crw`) so far; demosaicing and everything after stay with the develop
pipeline.
- [`craft-render-parity`](crates/render-parity/README.md): test support for the shared develop
pipeline (#587): how far two renders of the same photo are apart (per-channel error, CIEDE2000,
the worst pixel) and whether that is within a stated tolerance.
- [`craft-tga`](crates/tga/src/lib.rs): Truevision Targa, from FilmCraft (reader) and PhotoCraft
(writer): true-colour, grayscale and colour-mapped pictures, raw or run-length encoded, any
origin, to RGBA8, and a run-length writer whose packets end with their scan line. Bounded on
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "craft-render-parity"
description = "Compare two renders of the same photo (per-channel error and CIEDE2000) against a stated tolerance: the parity check for the shared develop pipeline. Test support only."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
publish = false
[lints]
workspace = true
[dependencies]
craft-cms = { workspace = true }
+37
View File
@@ -0,0 +1,37 @@
# craft-render-parity
Test support for the shared develop pipeline ([#587](https://github.com/storytold/craft-libs/issues/587)):
how far two renders of the same photo are apart, and whether that is within a stated tolerance.
A develop stage that moves from an app, or an app that switches to the shared pipeline, renders
the same fixture both ways and compares the two here.
```toml
[dev-dependencies]
craft-render-parity = { git = "https://github.com/storytold/craft-libs", rev = "<commit>" }
```
```rust
use craft_render_parity::{Image, Tolerance, compare};
let expected = Image::new(width, height, &app_render)?;
let actual = Image::new(width, height, &shared_render)?;
let report = compare(&expected, &actual)?;
assert!(report.within(&Tolerance::BASELINE), "{report}: {:?}", report.exceeded(&Tolerance::BASELINE));
```
## Input
Scene-linear RGB with Rec. 709 / sRGB primaries and a D65 white, three interleaved `f32` samples
per pixel, row by row, with 1.0 as diffuse white. Values above 1.0 are compared as they are. A NaN
or infinite sample is an error, never a pass.
## Measures
| Measure | Meaning |
|---|---|
| `max_delta_e`, `mean_delta_e` | CIEDE2000 (kL = kC = kH = 1) in CIELAB relative to D65, from `craft_cms::math` |
| `max_channel`, `mean_channel` | Absolute difference of the linear samples, overall and per channel |
| `worst` | The first pixel with the largest CIEDE2000 difference, or the first that differs at all |
`Tolerance::BASELINE`, the acceptance bar of #587, is CIEDE2000 at most 0.5 at any pixel and 0.05
on average (1.0 is about the smallest visible difference), and at most 1/1024 in any linear sample.
+205
View File
@@ -0,0 +1,205 @@
//! Render parity: how far two renders of the same photo are apart, and whether that is within a
//! stated [`Tolerance`].
//!
//! The shared develop pipeline (craft-libs#587) has to give LightCraft and PhotoCraft the same
//! picture. A stage moved from an app, or an app switched to the shared crate, is checked by
//! rendering the same fixture both ways and comparing them here. The [`Report`] names the worst
//! pixel, so a failure points at where to look.
//!
//! Renders are scene-linear RGB with Rec. 709 / sRGB primaries and a D65 white, interleaved, with
//! 1.0 as diffuse white. Colour difference is CIEDE2000 in CIELAB relative to D65 (from
//! [`craft_cms::math`]); values above 1.0 are allowed and compared as they are.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
use std::fmt;
use craft_cms::math::{delta_e2000, xyz_to_lab};
/// Linear Rec. 709 RGB → CIE XYZ (D65), IEC 61966-2-1.
const RGB_TO_XYZ: [[f64; 3]; 3] = [[0.4124, 0.3576, 0.1805], [0.2126, 0.7152, 0.0722], [0.0193, 0.1192, 0.9505]];
/// The D65 white in XYZ, from the matrix above applied to RGB (1, 1, 1).
const D65: [f64; 3] = [0.9505, 1.0, 1.089];
/// A render to compare: `width` × `height` interleaved RGB samples, row by row.
#[derive(Clone, Copy, Debug)]
pub struct Image<'a> {
width: u32,
height: u32,
rgb: &'a [f32],
}
impl<'a> Image<'a> {
/// A render of `width` × `height` pixels whose `rgb` holds exactly three samples per pixel.
pub fn new(width: u32, height: u32, rgb: &'a [f32]) -> Result<Self, ParityError> {
let expected = (width as usize).checked_mul(height as usize).and_then(|n| n.checked_mul(3));
if expected != Some(rgb.len()) {
return Err(ParityError::Length { width, height, samples: rgb.len() });
}
Ok(Self { width, height, rgb })
}
pub fn width(&self) -> u32 {
self.width
}
pub fn height(&self) -> u32 {
self.height
}
}
/// Why two renders could not be compared.
#[derive(Clone, Debug, PartialEq)]
pub enum ParityError {
/// The sample count is not three per pixel.
Length { width: u32, height: u32, samples: usize },
/// The renders differ in size, so a pixel of one has no counterpart in the other.
Size { expected: (u32, u32), actual: (u32, u32) },
/// A sample is NaN or infinite: no difference can be stated for it.
NotFinite { x: u32, y: u32 },
}
impl fmt::Display for ParityError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Length { width, height, samples } => write!(f, "{samples} samples is not RGB for {width} × {height} pixels"),
Self::Size { expected, actual } => write!(f, "render is {} × {}, expected {} × {}", actual.0, actual.1, expected.0, expected.1),
Self::NotFinite { x, y } => write!(f, "pixel ({x}, {y}) is not a finite colour"),
}
}
}
impl std::error::Error for ParityError {}
/// How far two renders may be apart and still count as the same picture.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Tolerance {
/// The largest CIEDE2000 difference at any pixel.
pub max_delta_e: f64,
/// The largest mean CIEDE2000 difference over the picture.
pub mean_delta_e: f64,
/// The largest difference in any one linear channel sample.
pub max_channel: f32,
}
impl Tolerance {
/// The baseline parity of craft-libs#587: well below a visible difference (CIEDE2000 1.0), and
/// within a 10-bit code value in any channel.
pub const BASELINE: Self = Self { max_delta_e: 0.5, mean_delta_e: 0.05, max_channel: 1.0 / 1024.0 };
}
impl Default for Tolerance {
fn default() -> Self {
Self::BASELINE
}
}
/// The difference between two renders of the same size.
#[derive(Clone, Debug, PartialEq)]
pub struct Report {
/// Pixels compared.
pub pixels: u64,
/// The largest CIEDE2000 difference, at [`Self::worst`].
pub max_delta_e: f64,
/// The mean CIEDE2000 difference.
pub mean_delta_e: f64,
/// The largest absolute difference in any channel sample.
pub max_channel: f32,
/// The mean absolute difference per channel (R, G, B).
pub mean_channel: [f64; 3],
/// The first pixel with the largest CIEDE2000 difference, when any pixel differs.
pub worst: Option<(u32, u32)>,
}
impl Report {
/// Whether every measure is within `tolerance`. A NaN limit is never met.
pub fn within(&self, tolerance: &Tolerance) -> bool {
self.exceeded(tolerance).is_empty()
}
/// The measures over `tolerance`, each with its value and limit, for a test failure message.
pub fn exceeded(&self, tolerance: &Tolerance) -> Vec<String> {
let mut over = Vec::new();
if exceeds(self.max_delta_e, tolerance.max_delta_e) {
over.push(format!("max ΔE2000 {:.4} > {}", self.max_delta_e, tolerance.max_delta_e));
}
if exceeds(self.mean_delta_e, tolerance.mean_delta_e) {
over.push(format!("mean ΔE2000 {:.4} > {}", self.mean_delta_e, tolerance.mean_delta_e));
}
if exceeds(f64::from(self.max_channel), f64::from(tolerance.max_channel)) {
over.push(format!("max channel difference {:.6} > {}", self.max_channel, tolerance.max_channel));
}
over
}
}
impl fmt::Display for Report {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} px: ΔE2000 max {:.4} mean {:.4}, channel max {:.6}", self.pixels, self.max_delta_e, self.mean_delta_e, self.max_channel)?;
match self.worst {
Some((x, y)) => write!(f, ", worst at ({x}, {y})"),
None => write!(f, ", identical"),
}
}
}
/// Compare `actual` with `expected`, pixel by pixel.
pub fn compare(expected: &Image<'_>, actual: &Image<'_>) -> Result<Report, ParityError> {
if (expected.width, expected.height) != (actual.width, actual.height) {
return Err(ParityError::Size { expected: (expected.width, expected.height), actual: (actual.width, actual.height) });
}
let width = expected.width.max(1) as usize;
let mut report = Report { pixels: 0, max_delta_e: 0.0, mean_delta_e: 0.0, max_channel: 0.0, mean_channel: [0.0; 3], worst: None };
let mut sum_delta_e = 0.0;
for (i, (e, a)) in expected.rgb.as_chunks::<3>().0.iter().zip(actual.rgb.as_chunks::<3>().0).enumerate() {
let (x, y) = ((i % width) as u32, (i / width) as u32);
let (Some(e), Some(a)) = (finite(*e), finite(*a)) else { return Err(ParityError::NotFinite { x, y }) };
for c in 0..3 {
let d = (e[c] - a[c]).abs();
report.max_channel = report.max_channel.max(d);
report.mean_channel[c] += f64::from(d);
}
let delta_e = delta_e2000(lab(e), lab(a));
sum_delta_e += delta_e;
if delta_e > report.max_delta_e {
report.max_delta_e = delta_e;
report.worst = Some((x, y));
}
report.pixels += 1;
}
if report.pixels > 0 {
let n = report.pixels as f64;
report.mean_delta_e = sum_delta_e / n;
report.mean_channel = report.mean_channel.map(|s| s / n);
}
if report.worst.is_none() && report.max_channel > 0.0 {
// channel differences too small to move CIELAB still name a pixel
report.worst = first_difference(expected, actual);
}
Ok(report)
}
/// Whether `value` exceeds `limit`. A NaN limit (or value) exceeds it: an unordered comparison never
/// passes, so an invalid tolerance cannot switch a check off.
fn exceeds(value: f64, limit: f64) -> bool {
!matches!(value.partial_cmp(&limit), Some(std::cmp::Ordering::Less | std::cmp::Ordering::Equal))
}
fn finite(px: [f32; 3]) -> Option<[f32; 3]> {
px.iter().all(|v| v.is_finite()).then_some(px)
}
fn first_difference(expected: &Image<'_>, actual: &Image<'_>) -> Option<(u32, u32)> {
let width = expected.width.max(1) as usize;
let i = expected.rgb.as_chunks::<3>().0.iter().zip(actual.rgb.as_chunks::<3>().0).position(|(e, a)| e != a)?;
Some(((i % width) as u32, (i / width) as u32))
}
fn lab(rgb: [f32; 3]) -> [f64; 3] {
let rgb = rgb.map(f64::from);
let xyz = RGB_TO_XYZ.map(|row| row[0] * rgb[0] + row[1] * rgb[1] + row[2] * rgb[2]);
xyz_to_lab(xyz, D65)
}
#[cfg(test)]
mod tests;
+100
View File
@@ -0,0 +1,100 @@
use super::*;
fn gray_ramp(width: u32, height: u32) -> Vec<f32> {
(0..width * height).flat_map(|i| [i as f32 / (width * height) as f32; 3]).collect()
}
#[test]
fn identical_renders_are_within_any_tolerance() {
let rgb = gray_ramp(4, 3);
let image = Image::new(4, 3, &rgb).unwrap();
let report = compare(&image, &image).unwrap();
assert_eq!((report.pixels, report.max_delta_e, report.max_channel, report.worst), (12, 0.0, 0.0, None));
assert!(report.within(&Tolerance { max_delta_e: 0.0, mean_delta_e: 0.0, max_channel: 0.0 }));
assert_eq!(report.to_string(), "12 px: ΔE2000 max 0.0000 mean 0.0000, channel max 0.000000, identical");
}
#[test]
fn white_in_the_matrix_is_neutral() {
// RGB (1, 1, 1) is the D65 white: L* 100, a* = b* = 0
let [l, a, b] = lab([1.0; 3]);
assert!((l - 100.0).abs() < 1e-9 && a.abs() < 1e-9 && b.abs() < 1e-9, "{l} {a} {b}");
}
#[test]
fn the_worst_pixel_and_the_means_are_reported() {
let expected = gray_ramp(3, 2);
let mut actual = expected.clone();
// pixel (1, 1): red up by 0.1, a visible difference
actual[4 * 3] += 0.1;
let report = compare(&Image::new(3, 2, &expected).unwrap(), &Image::new(3, 2, &actual).unwrap()).unwrap();
assert_eq!(report.worst, Some((1, 1)));
assert!(report.max_delta_e > 1.0, "{report}");
assert!((report.mean_delta_e - report.max_delta_e / 6.0).abs() < 1e-9);
assert!((report.max_channel - 0.1).abs() < 1e-6);
assert!((report.mean_channel[0] - 0.1 / 6.0).abs() < 1e-6 && report.mean_channel[1] == 0.0);
let over = report.exceeded(&Tolerance::BASELINE);
assert_eq!(over.len(), 3, "{over:?}");
assert!(over[0].starts_with("max ΔE2000") && over[1].starts_with("mean ΔE2000") && over[2].starts_with("max channel"));
assert!(!report.within(&Tolerance::default()));
assert!(report.to_string().ends_with("worst at (1, 1)"));
}
#[test]
fn a_tenth_of_a_10_bit_code_value_passes_the_baseline() {
let expected = vec![0.18f32; 2 * 2 * 3];
let actual: Vec<f32> = expected.iter().map(|v| v + 0.1 / 1024.0).collect();
let report = compare(&Image::new(2, 2, &expected).unwrap(), &Image::new(2, 2, &actual).unwrap()).unwrap();
assert!(report.within(&Tolerance::BASELINE), "{report}");
assert!(report.worst.is_some());
}
#[test]
fn a_channel_difference_too_small_for_cielab_still_names_a_pixel() {
let expected = vec![0.5f32; 6];
let mut actual = expected.clone();
actual[5] = f32::from_bits(0.5f32.to_bits() + 1);
let report = compare(&Image::new(2, 1, &expected).unwrap(), &Image::new(2, 1, &actual).unwrap()).unwrap();
assert!(report.max_channel > 0.0);
assert_eq!(report.worst, Some((1, 0)));
}
#[test]
fn an_empty_render_compares_as_identical() {
let image = Image::new(0, 5, &[]).unwrap();
let report = compare(&image, &image).unwrap();
assert_eq!((report.pixels, report.mean_delta_e, report.worst), (0, 0.0, None));
assert_eq!((image.width(), image.height()), (0, 5));
}
#[test]
fn bad_inputs_are_errors_not_panics() {
let e = Image::new(2, 2, &[0.0; 11]).unwrap_err();
assert_eq!(e, ParityError::Length { width: 2, height: 2, samples: 11 });
assert_eq!(e.to_string(), "11 samples is not RGB for 2 × 2 pixels");
assert!(Image::new(u32::MAX, u32::MAX, &[]).is_err());
let a = vec![0.0f32; 6];
let b = vec![0.0f32; 6];
let e = compare(&Image::new(2, 1, &a).unwrap(), &Image::new(1, 2, &b).unwrap()).unwrap_err();
assert_eq!(e.to_string(), "render is 1 × 2, expected 2 × 1");
let mut nan = a.clone();
nan[4] = f32::NAN;
let e = compare(&Image::new(2, 1, &a).unwrap(), &Image::new(2, 1, &nan).unwrap()).unwrap_err();
assert_eq!(e, ParityError::NotFinite { x: 1, y: 0 });
assert_eq!(e.to_string(), "pixel (1, 0) is not a finite colour");
let e: Box<dyn std::error::Error> = Box::new(e);
assert!(e.source().is_none());
}
#[test]
fn a_nan_tolerance_fails_every_check() {
let rgb = gray_ramp(2, 2);
let image = Image::new(2, 2, &rgb).unwrap();
let report = compare(&image, &image).unwrap();
let nan = Tolerance { max_delta_e: f64::NAN, mean_delta_e: f64::NAN, max_channel: f32::NAN };
assert_eq!(report.exceeded(&nan).len(), 3);
assert!(!report.within(&nan));
assert!(report.within(&Tolerance { max_delta_e: f64::INFINITY, ..Tolerance::BASELINE }));
}