diff --git a/Cargo.lock b/Cargo.lock index 1d5c223cf..3981cb49c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2658,6 +2658,13 @@ dependencies = [ "zune-jpegxl", ] +[[package]] +name = "craft-render-parity" +version = "0.1.0" +dependencies = [ + "craft-cms", +] + [[package]] name = "craft-rtf" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f1a09e32a..07aebdc37 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ craft-usd = { path = "crates/usd" } craft-smime = { path = "crates/smime" } craft-ber = { path = "crates/ber" } craft-proto = { path = "crates/proto" } +craft-render-parity = { path = "crates/render-parity" } acadrust = "=0.6.6" log = "0.4" # internal: add each crate here as `craft- = { path = "crates/" }` diff --git a/README.md b/README.md index 2709855d2..77a4a3ae4 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit Hasselblad 3FR/FFF, Minolta MRW, Epson ERF, Mamiya MEF, Kodak DCS, Pentax PEF, Samsung SRW, Canon CR2, Nikon NEF/NRW, Panasonic RW2, Fujifilm RAF, Olympus ORF (uncompressed), Sony ARW, Canon CR3 and Canon CRW (through `craft-crw`) so far; demosaicing and everything after stay with the develop pipeline. +- [`craft-render-parity`](crates/render-parity/README.md): test support for the shared develop + pipeline (#587): how far two renders of the same photo are apart (per-channel error, CIEDE2000, + the worst pixel) and whether that is within a stated tolerance. - [`craft-tga`](crates/tga/src/lib.rs): Truevision Targa, from FilmCraft (reader) and PhotoCraft (writer): true-colour, grayscale and colour-mapped pictures, raw or run-length encoded, any origin, to RGBA8, and a run-length writer whose packets end with their scan line. Bounded on diff --git a/crates/render-parity/Cargo.toml b/crates/render-parity/Cargo.toml new file mode 100644 index 000000000..087ff748b --- /dev/null +++ b/crates/render-parity/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-render-parity" +description = "Compare two renders of the same photo (per-channel error and CIEDE2000) against a stated tolerance: the parity check for the shared develop pipeline. Test support only." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +craft-cms = { workspace = true } diff --git a/crates/render-parity/README.md b/crates/render-parity/README.md new file mode 100644 index 000000000..0924d891a --- /dev/null +++ b/crates/render-parity/README.md @@ -0,0 +1,37 @@ +# craft-render-parity + +Test support for the shared develop pipeline ([#587](https://github.com/storytold/craft-libs/issues/587)): +how far two renders of the same photo are apart, and whether that is within a stated tolerance. +A develop stage that moves from an app, or an app that switches to the shared pipeline, renders +the same fixture both ways and compares the two here. + +```toml +[dev-dependencies] +craft-render-parity = { git = "https://github.com/storytold/craft-libs", rev = "" } +``` + +```rust +use craft_render_parity::{Image, Tolerance, compare}; + +let expected = Image::new(width, height, &app_render)?; +let actual = Image::new(width, height, &shared_render)?; +let report = compare(&expected, &actual)?; +assert!(report.within(&Tolerance::BASELINE), "{report}: {:?}", report.exceeded(&Tolerance::BASELINE)); +``` + +## Input + +Scene-linear RGB with Rec. 709 / sRGB primaries and a D65 white, three interleaved `f32` samples +per pixel, row by row, with 1.0 as diffuse white. Values above 1.0 are compared as they are. A NaN +or infinite sample is an error, never a pass. + +## Measures + +| Measure | Meaning | +|---|---| +| `max_delta_e`, `mean_delta_e` | CIEDE2000 (kL = kC = kH = 1) in CIELAB relative to D65, from `craft_cms::math` | +| `max_channel`, `mean_channel` | Absolute difference of the linear samples, overall and per channel | +| `worst` | The first pixel with the largest CIEDE2000 difference, or the first that differs at all | + +`Tolerance::BASELINE`, the acceptance bar of #587, is CIEDE2000 at most 0.5 at any pixel and 0.05 +on average (1.0 is about the smallest visible difference), and at most 1/1024 in any linear sample. diff --git a/crates/render-parity/src/lib.rs b/crates/render-parity/src/lib.rs new file mode 100644 index 000000000..aedac7f0f --- /dev/null +++ b/crates/render-parity/src/lib.rs @@ -0,0 +1,205 @@ +//! Render parity: how far two renders of the same photo are apart, and whether that is within a +//! stated [`Tolerance`]. +//! +//! The shared develop pipeline (craft-libs#587) has to give LightCraft and PhotoCraft the same +//! picture. A stage moved from an app, or an app switched to the shared crate, is checked by +//! rendering the same fixture both ways and comparing them here. The [`Report`] names the worst +//! pixel, so a failure points at where to look. +//! +//! Renders are scene-linear RGB with Rec. 709 / sRGB primaries and a D65 white, interleaved, with +//! 1.0 as diffuse white. Colour difference is CIEDE2000 in CIELAB relative to D65 (from +//! [`craft_cms::math`]); values above 1.0 are allowed and compared as they are. +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::fmt; + +use craft_cms::math::{delta_e2000, xyz_to_lab}; + +/// Linear Rec. 709 RGB → CIE XYZ (D65), IEC 61966-2-1. +const RGB_TO_XYZ: [[f64; 3]; 3] = [[0.4124, 0.3576, 0.1805], [0.2126, 0.7152, 0.0722], [0.0193, 0.1192, 0.9505]]; +/// The D65 white in XYZ, from the matrix above applied to RGB (1, 1, 1). +const D65: [f64; 3] = [0.9505, 1.0, 1.089]; + +/// A render to compare: `width` × `height` interleaved RGB samples, row by row. +#[derive(Clone, Copy, Debug)] +pub struct Image<'a> { + width: u32, + height: u32, + rgb: &'a [f32], +} + +impl<'a> Image<'a> { + /// A render of `width` × `height` pixels whose `rgb` holds exactly three samples per pixel. + pub fn new(width: u32, height: u32, rgb: &'a [f32]) -> Result { + let expected = (width as usize).checked_mul(height as usize).and_then(|n| n.checked_mul(3)); + if expected != Some(rgb.len()) { + return Err(ParityError::Length { width, height, samples: rgb.len() }); + } + Ok(Self { width, height, rgb }) + } + + pub fn width(&self) -> u32 { + self.width + } + + pub fn height(&self) -> u32 { + self.height + } +} + +/// Why two renders could not be compared. +#[derive(Clone, Debug, PartialEq)] +pub enum ParityError { + /// The sample count is not three per pixel. + Length { width: u32, height: u32, samples: usize }, + /// The renders differ in size, so a pixel of one has no counterpart in the other. + Size { expected: (u32, u32), actual: (u32, u32) }, + /// A sample is NaN or infinite: no difference can be stated for it. + NotFinite { x: u32, y: u32 }, +} + +impl fmt::Display for ParityError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Length { width, height, samples } => write!(f, "{samples} samples is not RGB for {width} × {height} pixels"), + Self::Size { expected, actual } => write!(f, "render is {} × {}, expected {} × {}", actual.0, actual.1, expected.0, expected.1), + Self::NotFinite { x, y } => write!(f, "pixel ({x}, {y}) is not a finite colour"), + } + } +} + +impl std::error::Error for ParityError {} + +/// How far two renders may be apart and still count as the same picture. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Tolerance { + /// The largest CIEDE2000 difference at any pixel. + pub max_delta_e: f64, + /// The largest mean CIEDE2000 difference over the picture. + pub mean_delta_e: f64, + /// The largest difference in any one linear channel sample. + pub max_channel: f32, +} + +impl Tolerance { + /// The baseline parity of craft-libs#587: well below a visible difference (CIEDE2000 1.0), and + /// within a 10-bit code value in any channel. + pub const BASELINE: Self = Self { max_delta_e: 0.5, mean_delta_e: 0.05, max_channel: 1.0 / 1024.0 }; +} + +impl Default for Tolerance { + fn default() -> Self { + Self::BASELINE + } +} + +/// The difference between two renders of the same size. +#[derive(Clone, Debug, PartialEq)] +pub struct Report { + /// Pixels compared. + pub pixels: u64, + /// The largest CIEDE2000 difference, at [`Self::worst`]. + pub max_delta_e: f64, + /// The mean CIEDE2000 difference. + pub mean_delta_e: f64, + /// The largest absolute difference in any channel sample. + pub max_channel: f32, + /// The mean absolute difference per channel (R, G, B). + pub mean_channel: [f64; 3], + /// The first pixel with the largest CIEDE2000 difference, when any pixel differs. + pub worst: Option<(u32, u32)>, +} + +impl Report { + /// Whether every measure is within `tolerance`. A NaN limit is never met. + pub fn within(&self, tolerance: &Tolerance) -> bool { + self.exceeded(tolerance).is_empty() + } + + /// The measures over `tolerance`, each with its value and limit, for a test failure message. + pub fn exceeded(&self, tolerance: &Tolerance) -> Vec { + let mut over = Vec::new(); + if exceeds(self.max_delta_e, tolerance.max_delta_e) { + over.push(format!("max ΔE2000 {:.4} > {}", self.max_delta_e, tolerance.max_delta_e)); + } + if exceeds(self.mean_delta_e, tolerance.mean_delta_e) { + over.push(format!("mean ΔE2000 {:.4} > {}", self.mean_delta_e, tolerance.mean_delta_e)); + } + if exceeds(f64::from(self.max_channel), f64::from(tolerance.max_channel)) { + over.push(format!("max channel difference {:.6} > {}", self.max_channel, tolerance.max_channel)); + } + over + } +} + +impl fmt::Display for Report { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} px: ΔE2000 max {:.4} mean {:.4}, channel max {:.6}", self.pixels, self.max_delta_e, self.mean_delta_e, self.max_channel)?; + match self.worst { + Some((x, y)) => write!(f, ", worst at ({x}, {y})"), + None => write!(f, ", identical"), + } + } +} + +/// Compare `actual` with `expected`, pixel by pixel. +pub fn compare(expected: &Image<'_>, actual: &Image<'_>) -> Result { + if (expected.width, expected.height) != (actual.width, actual.height) { + return Err(ParityError::Size { expected: (expected.width, expected.height), actual: (actual.width, actual.height) }); + } + let width = expected.width.max(1) as usize; + let mut report = Report { pixels: 0, max_delta_e: 0.0, mean_delta_e: 0.0, max_channel: 0.0, mean_channel: [0.0; 3], worst: None }; + let mut sum_delta_e = 0.0; + for (i, (e, a)) in expected.rgb.as_chunks::<3>().0.iter().zip(actual.rgb.as_chunks::<3>().0).enumerate() { + let (x, y) = ((i % width) as u32, (i / width) as u32); + let (Some(e), Some(a)) = (finite(*e), finite(*a)) else { return Err(ParityError::NotFinite { x, y }) }; + for c in 0..3 { + let d = (e[c] - a[c]).abs(); + report.max_channel = report.max_channel.max(d); + report.mean_channel[c] += f64::from(d); + } + let delta_e = delta_e2000(lab(e), lab(a)); + sum_delta_e += delta_e; + if delta_e > report.max_delta_e { + report.max_delta_e = delta_e; + report.worst = Some((x, y)); + } + report.pixels += 1; + } + if report.pixels > 0 { + let n = report.pixels as f64; + report.mean_delta_e = sum_delta_e / n; + report.mean_channel = report.mean_channel.map(|s| s / n); + } + if report.worst.is_none() && report.max_channel > 0.0 { + // channel differences too small to move CIELAB still name a pixel + report.worst = first_difference(expected, actual); + } + Ok(report) +} + +/// Whether `value` exceeds `limit`. A NaN limit (or value) exceeds it: an unordered comparison never +/// passes, so an invalid tolerance cannot switch a check off. +fn exceeds(value: f64, limit: f64) -> bool { + !matches!(value.partial_cmp(&limit), Some(std::cmp::Ordering::Less | std::cmp::Ordering::Equal)) +} + +fn finite(px: [f32; 3]) -> Option<[f32; 3]> { + px.iter().all(|v| v.is_finite()).then_some(px) +} + +fn first_difference(expected: &Image<'_>, actual: &Image<'_>) -> Option<(u32, u32)> { + let width = expected.width.max(1) as usize; + let i = expected.rgb.as_chunks::<3>().0.iter().zip(actual.rgb.as_chunks::<3>().0).position(|(e, a)| e != a)?; + Some(((i % width) as u32, (i / width) as u32)) +} + +fn lab(rgb: [f32; 3]) -> [f64; 3] { + let rgb = rgb.map(f64::from); + let xyz = RGB_TO_XYZ.map(|row| row[0] * rgb[0] + row[1] * rgb[1] + row[2] * rgb[2]); + xyz_to_lab(xyz, D65) +} + +#[cfg(test)] +mod tests; diff --git a/crates/render-parity/src/tests.rs b/crates/render-parity/src/tests.rs new file mode 100644 index 000000000..a5c81f24f --- /dev/null +++ b/crates/render-parity/src/tests.rs @@ -0,0 +1,100 @@ +use super::*; + +fn gray_ramp(width: u32, height: u32) -> Vec { + (0..width * height).flat_map(|i| [i as f32 / (width * height) as f32; 3]).collect() +} + +#[test] +fn identical_renders_are_within_any_tolerance() { + let rgb = gray_ramp(4, 3); + let image = Image::new(4, 3, &rgb).unwrap(); + let report = compare(&image, &image).unwrap(); + assert_eq!((report.pixels, report.max_delta_e, report.max_channel, report.worst), (12, 0.0, 0.0, None)); + assert!(report.within(&Tolerance { max_delta_e: 0.0, mean_delta_e: 0.0, max_channel: 0.0 })); + assert_eq!(report.to_string(), "12 px: ΔE2000 max 0.0000 mean 0.0000, channel max 0.000000, identical"); +} + +#[test] +fn white_in_the_matrix_is_neutral() { + // RGB (1, 1, 1) is the D65 white: L* 100, a* = b* = 0 + let [l, a, b] = lab([1.0; 3]); + assert!((l - 100.0).abs() < 1e-9 && a.abs() < 1e-9 && b.abs() < 1e-9, "{l} {a} {b}"); +} + +#[test] +fn the_worst_pixel_and_the_means_are_reported() { + let expected = gray_ramp(3, 2); + let mut actual = expected.clone(); + // pixel (1, 1): red up by 0.1, a visible difference + actual[4 * 3] += 0.1; + let report = compare(&Image::new(3, 2, &expected).unwrap(), &Image::new(3, 2, &actual).unwrap()).unwrap(); + assert_eq!(report.worst, Some((1, 1))); + assert!(report.max_delta_e > 1.0, "{report}"); + assert!((report.mean_delta_e - report.max_delta_e / 6.0).abs() < 1e-9); + assert!((report.max_channel - 0.1).abs() < 1e-6); + assert!((report.mean_channel[0] - 0.1 / 6.0).abs() < 1e-6 && report.mean_channel[1] == 0.0); + let over = report.exceeded(&Tolerance::BASELINE); + assert_eq!(over.len(), 3, "{over:?}"); + assert!(over[0].starts_with("max ΔE2000") && over[1].starts_with("mean ΔE2000") && over[2].starts_with("max channel")); + assert!(!report.within(&Tolerance::default())); + assert!(report.to_string().ends_with("worst at (1, 1)")); +} + +#[test] +fn a_tenth_of_a_10_bit_code_value_passes_the_baseline() { + let expected = vec![0.18f32; 2 * 2 * 3]; + let actual: Vec = expected.iter().map(|v| v + 0.1 / 1024.0).collect(); + let report = compare(&Image::new(2, 2, &expected).unwrap(), &Image::new(2, 2, &actual).unwrap()).unwrap(); + assert!(report.within(&Tolerance::BASELINE), "{report}"); + assert!(report.worst.is_some()); +} + +#[test] +fn a_channel_difference_too_small_for_cielab_still_names_a_pixel() { + let expected = vec![0.5f32; 6]; + let mut actual = expected.clone(); + actual[5] = f32::from_bits(0.5f32.to_bits() + 1); + let report = compare(&Image::new(2, 1, &expected).unwrap(), &Image::new(2, 1, &actual).unwrap()).unwrap(); + assert!(report.max_channel > 0.0); + assert_eq!(report.worst, Some((1, 0))); +} + +#[test] +fn an_empty_render_compares_as_identical() { + let image = Image::new(0, 5, &[]).unwrap(); + let report = compare(&image, &image).unwrap(); + assert_eq!((report.pixels, report.mean_delta_e, report.worst), (0, 0.0, None)); + assert_eq!((image.width(), image.height()), (0, 5)); +} + +#[test] +fn bad_inputs_are_errors_not_panics() { + let e = Image::new(2, 2, &[0.0; 11]).unwrap_err(); + assert_eq!(e, ParityError::Length { width: 2, height: 2, samples: 11 }); + assert_eq!(e.to_string(), "11 samples is not RGB for 2 × 2 pixels"); + assert!(Image::new(u32::MAX, u32::MAX, &[]).is_err()); + + let a = vec![0.0f32; 6]; + let b = vec![0.0f32; 6]; + let e = compare(&Image::new(2, 1, &a).unwrap(), &Image::new(1, 2, &b).unwrap()).unwrap_err(); + assert_eq!(e.to_string(), "render is 1 × 2, expected 2 × 1"); + + let mut nan = a.clone(); + nan[4] = f32::NAN; + let e = compare(&Image::new(2, 1, &a).unwrap(), &Image::new(2, 1, &nan).unwrap()).unwrap_err(); + assert_eq!(e, ParityError::NotFinite { x: 1, y: 0 }); + assert_eq!(e.to_string(), "pixel (1, 0) is not a finite colour"); + let e: Box = Box::new(e); + assert!(e.source().is_none()); +} + +#[test] +fn a_nan_tolerance_fails_every_check() { + let rgb = gray_ramp(2, 2); + let image = Image::new(2, 2, &rgb).unwrap(); + let report = compare(&image, &image).unwrap(); + let nan = Tolerance { max_delta_e: f64::NAN, mean_delta_e: f64::NAN, max_channel: f32::NAN }; + assert_eq!(report.exceeded(&nan).len(), 3); + assert!(!report.within(&nan)); + assert!(report.within(&Tolerance { max_delta_e: f64::INFINITY, ..Tolerance::BASELINE })); +}