From b8b8483bf9b5af858bb44ead80dafebc31f42d7a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 18:08:45 +0000 Subject: [PATCH] fix(algo): seam carving never panics on degenerate input; NaN never steers seams From an adversarial review of the Content-Aware Scale speed-up: - resize_bilinear with zero channels panicked (chunk size 0) since the previous commit; it, carve_with and carve_width_with now return early when there are no pixels or channels, or fewer values than the size says (sizes checked with checked_mul). carve with an empty width and a height change, and a protection map shorter than the image (now ignored), panicked before this series too. - With NaN in the energy map, which seam was chosen depended on the NaN payload, and that differed between builds. NaN samples now give the highest energy and NaN protection protects nothing, so costs are never NaN; outputs without NaN are unchanged (all pins match). Tests: degenerate_inputs_do_not_panic and nan_payloads_do_not_steer_seams (both failed before), proxy_rebuild_observes_cancellation (a cancel during the tall-image rebuild), and NaN/inf cases in incremental_dp_matches_full_dp. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LELc5BQzgV5dA7dxdMejkK --- crates/algo/src/seam.rs | 121 ++++++++++++++++++++++++++++++++++------ 1 file changed, 105 insertions(+), 16 deletions(-) diff --git a/crates/algo/src/seam.rs b/crates/algo/src/seam.rs index 8b5befda80..59e235bc05 100644 --- a/crates/algo/src/seam.rs +++ b/crates/algo/src/seam.rs @@ -9,13 +9,21 @@ //! //! A protect map (`0..=1`, e.g. an alpha channel or detected skin tones) adds a large energy so //! seams avoid protected pixels. Buffers are interleaved `w × h × ch` normalised floats of any -//! colour model and depth (alpha is carried like any other channel). Deterministic. +//! colour model and depth (alpha is carried like any other channel). Deterministic: a NaN sample +//! gives its pixels the highest energy and NaN protection protects nothing, so no NaN (whose +//! payload arithmetic does not keep consistently across builds) ever reaches the seam search. use photocraft_raster::{Cancelled, Interrupt}; /// Energy added per unit of protection: far above any gradient magnitude. const PROTECT_ENERGY: f32 = 1.0e4; +/// The energy protection `p` (`0..=1`) adds. NaN protects nothing. +#[inline] +fn protection(p: f32) -> f32 { + if p.is_nan() { 0.0 } else { p.clamp(0.0, 1.0) * PROTECT_ENERGY } +} + /// Keep full horizontal detail while reducing the height used to find vertical seams. At a /// 24 MP working size this cuts the dynamic-programming work by about four times without /// quantising the output pixels or changing the requested output dimensions. @@ -48,7 +56,7 @@ pub fn energy(w: usize, h: usize, ch: usize, img: &[f32]) -> Vec { /// [`energy`] of pixel `x` of row `y` of a `cw`-wide, `h`-tall image whose pixel `xx` of row `yy` /// is column `col(xx, yy)` of `img` (`w` pixels per row). Shared by the full map and the carver's -/// per-seam updates, so both give bit-identical values. +/// per-seam updates, so both give bit-identical values. NaN samples give the highest energy. #[allow(clippy::too_many_arguments)] #[inline] fn energy_at(img: &[f32], w: usize, cw: usize, h: usize, ch: usize, x: usize, y: usize, col: impl Fn(usize, usize) -> usize) -> f32 { @@ -62,7 +70,8 @@ fn energy_at(img: &[f32], w: usize, cw: usize, h: usize, ch: usize, x: usize, y: s += (img[right + c] - v).abs() + (v - img[left + c]).abs(); s += (img[down + c] - v).abs() + (v - img[up + c]).abs(); } - s * 0.5 + let e = s * 0.5; + if e.is_nan() { f32::INFINITY } else { e } } /// The lowest-energy vertical seam: one x per row. @@ -114,7 +123,8 @@ struct Dp { /// the pixel straight up, then up-left, then up-right. The cost doesn't depend on the tie-break /// and the direction doesn't depend on `e`, so the seam can be traced back from the costs alone. /// The full pass's edges, [`Dp::update`] and [`Dp::seam`] use it, and the full pass's interior -/// repeats it exactly, so every way of filling the map gives the same bits. +/// repeats it exactly, so every way of filling the map gives the same bits (costs are never NaN: +/// see [`energy_at`] and [`protection`]). #[inline] fn cell(up: &[f32], cw: usize, x: usize, e: f32) -> (f32, u8) { let (b, d) = if x == 0 { @@ -279,8 +289,8 @@ impl<'a> Carver<'a> { fn new(w: usize, h: usize, ch: usize, src: &'a [f32], prot: Option<&'a [f32]>) -> Self { let mut e = energy(w, h, ch, src); if let Some(p) = prot { - for (e, p) in e.iter_mut().zip(p) { - *e += p.clamp(0.0, 1.0) * PROTECT_ENERGY; + for (e, &p) in e.iter_mut().zip(p) { + *e += protection(p); } } let idx = (0..h).flat_map(|_| (0..w).map(|x| u32::try_from(x).unwrap_or(u32::MAX))).collect(); @@ -345,7 +355,7 @@ impl<'a> Carver<'a> { for x in lo..=hi { let mut v = energy_at(src, w, cw, h, ch, x, y, col); if let Some(p) = prot { - v += p[y * w + col(x, y)].clamp(0.0, 1.0) * PROTECT_ENERGY; + v += protection(p[y * w + col(x, y)]); } e[off[y] + x] = v; } @@ -458,6 +468,11 @@ fn average_rows(src: &[f32], row_len: usize, h: usize, step: usize) -> Vec out } +/// No pixels or channels to work with, or `img` holds fewer than `w × h × ch` values. +fn unusable(w: usize, h: usize, ch: usize, img: &[f32]) -> bool { + w == 0 || h == 0 || ch == 0 || w.checked_mul(h).and_then(|n| n.checked_mul(ch)).is_none_or(|n| img.len() < n) +} + /// Change the width of `img` to `new_w` by removing or duplicating vertical seams. pub fn carve_width(w: usize, h: usize, ch: usize, img: &[f32], protect: Option<&[f32]>, new_w: usize) -> Vec { // Never cancelled; the fallback is unreachable. @@ -467,10 +482,12 @@ pub fn carve_width(w: usize, h: usize, ch: usize, img: &[f32], protect: Option<& /// [`carve_width`] that checks `ctl` before every seam and reports progress per seam. pub fn carve_width_with(w: usize, h: usize, ch: usize, img: &[f32], protect: Option<&[f32]>, new_w: usize, ctl: &Interrupt) -> Result, Cancelled> { let new_w = new_w.max(1); - // No channels: nothing to carve (and no pixels to size the work by). - if new_w == w || w == 0 || h == 0 || ch == 0 { + // Nothing to carve: no pixels or channels, or fewer values than the size says. + if new_w == w || unusable(w, h, ch, img) { return Ok(img.to_vec()); } + // A protection map that doesn't cover the image protects nothing. + let protect = protect.filter(|p| p.len() >= w * h); let seams = new_w.abs_diff(w).max(1) as f32; if new_w < w { if h >= PROXY_SHRINK_MIN_HEIGHT && w - new_w >= PROXY_SHRINK_MIN_SEAMS { @@ -628,6 +645,10 @@ pub fn carve_with( ctl: &Interrupt, ) -> Result, Cancelled> { let (new_w, new_h) = (new_w.max(1), new_h.max(1)); + if unusable(w, h, ch, img) { + return Ok(img.to_vec()); + } + let protect = protect.filter(|p| p.len() >= w * h); // Split progress between the passes by their seam counts. let (sw, sh) = (new_w.abs_diff(w) as f32, new_h.abs_diff(h) as f32); let split = if sw + sh > 0.0 { sw / (sw + sh) } else { 1.0 }; @@ -647,8 +668,11 @@ pub fn carve_with( /// Plain bilinear resize (the non-content-aware part of the Amount blend). pub fn resize_bilinear(w: usize, h: usize, ch: usize, img: &[f32], nw: usize, nh: usize) -> Vec { - let mut out = vec![0.0f32; nw * nh * ch]; - if w == 0 || h == 0 || nw == 0 || nh == 0 { + let Some(len) = nw.checked_mul(nh).and_then(|n| n.checked_mul(ch)) else { + return Vec::new(); + }; + let mut out = vec![0.0f32; len]; + if len == 0 || unusable(w, h, ch, img) { return out; } let row = |(y, out): (usize, &mut [f32])| { @@ -702,7 +726,7 @@ mod tests { /// The straightforward implementation the per-seam carver replaced (full energy map, fresh /// seam search and a new buffer per seam), kept as the oracle for `carver_matches_reference`. mod reference { - use super::super::{PROTECT_ENERGY, resize_bilinear, transpose}; + use super::super::{protection, resize_bilinear, transpose}; pub(super) fn energy(w: usize, h: usize, ch: usize, img: &[f32]) -> Vec { let mut e = vec![0.0f32; w * h]; @@ -717,7 +741,8 @@ mod tests { s += (at(xr, y, c) - v).abs() + (v - at(xl, y, c)).abs(); s += (at(x, yd, c) - v).abs() + (v - at(x, yu, c)).abs(); } - e[y * w + x] = s * 0.5; + let v = s * 0.5; + e[y * w + x] = if v.is_nan() { f32::INFINITY } else { v }; } } e @@ -770,8 +795,8 @@ mod tests { } let protect_e = |e: &mut [f32], p: Option<&Vec>| { if let Some(p) = p { - for (e, p) in e.iter_mut().zip(p) { - *e += p.clamp(0.0, 1.0) * PROTECT_ENERGY; + for (e, &p) in e.iter_mut().zip(p) { + *e += protection(p); } } }; @@ -906,6 +931,12 @@ mod tests { let (x, y) = ((i % 120) as f32, (i / 120) as f32); *v = ((x * 0.07).sin() * (y * 0.05).cos() * 0.5 + 0.5).clamp(0.0, 1.0); } + // NaN and infinite samples and NaN protection: infinite energies, never NaN costs. + let mut odd = noise(40 * 30 * 3, 255, 11); + odd.iter_mut().step_by(13).for_each(|v| *v = f32::NAN); + odd.iter_mut().skip(5).step_by(17).for_each(|v| *v = f32::INFINITY); + let mut odd_prot = noise(40 * 30, 3, 12); + odd_prot.iter_mut().step_by(7).for_each(|v| *v = -f32::NAN); // (w, h, channels, image, protection, seams to remove) type Case = (usize, usize, usize, Vec, Option>, usize); let cases: Vec = vec![ @@ -916,6 +947,7 @@ mod tests { (33, 1, 1, noise(33, 5, 8), None, 32), (2, 40, 3, noise(2 * 40 * 3, 9, 9), None, 2), (50, 40, 1, vec![0.5; 50 * 40], Some(noise(50 * 40, 2, 10)), 49), + (40, 30, 3, odd, Some(odd_prot), 25), ]; let bits = |v: &[f32]| v.iter().map(|f| f.to_bits()).collect::>(); for (case, (w, h, ch, img, prot, seams)) in cases.into_iter().enumerate() { @@ -937,7 +969,7 @@ mod tests { if let Some(p) = &prot { let kept = (0..h).flat_map(|y| c.idx[rows.of(y)].iter().map(move |&x| y * w + x as usize)); for (e, i) in want.iter_mut().zip(kept) { - *e += p[i].clamp(0.0, 1.0) * PROTECT_ENERGY; + *e += protection(p[i]); } } let have: Vec = (0..h).flat_map(|y| c.e[rows.of(y)].to_vec()).collect(); @@ -951,6 +983,63 @@ mod tests { } } + /// Hostile but well-formed shapes return early instead of panicking: no channels, empty + /// sizes (also through the height pass), and images or protection maps shorter than their + /// size says (a short map protects nothing). + #[test] + fn degenerate_inputs_do_not_panic() { + assert!(carve(3, 3, 0, &[], None, 2, 2).is_empty()); + assert!(carve(3, 3, 0, &[], None, 5, 5).is_empty()); + assert!(transpose(3, 3, 0, &[]).is_empty()); + assert!(resize_bilinear(3, 3, 0, &[], 2, 2).is_empty()); + assert!(resize_bilinear(3, 3, 1, &[0.5; 9], 0, 2).is_empty()); + assert!(resize_bilinear(3, 3, 1, &[0.5; 9], 2, 0).is_empty()); + assert_eq!(resize_bilinear(3, 3, 1, &[0.5; 4], 2, 2), vec![0.0; 4]); + assert!(carve(0, 5, 3, &[], None, 3, 7).is_empty()); + assert!(carve(0, 5, 3, &[], None, 3, 2).is_empty()); + let img = noise(10 * 5, 7, 3); + assert_eq!(carve(10, 5, 1, &img, Some(&[0.0; 3]), 4, 5), carve(10, 5, 1, &img, None, 4, 5)); + assert_eq!(carve(10, 5, 1, &img, Some(&[0.0; 3]), 4, 3), carve(10, 5, 1, &img, None, 4, 3)); + assert_eq!(carve(10, 5, 3, &[0.5; 20], None, 4, 5), vec![0.5; 20]); + assert_eq!(carve_width(usize::MAX, 2, 4, &[0.5; 8], None, 1), vec![0.5; 8]); + } + + /// A cancel that arrives after the seam search, during the tall-image rebuild, still stops + /// the carve. + #[test] + fn proxy_rebuild_observes_cancellation() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let (w, h, ch, new_w) = (128, PROXY_SHRINK_MIN_HEIGHT, 3, 64); + let img = noise(w * h * ch, 7, 5); + let calls = AtomicUsize::new(0); + // One check per seam passes; the first one after the search cancels. + let cancel = || calls.fetch_add(1, Ordering::Relaxed) >= w - new_w; + assert_eq!(carve_width_with(w, h, ch, &img, None, new_w, &Interrupt::cancel_only(&cancel)), Err(Cancelled)); + assert!(calls.load(Ordering::Relaxed) > w - new_w, "the rebuild checked for cancellation"); + } + + /// NaN samples and NaN protection steer seams the same whatever their sign and payload + /// (which arithmetic does not preserve consistently across builds): the same pixels are kept. + #[test] + fn nan_payloads_do_not_steer_seams() { + let (w, h, ch) = (24, 18, 3); + let (base, prot_base) = (noise(w * h * ch, 255, 21), noise(w * h, 3, 22)); + let carve_with_nans = |pixel: f32, prot: f32| { + let (mut img, mut p) = (base.clone(), prot_base.clone()); + img.iter_mut().step_by(37).for_each(|v| *v = pixel); + p.iter_mut().step_by(29).for_each(|v| *v = prot); + carve(w, h, ch, &img, Some(&p), 15, 11) + }; + let canonical = |v: Vec| v.into_iter().map(|f| if f.is_nan() { u32::MAX } else { f.to_bits() }).collect::>(); + let nans = [f32::NAN, -f32::NAN, f32::from_bits(0x7fc0_1234), f32::from_bits(0xffc0_0001)]; + let want = canonical(carve_with_nans(nans[0], nans[0])); + for &a in &nans { + for &b in &nans { + assert_eq!(canonical(carve_with_nans(a, b)), want, "pixel NaN {:#x}, protection NaN {:#x}", a.to_bits(), b.to_bits()); + } + } + } + #[test] fn cancelled_carve_stops() { let (w, h) = (40, 20);