From 81d5c514e2a8a4f26bfef8302279deb5307b5ba3 Mon Sep 17 00:00:00 2001 From: Khawar Zamman Wani Date: Mon, 19 Oct 2026 23:01:33 +0530 Subject: [PATCH] perf(photomerge): build multi-band bands in bounded batches (#7758) (#7763) * perf(photomerge): build multi-band bands in bounded batches (#7758) panorama::multiband built every source's Laplacian and weight pyramids at once and kept them all until accumulation, so Photomerge's peak memory grew with the image count (~5 GB for four 24 MP sources). Build the bands in batches under a 2 GiB estimate (at least one image) and add each batch before building the next. Each pixel still adds the bands in image order, so the output is bit-identical. Planes now move into the pyramids instead of being cloned, and the coverage and push-pull masks are freed once used: with one image built at a time those clones were serial passes. seam_blend shifts its ROIs in place instead of cloning every source for the blend. Peak during the blend: 4x24 MP 5.04 -> 2.81 GB, 8x12 MP 5.50 -> 2.61 GB, at +5% time (2x24 MP unchanged). New example bench_multiband measures it. Co-Authored-By: Claude Opus 5.5 * perf(photomerge): let up to four 24 MP sources build their bands together A 2 GiB band budget built every 24 MP source alone, which made the usual 2-4 image Photomerge 12-36% slower on a 32-thread desktop. The native budget is now 5 GiB, so those still build side by side and only larger stacks are batched. wasm32 has no threads, so it adds each band as soon as it is built. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- crates/algo/examples/bench_multiband.rs | 42 +++++++ crates/algo/src/panorama.rs | 156 +++++++++++++++++------- crates/engine/src/photo_cmds.rs | 14 ++- 3 files changed, 164 insertions(+), 48 deletions(-) create mode 100644 crates/algo/examples/bench_multiband.rs diff --git a/crates/algo/examples/bench_multiband.rs b/crates/algo/examples/bench_multiband.rs new file mode 100644 index 0000000000..9d58b81a54 --- /dev/null +++ b/crates/algo/examples/bench_multiband.rs @@ -0,0 +1,42 @@ +//! Time and peak memory of the Photomerge multi-band blend: `n` RGB sources of `mp` megapixels +//! (3:2), each overlapping the next by half its width. +//! `cargo run --release -p photocraft-algo --example bench_multiband [n] [mp]` (default 4 × 24 MP). +//! Peak memory is the growth of the process's high-water mark during the blend (Linux only). +use std::time::Instant; + +use photocraft_algo::panorama::{self, RoiImage}; + +/// The process's peak resident set in MiB (`VmHWM`), where `/proc` has it. +fn peak_mib() -> Option { + let s = std::fs::read_to_string("/proc/self/status").ok()?; + let kb: f64 = s.lines().find(|l| l.starts_with("VmHWM:"))?.split_whitespace().nth(1)?.parse().ok()?; + Some(kb / 1024.0) +} + +fn main() { + let mut args = std::env::args().skip(1); + let n: usize = args.next().and_then(|a| a.parse().ok()).unwrap_or(4).max(1); + let mp: f64 = args.next().and_then(|a| a.parse().ok()).unwrap_or(24.0); + let sh = ((mp * 1e6 / 1.5).sqrt().round() as usize).max(2); + let sw = sh * 3 / 2; + let step = sw / 2; + let (cw, ch) = (step * (n - 1) + sw, sh); + let rois: Vec = (0..n) + .map(|k| { + let px = (0..sw * sh * 3).map(|i| ((i / 3 % sw * 7 + i / 3 / sw * 3 + k * 31 + i % 3 * 50) % 251) as f32 / 251.0).collect(); + RoiImage { x0: (step * k) as i32, y0: 0, w: sw, h: sh, ch: 3, px, alpha: vec![1.0; sw * sh] } + }) + .collect(); + // Each canvas column belongs to the image whose centre is nearest, as a seam would split it. + let wts: Vec> = (0..n) + .map(|k| (0..sw * sh).map(|i| if ((step * k + i % sw + step / 2) / step).saturating_sub(1).min(n - 1) == k { 1.0 } else { 0.0 }).collect()) + .collect(); + let levels = panorama::blend_levels(cw, ch).min(7); + let before = peak_mib(); + let t0 = Instant::now(); + let (px, cov) = panorama::multiband(cw, ch, &rois, &wts, levels); + let ms = t0.elapsed().as_secs_f64() * 1000.0; + let peak = peak_mib().zip(before).map_or("n/a".to_string(), |(a, b)| format!("{:.0} MiB", a - b)); + println!("{n} × {sw}×{sh} ({mp} MP) RGB, canvas {cw}×{ch}, {levels} levels: {ms:.0} ms, peak +{peak} during the blend"); + std::hint::black_box((px, cov)); +} diff --git a/crates/algo/src/panorama.rs b/crates/algo/src/panorama.rs index abe17db395..75fa9a4bfb 100644 --- a/crates/algo/src/panorama.rs +++ b/crates/algo/src/panorama.rs @@ -1170,10 +1170,28 @@ fn band_row(band: &PyramidBand, l: usize, ch: usize, py: usize) -> Option<(usize }) } +/// Estimated bytes of band construction [`multiband`] keeps in flight at once. Images are built +/// and accumulated in batches under it, so its peak memory doesn't grow with the image count +/// (#7758). Four 24 MP RGB sources (~1.2 GB each) fit in one batch: building bands side by side +/// uses a many-core machine better than one band's row-parallel work, and one at a time made the +/// usual 2–4 image Photomerge 12–36% slower on a 32-thread desktop. +#[cfg(not(target_arch = "wasm32"))] +const BAND_BUDGET: usize = 5 << 30; +/// Without threads the bands of a batch are built one after another anyway, so each is added as +/// soon as it is built (the browser's address space is 4 GB). +#[cfg(target_arch = "wasm32")] +const BAND_BUDGET: usize = 0; + /// Multi-band blend of `imgs` into a `w × h` panorama (origin 0,0) with one weight map per image /// over the same grid as its ROI (`weights[i]` has `w_i × h_i` entries). Returns the colour /// channels (`ch` per pixel) and the total coverage. pub fn multiband(w: usize, h: usize, imgs: &[RoiImage], weights: &[Vec], levels: usize) -> (Vec, Vec) { + multiband_batched(w, h, imgs, weights, levels, BAND_BUDGET) +} + +/// [`multiband`], building at most `budget` estimated bytes of bands at a time (at least one +/// image). Each pixel still adds the bands in image order, so the result doesn't depend on it. +fn multiband_batched(w: usize, h: usize, imgs: &[RoiImage], weights: &[Vec], levels: usize, budget: usize) -> (Vec, Vec) { let ch = imgs.first().map_or(1, |i| i.ch); let levels = levels.max(1); let align = 1i32 << (levels - 1); @@ -1187,13 +1205,22 @@ pub fn multiband(w: usize, h: usize, imgs: &[RoiImage], weights: &[Vec], le let mut wsum: Vec> = sizes.iter().map(|&(lw, lh)| vec![0.0; lw * lh]).collect(); let margin = align * 4; // Each image on its ROI grown by a margin and aligned to the coarsest level's grid. - let bands = par_map(imgs.len(), |k| { - let im = &imgs[k]; + let grid = |im: &RoiImage| { let gx0 = ((im.x0 - margin).max(0) / align) * align; let gy0 = ((im.y0 - margin).max(0) / align) * align; let gx1 = (im.x0 + im.w as i32 + margin).min(w as i32); let gy1 = (im.y0 + im.h as i32 + margin).min(h as i32); - let (gw, gh) = ((gx1 - gx0).max(1) as usize, (gy1 - gy0).max(1) as usize); + (gx0, gy0, (gx1 - gx0).max(1) as usize, (gy1 - gy0).max(1) as usize) + }; + // Building one band holds its grid's colour and weight planes about three times over: the + // pyramids are 4/3 of a plane, and the grid copy, coverage and `expand` scratch come on top. + let cost = |im: &RoiImage| { + let (_, _, gw, gh) = grid(im); + gw.saturating_mul(gh).saturating_mul(ch + 1).saturating_mul(4 * 3) + }; + let band = |k: usize| -> PyramidBand { + let im = &imgs[k]; + let (gx0, gy0, gw, gh) = grid(im); // Each grid pixel takes the image pixel that lands on it (every image pixel lands on at // most one), so rows of the grid are independent and fill in parallel. let src_of = |gx: usize, gy: usize| -> Option { @@ -1225,68 +1252,85 @@ pub fn multiband(w: usize, h: usize, imgs: &[RoiImage], weights: &[Vec], le } }); let bin: Vec = par_map1(&cov, |a| if a > 0.5 { 1.0 } else { 0.0 }); + drop(cov); push_pull(gw, gh, ch, &mut px, &bin); - // Laplacian of the colours, Gaussian of the weights. + drop(bin); + // Laplacian of the colours, Gaussian of the weights. Planes move into the pyramids rather + // than being copied: with one image built at a time, a copy is a serial pass over it. let mut lap: Vec<(usize, usize, Vec)> = Vec::new(); let mut gau: Vec> = Vec::new(); let (mut cw, mut chh, mut cur, mut cwt) = (gw, gh, px, wt); for l in 0..levels { if l + 1 == levels { - lap.push((cw, chh, cur.clone())); - gau.push(cwt.clone()); + lap.push((cw, chh, cur)); + gau.push(cwt); break; } let (nw, nh, down) = reduce(cw, chh, ch, &cur); let up = expand(nw, nh, ch, &down, cw, chh); lap.push((cw, chh, par_zip(&cur, &up, |a, b| a - b))); - gau.push(cwt.clone()); let (_, _, wdown) = reduce(cw, chh, 1, &cwt); + gau.push(cwt); cw = nw; chh = nh; cur = down; cwt = wdown; } (gx0, gy0, lap, gau) - }); - // Accumulate the bands. Each pixel adds the bands in the same order as the serial loop (per - // level, bands one after another), but rows of a level are independent, so they run in - // parallel: every row walks the bands itself. - for l in 0..levels { - let pw = sizes[l].0; - par_rows(&mut wsum[l], pw, 1, |py, row| { - for b in &bands { - let Some((ox, lw, g, _)) = band_row(b, l, ch, py) else { continue }; - for x in 0..lw { - let px = ox + x; - if px >= pw { - break; - } - let k = g[x]; - if k <= 0.0 { - continue; - } - row[px] += k; - } - } - }); - par_rows(&mut acc[l], pw, ch, |py, row| { - for b in &bands { - let Some((ox, lw, g, band)) = band_row(b, l, ch, py) else { continue }; - for x in 0..lw { - let px = ox + x; - if px >= pw { - break; - } - let k = g[x]; - if k <= 0.0 { - continue; - } - for c in 0..ch { - row[px * ch + c] += k * band[x * ch + c]; + }; + // Build the bands a batch at a time and add each batch before building the next. Each pixel + // adds the bands in the same order as the serial loop (per level, bands one after another; + // batches follow each other in image order), but rows of a level are independent, so they + // run in parallel: every row walks the batch's bands itself. + let mut start = 0; + while start < imgs.len() { + let mut end = start + 1; + let mut bytes = cost(&imgs[start]); + while let Some(im) = imgs.get(end) + && bytes.saturating_add(cost(im)) <= budget + { + bytes = bytes.saturating_add(cost(im)); + end += 1; + } + let bands = par_map(end - start, |j| band(start + j)); + for l in 0..levels { + let pw = sizes[l].0; + par_rows(&mut wsum[l], pw, 1, |py, row| { + for b in &bands { + let Some((ox, lw, g, _)) = band_row(b, l, ch, py) else { continue }; + for x in 0..lw { + let px = ox + x; + if px >= pw { + break; + } + let k = g[x]; + if k <= 0.0 { + continue; + } + row[px] += k; } } - } - }); + }); + par_rows(&mut acc[l], pw, ch, |py, row| { + for b in &bands { + let Some((ox, lw, g, band)) = band_row(b, l, ch, py) else { continue }; + for x in 0..lw { + let px = ox + x; + if px >= pw { + break; + } + let k = g[x]; + if k <= 0.0 { + continue; + } + for c in 0..ch { + row[px * ch + c] += k * band[x * ch + c]; + } + } + } + }); + } + start = end; } for l in 0..levels { let pw = sizes[l].0; @@ -1910,6 +1954,28 @@ mod parallel_tests { assert_eq!(multiband(9, 7, &[], &[], 3).0.len(), ref_multiband(9, 7, &[], &[], 3).0.len()); } + /// Batching the bands (#7758) changes when they are built, not the result. Budgets from none + /// (one image per batch) up to everything in steps of a fifth pass through every split of the + /// three images. + #[test] + fn multiband_batches_match_the_serial_code() { + let budgets: Vec = + std::iter::successors(Some(1024.0f64), |b| Some(b * 1.2)).take_while(|b| *b < 4e6).map(|b| b as usize).chain([0, usize::MAX]).collect(); + for (n, &(w, h)) in [(40usize, 30usize), (97, 61), (201, 129)].iter().enumerate() { + for ch in [1usize, 3] { + let (imgs, wts) = rois(w, h, ch, n as u64 * 17 + ch as u64); + for levels in [1usize, 3, 5] { + let want = ref_multiband(w, h, &imgs, &wts, levels); + for &budget in &budgets { + let got = multiband_batched(w, h, &imgs, &wts, levels, budget); + assert_eq!(bits(&got.0), bits(&want.0), "colours {w}x{h}x{ch} levels {levels} budget {budget}"); + assert_eq!(bits(&got.1), bits(&want.1), "coverage {w}x{h}x{ch} levels {levels} budget {budget}"); + } + } + } + } + } + /// Hashes recorded from the code before it was made parallel (plain f32 arithmetic only, so /// they hold on every platform). #[test] diff --git a/crates/engine/src/photo_cmds.rs b/crates/engine/src/photo_cmds.rs index 72851b3679..c729afd0d5 100644 --- a/crates/engine/src/photo_cmds.rs +++ b/crates/engine/src/photo_cmds.rs @@ -402,10 +402,18 @@ pub(crate) fn seam_blend(warped: &[Surface], canvas: Rect, order: &[usize], seam .collect() }) .collect(); - // Shift ROIs to canvas-relative coordinates for the blend. - let local: Vec = rois.iter().map(|r| RoiImage { x0: r.x0 - canvas.x0, y0: r.y0 - canvas.y0, ..r.clone() }).collect(); + // Shift ROIs to canvas-relative coordinates for the blend, and back after: a shifted copy + // would hold every source's pixels twice (#7758). + for r in &mut rois { + r.x0 -= canvas.x0; + r.y0 -= canvas.y0; + } let levels = if seamless { panorama::blend_levels(cw, chh).min(7) } else { 1 }; - let (blend, cov) = panorama::multiband(cw, chh, &local, &weights, levels); + let (blend, cov) = panorama::multiband(cw, chh, &rois, &weights, levels); + for r in &mut rois { + r.x0 += canvas.x0; + r.y0 += canvas.y0; + } let cc = fmt.mode.color_channels(); let nch = fmt.channels(); let mut composite = Surface::new(fmt);