mirror of
https://github.com/storytold/photocraft.git
synced 2026-11-16 23:24:22 +00:00
* perf(photomerge): build multi-band bands in bounded batches (#7758) panorama::multiband built every source's Laplacian and weight pyramids at once and kept them all until accumulation, so Photomerge's peak memory grew with the image count (~5 GB for four 24 MP sources). Build the bands in batches under a 2 GiB estimate (at least one image) and add each batch before building the next. Each pixel still adds the bands in image order, so the output is bit-identical. Planes now move into the pyramids instead of being cloned, and the coverage and push-pull masks are freed once used: with one image built at a time those clones were serial passes. seam_blend shifts its ROIs in place instead of cloning every source for the blend. Peak during the blend: 4x24 MP 5.04 -> 2.81 GB, 8x12 MP 5.50 -> 2.61 GB, at +5% time (2x24 MP unchanged). New example bench_multiband measures it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * perf(photomerge): let up to four 24 MP sources build their bands together A 2 GiB band budget built every 24 MP source alone, which made the usual 2-4 image Photomerge 12-36% slower on a 32-thread desktop. The native budget is now 5 GiB, so those still build side by side and only larger stacks are batched. wasm32 has no threads, so it adds each band as soon as it is built. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5b1f937287
commit
81d5c514e2
3 files changed
+164
-48
No files matched your search
@@ -0,0 +1,42 @@
|
||||
//! Time and peak memory of the Photomerge multi-band blend: `n` RGB sources of `mp` megapixels
|
||||
//! (3:2), each overlapping the next by half its width.
|
||||
//! `cargo run --release -p photocraft-algo --example bench_multiband [n] [mp]` (default 4 × 24 MP).
|
||||
//! Peak memory is the growth of the process's high-water mark during the blend (Linux only).
|
||||
use std::time::Instant;
|
||||
|
||||
use photocraft_algo::panorama::{self, RoiImage};
|
||||
|
||||
/// The process's peak resident set in MiB (`VmHWM`), where `/proc` has it.
|
||||
fn peak_mib() -> Option<f64> {
|
||||
let s = std::fs::read_to_string("/proc/self/status").ok()?;
|
||||
let kb: f64 = s.lines().find(|l| l.starts_with("VmHWM:"))?.split_whitespace().nth(1)?.parse().ok()?;
|
||||
Some(kb / 1024.0)
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let mut args = std::env::args().skip(1);
|
||||
let n: usize = args.next().and_then(|a| a.parse().ok()).unwrap_or(4).max(1);
|
||||
let mp: f64 = args.next().and_then(|a| a.parse().ok()).unwrap_or(24.0);
|
||||
let sh = ((mp * 1e6 / 1.5).sqrt().round() as usize).max(2);
|
||||
let sw = sh * 3 / 2;
|
||||
let step = sw / 2;
|
||||
let (cw, ch) = (step * (n - 1) + sw, sh);
|
||||
let rois: Vec<RoiImage> = (0..n)
|
||||
.map(|k| {
|
||||
let px = (0..sw * sh * 3).map(|i| ((i / 3 % sw * 7 + i / 3 / sw * 3 + k * 31 + i % 3 * 50) % 251) as f32 / 251.0).collect();
|
||||
RoiImage { x0: (step * k) as i32, y0: 0, w: sw, h: sh, ch: 3, px, alpha: vec![1.0; sw * sh] }
|
||||
})
|
||||
.collect();
|
||||
// Each canvas column belongs to the image whose centre is nearest, as a seam would split it.
|
||||
let wts: Vec<Vec<f32>> = (0..n)
|
||||
.map(|k| (0..sw * sh).map(|i| if ((step * k + i % sw + step / 2) / step).saturating_sub(1).min(n - 1) == k { 1.0 } else { 0.0 }).collect())
|
||||
.collect();
|
||||
let levels = panorama::blend_levels(cw, ch).min(7);
|
||||
let before = peak_mib();
|
||||
let t0 = Instant::now();
|
||||
let (px, cov) = panorama::multiband(cw, ch, &rois, &wts, levels);
|
||||
let ms = t0.elapsed().as_secs_f64() * 1000.0;
|
||||
let peak = peak_mib().zip(before).map_or("n/a".to_string(), |(a, b)| format!("{:.0} MiB", a - b));
|
||||
println!("{n} × {sw}×{sh} ({mp} MP) RGB, canvas {cw}×{ch}, {levels} levels: {ms:.0} ms, peak +{peak} during the blend");
|
||||
std::hint::black_box((px, cov));
|
||||
}
|
||||
+111
-45
@@ -1170,10 +1170,28 @@ fn band_row(band: &PyramidBand, l: usize, ch: usize, py: usize) -> Option<(usize
|
||||
})
|
||||
}
|
||||
|
||||
/// Estimated bytes of band construction [`multiband`] keeps in flight at once. Images are built
|
||||
/// and accumulated in batches under it, so its peak memory doesn't grow with the image count
|
||||
/// (#7758). Four 24 MP RGB sources (~1.2 GB each) fit in one batch: building bands side by side
|
||||
/// uses a many-core machine better than one band's row-parallel work, and one at a time made the
|
||||
/// usual 2–4 image Photomerge 12–36% slower on a 32-thread desktop.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
const BAND_BUDGET: usize = 5 << 30;
|
||||
/// Without threads the bands of a batch are built one after another anyway, so each is added as
|
||||
/// soon as it is built (the browser's address space is 4 GB).
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
const BAND_BUDGET: usize = 0;
|
||||
|
||||
/// Multi-band blend of `imgs` into a `w × h` panorama (origin 0,0) with one weight map per image
|
||||
/// over the same grid as its ROI (`weights[i]` has `w_i × h_i` entries). Returns the colour
|
||||
/// channels (`ch` per pixel) and the total coverage.
|
||||
pub fn multiband(w: usize, h: usize, imgs: &[RoiImage], weights: &[Vec<f32>], levels: usize) -> (Vec<f32>, Vec<f32>) {
|
||||
multiband_batched(w, h, imgs, weights, levels, BAND_BUDGET)
|
||||
}
|
||||
|
||||
/// [`multiband`], building at most `budget` estimated bytes of bands at a time (at least one
|
||||
/// image). Each pixel still adds the bands in image order, so the result doesn't depend on it.
|
||||
fn multiband_batched(w: usize, h: usize, imgs: &[RoiImage], weights: &[Vec<f32>], levels: usize, budget: usize) -> (Vec<f32>, Vec<f32>) {
|
||||
let ch = imgs.first().map_or(1, |i| i.ch);
|
||||
let levels = levels.max(1);
|
||||
let align = 1i32 << (levels - 1);
|
||||
@@ -1187,13 +1205,22 @@ pub fn multiband(w: usize, h: usize, imgs: &[RoiImage], weights: &[Vec<f32>], le
|
||||
let mut wsum: Vec<Vec<f32>> = sizes.iter().map(|&(lw, lh)| vec![0.0; lw * lh]).collect();
|
||||
let margin = align * 4;
|
||||
// Each image on its ROI grown by a margin and aligned to the coarsest level's grid.
|
||||
let bands = par_map(imgs.len(), |k| {
|
||||
let im = &imgs[k];
|
||||
let grid = |im: &RoiImage| {
|
||||
let gx0 = ((im.x0 - margin).max(0) / align) * align;
|
||||
let gy0 = ((im.y0 - margin).max(0) / align) * align;
|
||||
let gx1 = (im.x0 + im.w as i32 + margin).min(w as i32);
|
||||
let gy1 = (im.y0 + im.h as i32 + margin).min(h as i32);
|
||||
let (gw, gh) = ((gx1 - gx0).max(1) as usize, (gy1 - gy0).max(1) as usize);
|
||||
(gx0, gy0, (gx1 - gx0).max(1) as usize, (gy1 - gy0).max(1) as usize)
|
||||
};
|
||||
// Building one band holds its grid's colour and weight planes about three times over: the
|
||||
// pyramids are 4/3 of a plane, and the grid copy, coverage and `expand` scratch come on top.
|
||||
let cost = |im: &RoiImage| {
|
||||
let (_, _, gw, gh) = grid(im);
|
||||
gw.saturating_mul(gh).saturating_mul(ch + 1).saturating_mul(4 * 3)
|
||||
};
|
||||
let band = |k: usize| -> PyramidBand {
|
||||
let im = &imgs[k];
|
||||
let (gx0, gy0, gw, gh) = grid(im);
|
||||
// Each grid pixel takes the image pixel that lands on it (every image pixel lands on at
|
||||
// most one), so rows of the grid are independent and fill in parallel.
|
||||
let src_of = |gx: usize, gy: usize| -> Option<usize> {
|
||||
@@ -1225,68 +1252,85 @@ pub fn multiband(w: usize, h: usize, imgs: &[RoiImage], weights: &[Vec<f32>], le
|
||||
}
|
||||
});
|
||||
let bin: Vec<f32> = par_map1(&cov, |a| if a > 0.5 { 1.0 } else { 0.0 });
|
||||
drop(cov);
|
||||
push_pull(gw, gh, ch, &mut px, &bin);
|
||||
// Laplacian of the colours, Gaussian of the weights.
|
||||
drop(bin);
|
||||
// Laplacian of the colours, Gaussian of the weights. Planes move into the pyramids rather
|
||||
// than being copied: with one image built at a time, a copy is a serial pass over it.
|
||||
let mut lap: Vec<(usize, usize, Vec<f32>)> = Vec::new();
|
||||
let mut gau: Vec<Vec<f32>> = Vec::new();
|
||||
let (mut cw, mut chh, mut cur, mut cwt) = (gw, gh, px, wt);
|
||||
for l in 0..levels {
|
||||
if l + 1 == levels {
|
||||
lap.push((cw, chh, cur.clone()));
|
||||
gau.push(cwt.clone());
|
||||
lap.push((cw, chh, cur));
|
||||
gau.push(cwt);
|
||||
break;
|
||||
}
|
||||
let (nw, nh, down) = reduce(cw, chh, ch, &cur);
|
||||
let up = expand(nw, nh, ch, &down, cw, chh);
|
||||
lap.push((cw, chh, par_zip(&cur, &up, |a, b| a - b)));
|
||||
gau.push(cwt.clone());
|
||||
let (_, _, wdown) = reduce(cw, chh, 1, &cwt);
|
||||
gau.push(cwt);
|
||||
cw = nw;
|
||||
chh = nh;
|
||||
cur = down;
|
||||
cwt = wdown;
|
||||
}
|
||||
(gx0, gy0, lap, gau)
|
||||
});
|
||||
// Accumulate the bands. Each pixel adds the bands in the same order as the serial loop (per
|
||||
// level, bands one after another), but rows of a level are independent, so they run in
|
||||
// parallel: every row walks the bands itself.
|
||||
for l in 0..levels {
|
||||
let pw = sizes[l].0;
|
||||
par_rows(&mut wsum[l], pw, 1, |py, row| {
|
||||
for b in &bands {
|
||||
let Some((ox, lw, g, _)) = band_row(b, l, ch, py) else { continue };
|
||||
for x in 0..lw {
|
||||
let px = ox + x;
|
||||
if px >= pw {
|
||||
break;
|
||||
}
|
||||
let k = g[x];
|
||||
if k <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
row[px] += k;
|
||||
}
|
||||
}
|
||||
});
|
||||
par_rows(&mut acc[l], pw, ch, |py, row| {
|
||||
for b in &bands {
|
||||
let Some((ox, lw, g, band)) = band_row(b, l, ch, py) else { continue };
|
||||
for x in 0..lw {
|
||||
let px = ox + x;
|
||||
if px >= pw {
|
||||
break;
|
||||
}
|
||||
let k = g[x];
|
||||
if k <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
for c in 0..ch {
|
||||
row[px * ch + c] += k * band[x * ch + c];
|
||||
};
|
||||
// Build the bands a batch at a time and add each batch before building the next. Each pixel
|
||||
// adds the bands in the same order as the serial loop (per level, bands one after another;
|
||||
// batches follow each other in image order), but rows of a level are independent, so they
|
||||
// run in parallel: every row walks the batch's bands itself.
|
||||
let mut start = 0;
|
||||
while start < imgs.len() {
|
||||
let mut end = start + 1;
|
||||
let mut bytes = cost(&imgs[start]);
|
||||
while let Some(im) = imgs.get(end)
|
||||
&& bytes.saturating_add(cost(im)) <= budget
|
||||
{
|
||||
bytes = bytes.saturating_add(cost(im));
|
||||
end += 1;
|
||||
}
|
||||
let bands = par_map(end - start, |j| band(start + j));
|
||||
for l in 0..levels {
|
||||
let pw = sizes[l].0;
|
||||
par_rows(&mut wsum[l], pw, 1, |py, row| {
|
||||
for b in &bands {
|
||||
let Some((ox, lw, g, _)) = band_row(b, l, ch, py) else { continue };
|
||||
for x in 0..lw {
|
||||
let px = ox + x;
|
||||
if px >= pw {
|
||||
break;
|
||||
}
|
||||
let k = g[x];
|
||||
if k <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
row[px] += k;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
par_rows(&mut acc[l], pw, ch, |py, row| {
|
||||
for b in &bands {
|
||||
let Some((ox, lw, g, band)) = band_row(b, l, ch, py) else { continue };
|
||||
for x in 0..lw {
|
||||
let px = ox + x;
|
||||
if px >= pw {
|
||||
break;
|
||||
}
|
||||
let k = g[x];
|
||||
if k <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
for c in 0..ch {
|
||||
row[px * ch + c] += k * band[x * ch + c];
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
for l in 0..levels {
|
||||
let pw = sizes[l].0;
|
||||
@@ -1910,6 +1954,28 @@ mod parallel_tests {
|
||||
assert_eq!(multiband(9, 7, &[], &[], 3).0.len(), ref_multiband(9, 7, &[], &[], 3).0.len());
|
||||
}
|
||||
|
||||
/// Batching the bands (#7758) changes when they are built, not the result. Budgets from none
|
||||
/// (one image per batch) up to everything in steps of a fifth pass through every split of the
|
||||
/// three images.
|
||||
#[test]
|
||||
fn multiband_batches_match_the_serial_code() {
|
||||
let budgets: Vec<usize> =
|
||||
std::iter::successors(Some(1024.0f64), |b| Some(b * 1.2)).take_while(|b| *b < 4e6).map(|b| b as usize).chain([0, usize::MAX]).collect();
|
||||
for (n, &(w, h)) in [(40usize, 30usize), (97, 61), (201, 129)].iter().enumerate() {
|
||||
for ch in [1usize, 3] {
|
||||
let (imgs, wts) = rois(w, h, ch, n as u64 * 17 + ch as u64);
|
||||
for levels in [1usize, 3, 5] {
|
||||
let want = ref_multiband(w, h, &imgs, &wts, levels);
|
||||
for &budget in &budgets {
|
||||
let got = multiband_batched(w, h, &imgs, &wts, levels, budget);
|
||||
assert_eq!(bits(&got.0), bits(&want.0), "colours {w}x{h}x{ch} levels {levels} budget {budget}");
|
||||
assert_eq!(bits(&got.1), bits(&want.1), "coverage {w}x{h}x{ch} levels {levels} budget {budget}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hashes recorded from the code before it was made parallel (plain f32 arithmetic only, so
|
||||
/// they hold on every platform).
|
||||
#[test]
|
||||
|
||||
@@ -402,10 +402,18 @@ pub(crate) fn seam_blend(warped: &[Surface], canvas: Rect, order: &[usize], seam
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
// Shift ROIs to canvas-relative coordinates for the blend.
|
||||
let local: Vec<RoiImage> = rois.iter().map(|r| RoiImage { x0: r.x0 - canvas.x0, y0: r.y0 - canvas.y0, ..r.clone() }).collect();
|
||||
// Shift ROIs to canvas-relative coordinates for the blend, and back after: a shifted copy
|
||||
// would hold every source's pixels twice (#7758).
|
||||
for r in &mut rois {
|
||||
r.x0 -= canvas.x0;
|
||||
r.y0 -= canvas.y0;
|
||||
}
|
||||
let levels = if seamless { panorama::blend_levels(cw, chh).min(7) } else { 1 };
|
||||
let (blend, cov) = panorama::multiband(cw, chh, &local, &weights, levels);
|
||||
let (blend, cov) = panorama::multiband(cw, chh, &rois, &weights, levels);
|
||||
for r in &mut rois {
|
||||
r.x0 += canvas.x0;
|
||||
r.y0 += canvas.y0;
|
||||
}
|
||||
let cc = fmt.mode.color_channels();
|
||||
let nch = fmt.channels();
|
||||
let mut composite = Surface::new(fmt);
|
||||
|
||||
Reference in new issue
Block a user