mirror of
https://github.com/storytold/photocraft.git
synced 2026-11-16 07:02:42 +00:00
Integration: per-band AHD balancing, memory-capped fused bands (#9455)
`for_bands_grouped` always split the bands into `min(workers, jobs)` static chunks, so on the default develop (jobs == workers) a worker held up on the shared pool kept a fixed share of the bands waiting, where main's `chunks_mut_limited` runs one rayon task per band. Now, when every worker may hold a scratch, each band is its own task and borrows a scratch from a pool (`par::chunks_mut_pooled`, at most one per worker, dropped as soon as no waiting band can use it); the grouping stays for memory-limited jobs. The fused bilinear/MHC, X-Trans and LinearRaw paths ran one band per worker with no memory cap: a 262144 x 455 Bayer frame with bilinear needed 2.92 GB at 15+ workers and failed with LimitExceeded under the default 2 GiB. `working_memory` now caps those bands like AHD's and the fused paths use the returned jobs. The fused LinearRaw and X-Trans band buffers are `band.min(h)` rows, as charged (CodeRabbit, develop.rs:414). Tests: a_slow_band_does_not_hold_up_the_bands_after_it and fused_bands_at_once_are_capped_by_the_memory_limit fail without the fix; for_bands_keeps_at_most_jobs_scratches_alive, chunks_mut_pooled_makes_one_state_per_worker_at_most, multi-band and short X-Trans/LinearRaw fused-vs-full cases. Co-authored-by: Valentin Klein <271991477+vangel76@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3e144a381a
commit
0b09eb43a5
3 files changed
+244
-48
No files matched your search
+126
-30
@@ -232,7 +232,7 @@ pub(crate) fn demosaic_fused(
|
||||
phase: [u8; 4],
|
||||
method: Demosaic,
|
||||
to_xyz: &Mat3,
|
||||
ahd_jobs: usize,
|
||||
jobs: usize,
|
||||
fill: &(impl Fn(usize, &mut [f32]) + Sync),
|
||||
out: &mut [u16],
|
||||
emit: &(impl Fn(&[f32], &mut [u16]) + Sync),
|
||||
@@ -257,7 +257,7 @@ pub(crate) fn demosaic_fused(
|
||||
// The rows of the tallest band, as `develop`'s memory check counts them.
|
||||
let rows = h.min(AHD_BAND);
|
||||
let init = || (AhdScratch::for_image(h, s), vec![0.0f32; rows * w * 3], vec![0.0f32; (rows + 2 * PAD) * s]);
|
||||
for_bands_grouped(out, AHD_BAND * w * 3, ahd_bands(h), ahd_jobs, 1, init, |band, chunk, (sc, buf, win)| {
|
||||
for_bands_grouped(out, AHD_BAND * w * 3, ahd_bands(h), jobs, 1, init, |band, chunk, (sc, buf, win)| {
|
||||
load(win, band * AHD_BAND, chunk.len() / (w * 3));
|
||||
let buf = &mut buf[..chunk.len()];
|
||||
// The window starts at a multiple of AHD_BAND (even), so it is band 0 of its own rows.
|
||||
@@ -269,7 +269,7 @@ pub(crate) fn demosaic_fused(
|
||||
let ks = kernels(phase, method, s);
|
||||
let rows = h.min(FUSED_ROWS);
|
||||
let init = || (vec![0.0f32; rows * w * 3], vec![0.0f32; (rows + 2 * PAD) * s]);
|
||||
for_bands(out, FUSED_ROWS * w * 3, h.div_ceil(FUSED_ROWS), usize::MAX, init, |band, chunk, (buf, win)| {
|
||||
for_bands(out, FUSED_ROWS * w * 3, h.div_ceil(FUSED_ROWS), jobs, init, |band, chunk, (buf, win)| {
|
||||
load(win, band * FUSED_ROWS, chunk.len() / (w * 3));
|
||||
let buf = &mut buf[..chunk.len()];
|
||||
linear_band(win, w, s, phase, method, &ks, band * FUSED_ROWS, band * FUSED_ROWS, buf);
|
||||
@@ -318,15 +318,16 @@ fn linear_band(d: &[f32], w: usize, s: usize, phase: [u8; 4], method: Demosaic,
|
||||
}
|
||||
|
||||
/// Runs `f(band, chunk, scratch)` over the `unit`-sized chunks of `out` (`bands` of them), with
|
||||
/// at most `jobs` scratch values alive: a few groups of consecutive bands per worker (for
|
||||
/// balance), each group building its scratch once. For a scratch that is cheap to build.
|
||||
pub(crate) fn for_bands<T: Send, S>(
|
||||
/// at most `jobs` scratch values alive. When every worker may hold one, each band is its own
|
||||
/// rayon task and borrows a scratch from a pool (one per worker at most); otherwise a few groups
|
||||
/// of consecutive bands per job run, `jobs` at a time, each building its scratch once.
|
||||
pub(crate) fn for_bands<T: Send, S: Send>(
|
||||
out: &mut [T],
|
||||
unit: usize,
|
||||
bands: usize,
|
||||
jobs: usize,
|
||||
init: impl Fn() -> S + Sync,
|
||||
f: impl Fn(usize, &mut [T], &mut S) + Sync,
|
||||
init: impl Fn() -> S + Sync + Send,
|
||||
f: impl Fn(usize, &mut [T], &mut S) + Sync + Send,
|
||||
) {
|
||||
for_bands_grouped(out, unit, bands, jobs, 4, init, f);
|
||||
}
|
||||
@@ -337,18 +338,24 @@ fn bands_per_group(bands: usize, jobs: usize, per_worker: usize) -> usize {
|
||||
bands.div_ceil(groups).max(1)
|
||||
}
|
||||
|
||||
/// [`for_bands`] with `per_worker` groups of consecutive bands for each worker. A large scratch
|
||||
/// (AHD: 16 planes) wants one group per worker: with more groups than workers on a many-core
|
||||
/// machine every group is a single band and each band would build and zero its own scratch.
|
||||
fn for_bands_grouped<T: Send, S>(
|
||||
/// [`for_bands`] with `per_worker` groups of consecutive bands for each of the `jobs` workers when
|
||||
/// fewer than all may hold a scratch. A large scratch (AHD: 16 planes) wants one group per job:
|
||||
/// with more groups each would build and zero its own scratch for a band or two.
|
||||
fn for_bands_grouped<T: Send, S: Send>(
|
||||
out: &mut [T],
|
||||
unit: usize,
|
||||
bands: usize,
|
||||
jobs: usize,
|
||||
per_worker: usize,
|
||||
init: impl Fn() -> S + Sync,
|
||||
f: impl Fn(usize, &mut [T], &mut S) + Sync,
|
||||
init: impl Fn() -> S + Sync + Send,
|
||||
f: impl Fn(usize, &mut [T], &mut S) + Sync + Send,
|
||||
) {
|
||||
// One task per band keeps rayon's stealing, so a busy or slow worker doesn't hold up a
|
||||
// static share of the bands (as `par::chunks_mut_limited` does for the same reason).
|
||||
if jobs >= par::workers() {
|
||||
par::chunks_mut_pooled(out, unit, init, |sc, band, chunk| f(band, chunk, sc));
|
||||
return;
|
||||
}
|
||||
let per_group = bands_per_group(bands, jobs, per_worker);
|
||||
par::chunks_mut_limited(out, per_group * unit, jobs, |group, gchunk| {
|
||||
let mut sc = init();
|
||||
@@ -447,15 +454,15 @@ pub(crate) fn demosaic_cfa(p: &Padded, pat: &Pattern) -> Vec<f32> {
|
||||
}
|
||||
|
||||
/// [`demosaic_cfa`] without the full RGB frame (the green plane stays): see [`demosaic_fused`].
|
||||
pub(crate) fn demosaic_cfa_fused(p: &Padded, pat: &Pattern, out: &mut [u16], emit: &(impl Fn(&[f32], &mut [u16]) + Sync)) {
|
||||
pub(crate) fn demosaic_cfa_fused(p: &Padded, pat: &Pattern, jobs: usize, out: &mut [u16], emit: &(impl Fn(&[f32], &mut [u16]) + Sync)) {
|
||||
let (w, h) = (p.w, p.h);
|
||||
if out.len() < w * h * 3 {
|
||||
return;
|
||||
}
|
||||
let band = par::band_rows(w);
|
||||
let init = || vec![0.0f32; band * w * 3];
|
||||
let init = || vec![0.0f32; band.min(h) * w * 3];
|
||||
cfa_with(p, pat, |rows| {
|
||||
for_bands(&mut out[..w * h * 3], band * w * 3, h.div_ceil(band), usize::MAX, init, |b, chunk, buf| {
|
||||
for_bands(&mut out[..w * h * 3], band * w * 3, h.div_ceil(band), jobs, init, |b, chunk, buf| {
|
||||
let buf = &mut buf[..chunk.len()];
|
||||
rows(b * band, buf);
|
||||
emit(buf, chunk);
|
||||
@@ -944,6 +951,8 @@ fn ahd_ref(p: &Padded, phase: [u8; 4], to_xyz: &Mat3, out: &mut [f32], jobs: usi
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::color::IDENTITY;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[test]
|
||||
fn ahd_band_concurrency_does_not_change_pixels() {
|
||||
@@ -1037,8 +1046,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// One scratch serves many bands (1100 rows are 35 bands, the last one short, over 4 scratches
|
||||
/// with jobs 1 and at most 8 with jobs 2), so a stage reading a cell left over from an earlier band would show.
|
||||
/// One scratch serves many bands (1100 rows are 35 bands, the last one short, over one scratch
|
||||
/// with jobs 1 and two with jobs 2 on a bigger pool), so a stage reading a cell left over from
|
||||
/// an earlier band would show.
|
||||
#[test]
|
||||
fn ahd_matches_reference_with_reused_scratch() {
|
||||
let m = [[0.41, 0.36, 0.18], [0.21, 0.72, 0.07], [0.02, 0.12, 0.95]];
|
||||
@@ -1057,9 +1067,10 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// A 4-worker pool splits 35 bands into groups of 9 (the last of 8), so groups after the first
|
||||
/// start at band `group * 9` and every scratch is reused for several bands: a wrong band index
|
||||
/// or a stale scratch cell would change pixels. The group size does not depend on the machine.
|
||||
/// On a 4-worker pool with room for 3 scratches, 35 bands go in groups of 12 (the last of 11),
|
||||
/// so groups after the first start at band `group * 12` and every scratch is reused for several
|
||||
/// bands; with room for 4, every band is its own task and most borrow a used scratch. A wrong band
|
||||
/// index or a stale scratch cell would change pixels. The split does not depend on the machine.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[test]
|
||||
fn ahd_groups_of_several_bands_match_reference() {
|
||||
@@ -1070,18 +1081,103 @@ mod tests {
|
||||
let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap();
|
||||
pool.install(|| {
|
||||
assert_eq!(par::workers(), 4);
|
||||
assert_eq!(bands_per_group(ahd_bands(h), 4, 1), 9);
|
||||
// One worker per group, never more groups than workers, however many bands there are.
|
||||
assert_eq!(bands_per_group(126, 1000, 1), 32);
|
||||
for phase in [[0, 1, 1, 2], [1, 2, 0, 1]] {
|
||||
let mut want = vec![0.0f32; w * h * 3];
|
||||
ahd_ref(&p, phase, &m, &mut want, 4);
|
||||
let got = demosaic(&p, phase, Demosaic::Ahd, &m, 4);
|
||||
assert!(want.iter().zip(&got).all(|(a, b)| a.to_bits() == b.to_bits()), "{phase:?}");
|
||||
assert_eq!(bands_per_group(ahd_bands(h), 3, 1), 12);
|
||||
// One group per job, never more groups than jobs, however many bands there are.
|
||||
assert_eq!(bands_per_group(126, 3, 1), 42);
|
||||
for jobs in [3, 4] {
|
||||
for phase in [[0, 1, 1, 2], [1, 2, 0, 1]] {
|
||||
let mut want = vec![0.0f32; w * h * 3];
|
||||
ahd_ref(&p, phase, &m, &mut want, jobs);
|
||||
let got = demosaic(&p, phase, Demosaic::Ahd, &m, jobs);
|
||||
assert!(want.iter().zip(&got).all(|(a, b)| a.to_bits() == b.to_bits()), "{phase:?} jobs {jobs}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// A scratch that counts how many are alive at once.
|
||||
struct Live<'a>(&'a AtomicUsize);
|
||||
|
||||
impl<'a> Live<'a> {
|
||||
fn new(now: &'a AtomicUsize, peak: &AtomicUsize) -> Self {
|
||||
peak.fetch_max(now.fetch_add(1, SeqCst) + 1, SeqCst);
|
||||
Live(now)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Live<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
/// Every band runs once with its own index, and no more than `jobs` scratches (what `develop`'s
|
||||
/// memory check admits) or one per worker are alive at once, on either side of `jobs == workers`.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[test]
|
||||
fn for_bands_keeps_at_most_jobs_scratches_alive() {
|
||||
let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap();
|
||||
pool.install(|| {
|
||||
for jobs in [1, 2, 3, 7, 8, 9, usize::MAX] {
|
||||
for per_worker in [1, 4] {
|
||||
let (now, peak) = (AtomicUsize::new(0), AtomicUsize::new(0));
|
||||
let mut out = vec![0usize; 37 * 10 - 4];
|
||||
for_bands_grouped(
|
||||
&mut out,
|
||||
10,
|
||||
37,
|
||||
jobs,
|
||||
per_worker,
|
||||
|| Live::new(&now, &peak),
|
||||
|band, chunk, _| {
|
||||
std::thread::sleep(Duration::from_millis(1));
|
||||
chunk.iter_mut().for_each(|v| *v += band + 1);
|
||||
},
|
||||
);
|
||||
assert!(out.iter().enumerate().all(|(i, &v)| v == i / 10 + 1), "jobs {jobs}");
|
||||
let peak = peak.load(SeqCst);
|
||||
assert!((1..=jobs.min(8)).contains(&peak), "jobs {jobs} per_worker {per_worker}: {peak} scratches alive");
|
||||
assert_eq!(now.load(SeqCst), 0);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// With room for a scratch on every worker, the bands are separate rayon tasks: a worker held up
|
||||
/// on band 0 (by other work on the shared pool, or a slow core) leaves the bands after it to
|
||||
/// the others. With a fixed share of 9 bands per worker, 8 of them would wait for band 0, so at
|
||||
/// most 26 of the other 34 could finish meanwhile.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[test]
|
||||
fn a_slow_band_does_not_hold_up_the_bands_after_it() {
|
||||
let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap();
|
||||
pool.install(|| {
|
||||
let (done, seen) = (AtomicUsize::new(0), AtomicUsize::new(0));
|
||||
let mut out = vec![0u8; 35];
|
||||
for_bands_grouped(
|
||||
&mut out,
|
||||
1,
|
||||
35,
|
||||
4,
|
||||
1,
|
||||
|| (),
|
||||
|band, _, _| {
|
||||
if band > 0 {
|
||||
done.fetch_add(1, SeqCst);
|
||||
return;
|
||||
}
|
||||
let start = Instant::now();
|
||||
while done.load(SeqCst) <= 26 && start.elapsed() < Duration::from_secs(10) {
|
||||
std::thread::sleep(Duration::from_millis(1));
|
||||
}
|
||||
seen.store(done.load(SeqCst), SeqCst);
|
||||
},
|
||||
);
|
||||
assert_eq!(done.load(SeqCst), 34);
|
||||
assert!(seen.load(SeqCst) > 26, "only {} bands ran while band 0 was held up", seen.load(SeqCst));
|
||||
});
|
||||
}
|
||||
|
||||
/// Mosaics an RGB image with the given phase.
|
||||
fn mosaic(rgb: &[[f32; 3]], w: usize, h: usize, phase: [u8; 4]) -> Vec<f32> {
|
||||
(0..w * h).map(|i| rgb[i][phase[((i / w) & 1) * 2 + ((i % w) & 1)] as usize]).collect()
|
||||
|
||||
+52
-18
@@ -230,7 +230,7 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result<Devel
|
||||
// condition under which `Haze::new` and `ToneMap::new` return `None`).
|
||||
let fused = allow_fused && opts.tone.is_neutral() && s.black_point <= 0.0;
|
||||
let orient_copy = opts.orient && (2..=8).contains(&s.orientation);
|
||||
let ahd_jobs = working_memory(c.width, c.height, s.cfa.is_some(), bayer, Shape { fused, orient_copy }, opts, par::workers())?;
|
||||
let jobs = working_memory(c.width, c.height, s.cfa.is_some(), bayer, Shape { fused, orient_copy }, opts, par::workers())?;
|
||||
let mut warnings = s.warnings.clone();
|
||||
let sc = Scale::new(s);
|
||||
|
||||
@@ -374,7 +374,7 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result<Devel
|
||||
let p = cfa.phase(c.x, c.y);
|
||||
pattern = Some(names(&p));
|
||||
let mut o = vec![0u16; w * h * 3];
|
||||
demosaic_fused(w, h, p, opts.demosaic, &to_xyz, ahd_jobs, &fill_row, &mut o, &encode_plain);
|
||||
demosaic_fused(w, h, p, opts.demosaic, &to_xyz, jobs, &fill_row, &mut o, &encode_plain);
|
||||
fused_out = Some(o);
|
||||
} else {
|
||||
let mut plane = Padded::new(w, h);
|
||||
@@ -388,13 +388,13 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result<Devel
|
||||
if bayer {
|
||||
let p = cfa.phase(c.x, c.y);
|
||||
pattern = Some(names(&p));
|
||||
rgb = demosaic(&plane, p, opts.demosaic, &to_xyz, ahd_jobs);
|
||||
rgb = demosaic(&plane, p, opts.demosaic, &to_xyz, jobs);
|
||||
} else {
|
||||
let pat = Pattern::of(cfa, c.x, c.y);
|
||||
pattern = Some(pat.colors.chunks(pat.w.max(1)).map(names).collect::<Vec<_>>().join("/"));
|
||||
if fused {
|
||||
let mut o = vec![0u16; w * h * 3];
|
||||
demosaic_cfa_fused(&plane, &pat, &mut o, &encode_plain);
|
||||
demosaic_cfa_fused(&plane, &pat, jobs, &mut o, &encode_plain);
|
||||
fused_out = Some(o);
|
||||
} else {
|
||||
rgb = demosaic_cfa(&plane, &pat);
|
||||
@@ -410,8 +410,8 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result<Devel
|
||||
&mut o,
|
||||
band * w * 3,
|
||||
h.div_ceil(band),
|
||||
usize::MAX,
|
||||
|| vec![0.0f32; band * w * 3],
|
||||
jobs,
|
||||
|| vec![0.0f32; band.min(h) * w * 3],
|
||||
|b, chunk, buf| {
|
||||
let buf = &mut buf[..chunk.len()];
|
||||
linear_rows(b * band, buf);
|
||||
@@ -509,10 +509,10 @@ fn fused_band_bytes(w: usize, h: usize, band_rows: usize, window: bool) -> u64 {
|
||||
}
|
||||
|
||||
/// Checks the peak of the two development phases. The CFA and non-Bayer green
|
||||
/// planes are freed before the u16 output is allocated; AHD scratch is bounded
|
||||
/// by reducing simultaneous bands when needed. The fused path never holds the f32 RGB frame:
|
||||
/// it has the u16 output (6 bytes a pixel) next to the CFA plane, plus a band buffer and the
|
||||
/// scratch per worker, and an orientation copy of the output afterwards. Sensor storage belongs
|
||||
/// planes are freed before the u16 output is allocated; the AHD scratch and the fused band
|
||||
/// buffers are bounded by reducing simultaneous bands when needed. The fused path never holds
|
||||
/// the f32 RGB frame: it has the u16 output (6 bytes a pixel) next to the CFA plane, plus a band
|
||||
/// buffer and the scratch per simultaneous band, and an orientation copy of the output afterwards. Sensor storage belongs
|
||||
/// to the caller and is checked separately by the decoder.
|
||||
fn working_memory(w: usize, h: usize, cfa: bool, bayer: bool, shape: Shape, opts: &DevelopOptions, workers: usize) -> Result<usize> {
|
||||
let limits = &opts.limits;
|
||||
@@ -530,7 +530,12 @@ fn working_memory(w: usize, h: usize, cfa: bool, bayer: bool, shape: Shape, opts
|
||||
let mut jobs = workers.max(1);
|
||||
// Fused: the u16 output and, per worker running a band, one band of f32 RGB.
|
||||
let band_of = |rows: usize, window: bool| if fused { fused_band_bytes(w, h, rows, window) } else { 0 };
|
||||
let band_jobs = |rows: usize| workers.max(1).min(h.div_ceil(rows.max(1))) as u64;
|
||||
// Fewer bands at once when the bands of every worker don't fit next to `base` (at least one).
|
||||
let with_bands = |jobs: &mut usize, base: u64, rows: usize, band: u64| {
|
||||
let available = limits.max_alloc.saturating_sub(base) / band.max(1);
|
||||
*jobs = (*jobs).min(h.div_ceil(rows.max(1))).min(usize::try_from(available).unwrap_or(usize::MAX)).max(1);
|
||||
base.saturating_add(band.saturating_mul(*jobs as u64))
|
||||
};
|
||||
if cfa {
|
||||
let plane = (w as u64).saturating_add(2 * PAD as u64).saturating_mul((h as u64).saturating_add(2 * PAD as u64)).saturating_mul(4);
|
||||
// Bytes a pixel next to the plane: the f32 RGB frame (and the non-Bayer green plane), or
|
||||
@@ -547,19 +552,17 @@ fn working_memory(w: usize, h: usize, cfa: bool, bayer: bool, shape: Shape, opts
|
||||
if bayer && opts.demosaic == Demosaic::Ahd {
|
||||
mosaic = mosaic.saturating_add(crate::demosaic::AHD_TABLE_BYTES);
|
||||
let scratch = crate::demosaic::ahd_scratch(w as u64, h as u64).saturating_add(band_of(crate::demosaic::AHD_BAND, true));
|
||||
let available = limits.max_alloc.saturating_sub(mosaic) / scratch.max(1);
|
||||
jobs = jobs.min(crate::demosaic::ahd_bands(h)).min(usize::try_from(available).unwrap_or(usize::MAX));
|
||||
// At least one band must fit, even when no parallelism is available.
|
||||
mosaic = mosaic.saturating_add(scratch.saturating_mul(jobs.max(1) as u64));
|
||||
mosaic = with_bands(&mut jobs, mosaic, crate::demosaic::AHD_BAND, scratch);
|
||||
} else {
|
||||
let rows = if bayer { crate::demosaic::FUSED_ROWS } else { par::band_rows(w) };
|
||||
mosaic = mosaic.saturating_add(band_of(rows, bayer).saturating_mul(band_jobs(rows)));
|
||||
mosaic = with_bands(&mut jobs, mosaic, rows, band_of(rows, bayer));
|
||||
}
|
||||
peak = peak.max(mosaic);
|
||||
} else if fused {
|
||||
// LinearRaw: the output and the bands.
|
||||
let rows = par::band_rows(w);
|
||||
peak = peak.max(pixels.saturating_mul(6).saturating_add(band_of(rows, false).saturating_mul(band_jobs(rows))));
|
||||
peak = peak.max(with_bands(&mut jobs, pixels.saturating_mul(6), rows, band_of(rows, false)));
|
||||
}
|
||||
if peak > limits.max_alloc || usize::try_from(peak).is_err() {
|
||||
return Err(RawError::LimitExceeded(format!("{peak} bytes of RAW working memory exceed {}", limits.max_alloc)));
|
||||
@@ -836,13 +839,15 @@ mod tests {
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
for (w, h) in [(6, 6), (37, 41), (80, 150)] {
|
||||
// 600 samples wide: bands of 109 rows, so 350 rows are four (the last short) and 50 rows
|
||||
// are less than one.
|
||||
for (w, h) in [(6, 6), (37, 41), (80, 150), (600, 350), (600, 50)] {
|
||||
let mut s = test_sensor(w, h, Some(Cfa { width: 6, height: 6, colors: colors.clone(), origin_x: 0, origin_y: 0 }), 1);
|
||||
s.orientation = 8;
|
||||
assert_fused_matches_full(&s, &DevelopOptions::default(), &format!("xtrans {w}x{h}"));
|
||||
}
|
||||
for samples in [1, 3] {
|
||||
for (w, h) in [(1, 1), (7, 5), (90, 111)] {
|
||||
for (w, h) in [(1, 1), (7, 5), (90, 111), (600, 350), (600, 50)] {
|
||||
let mut s = test_sensor(w, h, None, samples);
|
||||
s.orientation = 3;
|
||||
assert_fused_matches_full(&s, &DevelopOptions::default(), &format!("linear {samples} {w}x{h}"));
|
||||
@@ -902,6 +907,35 @@ mod tests {
|
||||
assert!(working_memory(11648, 8736, true, true, FUSED, &opts, 1).is_ok());
|
||||
}
|
||||
|
||||
/// The fused bilinear / MHC, X-Trans and LinearRaw paths run fewer bands at once when the bands
|
||||
/// of every worker don't fit next to the output, as AHD does. A very wide, short Bayer frame
|
||||
/// (262144 x 455: 15 bands of 32 rows, about 147 MB a band with its window) needs 2.92 GB
|
||||
/// with all 15 at once against the default 2 GiB, and develops with 9.
|
||||
#[test]
|
||||
fn fused_bands_at_once_are_capped_by_the_memory_limit() {
|
||||
let opts = DevelopOptions { demosaic: Demosaic::Bilinear, ..Default::default() };
|
||||
let table = (GAMMA_STEPS as u64 + 2) * 4;
|
||||
let (w, h) = (262144usize, 455usize);
|
||||
let band = 32 * 262144 * 12 + (32 + 2 * PAD as u64) * (262144 + 2 * PAD as u64) * 4;
|
||||
for (workers, want) in [(1, 1), (8, 8), (9, 9), (10, 9), (16, 9), (32, 9)] {
|
||||
assert_eq!(working_memory(w, h, true, true, FUSED, &opts, workers).unwrap(), want, "{workers} workers");
|
||||
}
|
||||
// Never more jobs than bands, and one when only one band fits.
|
||||
assert_eq!(working_memory(1000, 40, true, true, FUSED, &opts, 32).unwrap(), 2);
|
||||
let mut tight = opts.clone();
|
||||
tight.limits.max_alloc = 262144 * 455 * 6 + table + band;
|
||||
assert_eq!(working_memory(w, h, true, true, FUSED, &tight, 32).unwrap(), 1);
|
||||
tight.limits.max_alloc -= 1;
|
||||
assert!(working_memory(w, h, true, true, FUSED, &tight, 32).is_err());
|
||||
// X-Trans (bands of `band_rows(1000)` = 65 rows next to its planes) and LinearRaw likewise.
|
||||
let plane = (1000 + 2 * PAD as u64) * (2000 + 2 * PAD as u64) * 4;
|
||||
let mut x = opts.clone();
|
||||
x.limits.max_alloc = 1000 * 2000 * 10 + plane + table + 3 * 65 * 1000 * 12;
|
||||
assert_eq!(working_memory(1000, 2000, true, false, FUSED, &x, 32).unwrap(), 3);
|
||||
x.limits.max_alloc = 1000 * 2000 * 6 + 5 * 65 * 1000 * 12;
|
||||
assert_eq!(working_memory(1000, 2000, false, true, FUSED, &x, 32).unwrap(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn working_memory_accounts_for_padding_and_at_least_one_ahd_band() {
|
||||
let mut opts = DevelopOptions { demosaic: Demosaic::Ahd, ..Default::default() };
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! Data-parallel helpers: rayon on native targets, sequential on wasm.
|
||||
|
||||
use std::sync::{Mutex, PoisonError};
|
||||
|
||||
/// Calls `f(index, chunk)` for each `chunk`-sized piece of `buf`.
|
||||
pub(crate) fn chunks_mut<T: Send>(buf: &mut [T], chunk: usize, f: impl Fn(usize, &mut [T]) + Sync + Send) {
|
||||
let chunk = chunk.max(1);
|
||||
@@ -12,6 +14,36 @@ pub(crate) fn chunks_mut<T: Send>(buf: &mut [T], chunk: usize, f: impl Fn(usize,
|
||||
buf.chunks_mut(chunk).enumerate().for_each(|(i, c)| f(i, c));
|
||||
}
|
||||
|
||||
/// Like [`chunks_mut`], with `f(state, index, chunk)` borrowing a state: a chunk takes a free one
|
||||
/// or makes one with `init`, and gives it back when done, so rayon keeps its fine-grained
|
||||
/// splitting and stealing while at most one state per worker is ever made (`f` must not itself
|
||||
/// wait on rayon work). Rayon's `for_each_init` would make one per stolen piece, which on an idle
|
||||
/// pool is often every chunk. A state no chunk still waiting can take is dropped at once, so
|
||||
/// near the end they go as their workers finish rather than all at the very end.
|
||||
pub(crate) fn chunks_mut_pooled<T: Send, S: Send>(
|
||||
buf: &mut [T],
|
||||
chunk: usize,
|
||||
init: impl Fn() -> S + Sync + Send,
|
||||
f: impl Fn(&mut S, usize, &mut [T]) + Sync + Send,
|
||||
) {
|
||||
let chunk = chunk.max(1);
|
||||
// The free states and the number of chunks that have not taken one yet.
|
||||
let pool = Mutex::new((Vec::new(), buf.len().div_ceil(chunk)));
|
||||
chunks_mut(buf, chunk, |i, c| {
|
||||
let taken = {
|
||||
let mut pool = pool.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
pool.1 = pool.1.saturating_sub(1);
|
||||
pool.0.pop()
|
||||
};
|
||||
let mut state = taken.unwrap_or_else(&init);
|
||||
f(&mut state, i, c);
|
||||
let mut pool = pool.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
if pool.0.len() < pool.1 {
|
||||
pool.0.push(state);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Native worker count (the browser builds run sequentially).
|
||||
pub(crate) fn workers() -> usize {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
@@ -56,3 +88,37 @@ pub(crate) fn map<T: Send>(n: usize, f: impl Fn(usize) -> T + Sync + Send) -> Ve
|
||||
pub(crate) fn band_rows(width: usize) -> usize {
|
||||
(65536 / width.max(1)).clamp(1, 256)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
/// Every chunk is visited once with its own index (the last one short), and the chunks share
|
||||
/// the states: a 4-worker pool makes at most 4 for 101 chunks, an empty buffer none.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[test]
|
||||
fn chunks_mut_pooled_makes_one_state_per_worker_at_most() {
|
||||
let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap();
|
||||
pool.install(|| {
|
||||
let mut buf = vec![0usize; 1003];
|
||||
let inits = AtomicUsize::new(0);
|
||||
let runs = AtomicUsize::new(0);
|
||||
let init = || {
|
||||
inits.fetch_add(1, Ordering::Relaxed);
|
||||
0usize
|
||||
};
|
||||
chunks_mut_pooled(&mut buf, 10, init, |used, i, c| {
|
||||
*used += 1;
|
||||
runs.fetch_add(1, Ordering::Relaxed);
|
||||
c.iter_mut().for_each(|v| *v += i + 1);
|
||||
});
|
||||
assert!(buf.iter().enumerate().all(|(k, &v)| v == k / 10 + 1));
|
||||
assert_eq!(runs.load(Ordering::Relaxed), 101);
|
||||
let n = inits.load(Ordering::Relaxed);
|
||||
assert!((1..=4).contains(&n), "{n} states for 101 chunks on 4 workers");
|
||||
chunks_mut_pooled(&mut [0u8; 0], 4, init, |_, _, _| {});
|
||||
assert_eq!(inits.load(Ordering::Relaxed), n, "no state for an empty buffer");
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user