Integration: per-band AHD balancing, memory-capped fused bands (#9455)

`for_bands_grouped` always split the bands into `min(workers, jobs)` static
chunks, so on the default develop (jobs == workers) a worker held up on the
shared pool kept a fixed share of the bands waiting, where main's
`chunks_mut_limited` runs one rayon task per band. Now, when every worker may
hold a scratch, each band is its own task and borrows a scratch from a pool
(`par::chunks_mut_pooled`, at most one per worker, dropped as soon as no
waiting band can use it); the grouping stays for memory-limited jobs.

The fused bilinear/MHC, X-Trans and LinearRaw paths ran one band per worker
with no memory cap: a 262144 x 455 Bayer frame with bilinear needed 2.92 GB
at 15+ workers and failed with LimitExceeded under the default 2 GiB.
`working_memory` now caps those bands like AHD's and the fused paths use the
returned jobs. The fused LinearRaw and X-Trans band buffers are `band.min(h)`
rows, as charged (CodeRabbit, develop.rs:414).

Tests: a_slow_band_does_not_hold_up_the_bands_after_it and
fused_bands_at_once_are_capped_by_the_memory_limit fail without the fix;
for_bands_keeps_at_most_jobs_scratches_alive, chunks_mut_pooled_makes_one_state_per_worker_at_most,
multi-band and short X-Trans/LinearRaw fused-vs-full cases.

Co-authored-by: Valentin Klein <271991477+vangel76@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
committed 2026-10-27 14:24:57 +01:00
1 parent 3e144a381a
commit 0b09eb43a5
3 files changed
+244 -48

No files matched your search

+126 -30
View File
@@ -232,7 +232,7 @@ pub(crate) fn demosaic_fused(
phase: [u8; 4],
method: Demosaic,
to_xyz: &Mat3,
ahd_jobs: usize,
jobs: usize,
fill: &(impl Fn(usize, &mut [f32]) + Sync),
out: &mut [u16],
emit: &(impl Fn(&[f32], &mut [u16]) + Sync),
@@ -257,7 +257,7 @@ pub(crate) fn demosaic_fused(
// The rows of the tallest band, as `develop`'s memory check counts them.
let rows = h.min(AHD_BAND);
let init = || (AhdScratch::for_image(h, s), vec![0.0f32; rows * w * 3], vec![0.0f32; (rows + 2 * PAD) * s]);
for_bands_grouped(out, AHD_BAND * w * 3, ahd_bands(h), ahd_jobs, 1, init, |band, chunk, (sc, buf, win)| {
for_bands_grouped(out, AHD_BAND * w * 3, ahd_bands(h), jobs, 1, init, |band, chunk, (sc, buf, win)| {
load(win, band * AHD_BAND, chunk.len() / (w * 3));
let buf = &mut buf[..chunk.len()];
// The window starts at a multiple of AHD_BAND (even), so it is band 0 of its own rows.
@@ -269,7 +269,7 @@ pub(crate) fn demosaic_fused(
let ks = kernels(phase, method, s);
let rows = h.min(FUSED_ROWS);
let init = || (vec![0.0f32; rows * w * 3], vec![0.0f32; (rows + 2 * PAD) * s]);
for_bands(out, FUSED_ROWS * w * 3, h.div_ceil(FUSED_ROWS), usize::MAX, init, |band, chunk, (buf, win)| {
for_bands(out, FUSED_ROWS * w * 3, h.div_ceil(FUSED_ROWS), jobs, init, |band, chunk, (buf, win)| {
load(win, band * FUSED_ROWS, chunk.len() / (w * 3));
let buf = &mut buf[..chunk.len()];
linear_band(win, w, s, phase, method, &ks, band * FUSED_ROWS, band * FUSED_ROWS, buf);
@@ -318,15 +318,16 @@ fn linear_band(d: &[f32], w: usize, s: usize, phase: [u8; 4], method: Demosaic,
}
/// Runs `f(band, chunk, scratch)` over the `unit`-sized chunks of `out` (`bands` of them), with
/// at most `jobs` scratch values alive: a few groups of consecutive bands per worker (for
/// balance), each group building its scratch once. For a scratch that is cheap to build.
pub(crate) fn for_bands<T: Send, S>(
/// at most `jobs` scratch values alive. When every worker may hold one, each band is its own
/// rayon task and borrows a scratch from a pool (one per worker at most); otherwise a few groups
/// of consecutive bands per job run, `jobs` at a time, each building its scratch once.
pub(crate) fn for_bands<T: Send, S: Send>(
out: &mut [T],
unit: usize,
bands: usize,
jobs: usize,
init: impl Fn() -> S + Sync,
f: impl Fn(usize, &mut [T], &mut S) + Sync,
init: impl Fn() -> S + Sync + Send,
f: impl Fn(usize, &mut [T], &mut S) + Sync + Send,
) {
for_bands_grouped(out, unit, bands, jobs, 4, init, f);
}
@@ -337,18 +338,24 @@ fn bands_per_group(bands: usize, jobs: usize, per_worker: usize) -> usize {
bands.div_ceil(groups).max(1)
}
/// [`for_bands`] with `per_worker` groups of consecutive bands for each worker. A large scratch
/// (AHD: 16 planes) wants one group per worker: with more groups than workers on a many-core
/// machine every group is a single band and each band would build and zero its own scratch.
fn for_bands_grouped<T: Send, S>(
/// [`for_bands`] with `per_worker` groups of consecutive bands for each of the `jobs` workers when
/// fewer than all may hold a scratch. A large scratch (AHD: 16 planes) wants one group per job:
/// with more groups each would build and zero its own scratch for a band or two.
fn for_bands_grouped<T: Send, S: Send>(
out: &mut [T],
unit: usize,
bands: usize,
jobs: usize,
per_worker: usize,
init: impl Fn() -> S + Sync,
f: impl Fn(usize, &mut [T], &mut S) + Sync,
init: impl Fn() -> S + Sync + Send,
f: impl Fn(usize, &mut [T], &mut S) + Sync + Send,
) {
// One task per band keeps rayon's stealing, so a busy or slow worker doesn't hold up a
// static share of the bands (as `par::chunks_mut_limited` does for the same reason).
if jobs >= par::workers() {
par::chunks_mut_pooled(out, unit, init, |sc, band, chunk| f(band, chunk, sc));
return;
}
let per_group = bands_per_group(bands, jobs, per_worker);
par::chunks_mut_limited(out, per_group * unit, jobs, |group, gchunk| {
let mut sc = init();
@@ -447,15 +454,15 @@ pub(crate) fn demosaic_cfa(p: &Padded, pat: &Pattern) -> Vec<f32> {
}
/// [`demosaic_cfa`] without the full RGB frame (the green plane stays): see [`demosaic_fused`].
pub(crate) fn demosaic_cfa_fused(p: &Padded, pat: &Pattern, out: &mut [u16], emit: &(impl Fn(&[f32], &mut [u16]) + Sync)) {
pub(crate) fn demosaic_cfa_fused(p: &Padded, pat: &Pattern, jobs: usize, out: &mut [u16], emit: &(impl Fn(&[f32], &mut [u16]) + Sync)) {
let (w, h) = (p.w, p.h);
if out.len() < w * h * 3 {
return;
}
let band = par::band_rows(w);
let init = || vec![0.0f32; band * w * 3];
let init = || vec![0.0f32; band.min(h) * w * 3];
cfa_with(p, pat, |rows| {
for_bands(&mut out[..w * h * 3], band * w * 3, h.div_ceil(band), usize::MAX, init, |b, chunk, buf| {
for_bands(&mut out[..w * h * 3], band * w * 3, h.div_ceil(band), jobs, init, |b, chunk, buf| {
let buf = &mut buf[..chunk.len()];
rows(b * band, buf);
emit(buf, chunk);
@@ -944,6 +951,8 @@ fn ahd_ref(p: &Padded, phase: [u8; 4], to_xyz: &Mat3, out: &mut [f32], jobs: usi
mod tests {
use super::*;
use crate::color::IDENTITY;
use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
use std::time::{Duration, Instant};
#[test]
fn ahd_band_concurrency_does_not_change_pixels() {
@@ -1037,8 +1046,9 @@ mod tests {
}
}
/// One scratch serves many bands (1100 rows are 35 bands, the last one short, over 4 scratches
/// with jobs 1 and at most 8 with jobs 2), so a stage reading a cell left over from an earlier band would show.
/// One scratch serves many bands (1100 rows are 35 bands, the last one short, over one scratch
/// with jobs 1 and two with jobs 2 on a bigger pool), so a stage reading a cell left over from
/// an earlier band would show.
#[test]
fn ahd_matches_reference_with_reused_scratch() {
let m = [[0.41, 0.36, 0.18], [0.21, 0.72, 0.07], [0.02, 0.12, 0.95]];
@@ -1057,9 +1067,10 @@ mod tests {
}
}
/// A 4-worker pool splits 35 bands into groups of 9 (the last of 8), so groups after the first
/// start at band `group * 9` and every scratch is reused for several bands: a wrong band index
/// or a stale scratch cell would change pixels. The group size does not depend on the machine.
/// On a 4-worker pool with room for 3 scratches, 35 bands go in groups of 12 (the last of 11),
/// so groups after the first start at band `group * 12` and every scratch is reused for several
/// bands; with room for 4, every band is its own task and most borrow a used scratch. A wrong band
/// index or a stale scratch cell would change pixels. The split does not depend on the machine.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn ahd_groups_of_several_bands_match_reference() {
@@ -1070,18 +1081,103 @@ mod tests {
let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap();
pool.install(|| {
assert_eq!(par::workers(), 4);
assert_eq!(bands_per_group(ahd_bands(h), 4, 1), 9);
// One worker per group, never more groups than workers, however many bands there are.
assert_eq!(bands_per_group(126, 1000, 1), 32);
for phase in [[0, 1, 1, 2], [1, 2, 0, 1]] {
let mut want = vec![0.0f32; w * h * 3];
ahd_ref(&p, phase, &m, &mut want, 4);
let got = demosaic(&p, phase, Demosaic::Ahd, &m, 4);
assert!(want.iter().zip(&got).all(|(a, b)| a.to_bits() == b.to_bits()), "{phase:?}");
assert_eq!(bands_per_group(ahd_bands(h), 3, 1), 12);
// One group per job, never more groups than jobs, however many bands there are.
assert_eq!(bands_per_group(126, 3, 1), 42);
for jobs in [3, 4] {
for phase in [[0, 1, 1, 2], [1, 2, 0, 1]] {
let mut want = vec![0.0f32; w * h * 3];
ahd_ref(&p, phase, &m, &mut want, jobs);
let got = demosaic(&p, phase, Demosaic::Ahd, &m, jobs);
assert!(want.iter().zip(&got).all(|(a, b)| a.to_bits() == b.to_bits()), "{phase:?} jobs {jobs}");
}
}
});
}
/// A scratch that counts how many are alive at once.
struct Live<'a>(&'a AtomicUsize);
impl<'a> Live<'a> {
fn new(now: &'a AtomicUsize, peak: &AtomicUsize) -> Self {
peak.fetch_max(now.fetch_add(1, SeqCst) + 1, SeqCst);
Live(now)
}
}
impl Drop for Live<'_> {
fn drop(&mut self) {
self.0.fetch_sub(1, SeqCst);
}
}
/// Every band runs once with its own index, and no more than `jobs` scratches (what `develop`'s
/// memory check admits) or one per worker are alive at once, on either side of `jobs == workers`.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn for_bands_keeps_at_most_jobs_scratches_alive() {
let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap();
pool.install(|| {
for jobs in [1, 2, 3, 7, 8, 9, usize::MAX] {
for per_worker in [1, 4] {
let (now, peak) = (AtomicUsize::new(0), AtomicUsize::new(0));
let mut out = vec![0usize; 37 * 10 - 4];
for_bands_grouped(
&mut out,
10,
37,
jobs,
per_worker,
|| Live::new(&now, &peak),
|band, chunk, _| {
std::thread::sleep(Duration::from_millis(1));
chunk.iter_mut().for_each(|v| *v += band + 1);
},
);
assert!(out.iter().enumerate().all(|(i, &v)| v == i / 10 + 1), "jobs {jobs}");
let peak = peak.load(SeqCst);
assert!((1..=jobs.min(8)).contains(&peak), "jobs {jobs} per_worker {per_worker}: {peak} scratches alive");
assert_eq!(now.load(SeqCst), 0);
}
}
});
}
/// With room for a scratch on every worker, the bands are separate rayon tasks: a worker held up
/// on band 0 (by other work on the shared pool, or a slow core) leaves the bands after it to
/// the others. With a fixed share of 9 bands per worker, 8 of them would wait for band 0, so at
/// most 26 of the other 34 could finish meanwhile.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn a_slow_band_does_not_hold_up_the_bands_after_it() {
let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap();
pool.install(|| {
let (done, seen) = (AtomicUsize::new(0), AtomicUsize::new(0));
let mut out = vec![0u8; 35];
for_bands_grouped(
&mut out,
1,
35,
4,
1,
|| (),
|band, _, _| {
if band > 0 {
done.fetch_add(1, SeqCst);
return;
}
let start = Instant::now();
while done.load(SeqCst) <= 26 && start.elapsed() < Duration::from_secs(10) {
std::thread::sleep(Duration::from_millis(1));
}
seen.store(done.load(SeqCst), SeqCst);
},
);
assert_eq!(done.load(SeqCst), 34);
assert!(seen.load(SeqCst) > 26, "only {} bands ran while band 0 was held up", seen.load(SeqCst));
});
}
/// Mosaics an RGB image with the given phase.
fn mosaic(rgb: &[[f32; 3]], w: usize, h: usize, phase: [u8; 4]) -> Vec<f32> {
(0..w * h).map(|i| rgb[i][phase[((i / w) & 1) * 2 + ((i % w) & 1)] as usize]).collect()
+52 -18
View File
@@ -230,7 +230,7 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result<Devel
// condition under which `Haze::new` and `ToneMap::new` return `None`).
let fused = allow_fused && opts.tone.is_neutral() && s.black_point <= 0.0;
let orient_copy = opts.orient && (2..=8).contains(&s.orientation);
let ahd_jobs = working_memory(c.width, c.height, s.cfa.is_some(), bayer, Shape { fused, orient_copy }, opts, par::workers())?;
let jobs = working_memory(c.width, c.height, s.cfa.is_some(), bayer, Shape { fused, orient_copy }, opts, par::workers())?;
let mut warnings = s.warnings.clone();
let sc = Scale::new(s);
@@ -374,7 +374,7 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result<Devel
let p = cfa.phase(c.x, c.y);
pattern = Some(names(&p));
let mut o = vec![0u16; w * h * 3];
demosaic_fused(w, h, p, opts.demosaic, &to_xyz, ahd_jobs, &fill_row, &mut o, &encode_plain);
demosaic_fused(w, h, p, opts.demosaic, &to_xyz, jobs, &fill_row, &mut o, &encode_plain);
fused_out = Some(o);
} else {
let mut plane = Padded::new(w, h);
@@ -388,13 +388,13 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result<Devel
if bayer {
let p = cfa.phase(c.x, c.y);
pattern = Some(names(&p));
rgb = demosaic(&plane, p, opts.demosaic, &to_xyz, ahd_jobs);
rgb = demosaic(&plane, p, opts.demosaic, &to_xyz, jobs);
} else {
let pat = Pattern::of(cfa, c.x, c.y);
pattern = Some(pat.colors.chunks(pat.w.max(1)).map(names).collect::<Vec<_>>().join("/"));
if fused {
let mut o = vec![0u16; w * h * 3];
demosaic_cfa_fused(&plane, &pat, &mut o, &encode_plain);
demosaic_cfa_fused(&plane, &pat, jobs, &mut o, &encode_plain);
fused_out = Some(o);
} else {
rgb = demosaic_cfa(&plane, &pat);
@@ -410,8 +410,8 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result<Devel
&mut o,
band * w * 3,
h.div_ceil(band),
usize::MAX,
|| vec![0.0f32; band * w * 3],
jobs,
|| vec![0.0f32; band.min(h) * w * 3],
|b, chunk, buf| {
let buf = &mut buf[..chunk.len()];
linear_rows(b * band, buf);
@@ -509,10 +509,10 @@ fn fused_band_bytes(w: usize, h: usize, band_rows: usize, window: bool) -> u64 {
}
/// Checks the peak of the two development phases. The CFA and non-Bayer green
/// planes are freed before the u16 output is allocated; AHD scratch is bounded
/// by reducing simultaneous bands when needed. The fused path never holds the f32 RGB frame:
/// it has the u16 output (6 bytes a pixel) next to the CFA plane, plus a band buffer and the
/// scratch per worker, and an orientation copy of the output afterwards. Sensor storage belongs
/// planes are freed before the u16 output is allocated; the AHD scratch and the fused band
/// buffers are bounded by reducing simultaneous bands when needed. The fused path never holds
/// the f32 RGB frame: it has the u16 output (6 bytes a pixel) next to the CFA plane, plus a band
/// buffer and the scratch per simultaneous band, and an orientation copy of the output afterwards. Sensor storage belongs
/// to the caller and is checked separately by the decoder.
fn working_memory(w: usize, h: usize, cfa: bool, bayer: bool, shape: Shape, opts: &DevelopOptions, workers: usize) -> Result<usize> {
let limits = &opts.limits;
@@ -530,7 +530,12 @@ fn working_memory(w: usize, h: usize, cfa: bool, bayer: bool, shape: Shape, opts
let mut jobs = workers.max(1);
// Fused: the u16 output and, per worker running a band, one band of f32 RGB.
let band_of = |rows: usize, window: bool| if fused { fused_band_bytes(w, h, rows, window) } else { 0 };
let band_jobs = |rows: usize| workers.max(1).min(h.div_ceil(rows.max(1))) as u64;
// Fewer bands at once when the bands of every worker don't fit next to `base` (at least one).
let with_bands = |jobs: &mut usize, base: u64, rows: usize, band: u64| {
let available = limits.max_alloc.saturating_sub(base) / band.max(1);
*jobs = (*jobs).min(h.div_ceil(rows.max(1))).min(usize::try_from(available).unwrap_or(usize::MAX)).max(1);
base.saturating_add(band.saturating_mul(*jobs as u64))
};
if cfa {
let plane = (w as u64).saturating_add(2 * PAD as u64).saturating_mul((h as u64).saturating_add(2 * PAD as u64)).saturating_mul(4);
// Bytes a pixel next to the plane: the f32 RGB frame (and the non-Bayer green plane), or
@@ -547,19 +552,17 @@ fn working_memory(w: usize, h: usize, cfa: bool, bayer: bool, shape: Shape, opts
if bayer && opts.demosaic == Demosaic::Ahd {
mosaic = mosaic.saturating_add(crate::demosaic::AHD_TABLE_BYTES);
let scratch = crate::demosaic::ahd_scratch(w as u64, h as u64).saturating_add(band_of(crate::demosaic::AHD_BAND, true));
let available = limits.max_alloc.saturating_sub(mosaic) / scratch.max(1);
jobs = jobs.min(crate::demosaic::ahd_bands(h)).min(usize::try_from(available).unwrap_or(usize::MAX));
// At least one band must fit, even when no parallelism is available.
mosaic = mosaic.saturating_add(scratch.saturating_mul(jobs.max(1) as u64));
mosaic = with_bands(&mut jobs, mosaic, crate::demosaic::AHD_BAND, scratch);
} else {
let rows = if bayer { crate::demosaic::FUSED_ROWS } else { par::band_rows(w) };
mosaic = mosaic.saturating_add(band_of(rows, bayer).saturating_mul(band_jobs(rows)));
mosaic = with_bands(&mut jobs, mosaic, rows, band_of(rows, bayer));
}
peak = peak.max(mosaic);
} else if fused {
// LinearRaw: the output and the bands.
let rows = par::band_rows(w);
peak = peak.max(pixels.saturating_mul(6).saturating_add(band_of(rows, false).saturating_mul(band_jobs(rows))));
peak = peak.max(with_bands(&mut jobs, pixels.saturating_mul(6), rows, band_of(rows, false)));
}
if peak > limits.max_alloc || usize::try_from(peak).is_err() {
return Err(RawError::LimitExceeded(format!("{peak} bytes of RAW working memory exceed {}", limits.max_alloc)));
@@ -836,13 +839,15 @@ mod tests {
})
})
.collect();
for (w, h) in [(6, 6), (37, 41), (80, 150)] {
// 600 samples wide: bands of 109 rows, so 350 rows are four (the last short) and 50 rows
// are less than one.
for (w, h) in [(6, 6), (37, 41), (80, 150), (600, 350), (600, 50)] {
let mut s = test_sensor(w, h, Some(Cfa { width: 6, height: 6, colors: colors.clone(), origin_x: 0, origin_y: 0 }), 1);
s.orientation = 8;
assert_fused_matches_full(&s, &DevelopOptions::default(), &format!("xtrans {w}x{h}"));
}
for samples in [1, 3] {
for (w, h) in [(1, 1), (7, 5), (90, 111)] {
for (w, h) in [(1, 1), (7, 5), (90, 111), (600, 350), (600, 50)] {
let mut s = test_sensor(w, h, None, samples);
s.orientation = 3;
assert_fused_matches_full(&s, &DevelopOptions::default(), &format!("linear {samples} {w}x{h}"));
@@ -902,6 +907,35 @@ mod tests {
assert!(working_memory(11648, 8736, true, true, FUSED, &opts, 1).is_ok());
}
/// The fused bilinear / MHC, X-Trans and LinearRaw paths run fewer bands at once when the bands
/// of every worker don't fit next to the output, as AHD does. A very wide, short Bayer frame
/// (262144 x 455: 15 bands of 32 rows, about 147 MB a band with its window) needs 2.92 GB
/// with all 15 at once against the default 2 GiB, and develops with 9.
#[test]
fn fused_bands_at_once_are_capped_by_the_memory_limit() {
let opts = DevelopOptions { demosaic: Demosaic::Bilinear, ..Default::default() };
let table = (GAMMA_STEPS as u64 + 2) * 4;
let (w, h) = (262144usize, 455usize);
let band = 32 * 262144 * 12 + (32 + 2 * PAD as u64) * (262144 + 2 * PAD as u64) * 4;
for (workers, want) in [(1, 1), (8, 8), (9, 9), (10, 9), (16, 9), (32, 9)] {
assert_eq!(working_memory(w, h, true, true, FUSED, &opts, workers).unwrap(), want, "{workers} workers");
}
// Never more jobs than bands, and one when only one band fits.
assert_eq!(working_memory(1000, 40, true, true, FUSED, &opts, 32).unwrap(), 2);
let mut tight = opts.clone();
tight.limits.max_alloc = 262144 * 455 * 6 + table + band;
assert_eq!(working_memory(w, h, true, true, FUSED, &tight, 32).unwrap(), 1);
tight.limits.max_alloc -= 1;
assert!(working_memory(w, h, true, true, FUSED, &tight, 32).is_err());
// X-Trans (bands of `band_rows(1000)` = 65 rows next to its planes) and LinearRaw likewise.
let plane = (1000 + 2 * PAD as u64) * (2000 + 2 * PAD as u64) * 4;
let mut x = opts.clone();
x.limits.max_alloc = 1000 * 2000 * 10 + plane + table + 3 * 65 * 1000 * 12;
assert_eq!(working_memory(1000, 2000, true, false, FUSED, &x, 32).unwrap(), 3);
x.limits.max_alloc = 1000 * 2000 * 6 + 5 * 65 * 1000 * 12;
assert_eq!(working_memory(1000, 2000, false, true, FUSED, &x, 32).unwrap(), 5);
}
#[test]
fn working_memory_accounts_for_padding_and_at_least_one_ahd_band() {
let mut opts = DevelopOptions { demosaic: Demosaic::Ahd, ..Default::default() };
+66
View File
@@ -1,5 +1,7 @@
//! Data-parallel helpers: rayon on native targets, sequential on wasm.
use std::sync::{Mutex, PoisonError};
/// Calls `f(index, chunk)` for each `chunk`-sized piece of `buf`.
pub(crate) fn chunks_mut<T: Send>(buf: &mut [T], chunk: usize, f: impl Fn(usize, &mut [T]) + Sync + Send) {
let chunk = chunk.max(1);
@@ -12,6 +14,36 @@ pub(crate) fn chunks_mut<T: Send>(buf: &mut [T], chunk: usize, f: impl Fn(usize,
buf.chunks_mut(chunk).enumerate().for_each(|(i, c)| f(i, c));
}
/// Like [`chunks_mut`], with `f(state, index, chunk)` borrowing a state: a chunk takes a free one
/// or makes one with `init`, and gives it back when done, so rayon keeps its fine-grained
/// splitting and stealing while at most one state per worker is ever made (`f` must not itself
/// wait on rayon work). Rayon's `for_each_init` would make one per stolen piece, which on an idle
/// pool is often every chunk. A state no chunk still waiting can take is dropped at once, so
/// near the end they go as their workers finish rather than all at the very end.
pub(crate) fn chunks_mut_pooled<T: Send, S: Send>(
buf: &mut [T],
chunk: usize,
init: impl Fn() -> S + Sync + Send,
f: impl Fn(&mut S, usize, &mut [T]) + Sync + Send,
) {
let chunk = chunk.max(1);
// The free states and the number of chunks that have not taken one yet.
let pool = Mutex::new((Vec::new(), buf.len().div_ceil(chunk)));
chunks_mut(buf, chunk, |i, c| {
let taken = {
let mut pool = pool.lock().unwrap_or_else(PoisonError::into_inner);
pool.1 = pool.1.saturating_sub(1);
pool.0.pop()
};
let mut state = taken.unwrap_or_else(&init);
f(&mut state, i, c);
let mut pool = pool.lock().unwrap_or_else(PoisonError::into_inner);
if pool.0.len() < pool.1 {
pool.0.push(state);
}
});
}
/// Native worker count (the browser builds run sequentially).
pub(crate) fn workers() -> usize {
#[cfg(not(target_arch = "wasm32"))]
@@ -56,3 +88,37 @@ pub(crate) fn map<T: Send>(n: usize, f: impl Fn(usize) -> T + Sync + Send) -> Ve
pub(crate) fn band_rows(width: usize) -> usize {
(65536 / width.max(1)).clamp(1, 256)
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
/// Every chunk is visited once with its own index (the last one short), and the chunks share
/// the states: a 4-worker pool makes at most 4 for 101 chunks, an empty buffer none.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn chunks_mut_pooled_makes_one_state_per_worker_at_most() {
let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap();
pool.install(|| {
let mut buf = vec![0usize; 1003];
let inits = AtomicUsize::new(0);
let runs = AtomicUsize::new(0);
let init = || {
inits.fetch_add(1, Ordering::Relaxed);
0usize
};
chunks_mut_pooled(&mut buf, 10, init, |used, i, c| {
*used += 1;
runs.fetch_add(1, Ordering::Relaxed);
c.iter_mut().for_each(|v| *v += i + 1);
});
assert!(buf.iter().enumerate().all(|(k, &v)| v == k / 10 + 1));
assert_eq!(runs.load(Ordering::Relaxed), 101);
let n = inits.load(Ordering::Relaxed);
assert!((1..=4).contains(&n), "{n} states for 101 chunks on 4 workers");
chunks_mut_pooled(&mut [0u8; 0], 4, init, |_, _, _| {});
assert_eq!(inits.load(Ordering::Relaxed), n, "no state for an empty buffer");
});
}
}