diff --git a/crates/raw/src/demosaic.rs b/crates/raw/src/demosaic.rs index 86bcee5f3c..6303931811 100644 --- a/crates/raw/src/demosaic.rs +++ b/crates/raw/src/demosaic.rs @@ -232,7 +232,7 @@ pub(crate) fn demosaic_fused( phase: [u8; 4], method: Demosaic, to_xyz: &Mat3, - ahd_jobs: usize, + jobs: usize, fill: &(impl Fn(usize, &mut [f32]) + Sync), out: &mut [u16], emit: &(impl Fn(&[f32], &mut [u16]) + Sync), @@ -257,7 +257,7 @@ pub(crate) fn demosaic_fused( // The rows of the tallest band, as `develop`'s memory check counts them. let rows = h.min(AHD_BAND); let init = || (AhdScratch::for_image(h, s), vec![0.0f32; rows * w * 3], vec![0.0f32; (rows + 2 * PAD) * s]); - for_bands_grouped(out, AHD_BAND * w * 3, ahd_bands(h), ahd_jobs, 1, init, |band, chunk, (sc, buf, win)| { + for_bands_grouped(out, AHD_BAND * w * 3, ahd_bands(h), jobs, 1, init, |band, chunk, (sc, buf, win)| { load(win, band * AHD_BAND, chunk.len() / (w * 3)); let buf = &mut buf[..chunk.len()]; // The window starts at a multiple of AHD_BAND (even), so it is band 0 of its own rows. @@ -269,7 +269,7 @@ pub(crate) fn demosaic_fused( let ks = kernels(phase, method, s); let rows = h.min(FUSED_ROWS); let init = || (vec![0.0f32; rows * w * 3], vec![0.0f32; (rows + 2 * PAD) * s]); - for_bands(out, FUSED_ROWS * w * 3, h.div_ceil(FUSED_ROWS), usize::MAX, init, |band, chunk, (buf, win)| { + for_bands(out, FUSED_ROWS * w * 3, h.div_ceil(FUSED_ROWS), jobs, init, |band, chunk, (buf, win)| { load(win, band * FUSED_ROWS, chunk.len() / (w * 3)); let buf = &mut buf[..chunk.len()]; linear_band(win, w, s, phase, method, &ks, band * FUSED_ROWS, band * FUSED_ROWS, buf); @@ -318,15 +318,16 @@ fn linear_band(d: &[f32], w: usize, s: usize, phase: [u8; 4], method: Demosaic, } /// Runs `f(band, chunk, scratch)` over the `unit`-sized chunks of `out` (`bands` of them), with -/// at most `jobs` scratch values alive: a few groups of consecutive bands per worker (for -/// balance), each group building its scratch once. For a scratch that is cheap to build. -pub(crate) fn for_bands( +/// at most `jobs` scratch values alive. When every worker may hold one, each band is its own +/// rayon task and borrows a scratch from a pool (one per worker at most); otherwise a few groups +/// of consecutive bands per job run, `jobs` at a time, each building its scratch once. +pub(crate) fn for_bands( out: &mut [T], unit: usize, bands: usize, jobs: usize, - init: impl Fn() -> S + Sync, - f: impl Fn(usize, &mut [T], &mut S) + Sync, + init: impl Fn() -> S + Sync + Send, + f: impl Fn(usize, &mut [T], &mut S) + Sync + Send, ) { for_bands_grouped(out, unit, bands, jobs, 4, init, f); } @@ -337,18 +338,24 @@ fn bands_per_group(bands: usize, jobs: usize, per_worker: usize) -> usize { bands.div_ceil(groups).max(1) } -/// [`for_bands`] with `per_worker` groups of consecutive bands for each worker. A large scratch -/// (AHD: 16 planes) wants one group per worker: with more groups than workers on a many-core -/// machine every group is a single band and each band would build and zero its own scratch. -fn for_bands_grouped( +/// [`for_bands`] with `per_worker` groups of consecutive bands for each of the `jobs` workers when +/// fewer than all may hold a scratch. A large scratch (AHD: 16 planes) wants one group per job: +/// with more groups each would build and zero its own scratch for a band or two. +fn for_bands_grouped( out: &mut [T], unit: usize, bands: usize, jobs: usize, per_worker: usize, - init: impl Fn() -> S + Sync, - f: impl Fn(usize, &mut [T], &mut S) + Sync, + init: impl Fn() -> S + Sync + Send, + f: impl Fn(usize, &mut [T], &mut S) + Sync + Send, ) { + // One task per band keeps rayon's stealing, so a busy or slow worker doesn't hold up a + // static share of the bands (as `par::chunks_mut_limited` does for the same reason). + if jobs >= par::workers() { + par::chunks_mut_pooled(out, unit, init, |sc, band, chunk| f(band, chunk, sc)); + return; + } let per_group = bands_per_group(bands, jobs, per_worker); par::chunks_mut_limited(out, per_group * unit, jobs, |group, gchunk| { let mut sc = init(); @@ -447,15 +454,15 @@ pub(crate) fn demosaic_cfa(p: &Padded, pat: &Pattern) -> Vec { } /// [`demosaic_cfa`] without the full RGB frame (the green plane stays): see [`demosaic_fused`]. -pub(crate) fn demosaic_cfa_fused(p: &Padded, pat: &Pattern, out: &mut [u16], emit: &(impl Fn(&[f32], &mut [u16]) + Sync)) { +pub(crate) fn demosaic_cfa_fused(p: &Padded, pat: &Pattern, jobs: usize, out: &mut [u16], emit: &(impl Fn(&[f32], &mut [u16]) + Sync)) { let (w, h) = (p.w, p.h); if out.len() < w * h * 3 { return; } let band = par::band_rows(w); - let init = || vec![0.0f32; band * w * 3]; + let init = || vec![0.0f32; band.min(h) * w * 3]; cfa_with(p, pat, |rows| { - for_bands(&mut out[..w * h * 3], band * w * 3, h.div_ceil(band), usize::MAX, init, |b, chunk, buf| { + for_bands(&mut out[..w * h * 3], band * w * 3, h.div_ceil(band), jobs, init, |b, chunk, buf| { let buf = &mut buf[..chunk.len()]; rows(b * band, buf); emit(buf, chunk); @@ -944,6 +951,8 @@ fn ahd_ref(p: &Padded, phase: [u8; 4], to_xyz: &Mat3, out: &mut [f32], jobs: usi mod tests { use super::*; use crate::color::IDENTITY; + use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + use std::time::{Duration, Instant}; #[test] fn ahd_band_concurrency_does_not_change_pixels() { @@ -1037,8 +1046,9 @@ mod tests { } } - /// One scratch serves many bands (1100 rows are 35 bands, the last one short, over 4 scratches - /// with jobs 1 and at most 8 with jobs 2), so a stage reading a cell left over from an earlier band would show. + /// One scratch serves many bands (1100 rows are 35 bands, the last one short, over one scratch + /// with jobs 1 and two with jobs 2 on a bigger pool), so a stage reading a cell left over from + /// an earlier band would show. #[test] fn ahd_matches_reference_with_reused_scratch() { let m = [[0.41, 0.36, 0.18], [0.21, 0.72, 0.07], [0.02, 0.12, 0.95]]; @@ -1057,9 +1067,10 @@ mod tests { } } - /// A 4-worker pool splits 35 bands into groups of 9 (the last of 8), so groups after the first - /// start at band `group * 9` and every scratch is reused for several bands: a wrong band index - /// or a stale scratch cell would change pixels. The group size does not depend on the machine. + /// On a 4-worker pool with room for 3 scratches, 35 bands go in groups of 12 (the last of 11), + /// so groups after the first start at band `group * 12` and every scratch is reused for several + /// bands; with room for 4, every band is its own task and most borrow a used scratch. A wrong band + /// index or a stale scratch cell would change pixels. The split does not depend on the machine. #[cfg(not(target_arch = "wasm32"))] #[test] fn ahd_groups_of_several_bands_match_reference() { @@ -1070,18 +1081,103 @@ mod tests { let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap(); pool.install(|| { assert_eq!(par::workers(), 4); - assert_eq!(bands_per_group(ahd_bands(h), 4, 1), 9); - // One worker per group, never more groups than workers, however many bands there are. - assert_eq!(bands_per_group(126, 1000, 1), 32); - for phase in [[0, 1, 1, 2], [1, 2, 0, 1]] { - let mut want = vec![0.0f32; w * h * 3]; - ahd_ref(&p, phase, &m, &mut want, 4); - let got = demosaic(&p, phase, Demosaic::Ahd, &m, 4); - assert!(want.iter().zip(&got).all(|(a, b)| a.to_bits() == b.to_bits()), "{phase:?}"); + assert_eq!(bands_per_group(ahd_bands(h), 3, 1), 12); + // One group per job, never more groups than jobs, however many bands there are. + assert_eq!(bands_per_group(126, 3, 1), 42); + for jobs in [3, 4] { + for phase in [[0, 1, 1, 2], [1, 2, 0, 1]] { + let mut want = vec![0.0f32; w * h * 3]; + ahd_ref(&p, phase, &m, &mut want, jobs); + let got = demosaic(&p, phase, Demosaic::Ahd, &m, jobs); + assert!(want.iter().zip(&got).all(|(a, b)| a.to_bits() == b.to_bits()), "{phase:?} jobs {jobs}"); + } } }); } + /// A scratch that counts how many are alive at once. + struct Live<'a>(&'a AtomicUsize); + + impl<'a> Live<'a> { + fn new(now: &'a AtomicUsize, peak: &AtomicUsize) -> Self { + peak.fetch_max(now.fetch_add(1, SeqCst) + 1, SeqCst); + Live(now) + } + } + + impl Drop for Live<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, SeqCst); + } + } + + /// Every band runs once with its own index, and no more than `jobs` scratches (what `develop`'s + /// memory check admits) or one per worker are alive at once, on either side of `jobs == workers`. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn for_bands_keeps_at_most_jobs_scratches_alive() { + let pool = rayon::ThreadPoolBuilder::new().num_threads(8).build().unwrap(); + pool.install(|| { + for jobs in [1, 2, 3, 7, 8, 9, usize::MAX] { + for per_worker in [1, 4] { + let (now, peak) = (AtomicUsize::new(0), AtomicUsize::new(0)); + let mut out = vec![0usize; 37 * 10 - 4]; + for_bands_grouped( + &mut out, + 10, + 37, + jobs, + per_worker, + || Live::new(&now, &peak), + |band, chunk, _| { + std::thread::sleep(Duration::from_millis(1)); + chunk.iter_mut().for_each(|v| *v += band + 1); + }, + ); + assert!(out.iter().enumerate().all(|(i, &v)| v == i / 10 + 1), "jobs {jobs}"); + let peak = peak.load(SeqCst); + assert!((1..=jobs.min(8)).contains(&peak), "jobs {jobs} per_worker {per_worker}: {peak} scratches alive"); + assert_eq!(now.load(SeqCst), 0); + } + } + }); + } + + /// With room for a scratch on every worker, the bands are separate rayon tasks: a worker held up + /// on band 0 (by other work on the shared pool, or a slow core) leaves the bands after it to + /// the others. With a fixed share of 9 bands per worker, 8 of them would wait for band 0, so at + /// most 26 of the other 34 could finish meanwhile. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_slow_band_does_not_hold_up_the_bands_after_it() { + let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap(); + pool.install(|| { + let (done, seen) = (AtomicUsize::new(0), AtomicUsize::new(0)); + let mut out = vec![0u8; 35]; + for_bands_grouped( + &mut out, + 1, + 35, + 4, + 1, + || (), + |band, _, _| { + if band > 0 { + done.fetch_add(1, SeqCst); + return; + } + let start = Instant::now(); + while done.load(SeqCst) <= 26 && start.elapsed() < Duration::from_secs(10) { + std::thread::sleep(Duration::from_millis(1)); + } + seen.store(done.load(SeqCst), SeqCst); + }, + ); + assert_eq!(done.load(SeqCst), 34); + assert!(seen.load(SeqCst) > 26, "only {} bands ran while band 0 was held up", seen.load(SeqCst)); + }); + } + /// Mosaics an RGB image with the given phase. fn mosaic(rgb: &[[f32; 3]], w: usize, h: usize, phase: [u8; 4]) -> Vec { (0..w * h).map(|i| rgb[i][phase[((i / w) & 1) * 2 + ((i % w) & 1)] as usize]).collect() diff --git a/crates/raw/src/develop.rs b/crates/raw/src/develop.rs index ed5248b67b..aaeb390eff 100644 --- a/crates/raw/src/develop.rs +++ b/crates/raw/src/develop.rs @@ -230,7 +230,7 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result Result Result>().join("/")); if fused { let mut o = vec![0u16; w * h * 3]; - demosaic_cfa_fused(&plane, &pat, &mut o, &encode_plain); + demosaic_cfa_fused(&plane, &pat, jobs, &mut o, &encode_plain); fused_out = Some(o); } else { rgb = demosaic_cfa(&plane, &pat); @@ -410,8 +410,8 @@ fn develop(s: &Sensor, opts: &DevelopOptions, allow_fused: bool) -> Result u64 { } /// Checks the peak of the two development phases. The CFA and non-Bayer green -/// planes are freed before the u16 output is allocated; AHD scratch is bounded -/// by reducing simultaneous bands when needed. The fused path never holds the f32 RGB frame: -/// it has the u16 output (6 bytes a pixel) next to the CFA plane, plus a band buffer and the -/// scratch per worker, and an orientation copy of the output afterwards. Sensor storage belongs +/// planes are freed before the u16 output is allocated; the AHD scratch and the fused band +/// buffers are bounded by reducing simultaneous bands when needed. The fused path never holds +/// the f32 RGB frame: it has the u16 output (6 bytes a pixel) next to the CFA plane, plus a band +/// buffer and the scratch per simultaneous band, and an orientation copy of the output afterwards. Sensor storage belongs /// to the caller and is checked separately by the decoder. fn working_memory(w: usize, h: usize, cfa: bool, bayer: bool, shape: Shape, opts: &DevelopOptions, workers: usize) -> Result { let limits = &opts.limits; @@ -530,7 +530,12 @@ fn working_memory(w: usize, h: usize, cfa: bool, bayer: bool, shape: Shape, opts let mut jobs = workers.max(1); // Fused: the u16 output and, per worker running a band, one band of f32 RGB. let band_of = |rows: usize, window: bool| if fused { fused_band_bytes(w, h, rows, window) } else { 0 }; - let band_jobs = |rows: usize| workers.max(1).min(h.div_ceil(rows.max(1))) as u64; + // Fewer bands at once when the bands of every worker don't fit next to `base` (at least one). + let with_bands = |jobs: &mut usize, base: u64, rows: usize, band: u64| { + let available = limits.max_alloc.saturating_sub(base) / band.max(1); + *jobs = (*jobs).min(h.div_ceil(rows.max(1))).min(usize::try_from(available).unwrap_or(usize::MAX)).max(1); + base.saturating_add(band.saturating_mul(*jobs as u64)) + }; if cfa { let plane = (w as u64).saturating_add(2 * PAD as u64).saturating_mul((h as u64).saturating_add(2 * PAD as u64)).saturating_mul(4); // Bytes a pixel next to the plane: the f32 RGB frame (and the non-Bayer green plane), or @@ -547,19 +552,17 @@ fn working_memory(w: usize, h: usize, cfa: bool, bayer: bool, shape: Shape, opts if bayer && opts.demosaic == Demosaic::Ahd { mosaic = mosaic.saturating_add(crate::demosaic::AHD_TABLE_BYTES); let scratch = crate::demosaic::ahd_scratch(w as u64, h as u64).saturating_add(band_of(crate::demosaic::AHD_BAND, true)); - let available = limits.max_alloc.saturating_sub(mosaic) / scratch.max(1); - jobs = jobs.min(crate::demosaic::ahd_bands(h)).min(usize::try_from(available).unwrap_or(usize::MAX)); // At least one band must fit, even when no parallelism is available. - mosaic = mosaic.saturating_add(scratch.saturating_mul(jobs.max(1) as u64)); + mosaic = with_bands(&mut jobs, mosaic, crate::demosaic::AHD_BAND, scratch); } else { let rows = if bayer { crate::demosaic::FUSED_ROWS } else { par::band_rows(w) }; - mosaic = mosaic.saturating_add(band_of(rows, bayer).saturating_mul(band_jobs(rows))); + mosaic = with_bands(&mut jobs, mosaic, rows, band_of(rows, bayer)); } peak = peak.max(mosaic); } else if fused { // LinearRaw: the output and the bands. let rows = par::band_rows(w); - peak = peak.max(pixels.saturating_mul(6).saturating_add(band_of(rows, false).saturating_mul(band_jobs(rows)))); + peak = peak.max(with_bands(&mut jobs, pixels.saturating_mul(6), rows, band_of(rows, false))); } if peak > limits.max_alloc || usize::try_from(peak).is_err() { return Err(RawError::LimitExceeded(format!("{peak} bytes of RAW working memory exceed {}", limits.max_alloc))); @@ -836,13 +839,15 @@ mod tests { }) }) .collect(); - for (w, h) in [(6, 6), (37, 41), (80, 150)] { + // 600 samples wide: bands of 109 rows, so 350 rows are four (the last short) and 50 rows + // are less than one. + for (w, h) in [(6, 6), (37, 41), (80, 150), (600, 350), (600, 50)] { let mut s = test_sensor(w, h, Some(Cfa { width: 6, height: 6, colors: colors.clone(), origin_x: 0, origin_y: 0 }), 1); s.orientation = 8; assert_fused_matches_full(&s, &DevelopOptions::default(), &format!("xtrans {w}x{h}")); } for samples in [1, 3] { - for (w, h) in [(1, 1), (7, 5), (90, 111)] { + for (w, h) in [(1, 1), (7, 5), (90, 111), (600, 350), (600, 50)] { let mut s = test_sensor(w, h, None, samples); s.orientation = 3; assert_fused_matches_full(&s, &DevelopOptions::default(), &format!("linear {samples} {w}x{h}")); @@ -902,6 +907,35 @@ mod tests { assert!(working_memory(11648, 8736, true, true, FUSED, &opts, 1).is_ok()); } + /// The fused bilinear / MHC, X-Trans and LinearRaw paths run fewer bands at once when the bands + /// of every worker don't fit next to the output, as AHD does. A very wide, short Bayer frame + /// (262144 x 455: 15 bands of 32 rows, about 147 MB a band with its window) needs 2.92 GB + /// with all 15 at once against the default 2 GiB, and develops with 9. + #[test] + fn fused_bands_at_once_are_capped_by_the_memory_limit() { + let opts = DevelopOptions { demosaic: Demosaic::Bilinear, ..Default::default() }; + let table = (GAMMA_STEPS as u64 + 2) * 4; + let (w, h) = (262144usize, 455usize); + let band = 32 * 262144 * 12 + (32 + 2 * PAD as u64) * (262144 + 2 * PAD as u64) * 4; + for (workers, want) in [(1, 1), (8, 8), (9, 9), (10, 9), (16, 9), (32, 9)] { + assert_eq!(working_memory(w, h, true, true, FUSED, &opts, workers).unwrap(), want, "{workers} workers"); + } + // Never more jobs than bands, and one when only one band fits. + assert_eq!(working_memory(1000, 40, true, true, FUSED, &opts, 32).unwrap(), 2); + let mut tight = opts.clone(); + tight.limits.max_alloc = 262144 * 455 * 6 + table + band; + assert_eq!(working_memory(w, h, true, true, FUSED, &tight, 32).unwrap(), 1); + tight.limits.max_alloc -= 1; + assert!(working_memory(w, h, true, true, FUSED, &tight, 32).is_err()); + // X-Trans (bands of `band_rows(1000)` = 65 rows next to its planes) and LinearRaw likewise. + let plane = (1000 + 2 * PAD as u64) * (2000 + 2 * PAD as u64) * 4; + let mut x = opts.clone(); + x.limits.max_alloc = 1000 * 2000 * 10 + plane + table + 3 * 65 * 1000 * 12; + assert_eq!(working_memory(1000, 2000, true, false, FUSED, &x, 32).unwrap(), 3); + x.limits.max_alloc = 1000 * 2000 * 6 + 5 * 65 * 1000 * 12; + assert_eq!(working_memory(1000, 2000, false, true, FUSED, &x, 32).unwrap(), 5); + } + #[test] fn working_memory_accounts_for_padding_and_at_least_one_ahd_band() { let mut opts = DevelopOptions { demosaic: Demosaic::Ahd, ..Default::default() }; diff --git a/crates/raw/src/par.rs b/crates/raw/src/par.rs index 6d23fd70c4..d966d94f52 100644 --- a/crates/raw/src/par.rs +++ b/crates/raw/src/par.rs @@ -1,5 +1,7 @@ //! Data-parallel helpers: rayon on native targets, sequential on wasm. +use std::sync::{Mutex, PoisonError}; + /// Calls `f(index, chunk)` for each `chunk`-sized piece of `buf`. pub(crate) fn chunks_mut(buf: &mut [T], chunk: usize, f: impl Fn(usize, &mut [T]) + Sync + Send) { let chunk = chunk.max(1); @@ -12,6 +14,36 @@ pub(crate) fn chunks_mut(buf: &mut [T], chunk: usize, f: impl Fn(usize, buf.chunks_mut(chunk).enumerate().for_each(|(i, c)| f(i, c)); } +/// Like [`chunks_mut`], with `f(state, index, chunk)` borrowing a state: a chunk takes a free one +/// or makes one with `init`, and gives it back when done, so rayon keeps its fine-grained +/// splitting and stealing while at most one state per worker is ever made (`f` must not itself +/// wait on rayon work). Rayon's `for_each_init` would make one per stolen piece, which on an idle +/// pool is often every chunk. A state no chunk still waiting can take is dropped at once, so +/// near the end they go as their workers finish rather than all at the very end. +pub(crate) fn chunks_mut_pooled( + buf: &mut [T], + chunk: usize, + init: impl Fn() -> S + Sync + Send, + f: impl Fn(&mut S, usize, &mut [T]) + Sync + Send, +) { + let chunk = chunk.max(1); + // The free states and the number of chunks that have not taken one yet. + let pool = Mutex::new((Vec::new(), buf.len().div_ceil(chunk))); + chunks_mut(buf, chunk, |i, c| { + let taken = { + let mut pool = pool.lock().unwrap_or_else(PoisonError::into_inner); + pool.1 = pool.1.saturating_sub(1); + pool.0.pop() + }; + let mut state = taken.unwrap_or_else(&init); + f(&mut state, i, c); + let mut pool = pool.lock().unwrap_or_else(PoisonError::into_inner); + if pool.0.len() < pool.1 { + pool.0.push(state); + } + }); +} + /// Native worker count (the browser builds run sequentially). pub(crate) fn workers() -> usize { #[cfg(not(target_arch = "wasm32"))] @@ -56,3 +88,37 @@ pub(crate) fn map(n: usize, f: impl Fn(usize) -> T + Sync + Send) -> Ve pub(crate) fn band_rows(width: usize) -> usize { (65536 / width.max(1)).clamp(1, 256) } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Every chunk is visited once with its own index (the last one short), and the chunks share + /// the states: a 4-worker pool makes at most 4 for 101 chunks, an empty buffer none. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn chunks_mut_pooled_makes_one_state_per_worker_at_most() { + let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap(); + pool.install(|| { + let mut buf = vec![0usize; 1003]; + let inits = AtomicUsize::new(0); + let runs = AtomicUsize::new(0); + let init = || { + inits.fetch_add(1, Ordering::Relaxed); + 0usize + }; + chunks_mut_pooled(&mut buf, 10, init, |used, i, c| { + *used += 1; + runs.fetch_add(1, Ordering::Relaxed); + c.iter_mut().for_each(|v| *v += i + 1); + }); + assert!(buf.iter().enumerate().all(|(k, &v)| v == k / 10 + 1)); + assert_eq!(runs.load(Ordering::Relaxed), 101); + let n = inits.load(Ordering::Relaxed); + assert!((1..=4).contains(&n), "{n} states for 101 chunks on 4 workers"); + chunks_mut_pooled(&mut [0u8; 0], 4, init, |_, _, _| {}); + assert_eq!(inits.load(Ordering::Relaxed), n, "no state for an empty buffer"); + }); + } +}