Files
effectcraft/crates/gpu/src/inpaint.rs
T
JosephandFredApps 6572514eb1 GPU: size the memory cap from the card's video memory on Linux (fixes #1487) (#1706)
On Linux, the discrete-GPU memory cap now comes from the card's video memory: min(3 GiB, ¾ of amdgpu's `mem_info_vram_total`), so a 2 GB RX 550 gets 1.5 GiB. The card is matched by PCI address, falling back to the smallest card with the same ids.

Also folds in CodeRabbit follow-ups from #1856 (PatchMatch flags buffer limit preflight), #1490 (arbitrary-map index wrap), #1836 (round_away for |x| >= 2^23) and #3025 (saturating FrameCache byte accounting).

Fixes #1487

Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com>
2026-11-04 01:55:02 +01:00

765 lines
36 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Content-Aware Fill's per-frame synthesis and flow on the GPU: PatchMatch's multi-scale EM
//! loop, propagation, random search and vote of `effectcraft_raster::inpaint`, and the block
//! matching of `effectcraft_raster::flow::block_flow`, as compute kernels following the CPU's
//! steps (`shaders/inpaint.wgsl`).
//!
//! The fill's input frames are composited on the CPU by design (analyses do not use the job
//! worker's GPU: `effectcraft_engine::background::offload::job_accel`), so this module is about the fill
//! itself — the part that runs for minutes on a large mask, and whose parallel search is what a
//! device wins on. The membrane (harmonic) solve, the flow's hole completion, the orchestration
//! ([`effectcraft_raster::inpaint::fill_sequence_with`]), the flow-guided propagation and the
//! per-frame flow control stay on the CPU: the membrane is bandwidth-bound (see
//! [`FillBackend::patch`]), and the propagation's coupling is sequential, needing every frame of
//! the range resident at once.
//!
//! Parity with the CPU reference is a quality contract, not bit-exactness: the CPU's
//! scan-order propagation becomes a two-pass checkerboard, and its in-place relaxation becomes
//! Jacobi reads. Both solve the same problem to the same quality (`tests/inpaint.rs`).
use effectcraft_raster::Image;
use effectcraft_raster::flow::{Flow, block_flow};
use effectcraft_raster::inpaint::{FillBackend, complete_flow, dilate};
use crate::Gpu;
use crate::context::{Enc, GpuContext, GpuImage, KernelImages, Params};
/// The kernels this module adds to the shared pipeline registry (`context.rs`).
pub(crate) const KERNELS: &[&str] = &[
"imp_down",
"imp_up",
"imp_jacobi",
"imp_pm_down",
"imp_pm_up",
"imp_nnf_up",
"imp_pm_init",
"imp_pm_prop",
"imp_pm_rand",
"imp_nnf_store",
"imp_pm_vote",
"imp_hist",
"imp_fl_half",
"imp_fl_search",
"imp_fl_subpix",
"imp_fl_median",
];
/// Distance histogram bins (`imp_hist`); bin 0's `.g` holds the largest finite distance the
/// dispatch saw, for the HDR re-bin.
const BINS: usize = 1024;
/// The distance range `imp_hist`'s bins cover for SDR input. An image whose channels are all ≤ 1
/// has `Σ_c (a − b)² ≤ 4`, so no patch distance exceeds it and the bins need no rescaling.
const SDR_DISTANCE: f32 = 4.0;
/// `u`/`f` blocks of a kernel's parameter buffer (see `inpaint.wgsl`'s header).
fn params(u0: [u32; 4], u1: [u32; 4], u2: [u32; 4], f0: [f32; 4]) -> Params {
Params { u: [u0, u1, u2, [0; 4]], f: [f0, [0.0; 4], [0.0; 4], [0.0; 4], [0.0; 4], [0.0; 4], [0.0; 4], [0.0; 4], [0.0; 4], [0.0; 4], [0.0; 4], [0.0; 4]] }
}
/// `p` with this level's mask offset in `u[3].x` (`inpaint.wgsl`'s `data`).
fn with_off(p: Params, off: u32) -> Params {
Params { u: [p.u[0], p.u[1], p.u[2], [off, 0, 0, 0]], f: p.f }
}
/// A relaxation sweep count of `membrane_core`. The CPU uses 200 for the small levels and 40 for
/// the large ones; this uses 64 and 40. The small levels are where the difference lies: Jacobi
/// needs O(n²) sweeps to converge on an n-wide grid, so at the levels the coarse-to-fine schedule
/// actually reaches (n ≤ 16, where n² ≤ 256) 64 sweeps is already converged, and finer levels
/// start from a good guess. Matching the CPU's 200 measured ≈ 6% slower on the whole fill with no
/// change in the result to two decimals (67.0 dB either way), so the schedule is deliberately
/// not the CPU's — Jacobi and Gauss–Seidel do not converge alike, and the parity contract is
/// quality, not bit-exactness.
fn sweep_count(w: usize, h: usize) -> usize {
if w * h <= 4096 { 64 } else { 40 }
}
/// PatchMatch's patch radius at a level (`pm_level`).
fn radius(patch: usize, w: u32, h: u32) -> u32 {
(patch / 2).min(w.min(h).saturating_sub(1) as usize / 2).max(1) as u32
}
/// The bounding box of `mask`'s set pixels, `[x0, y0, x1, y1)`, or `None` when it is empty.
fn bbox(mask: &[bool], w: usize, h: usize) -> Option<[u32; 4]> {
let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0usize, 0usize);
for y in 0..h {
for x in 0..w {
if mask.get(y * w + x).copied().unwrap_or(false) {
x0 = x0.min(x);
y0 = y0.min(y);
x1 = x1.max(x + 1);
y1 = y1.max(y + 1);
}
}
}
if x0 == usize::MAX { None } else { Some([x0 as u32, y0 as u32, x1 as u32, y1 as u32]) }
}
/// [`bbox`] grown by `n` pixels, clipped to the image.
fn grow(b: [u32; 4], n: u32, w: u32, h: u32) -> [u32; 4] {
[b[0].saturating_sub(n), b[1].saturating_sub(n), b[2].saturating_add(n).min(w), b[3].saturating_add(n).min(h)]
}
/// The bounding box of the pixels a membrane level still has to solve (`known` false), or
/// `None` when every pixel is known.
fn unknown_bbox(known: &[f32], w: usize, h: usize) -> Option<[u32; 4]> {
let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0usize, 0usize);
for y in 0..h {
for x in 0..w {
if known.get(y * w + x).copied().unwrap_or(1.0) <= 0.5 {
x0 = x0.min(x);
y0 = y0.min(y);
x1 = x1.max(x + 1);
y1 = y1.max(y + 1);
}
}
}
if x0 == usize::MAX { None } else { Some([x0 as u32, y0 as u32, x1 as u32, y1 as u32]) }
}
/// One dispatch's images, its per-pixel `data` table and the region it covers. Grouped into a
/// struct because a kernel's bindings plus its bounds exceed clippy's argument limit, and the
/// workspace denies `too_many_arguments`.
struct Pass<'a> {
src: &'a GpuImage,
aux: Option<&'a GpuImage>,
out: &'a GpuImage,
data: Option<&'a wgpu::Buffer>,
bb: [u32; 4],
}
/// Dispatch a kernel over `pass.bb`. The box's origin is written into `P.u[1].xy` (the kernels
/// place their other `u[1]` fields in `.zw`): deriving it from the box the dispatch is sized from
/// keeps a dispatch's covered region and the origin its shader reads from drifting apart, which
/// is the subtle way an edge of the region goes unwritten.
fn over(e: &mut Enc, entry: &str, p: &Params, pass: Pass<'_>) {
let bb = pass.bb;
let p = Params { u: [p.u[0], [bb[0], bb[1], p.u[1][2], p.u[1][3]], p.u[2], p.u[3]], f: p.f };
let groups = ((bb[2] - bb[0]).div_ceil(16), (bb[3] - bb[1]).div_ceil(16));
e.dispatch(entry, &p, KernelImages { src: pass.src, aux: pass.aux, out: pass.out }, pass.data, groups);
}
// ------------------------------------------------------------------ membrane
/// Relax `fine`'s unknown pixels — the harmonic interpolation of their border — with the CPU's
/// coarse-to-fine schedule (`membrane_core`). `spare` is a free texture of the same size; the
/// result is returned (it may be `fine`, `spare` or one of the coarse levels).
fn membrane_tex(e: &mut Enc, fine: &GpuImage, spare: &GpuImage, hole: &[bool], w: u32, h: u32) -> Option<GpuImage> {
let (uw, uh) = (w as usize, h as usize);
let n = uw * uh;
if hole.len() < n || !hole.iter().take(n).any(|b| *b) || !hole.iter().take(n).any(|b| !*b) {
return Some(fine.clone());
}
// The recursion's levels: halve while the level is wider than 8.
let mut dims = vec![(uw, uh)];
while let Some(&(cw, ch)) = dims.last() {
if !(cw > 8 && ch > 8) || dims.len() >= 32 {
break;
}
dims.push((cw.div_ceil(2), ch.div_ceil(2)));
}
// The known mask of every level: a coarse pixel is known where any of its 2×2 is.
let mut knowns: Vec<Vec<f32>> = Vec::with_capacity(dims.len());
knowns.push(hole.iter().take(n).map(|b| if *b { 0.0 } else { 1.0 }).collect());
for k in 1..dims.len() {
let (pw, ph) = dims[k - 1];
let (cw, ch) = dims[k];
let prev = &knowns[k - 1];
let mut cur = vec![0.0f32; cw * ch];
for y in 0..ch {
for x in 0..cw {
for (dx, dy) in [(0usize, 0usize), (1, 0), (0, 1), (1, 1)] {
let (fx, fy) = (2 * x + dx, 2 * y + dy);
if fx < pw && fy < ph && prev[fy * pw + fx] > 0.5 {
cur[y * cw + x] = 1.0;
break;
}
}
}
}
knowns.push(cur);
}
// One buffer for every level's mask (a buffer per level costs a device allocation each).
let mut packed: Vec<f32> = Vec::with_capacity(knowns.iter().map(|k| k.len()).sum());
let mut offs: Vec<u32> = Vec::with_capacity(knowns.len());
for k in &knowns {
offs.push(packed.len() as u32);
packed.extend_from_slice(k);
}
let datas = e.data(&packed);
let moff = |k: usize| offs.get(k).copied().unwrap_or(0);
// The bounds of each level's unknown pixels: the relaxation only touches them (known pixels
// never move), which keeps a fill's cost proportional to the hole, not the frame.
let mut uns: Vec<Option<[u32; 4]>> = Vec::with_capacity(dims.len());
for (k, &(cw, ch)) in dims.iter().enumerate() {
uns.push(unknown_bbox(knowns.get(k)?, cw, ch));
}
// Two textures per coarse level, for the ping-pong sweeps.
let mut lv: Vec<GpuImage> = Vec::with_capacity(2 * dims.len());
for k in 1..dims.len() {
let (cw, ch) = dims[k];
lv.push(e.scratch(cw as u32, ch as u32));
lv.push(e.scratch(cw as u32, ch as u32));
}
// Fine to coarse.
for k in 1..dims.len() {
let (pw, ph) = dims[k - 1];
let (cw, ch) = dims[k];
let p = with_off(params([pw as u32, ph as u32, cw as u32, ch as u32], [0; 4], [0; 4], [0.0; 4]), moff(k - 1));
let src = if k == 1 { fine } else { lv.get(2 * (k - 2))? };
e.pixels("imp_down", &p, src, None, lv.get(2 * (k - 1))?, Some(&datas));
}
// Relax, coarse to fine. `first[k]` says whether level k's image is the first of its pair
// (level 0's pair is `fine`/`spare`).
let top = dims.len() - 1;
let mut first = vec![true; dims.len()];
if top >= 1 {
let (a, b) = (lv.get(2 * (top - 1))?, lv.get(2 * (top - 1) + 1)?);
// Both textures must hold the level before the sweeps: they now write only the hole.
e.copy_into(a, b, 0, 0);
first[top] = sweeps(e, &datas, moff(top), a, b, Region { w: dims[top].0 as u32, h: dims[top].1 as u32, bb: uns.get(top).copied().flatten() });
}
for k in (1..top).rev() {
let (cw, ch) = dims[k];
let (pw, ph) = dims[k + 1];
let coarse = lv.get(2 * k + usize::from(!first[k + 1]))?;
let (fa, fb) = (lv.get(2 * (k - 1))?, lv.get(2 * (k - 1) + 1)?);
e.copy_into(fa, fb, 0, 0);
if let Some(bb) = uns.get(k).copied().flatten() {
let p = with_off(params([cw as u32, ch as u32, pw as u32, ph as u32], [0; 4], [0; 4], [0.0; 4]), moff(k));
over(e, "imp_up", &p, Pass { src: coarse, aux: None, out: fb, data: Some(&datas), bb });
}
// `imp_up` wrote into `fb`, the pair's second.
first[k] = !sweeps(e, &datas, moff(k), fb, fa, Region { w: cw as u32, h: ch as u32, bb: uns.get(k).copied().flatten() });
}
if top >= 1 {
let coarse = lv.get(usize::from(!first[1]))?;
e.copy_into(fine, spare, 0, 0);
let bb = uns.first().copied().flatten()?;
let p = with_off(params([w, h, dims[1].0 as u32, dims[1].1 as u32], [0; 4], [0; 4], [0.0; 4]), moff(0));
over(e, "imp_up", &p, Pass { src: coarse, aux: None, out: spare, data: Some(&datas), bb });
return Some(if sweeps(e, &datas, moff(0), spare, fine, Region { w, h, bb: Some(bb) }) { spare.clone() } else { fine.clone() });
}
let bb = uns.first().copied().flatten()?;
e.copy_into(fine, spare, 0, 0);
Some(if sweeps(e, &datas, moff(0), fine, spare, Region { w, h, bb: Some(bb) }) { fine.clone() } else { spare.clone() })
}
/// A texture's dimensions and the region a dispatch covers (`None`: nothing to do).
struct Region {
w: u32,
h: u32,
bb: Option<[u32; 4]>,
}
/// Jacobi relaxation sweeps of a texture pair, over the unknown pixels' bounds `region.bb` (the
/// state starts in `a`); returns whether the result is in `a`.
fn sweeps(e: &mut Enc, data: &wgpu::Buffer, off: u32, a: &GpuImage, b: &GpuImage, region: Region) -> bool {
let Some(bb) = region.bb else { return true };
let p = with_off(params([region.w, region.h, 0, 0], [0; 4], [0; 4], [0.0; 4]), off);
let mut in_a = true;
for _ in 0..sweep_count(region.w as usize, region.h as usize) {
let (src, out) = if in_a { (a, b) } else { (b, a) };
over(e, "imp_jacobi", &p, Pass { src, aux: None, out, data: Some(data), bb });
in_a = !in_a;
}
in_a
}
/// `img` with the pixels `mask` marks inside `bb` replaced by `rect` — the tightly packed RGBA f32
/// rows of the rectangle that was read back. Pixels outside the mask stay the input's, bit for bit.
fn patch_back(img: &Image, mask: &[bool], bb: [u32; 4], rect: &[u8]) -> Image {
let (w, rw) = (img.width as usize, (bb[2] - bb[0]) as usize);
let mut out = img.clone();
for y in bb[1]..bb[3] {
for x in bb[0]..bb[2] {
let i = y as usize * w + x as usize;
if !mask.get(i).copied().unwrap_or(false) {
continue;
}
let off = ((y - bb[1]) as usize * rw + (x - bb[0]) as usize) * 16;
let Some(b) = rect.get(off..off + 16) else { continue };
let mut px = [0.0f32; 4];
for (c, v) in px.iter_mut().enumerate() {
*v = f32::from_le_bytes([b[c * 4], b[c * 4 + 1], b[c * 4 + 2], b[c * 4 + 3]]);
}
out.data[i] = px;
}
}
out
}
impl FillBackend for Gpu {
/// PatchMatch synthesis — the fill's dominant cost, and the stage a device wins on: two to
/// three times faster than the CPU's 18-rayon-thread loop at 1920×1080 with a 300×200 hole
/// on an Intel Arc iGPU, and far more on a discrete one.
///
/// The membrane ([`FillBackend::membrane`]) deliberately stays on the CPU: a harmonic solve
/// over a hole is only a few tens of milliseconds, and on the same measurement it is *slower*
/// on the device (≈ 90 ms against ≈ 55 ms). Most of that is the fill's full-frame data
/// movement — uploading the frame, copying it and building the known pyramid — which does not
/// shrink with the hole, while the relaxation itself does. The CPU's in-place Gauss–Seidel
/// over many cores is already competitive there, so the device only takes the stage where its
/// parallel search actually pays. (`membrane_tex` still runs on the device *inside* this
/// stage, where it is the coarsest level's start and never crosses back to the CPU.)
fn patch(&self, img: &Image, hole: &[bool], init: Option<&Image>, patch: usize, seed: u32) -> Image {
// The error scopes make an allocation or validation failure local to this frame: the
// CPU fills it instead, rather than an uncaptured error retiring the shared context.
if self.ctx.can_wait()
&& let Ok(Some(out)) = self.within_memory(|| patch_image(&self.ctx, img, hole, init, patch, seed))
{
return out;
}
effectcraft_raster::inpaint::patch_fill(img, hole, init, patch, seed)
}
/// The block matching that the sequence's flow-guided propagation starts from, which is the
/// flow's dominant cost. The field is a small `cols × rows` grid (8 px tiles, so 240 × 135 at
/// 1080p), and the search is texture-load-bound: at 1920×1080 the CPU's `block_flow` is
/// ≈ 65–70 ms against ≈ 55 ms here, most of it the coarsest level's window (`radius 6` over
/// up to six seeds) and the finest level's 8×8 blocks. That is a smaller margin than
/// [`patch`](Self::patch), but the flow runs twice per frame pair, so on a four-frame 1080p
/// sequence it is worth ≈ 0.15× of the fill's total (1.42× against 1.27× with the CPU flow).
///
/// The field is exhaustive, not heuristic: it matches the CPU's to a float rounding, so the
/// propagation sees the same motion. Both A's block per candidate (hoisted into registers)
/// and the CPU's `Plane::luma` (computed here by [`effectcraft_raster::flow::luma_pair`] and
/// uploaded, because a device may contract `a·b + c` into a multiply-add and round the luma
/// differently) are why it is that close.
///
/// The completion — marking the tiles that touch either hole and interpolating the surrounding
/// motion over them — stays on the CPU ([`complete_flow`]). It is a membrane solve over that
/// same tiny grid whose cost is the mask, not the solve, so there is little for the device to
/// take; keeping it here also keeps the completed field the CPU's to within the search's
/// rounding.
fn flow(&self, a: &Image, b: &Image, hole_a: &[bool], hole_b: &[bool]) -> Flow {
// Frame-local error scopes, as for the patch: a failed GPU search falls back to the CPU.
if self.ctx.can_wait()
&& let Ok(Some(f)) = self.within_memory(|| flow_image(&self.ctx, a, b))
{
return complete_flow(f, hole_a, hole_b, a.width as usize, a.height as usize);
}
complete_flow(block_flow(a, b, 8, 6), hole_a, hole_b, a.width as usize, a.height as usize)
}
}
// ------------------------------------------------------------------ block flow
/// `effectcraft_raster::flow::block_flow(a, b, 8, 6)` on the device: the two frames' luma
/// pyramids, the coarse-to-fine search, the sub-pixel fit and the 3×3 median, returning the raw
/// field ([`complete_flow`] does the hole completion). `None` where the device cannot run it and
/// the caller falls back to the CPU.
pub(crate) fn flow_image(g: &GpuContext, a: &Image, b: &Image) -> Option<Flow> {
let (w, h) = (a.width, a.height);
if w == 0 || h == 0 || w != b.width || h != b.height || !g.fits(w, h) {
return None;
}
let (tile, win, radius) = (8u32, 8u32, 6u32);
let cols = (w as usize).div_ceil(tile as usize).max(1);
let rows = (h as usize).div_ceil(tile as usize).max(1);
// block_flow_window's early out: a frame too small to hold one block and window.
if w < tile || h < tile || w < win || h < win {
return Some(Flow { width: w, height: h, block: tile, cols, rows, v: vec![[0.0; 2]; cols * rows] });
}
let mut e = Enc::new(g);
// The luma pyramid: `Plane::luma` (packed .x/.y), halved while both sides stay worth matching
// (`Plane::half`), at most four levels.
let mut dims = vec![(w, h)];
while dims.len() < 4 {
let (cw, ch) = *dims.last()?;
if cw / 2 < 16 || ch / 2 < 16 {
break;
}
dims.push(((cw / 2).max(1), (ch / 2).max(1)));
}
// The luma is the CPU's (`luma_pair`), so the field matches `block_flow`'s bit for bit; the
// halves below are exact additions and a multiply by a power of two.
let pair = g.upload_image(&effectcraft_raster::flow::luma_pair(a, b))?;
let mut luma: Vec<GpuImage> = Vec::with_capacity(dims.len());
luma.push(pair);
for k in 1..dims.len() {
let (pw, ph) = dims[k - 1];
let (cw, ch) = dims[k];
let dst = e.scratch(cw, ch);
e.pixels("imp_fl_half", &params([pw, ph, cw, ch], [0; 4], [0; 4], [0.0; 4]), luma.get(k - 1)?, None, &dst, None);
luma.push(dst);
}
// The coarse-to-fine search: the coarsest level from the zero vector, each finer level from
// the doubled vectors of its tile and its four neighbours.
let top = dims.len() - 1;
let groups = (cols.div_ceil(16) as u32, rows.div_ceil(16) as u32);
let mut field: Vec<GpuImage> = Vec::with_capacity(dims.len());
for _ in 0..dims.len() {
field.push(e.scratch(cols as u32, rows as u32));
}
for k in (0..dims.len()).rev() {
let (pw, ph) = dims[k];
let coarser = (k != top).then(|| field.get(k + 1)).flatten();
let p = params([pw, ph, cols as u32, rows as u32], [tile, win, 1u32 << k, if k == top { radius } else { 1 }], [u32::from(k != top), 0, 0, 0], [0.0; 4]);
e.dispatch("imp_fl_search", &p, KernelImages { src: luma.get(k)?, aux: coarser, out: field.get(k)? }, None, groups);
}
// The full-resolution sub-pixel fit, then the vector median.
let fine = e.scratch(cols as u32, rows as u32);
e.dispatch(
"imp_fl_subpix",
&params([w, h, cols as u32, rows as u32], [tile, win, 0, 0], [0; 4], [0.0; 4]),
KernelImages { src: luma.first()?, aux: Some(field.first()?), out: &fine },
None,
groups,
);
let out = e.scratch(cols as u32, rows as u32);
e.dispatch(
"imp_fl_median",
&params([w, h, cols as u32, rows as u32], [0; 4], [0; 4], [0.0; 4]),
KernelImages { src: &fine, aux: None, out: &out },
None,
groups,
);
if e.take_missing() {
return None;
}
// Only the field crosses back to the CPU: `cols × rows` entries.
let rect = e.read_rect(&out.texture, 0, 0, cols as u32, rows as u32, 16)?;
let mut v = Vec::with_capacity(cols * rows);
for i in 0..cols * rows {
let b = rect.get(i * 16..i * 16 + 16)?;
let f = |o: usize| b.get(o).copied().unwrap_or(0);
v.push([f32::from_le_bytes([f(0), f(1), f(2), f(3)]), f32::from_le_bytes([f(4), f(5), f(6), f(7)])]);
}
Some(Flow { width: w, height: h, block: tile, cols, rows, v })
}
// ------------------------------------------------------------------ PatchMatch
/// One PatchMatch level's plan.
struct Lv {
w: u32,
h: u32,
r: u32,
/// The target centres: pixels whose patch overlaps the hole (`near`).
near: [u32; 4],
/// The hole's own bounding box (the vote's dispatch).
holes: [u32; 4],
/// Where the nearest-neighbour field must be defined: the vote reads it `r` pixels out.
field: [u32; 4],
flags: wgpu::Buffer,
}
/// The multi-scale PatchMatch synthesis of `patch_fill`: `init` is the starting guess (the
/// previous frame's fill warped onto this one), which refines the finest level only.
pub(crate) fn patch_image(g: &GpuContext, img: &Image, hole: &[bool], init: Option<&Image>, patch: usize, seed: u32) -> Option<Image> {
let (w, h) = (img.width, img.height);
if w == 0 || h == 0 || !g.fits(w, h) {
return None;
}
let (uw, uh) = (w as usize, h as usize);
let n = uw * uh;
if hole.len() < n || !hole.iter().take(n).any(|b| *b) {
return Some(img.clone());
}
let hole: Vec<bool> = hole.iter().take(n).copied().collect();
let patch = patch.max(3) | 1;
let mut e = Enc::new(g);
// A refinement of one frame (`patch_fill`'s `init`): the previous frame's fill seeds the
// hole and only the finest level runs, from 2 EM rounds of 4 iterations.
if let Some(init) = init {
if init.width != w || init.height != h {
return None;
}
let mut start = img.clone();
for (i, v) in start.data.iter_mut().enumerate() {
if hole.get(i).copied().unwrap_or(false) {
*v = *init.data.get(i)?;
}
}
let lv = plan_level(&mut e, &hole, w, h, patch)?;
let image = g.upload_image(&start)?;
let spare = e.scratch(w, h);
let nnf = e.image(w, h);
let (out, _) = pm_level(&mut e, &lv, PmRun { seed, em: 2, iters: 4 }, &image, &spare, &nnf)?;
if e.take_missing() {
return None;
}
let bb = bbox(&hole, uw, uh)?;
let rect = e.read_rect(&out.texture, bb[0], bb[1], bb[2] - bb[0], bb[3] - bb[1], 16)?;
return Some(patch_back(img, &hole, bb, &rect));
}
// The pyramid: halve while the level is still a few patches across.
let mut dims = vec![(w, h)];
let mut holes = vec![hole.clone()];
while dims.len() < 6 {
let &(lw, lh) = dims.last()?;
let hl = holes.last()?;
if lw.min(lh) / 2 < patch as u32 * 4 || !hl.iter().any(|b| *b) {
break;
}
let (cw, ch) = (lw.div_ceil(2) as usize, lh.div_ceil(2) as usize);
let mut nh = vec![false; cw * ch];
for y in 0..ch {
for x in 0..cw {
for (dx, dy) in [(0usize, 0usize), (1, 0), (0, 1), (1, 1)] {
let (fx, fy) = ((2 * x + dx).min(lw as usize - 1), (2 * y + dy).min(lh as usize - 1));
if hl[fy * lw as usize + fx] {
nh[y * cw + x] = true;
break;
}
}
}
}
if !nh.iter().any(|b| !*b) {
break;
}
dims.push((cw as u32, ch as u32));
holes.push(nh);
}
let levels = dims.len();
let top = levels - 1;
let mut plan = Vec::with_capacity(levels);
for k in 0..levels {
plan.push(plan_level(&mut e, holes.get(k)?, dims[k].0, dims[k].1, patch)?);
}
// One image per level (the coarse-to-fine steps and the vote ping-pong in fresh textures).
let mut ima: Vec<GpuImage> = Vec::with_capacity(levels);
ima.push(g.upload_image(img)?);
for k in 1..levels {
let (cw, ch) = dims[k];
ima.push(e.scratch(cw, ch));
}
for k in 1..levels {
let (pw, ph) = dims[k - 1];
let (cw, ch) = dims[k];
e.pixels("imp_pm_down", &params([pw, ph, cw, ch], [0; 4], [0; 4], [0.0; 4]), &ima[k - 1], None, &ima[k], Some(&plan[k - 1].flags));
}
// The coarsest level starts as its membrane fill.
let (cw, ch) = dims[top];
let spare = e.scratch(cw, ch);
ima[top] = membrane_tex(&mut e, &ima[top], &spare, &holes[top], cw, ch)?;
// Coarse to fine: upsample, then the EM rounds of propagation and voting.
let mut nnf: Option<GpuImage> = None;
for k in (0..levels).rev() {
let lv = &plan[k];
nnf = Some(match nnf {
Some(coarse) => {
let (cw, ch) = dims[k + 1];
let fine = e.scratch(lv.w, lv.h);
e.pixels("imp_nnf_up", &params([lv.w, lv.h, cw, ch], [0; 4], [0; 4], [0.0; 4]), &coarse, None, &fine, None);
fine
}
None => e.image(lv.w, lv.h),
});
if k < top {
let (cw, ch) = dims[k + 1];
let dst = e.scratch(lv.w, lv.h);
e.copy_into(&ima[k], &dst, 0, 0);
e.pixels("imp_pm_up", &params([lv.w, lv.h, cw, ch], [0; 4], [0; 4], [0.0; 4]), &ima[k + 1], None, &dst, Some(&lv.flags));
ima[k] = dst;
}
let (em, iters) = if k == top { (8, 5) } else { (4, 5) };
let spare = e.scratch(lv.w, lv.h);
let run = PmRun { seed: seed.wrapping_add(k as u32 * 101), em, iters };
let (out, out_nnf) = pm_level(&mut e, lv, run, &ima[k], &spare, nnf.as_ref()?)?;
ima[k] = out;
nnf = Some(out_nnf);
}
if e.take_missing() {
return None;
}
// Only the hole changes, so only the hole's bounds cross back to the CPU.
let bb = bbox(&hole, uw, uh)?;
let rect = e.read_rect(&ima[0].texture, bb[0], bb[1], bb[2] - bb[0], bb[3] - bb[1], 16)?;
Some(patch_back(img, &hole, bb, &rect))
}
/// The plan of one level: its flags (`bit 0` hole, `bit 1` a target centre) and the dispatch
/// bounds of its targets, holes and field.
fn plan_level(e: &mut Enc, hole: &[bool], w: u32, h: u32, patch: usize) -> Option<Lv> {
// Texture and buffer limits are independent: a level that fits as a texture can still need a
// flags buffer over the device's storage binding limit, and that validation error would
// retire the context. Decline the GPU path instead (the fill runs on the CPU).
if !flags_fit(&e.g.device.limits(), w, h) {
return None;
}
let (iw, ih) = (w as usize, h as usize);
let r = radius(patch, w, h);
let near = dilate(hole, iw, ih, r as usize);
let near_bb = bbox(&near, iw, ih)?;
let flags: Vec<f32> = (0..iw * ih).map(|i| (u32::from(hole[i]) + 2 * u32::from(near[i])) as f32).collect();
Some(Lv { w, h, r, near: near_bb, holes: bbox(hole, iw, ih)?, field: grow(near_bb, r + 1, w, h), flags: e.data(&flags) })
}
/// Whether a level's `w × h` f32 flags buffer (at least the 16 bytes [`Enc::data`] allocates)
/// can be created and bound as one storage buffer on a device with `limits`.
fn flags_fit(limits: &wgpu::Limits, w: u32, h: u32) -> bool {
u64::from(w)
.checked_mul(u64::from(h))
.and_then(|n| n.checked_mul(4))
.map(|b| b.max(16))
.is_some_and(|b| b <= limits.max_buffer_size && b <= limits.max_storage_buffer_binding_size)
}
/// The PatchMatch run at one level: the random stream's seed and the EM round and iteration
/// counts.
struct PmRun {
seed: u32,
em: usize,
iters: usize,
}
/// `pm_level`: `run.em` EM rounds of propagation and voting at one level. `img` holds the level's
/// image and `img_spare` is a free texture of the same size; the nearest-neighbour field starts
/// at `nnf`. Returns the level's filled image and its field.
fn pm_level(e: &mut Enc, lv: &Lv, run: PmRun, img: &GpuImage, img_spare: &GpuImage, nnf: &GpuImage) -> Option<(GpuImage, GpuImage)> {
let (seed, em, iters) = (run.seed, run.em, run.iters);
let (r, w, h) = (lv.r, lv.w, lv.h);
let field_a = e.scratch(w, h);
let field_b = e.scratch(w, h);
let hist = e.scratch(BINS as u32, 1);
let (img_a, img_b) = (img.clone(), img_spare.clone());
let (mut field_nnf, mut nnf_spare) = (nnf.clone(), e.scratch(w, h));
// The propagation and random search write the targets and the one-pixel ring around them, so
// every neighbour they read has a defined alpha (`grow(near, 1)`). The dispatch origin must
// be the ring's, not the targets': `over` sizes the dispatch from the box it is given.
let ring = grow(lv.near, 1, w, h);
let mut in_a = true;
for round in 0..em {
// The round's initial field: the inherited source where it is still valid, else random.
// Every pixel of the field's bounds is written, so the previous round's field is free.
let img = if in_a { &img_a } else { &img_b };
let p = params([w, h, r, 0], [0; 4], [seed, 0, 0, round as u32], [0.0; 4]);
over(e, "imp_pm_init", &p, Pass { src: &field_nnf, aux: Some(img), out: &field_a, data: Some(&lv.flags), bb: lv.field });
let mut f_in_a = true;
for it in 0..iters {
for parity in 0..2u32 {
let (s, o) = if f_in_a { (&field_a, &field_b) } else { (&field_b, &field_a) };
let p = params([w, h, r, parity], [0; 4], [seed, it as u32, 0, round as u32], [0.0; 4]);
over(e, "imp_pm_prop", &p, Pass { src: s, aux: Some(img), out: o, data: Some(&lv.flags), bb: ring });
f_in_a = !f_in_a;
}
let (s, o) = if f_in_a { (&field_a, &field_b) } else { (&field_b, &field_a) };
let p = params([w, h, r, 0], [0; 4], [seed, it as u32, 0, round as u32], [0.0; 4]);
over(e, "imp_pm_rand", &p, Pass { src: s, aux: Some(img), out: o, data: Some(&lv.flags), bb: ring });
f_in_a = !f_in_a;
}
let field = if f_in_a { &field_a } else { &field_b };
// The vote's sigma²: the 75th percentile of the targets' match distances (`pm_level`).
let sigma2 = sigma2(e, field, lv, &hist)?;
let next_img = if in_a { &img_b } else { &img_a };
e.copy_into(img, next_img, 0, 0);
let p = params([w, h, r, 0], [0; 4], [0; 4], [sigma2, 0.0, 0.0, 0.0]);
over(e, "imp_pm_vote", &p, Pass { src: field, aux: Some(img), out: next_img, data: Some(&lv.flags), bb: lv.holes });
// Carry the round's sources into the nearest-neighbour field (targets only).
e.pixels("imp_nnf_store", &params([w, h, 0, 0], [0; 4], [0; 4], [0.0; 4]), field, Some(&field_nnf), &nnf_spare, Some(&lv.flags));
std::mem::swap(&mut field_nnf, &mut nnf_spare);
in_a = !in_a;
}
Some((if in_a { img_a } else { img_b }, field_nnf))
}
/// The 75th percentile of a 1024-bin distance histogram (`imp_hist`'s readback) covering
/// `0..range`, and the histogram's total. The bin's centre is the estimate: within half a bin
/// (`range/2048`) of the CPU's exact percentile.
fn percentile(bytes: &[u8], range: f32) -> (f32, f64) {
let bin = |i: usize| -> f64 {
match bytes.get(i * 16..i * 16 + 4) {
Some(b) => f32::from_le_bytes([b[0], b[1], b[2], b[3]]) as f64,
None => 0.0,
}
};
let total: f64 = (0..BINS).map(bin).sum();
if total < 1.0 {
return (1.0, total);
}
let rank = total * 3.0 / 4.0;
let mut cum = 0.0f64;
let mut at = BINS - 1;
for i in 0..BINS {
cum += bin(i);
if cum > rank {
at = i;
break;
}
}
let centre = ((at as f32) + 0.5) * range / BINS as f32;
(centre.max(1e-5), total)
}
/// The largest finite distance `imp_hist` saw, from bin 0's `.g` channel.
fn max_distance(bytes: &[u8]) -> f32 {
match bytes.get(4..8) {
Some(b) => f32::from_le_bytes([b[0], b[1], b[2], b[3]]),
None => 0.0,
}
}
/// `imp_hist` over `bb`, the targets' bounds: the 75th percentile of the finite distances and the
/// sample count. SDR input is binned over the fixed [`SDR_DISTANCE`] range; a frame with values
/// above 1 (32-bpc HDR) has larger distances, so the second dispatch bins over the observed
/// maximum instead. Clamping those to the SDR range would report a sigma² of ≈ 4 for a distance
/// of 1600, and `imp_pm_vote`'s weights would round to zero and keep the prior image.
///
/// One 16 KB readback per EM round (≈ 28 per 1080p patch) on the SDR path: a few percent of the
/// stage, and the obvious next thing to remove if the patch has to be faster (a device-side
/// percentile would need a reduction instead). HDR frames pay one more dispatch and readback.
fn histogram(e: &mut Enc, field: &GpuImage, flags: &wgpu::Buffer, w: u32, h: u32, bb: [u32; 4], hist: &GpuImage) -> Option<(f32, f64)> {
let (bw, bh) = (bb[2] - bb[0], bb[3] - bb[1]);
let mut p = params([w, h, 0, 0], [bb[0], bb[1], bw, bh], [0; 4], [0.0; 4]);
e.dispatch("imp_hist", &p, KernelImages { src: field, aux: None, out: hist }, Some(flags), (1, 1));
let bytes = e.read_texture(&hist.texture, BINS as u32, 1, 16)?;
let max = max_distance(&bytes);
if max <= SDR_DISTANCE {
return Some(percentile(&bytes, SDR_DISTANCE));
}
// The observed maximum itself has to be in the last bin, so widen the range by a hair rather
// than binning it to 1024.
let range = max * (1.0 + 1e-6);
p.f[0] = [range, 0.0, 0.0, 0.0];
e.dispatch("imp_hist", &p, KernelImages { src: field, aux: None, out: hist }, Some(flags), (1, 1));
let bytes = e.read_texture(&hist.texture, BINS as u32, 1, 16)?;
Some(percentile(&bytes, range))
}
/// The vote's sigma² at a level (`pm_level`): the 75th percentile of the targets' distances.
fn sigma2(e: &mut Enc, field: &GpuImage, lv: &Lv, hist: &GpuImage) -> Option<f32> {
Some(histogram(e, field, &lv.flags, lv.w, lv.h, lv.near, hist)?.0)
}
/// [`imp_hist`] on its own: `field`'s match distance (`.b`, `.a` > 0.5 where set) over the
/// pixels `near` marks, returning the 75th percentile and the sample count. Used by the test that
/// checks the histogram counts the whole target population.
#[cfg(test)]
pub(crate) fn distance_histogram(g: &GpuContext, field: &Image, near: &[bool], w: u32, h: u32) -> Option<(f32, f64)> {
let flags: Vec<f32> = near.iter().map(|b| if *b { 2.0 } else { 0.0 }).collect();
let mut e = Enc::new(g);
let data = e.data(&flags);
let img = g.upload_image(field)?;
let hist = e.scratch(BINS as u32, 1);
let out = histogram(&mut e, &img, &data, w, h, [0, 0, w, h], &hist);
if e.take_missing() {
return None;
}
out
}
#[cfg(test)]
mod limit_tests {
use super::flags_fit;
/// A level whose flags buffer is over the storage binding limit is declined before it is
/// created (the default 128 MiB binding holds 33,554,432 flags: 8192 × 4096).
#[test]
fn flags_buffer_is_checked_against_the_device_limits() {
let l = wgpu::Limits::default();
assert!(flags_fit(&l, 1, 1));
assert!(flags_fit(&l, 8192, 4096));
assert!(!flags_fit(&l, 8192, 4097));
assert!(!flags_fit(&l, 16384, 16384));
let small = wgpu::Limits { max_buffer_size: 1024, ..wgpu::Limits::default() };
assert!(flags_fit(&small, 16, 16));
assert!(!flags_fit(&small, 16, 17));
}
}