Merge PR #926
# Conflicts: # Cargo.lock # Cargo.toml # crates/ui-egui/locales/de.json # crates/ui-egui/locales/es.json # crates/ui-egui/locales/fr.json # crates/ui-egui/locales/ja.json # crates/ui-egui/locales/pt-br.json # crates/ui-egui/locales/ru.json # crates/ui-egui/locales/uk.json # crates/ui-egui/locales/zh-hans.json # crates/ui-egui/locales/zh-hant.json # xtask/src/layers.rs
This commit is contained in:
24 files changed
+1009
-14
No files matched your search
Generated
+8
@@ -3302,6 +3302,13 @@ dependencies = [
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-inpaint"
|
||||
version = "0.6.0"
|
||||
dependencies = [
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-lensdb"
|
||||
version = "0.6.0"
|
||||
@@ -3358,6 +3365,7 @@ dependencies = [
|
||||
"lightcraft-color",
|
||||
"lightcraft-develop",
|
||||
"lightcraft-geom",
|
||||
"lightcraft-inpaint",
|
||||
"lightcraft-raster",
|
||||
"lightcraft-raw",
|
||||
"lightcraft-scenes",
|
||||
|
||||
@@ -25,6 +25,7 @@ lightcraft-codecs = { path = "crates/codecs" }
|
||||
lightcraft-meta = { path = "crates/meta" }
|
||||
lightcraft-develop = { path = "crates/develop" }
|
||||
lightcraft-lensdb = { path = "crates/lensdb" }
|
||||
lightcraft-inpaint = { path = "crates/inpaint" }
|
||||
lightcraft-scenes = { path = "crates/scenes" }
|
||||
lightcraft-pipeline = { path = "crates/pipeline" }
|
||||
lightcraft-gpu = { path = "crates/gpu" }
|
||||
|
||||
@@ -22,6 +22,12 @@ Foundation project licensed under the Apache License 2.0, and from the CIE 1931
|
||||
functions it ships (CIE, CC-BY-4.0). The matrices were fitted by the LightCraft contributors
|
||||
(2026) with tools/spectral_camera_table.py. See assets/ATTRIBUTION.md.
|
||||
|
||||
crates/inpaint (lightcraft-inpaint) ports the inpainting (PatchMatch completion) and Poisson membrane
|
||||
fill code of PhotoCraft (https://github.com/storytold/photocraft, crates/algo/src/inpaint.rs and
|
||||
poisson.rs), Copyright the PhotoCraft contributors, licensed under the MIT License or the Apache
|
||||
License 2.0, at your option. The port was modified by the LightCraft contributors (2026); each
|
||||
ported file says so.
|
||||
|
||||
crates/segment (lightcraft-segment) ports the SAM 3 model code of Hugging Face Transformers
|
||||
(https://github.com/huggingface/transformers, src/transformers/models/sam3 and sam3_tracker),
|
||||
Copyright The HuggingFace Team and Meta Platforms, Inc., licensed under the Apache License 2.0.
|
||||
|
||||
@@ -963,6 +963,8 @@ pub enum SpotMode {
|
||||
Remove,
|
||||
Heal,
|
||||
Clone,
|
||||
/// Content-aware: the spot is rebuilt from patches of its surroundings (no single source).
|
||||
Fill,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
|
||||
@@ -752,7 +752,7 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
"Add Remove Spot",
|
||||
[],
|
||||
None,
|
||||
"{mode?: remove|heal|clone, points: [[x,y],…], size?: fraction of long edge, feather?, opacity?, source?: [dx,dy] (default: the best match nearby)} — selects the new spot; returns {index}",
|
||||
"{mode?: remove|heal|clone|fill (content-aware), points: [[x,y],…], size?: fraction of long edge, feather?, opacity?, source?: [dx,dy] (default: the best match nearby)} — selects the new spot; returns {index}",
|
||||
has_active,
|
||||
|s, p| {
|
||||
let mode: SpotMode =
|
||||
@@ -776,7 +776,7 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
};
|
||||
let id = s.active().ok_or_else(|| bad("spot.add", "no active photo"))?;
|
||||
let mut dd = (*s.develop_of(id).unwrap_or_default()).clone();
|
||||
if spot.source_offset.is_none() {
|
||||
if spot.source_offset.is_none() && spot.mode != SpotMode::Fill {
|
||||
// resolve the automatic source now: it gets a pin and stays put at every size
|
||||
spot.source_offset = pick_source(s, id, &dd, &spot, None);
|
||||
}
|
||||
|
||||
@@ -55,3 +55,13 @@ fn spots_select_move_edit_refresh_and_delete() {
|
||||
s.execute("library.next", &json!({})).unwrap();
|
||||
assert_eq!(s.active_spot, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_content_aware_spot_has_no_source() {
|
||||
let mut s = Session::with_demo();
|
||||
s.execute("spot.add", &json!({"mode": "fill", "points": [[0.4, 0.5]], "size": 0.03})).unwrap();
|
||||
let sp = &spots(&s)[0];
|
||||
assert_eq!((sp.mode, sp.source_offset), (lightcraft_develop::SpotMode::Fill, None));
|
||||
assert_eq!(serde_json::to_value(sp).unwrap()["mode"], "fill");
|
||||
assert!(s.execute("spot.update", &json!({"mode": "nonsense"})).is_err());
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "lightcraft-inpaint"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "Content-aware hole filling: PatchMatch-based patch completion and membrane fill (L1). Ported from PhotoCraft"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
rayon = { workspace = true }
|
||||
@@ -0,0 +1,61 @@
|
||||
//! Content-aware hole filling (inpainting) for the Remove tool.
|
||||
//!
|
||||
//! * [`complete`]: multi-scale patch-based completion (Wexler, Shechtman, Irani 2007) with the
|
||||
//! nearest-neighbour field found by PatchMatch (Barnes et al. 2009). See [`patchmatch`].
|
||||
//! * [`membrane_fill`]: smooth harmonic fill, used to seed the coarsest level and as a fallback.
|
||||
//!
|
||||
//! Buffers are interleaved `w × h × ch` floats in any colour model plus a `w × h` hole mask.
|
||||
//! Everything is deterministic (fixed-seed RNG, results independent of the thread count) and never
|
||||
//! panics on mismatched sizes: such calls return `None` or the input unchanged.
|
||||
//!
|
||||
//! Sources: C. Barnes, E. Shechtman, A. Finkelstein, D. B. Goldman, *PatchMatch: A Randomized
|
||||
//! Correspondence Algorithm for Structural Image Editing*, SIGGRAPH 2009; Y. Wexler, E. Shechtman,
|
||||
//! M. Irani, *Space-Time Completion of Video*, PAMI 2007.
|
||||
//!
|
||||
//! Ported from PhotoCraft (`crates/algo/src/inpaint.rs`, `poisson.rs`, MIT OR Apache-2.0, same
|
||||
//! organisation; see `NOTICE`).
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
pub mod membrane;
|
||||
pub mod patchmatch;
|
||||
|
||||
pub use membrane::membrane_fill;
|
||||
pub use patchmatch::{CompleteParams, complete, complete_with};
|
||||
|
||||
/// A cancelled [`complete_with`].
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Cancelled;
|
||||
|
||||
impl std::fmt::Display for Cancelled {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Cancelled {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn hostile_sizes_do_not_panic() {
|
||||
assert_eq!(complete(4, 4, 3, &[0.0; 5], &[true; 16], &CompleteParams::default()), None);
|
||||
assert_eq!(complete(4, 4, 0, &[], &[true; 16], &CompleteParams::default()), None);
|
||||
assert_eq!(complete(usize::MAX, 2, 3, &[0.0; 6], &[true; 2], &CompleteParams::default()), None);
|
||||
assert_eq!(membrane_fill(4, 4, 1, &[0.5; 3], &[true; 16]), vec![0.5; 3]);
|
||||
// A hole covering everything: nothing to copy from.
|
||||
assert_eq!(complete(16, 16, 1, &[0.5; 256], &[true; 256], &CompleteParams::default()), None);
|
||||
// A tiny image smaller than a patch.
|
||||
let _ = complete(2, 2, 1, &[0.5; 4], &[true, false, false, false], &CompleteParams::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_stops() {
|
||||
let img = vec![0.5f32; 64 * 64];
|
||||
let hole: Vec<bool> = (0..64 * 64usize).map(|i| (i % 64).abs_diff(32) < 4 && (i / 64).abs_diff(32) < 4).collect();
|
||||
assert_eq!(complete_with(64, 64, 1, &img, &hole, &CompleteParams::default(), &|| true), Err(Cancelled));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
//! Membrane (harmonic) fill: `Δv = 0` inside a hole with the known pixels as Dirichlet boundary.
|
||||
//! Used to initialise the coarsest level of [`crate::complete`] and as its fallback.
|
||||
//!
|
||||
//! Sources: the membrane interpolant of P. Pérez, M. Gangnet, A. Blake, *Poisson Image Editing*,
|
||||
//! ACM SIGGRAPH 2003 (§3), solved by successive over-relaxation on the 4-connected grid, accelerated
|
||||
//! cascadically (solve on a half-resolution grid, prolong bilinearly, a few SOR sweeps per level).
|
||||
//! Ported from PhotoCraft's `algo::poisson` (MIT OR Apache-2.0).
|
||||
|
||||
/// Convergence threshold (max per-sweep update) for the SOR sweeps.
|
||||
const TOL: f32 = 2e-5;
|
||||
/// SOR sweeps (cap) and relaxation factor on every level above the coarsest: tuned so a 96×72 disc
|
||||
/// is within ~1e-3 of a fully converged Gauss–Seidel solve (below one 8-bit step).
|
||||
const FINE_ITERS: usize = 30;
|
||||
const FINE_OMEGA: f32 = 1.7;
|
||||
|
||||
/// Solve `Δv = 0` on `unknown` pixels with Dirichlet values from the known pixels of `v`
|
||||
/// (out-of-grid neighbours are ignored, i.e. a Neumann boundary). `v` holds the known values on
|
||||
/// entry and the solution on exit. Cascadic multigrid + SOR.
|
||||
pub(crate) fn solve_membrane(w: usize, h: usize, unknown: &[bool], v: &mut [f32]) {
|
||||
if unknown.len() != w * h || v.len() != w * h {
|
||||
return;
|
||||
}
|
||||
solve_rec(w, h, unknown, v, 0, &|| false);
|
||||
}
|
||||
|
||||
fn solve_rec(w: usize, h: usize, unknown: &[bool], v: &mut [f32], depth: usize, stop: &(dyn Fn() -> bool + Sync)) {
|
||||
if stop() {
|
||||
return;
|
||||
}
|
||||
let n_unknown = unknown.iter().filter(|u| **u).count();
|
||||
if n_unknown == 0 {
|
||||
return;
|
||||
}
|
||||
let known: Vec<f32> = v.iter().zip(unknown).filter(|(_, u)| !**u).map(|(x, _)| *x).collect();
|
||||
if known.is_empty() {
|
||||
return; // nothing to interpolate from
|
||||
}
|
||||
if w >= 8 && h >= 8 && n_unknown > 64 && depth < 16 {
|
||||
// Restrict: a coarse pixel is known if any of its children is (value = mean of known children).
|
||||
let (cw, chh) = (w.div_ceil(2), h.div_ceil(2));
|
||||
let mut cu = vec![true; cw * chh];
|
||||
let mut cv = vec![0.0f32; cw * chh];
|
||||
for cy in 0..chh {
|
||||
for cx in 0..cw {
|
||||
let (mut s, mut n) = (0.0f32, 0u32);
|
||||
for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] {
|
||||
let (x, y) = (cx * 2 + dx, cy * 2 + dy);
|
||||
if x < w && y < h && !unknown[y * w + x] {
|
||||
s += v[y * w + x];
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
if n > 0 {
|
||||
cu[cy * cw + cx] = false;
|
||||
cv[cy * cw + cx] = s / n as f32;
|
||||
}
|
||||
}
|
||||
}
|
||||
solve_rec(cw, chh, &cu, &mut cv, depth + 1, stop);
|
||||
// Prolong: bilinear sample of the coarse solution as the initial guess.
|
||||
for y in 0..h {
|
||||
let fy = ((y as f32 + 0.5) / 2.0 - 0.5).clamp(0.0, (chh - 1) as f32);
|
||||
let y0 = fy.floor() as usize;
|
||||
let y1 = (y0 + 1).min(chh - 1);
|
||||
let ty = fy - y0 as f32;
|
||||
for x in 0..w {
|
||||
if !unknown[y * w + x] {
|
||||
continue;
|
||||
}
|
||||
let fx = ((x as f32 + 0.5) / 2.0 - 0.5).clamp(0.0, (cw - 1) as f32);
|
||||
let x0 = fx.floor() as usize;
|
||||
let x1 = (x0 + 1).min(cw - 1);
|
||||
let tx = fx - x0 as f32;
|
||||
let a = cv[y0 * cw + x0] + (cv[y0 * cw + x1] - cv[y0 * cw + x0]) * tx;
|
||||
let b = cv[y1 * cw + x0] + (cv[y1 * cw + x1] - cv[y1 * cw + x0]) * tx;
|
||||
v[y * w + x] = a + (b - a) * ty;
|
||||
}
|
||||
}
|
||||
sor(w, h, unknown, v, FINE_ITERS, FINE_OMEGA, stop);
|
||||
} else {
|
||||
let mean = known.iter().sum::<f32>() / known.len() as f32;
|
||||
for (x, u) in v.iter_mut().zip(unknown) {
|
||||
if *u {
|
||||
*x = mean;
|
||||
}
|
||||
}
|
||||
sor(w, h, unknown, v, 2000, 1.85, stop);
|
||||
}
|
||||
}
|
||||
|
||||
/// SOR sweeps over the unknown pixels until the largest update drops below [`TOL`].
|
||||
fn sor(w: usize, h: usize, unknown: &[bool], v: &mut [f32], max_iter: usize, omega: f32, stop: &(dyn Fn() -> bool + Sync)) {
|
||||
// Precompute the unknown pixels and which neighbours exist.
|
||||
let cells: Vec<(u32, u8)> = (0..w * h)
|
||||
.filter(|&i| unknown[i])
|
||||
.map(|i| {
|
||||
let (x, y) = (i % w, i / w);
|
||||
let m = u8::from(x > 0) | (u8::from(x + 1 < w) << 1) | (u8::from(y > 0) << 2) | (u8::from(y + 1 < h) << 3);
|
||||
(i as u32, m)
|
||||
})
|
||||
.collect();
|
||||
for _ in 0..max_iter {
|
||||
if stop() {
|
||||
return;
|
||||
}
|
||||
let mut max_d = 0.0f32;
|
||||
for &(i, m) in &cells {
|
||||
let i = i as usize;
|
||||
let (s, n) = if m == 0b1111 {
|
||||
(v[i - 1] + v[i + 1] + v[i - w] + v[i + w], 4.0)
|
||||
} else {
|
||||
let mut s = 0.0;
|
||||
let mut n = 0.0;
|
||||
if m & 1 != 0 {
|
||||
s += v[i - 1];
|
||||
n += 1.0;
|
||||
}
|
||||
if m & 2 != 0 {
|
||||
s += v[i + 1];
|
||||
n += 1.0;
|
||||
}
|
||||
if m & 4 != 0 {
|
||||
s += v[i - w];
|
||||
n += 1.0;
|
||||
}
|
||||
if m & 8 != 0 {
|
||||
s += v[i + w];
|
||||
n += 1.0;
|
||||
}
|
||||
if n == 0.0 {
|
||||
continue;
|
||||
}
|
||||
(s, n)
|
||||
};
|
||||
let d = omega * (s / n - v[i]);
|
||||
v[i] += d;
|
||||
max_d = max_d.max(d.abs());
|
||||
}
|
||||
if max_d < TOL {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Map `f` over the channels of an interleaved buffer as planar slices (parallel on native).
|
||||
fn per_channel(w: usize, h: usize, ch: usize, f: impl Fn(usize) -> Vec<f32> + Sync + Send) -> Vec<f32> {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let planes: Vec<Vec<f32>> = if w * h >= 64 * 64 {
|
||||
use rayon::prelude::*;
|
||||
(0..ch).into_par_iter().map(&f).collect()
|
||||
} else {
|
||||
(0..ch).map(&f).collect()
|
||||
};
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
let planes: Vec<Vec<f32>> = (0..ch).map(&f).collect();
|
||||
let mut out = vec![0.0f32; w * h * ch];
|
||||
for (c, p) in planes.iter().enumerate() {
|
||||
for (i, v) in p.iter().enumerate() {
|
||||
out[i * ch + c] = *v;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Membrane fill (no source texture): harmonic interpolation of the known pixels into `hole`.
|
||||
/// `img` is interleaved `w × h × ch`, `hole` has `w × h` entries; on a size mismatch `img` comes back
|
||||
/// unchanged.
|
||||
pub fn membrane_fill(w: usize, h: usize, ch: usize, img: &[f32], hole: &[bool]) -> Vec<f32> {
|
||||
if Some(hole.len()) != w.checked_mul(h) || Some(img.len()) != hole.len().checked_mul(ch) {
|
||||
return img.to_vec();
|
||||
}
|
||||
per_channel(w, h, ch, |c| {
|
||||
let mut v: Vec<f32> = (0..w * h).map(|i| if hole[i] { 0.0 } else { img[i * ch + c] }).collect();
|
||||
solve_membrane(w, h, hole, &mut v);
|
||||
v
|
||||
})
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn disc(w: usize, h: usize, cx: f32, cy: f32, r: f32) -> Vec<bool> {
|
||||
(0..w * h).map(|i| ((i % w) as f32 - cx).hypot((i / w) as f32 - cy) < r).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn membrane_reproduces_linear_boundary() {
|
||||
// A linear function is harmonic: the solve must reproduce it exactly inside the hole.
|
||||
let (w, h) = (80, 60);
|
||||
let unknown = disc(w, h, 40.0, 30.0, 20.0);
|
||||
let truth: Vec<f32> = (0..w * h).map(|i| 0.2 + 0.005 * (i % w) as f32 + 0.003 * (i / w) as f32).collect();
|
||||
let mut v: Vec<f32> = truth.iter().zip(&unknown).map(|(t, u)| if *u { 0.0 } else { *t }).collect();
|
||||
solve_membrane(w, h, &unknown, &mut v);
|
||||
let err = v.iter().zip(&truth).map(|(a, b)| (a - b).abs()).fold(0.0, f32::max);
|
||||
assert!(err < 2e-3, "{err}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,587 @@
|
||||
//! Hole filling for Spot Healing: patch-based completion, proximity match and texture synthesis.
|
||||
//!
|
||||
//! * [`complete`], content-aware fill: the multi-scale "space-time completion" EM scheme of
|
||||
//! Y. Wexler, E. Shechtman, M. Irani, *Space-Time Completion of Video*, IEEE TPAMI 2007, with the
|
||||
//! nearest-neighbour field computed by C. Barnes, E. Shechtman, A. Finkelstein, D. B. Goldman,
|
||||
//! *PatchMatch: A Randomized Correspondence Algorithm for Structural Image Editing*, SIGGRAPH 2009
|
||||
//! (random init, propagation, exponentially shrinking random search). Each EM iteration finds, for
|
||||
//! every patch overlapping the hole, its most similar fully-known patch, then re-estimates hole
|
||||
//! pixels by a similarity-weighted vote of the overlapping matches. Coarse-to-fine over an image
|
||||
//! pyramid (the coarsest level is initialised by membrane interpolation), so large holes stay cheap.
|
||||
//! * [`best_offset`], proximity match: the single displacement whose surrounding ring best matches the
|
||||
//! ring around the hole (sum of squared differences), i.e. an automatic Healing Brush source.
|
||||
//! * [`synthesize`], create texture: patch-based texture synthesis in the style of A. Efros,
|
||||
//! W. Freeman, *Image Quilting for Texture Synthesis and Transfer*, SIGGRAPH 2001 (blocks picked from
|
||||
//! random candidates by overlap error, feathered overlap instead of a min-cut seam).
|
||||
//!
|
||||
//! Ported from PhotoCraft's `algo::inpaint` (MIT OR Apache-2.0); its proximity match and texture
|
||||
//! synthesis are left out.
|
||||
//!
|
||||
//! All functions take interleaved `w × h × ch` normalised floats (any colour model/depth) and a
|
||||
//! `w × h` hole mask, and return a full buffer equal to the input outside the hole. They are
|
||||
//! deterministic (fixed-seed hash RNG, results independent of thread count).
|
||||
|
||||
use crate::Cancelled;
|
||||
use crate::membrane::membrane_fill;
|
||||
|
||||
/// Below this many grid cells a level runs single-threaded (thread hand-off costs more than it saves).
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
const PAR_MIN: usize = 128 * 128;
|
||||
|
||||
/// SplitMix64: tiny deterministic RNG.
|
||||
#[derive(Clone)]
|
||||
struct Rng(u64);
|
||||
impl Rng {
|
||||
fn new(seed: u64) -> Self {
|
||||
Rng(seed ^ 0x9E37_79B9_7F4A_7C15)
|
||||
}
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = self.0;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
/// Uniform integer in `lo..=hi`.
|
||||
fn range(&mut self, lo: i32, hi: i32) -> i32 {
|
||||
if hi <= lo {
|
||||
return lo;
|
||||
}
|
||||
lo + (self.next() % (hi - lo + 1) as u64) as i32
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters for [`complete`].
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct CompleteParams {
|
||||
/// Patch radius (patches are `2r+1` square).
|
||||
pub patch_radius: usize,
|
||||
/// EM iterations at the coarsest level (finer levels use fewer, down to `min_em_iters`).
|
||||
pub em_iters: usize,
|
||||
pub min_em_iters: usize,
|
||||
/// PatchMatch passes per EM iteration.
|
||||
pub pm_iters: usize,
|
||||
/// Levels where the hole is wider than this (in that level's pixels) only *refine* the upsampled
|
||||
/// match field (one local PatchMatch pass and one vote), which keeps big holes tractable: the
|
||||
/// structure is decided at coarse scale, fine levels just add detail.
|
||||
pub refine_extent: usize,
|
||||
pub seed: u64,
|
||||
}
|
||||
|
||||
impl Default for CompleteParams {
|
||||
fn default() -> Self {
|
||||
Self { patch_radius: 3, em_iters: 8, min_em_iters: 2, pm_iters: 2, refine_extent: 96, seed: 1 }
|
||||
}
|
||||
}
|
||||
|
||||
/// One pyramid level.
|
||||
#[derive(Clone)]
|
||||
struct Level {
|
||||
w: usize,
|
||||
h: usize,
|
||||
img: Vec<f32>,
|
||||
hole: Vec<bool>,
|
||||
}
|
||||
|
||||
fn downsample(l: &Level, ch: usize) -> Level {
|
||||
let (w, h) = (l.w.div_ceil(2), l.h.div_ceil(2));
|
||||
let mut img = vec![0.0f32; w * h * ch];
|
||||
let mut hole = vec![false; w * h];
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let mut n = 0.0f32;
|
||||
let o = (y * w + x) * ch;
|
||||
for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] {
|
||||
let (sx, sy) = (x * 2 + dx, y * 2 + dy);
|
||||
if sx >= l.w || sy >= l.h {
|
||||
continue;
|
||||
}
|
||||
let si = sy * l.w + sx;
|
||||
if l.hole[si] {
|
||||
hole[y * w + x] = true;
|
||||
} else {
|
||||
for c in 0..ch {
|
||||
img[o + c] += l.img[si * ch + c];
|
||||
}
|
||||
n += 1.0;
|
||||
}
|
||||
}
|
||||
if n > 0.0 {
|
||||
for c in 0..ch {
|
||||
img[o + c] /= n;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Level { w, h, img, hole }
|
||||
}
|
||||
|
||||
/// Summed-area table of a boolean grid (`(w+1) × (h+1)`).
|
||||
fn integral(w: usize, h: usize, m: &[bool]) -> Vec<u32> {
|
||||
let mut s = vec![0u32; (w + 1) * (h + 1)];
|
||||
for y in 0..h {
|
||||
let mut row = 0u32;
|
||||
for x in 0..w {
|
||||
row += u32::from(m[y * w + x]);
|
||||
s[(y + 1) * (w + 1) + x + 1] = s[y * (w + 1) + x + 1] + row;
|
||||
}
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// Count of set cells in the clipped window `[x0, x1) × [y0, y1)`.
|
||||
fn window_count(s: &[u32], gw: usize, gh: usize, x0: i32, y0: i32, x1: i32, y1: i32) -> u32 {
|
||||
let (x0, y0) = (x0.max(0) as usize, y0.max(0) as usize);
|
||||
let (x1, y1) = ((x1.max(0) as usize).min(gw), (y1.max(0) as usize).min(gh));
|
||||
if x1 <= x0 || y1 <= y0 {
|
||||
return 0;
|
||||
}
|
||||
let w1 = gw + 1;
|
||||
s[y1 * w1 + x1] + s[y0 * w1 + x0] - s[y0 * w1 + x1] - s[y1 * w1 + x0]
|
||||
}
|
||||
|
||||
fn hole_bbox(w: usize, h: usize, hole: &[bool]) -> Option<(usize, usize, usize, usize)> {
|
||||
let mut b: Option<(usize, usize, usize, usize)> = None;
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
if hole[y * w + x] {
|
||||
b = Some(match b {
|
||||
None => (x, y, x + 1, y + 1),
|
||||
Some((x0, y0, x1, y1)) => (x0.min(x), y0.min(y), x1.max(x + 1), y1.max(y + 1)),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// Per-level PatchMatch/EM state.
|
||||
struct Solver<'a> {
|
||||
w: usize,
|
||||
h: usize,
|
||||
ch: usize,
|
||||
r: i32,
|
||||
img: &'a mut Vec<f32>,
|
||||
hole: &'a [bool],
|
||||
/// Valid source centres: the whole patch is inside the grid and contains no hole pixel.
|
||||
valid: Vec<bool>,
|
||||
valid_list: Vec<(i32, i32)>,
|
||||
/// Target centres (patch overlaps the hole), row-major.
|
||||
targets: Vec<bool>,
|
||||
/// Cancellation: checked per row band; a cancelled solve's output is discarded.
|
||||
ctl: &'a (dyn Fn() -> bool + Sync),
|
||||
}
|
||||
|
||||
impl Solver<'_> {
|
||||
#[inline]
|
||||
fn is_valid(&self, x: i32, y: i32) -> bool {
|
||||
x >= 0 && y >= 0 && (x as usize) < self.w && (y as usize) < self.h && self.valid[y as usize * self.w + x as usize]
|
||||
}
|
||||
|
||||
/// SSD between the (clipped) target patch at `t` and the source patch at `s`, per sample.
|
||||
fn dist(&self, t: (i32, i32), s: (i32, i32), cutoff: f32) -> f32 {
|
||||
let (w, h, ch, r) = (self.w as i32, self.h as i32, self.ch, self.r);
|
||||
let full = ((2 * r + 1) * (2 * r + 1)) as f32 * ch as f32;
|
||||
let mut sum = 0.0f32;
|
||||
let mut n = 0usize;
|
||||
for dy in -r..=r {
|
||||
let ty = t.1 + dy;
|
||||
if ty < 0 || ty >= h {
|
||||
continue;
|
||||
}
|
||||
let sy = s.1 + dy;
|
||||
for dx in -r..=r {
|
||||
let tx = t.0 + dx;
|
||||
if tx < 0 || tx >= w {
|
||||
continue;
|
||||
}
|
||||
let ti = (ty as usize * self.w + tx as usize) * ch;
|
||||
let si = (sy as usize * self.w + (s.0 + dx) as usize) * ch;
|
||||
for c in 0..ch {
|
||||
let d = self.img[ti + c] - self.img[si + c];
|
||||
sum += d * d;
|
||||
}
|
||||
n += ch;
|
||||
}
|
||||
// Early out: the mean can only be ≥ sum / (full patch size).
|
||||
if sum > cutoff * full {
|
||||
return f32::INFINITY;
|
||||
}
|
||||
}
|
||||
if n == 0 { f32::INFINITY } else { sum / n as f32 }
|
||||
}
|
||||
|
||||
/// One PatchMatch pass over all target rows, in parallel bands (each band propagates within itself).
|
||||
fn patchmatch(&self, nnf: &mut [(i32, i32)], cost: &mut [f32], pass: usize, seed: u64, search_radius: i32) {
|
||||
const BAND: usize = 16;
|
||||
let w = self.w;
|
||||
let reverse = pass % 2 == 1;
|
||||
let band_fn = |bi: usize, nn: &mut [(i32, i32)], co: &mut [f32]| {
|
||||
if (self.ctl)() {
|
||||
return;
|
||||
}
|
||||
let mut rng = Rng::new(seed ^ (bi as u64).wrapping_mul(0x2545_F491_4F6C_DD1D) ^ ((pass as u64) << 40));
|
||||
let y0 = bi * BAND;
|
||||
let rows = nn.len() / w;
|
||||
let step: i32 = if reverse { -1 } else { 1 };
|
||||
for ry in 0..rows {
|
||||
let ly = if reverse { rows - 1 - ry } else { ry };
|
||||
let y = y0 + ly;
|
||||
for rx in 0..w {
|
||||
let x = if reverse { w - 1 - rx } else { rx };
|
||||
let li = ly * w + x;
|
||||
if !self.targets[y * w + x] {
|
||||
continue;
|
||||
}
|
||||
let t = (x as i32, y as i32);
|
||||
let (mut best, mut bc) = (nn[li], co[li]);
|
||||
// Propagation from the already-visited neighbours (within this band).
|
||||
let nx = x as i32 - step;
|
||||
if nx >= 0 && (nx as usize) < w {
|
||||
let c = nn[ly * w + nx as usize];
|
||||
let cand = (c.0 + step, c.1);
|
||||
if self.is_valid(cand.0, cand.1) && cand != best {
|
||||
let d = self.dist(t, cand, bc);
|
||||
if d < bc {
|
||||
(best, bc) = (cand, d);
|
||||
}
|
||||
}
|
||||
}
|
||||
let nly = ly as i32 - step;
|
||||
if nly >= 0 && (nly as usize) < rows {
|
||||
let c = nn[nly as usize * w + x];
|
||||
let cand = (c.0, c.1 + step);
|
||||
if self.is_valid(cand.0, cand.1) && cand != best {
|
||||
let d = self.dist(t, cand, bc);
|
||||
if d < bc {
|
||||
(best, bc) = (cand, d);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Random search in exponentially shrinking windows.
|
||||
let mut rad = search_radius;
|
||||
while rad >= 1 {
|
||||
let cand = (best.0 + rng.range(-rad, rad), best.1 + rng.range(-rad, rad));
|
||||
if self.is_valid(cand.0, cand.1) && cand != best {
|
||||
let d = self.dist(t, cand, bc);
|
||||
if d < bc {
|
||||
(best, bc) = (cand, d);
|
||||
}
|
||||
}
|
||||
rad /= 2;
|
||||
}
|
||||
nn[li] = best;
|
||||
co[li] = bc;
|
||||
}
|
||||
}
|
||||
};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if w * self.h >= PAR_MIN {
|
||||
use rayon::prelude::*;
|
||||
nnf.par_chunks_mut(BAND * w).zip(cost.par_chunks_mut(BAND * w)).enumerate().for_each(|(bi, (nn, co))| band_fn(bi, nn, co));
|
||||
return;
|
||||
}
|
||||
for (bi, (nn, co)) in nnf.chunks_mut(BAND * w).zip(cost.chunks_mut(BAND * w)).enumerate() {
|
||||
band_fn(bi, nn, co);
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-estimate hole pixels by a similarity-weighted vote of all overlapping patch matches.
|
||||
fn vote(&mut self, nnf: &[(i32, i32)], cost: &[f32]) {
|
||||
let (w, h, ch, r) = (self.w, self.h, self.ch, self.r);
|
||||
// σ² from the 75th percentile of match costs (Wexler et al. use a robust scale like this).
|
||||
let mut cs: Vec<f32> = cost.iter().zip(&self.targets).filter(|(c, t)| **t && c.is_finite()).map(|(c, _)| *c).collect();
|
||||
let sigma2 = if cs.is_empty() {
|
||||
1.0
|
||||
} else {
|
||||
let k = (cs.len() * 3 / 4).min(cs.len() - 1);
|
||||
let (_, v, _) = cs.select_nth_unstable_by(k, |a, b| a.total_cmp(b));
|
||||
v.max(1e-6)
|
||||
};
|
||||
let img = &*self.img;
|
||||
let row = |y: usize| -> Vec<(usize, Vec<f32>)> {
|
||||
let mut out = Vec::new();
|
||||
if (self.ctl)() {
|
||||
return out;
|
||||
}
|
||||
for x in 0..w {
|
||||
if !self.hole[y * w + x] {
|
||||
continue;
|
||||
}
|
||||
let mut acc = vec![0.0f32; ch];
|
||||
let mut wsum = 0.0f32;
|
||||
for dy in -r..=r {
|
||||
let ty = y as i32 + dy;
|
||||
if ty < 0 || ty as usize >= h {
|
||||
continue;
|
||||
}
|
||||
for dx in -r..=r {
|
||||
let tx = x as i32 + dx;
|
||||
if tx < 0 || tx as usize >= w {
|
||||
continue;
|
||||
}
|
||||
let ti = ty as usize * w + tx as usize;
|
||||
if !self.targets[ti] || !cost[ti].is_finite() {
|
||||
continue;
|
||||
}
|
||||
let s = nnf[ti];
|
||||
let (sx, sy) = (s.0 - dx, s.1 - dy);
|
||||
let wt = (-cost[ti] / (2.0 * sigma2)).exp().max(1e-8);
|
||||
let si = (sy as usize * w + sx as usize) * ch;
|
||||
for c in 0..ch {
|
||||
acc[c] += img[si + c] * wt;
|
||||
}
|
||||
wsum += wt;
|
||||
}
|
||||
}
|
||||
if wsum > 0.0 {
|
||||
acc.iter_mut().for_each(|v| *v /= wsum);
|
||||
out.push((x, acc));
|
||||
}
|
||||
}
|
||||
out
|
||||
};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let rows: Vec<Vec<(usize, Vec<f32>)>> = if w * h >= PAR_MIN {
|
||||
use rayon::prelude::*;
|
||||
(0..h).into_par_iter().map(row).collect()
|
||||
} else {
|
||||
(0..h).map(row).collect()
|
||||
};
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
let rows: Vec<Vec<(usize, Vec<f32>)>> = (0..h).map(row).collect();
|
||||
for (y, r) in rows.into_iter().enumerate() {
|
||||
for (x, v) in r {
|
||||
self.img[(y * w + x) * ch..(y * w + x + 1) * ch].copy_from_slice(&v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Content-aware completion of `hole` (see module docs). Returns `None` when there is no fully-known
|
||||
/// patch to copy from (the caller can fall back to [`membrane_fill`]).
|
||||
pub fn complete(w: usize, h: usize, ch: usize, img: &[f32], hole: &[bool], p: &CompleteParams) -> Option<Vec<f32>> {
|
||||
// Never cancelled, so never `Err`.
|
||||
complete_with(w, h, ch, img, hole, p, &|| false).unwrap_or(None)
|
||||
}
|
||||
|
||||
/// [`complete`] that can be cancelled (checked per EM iteration, PatchMatch pass and row band)
|
||||
/// (`cancelled` is polled per EM iteration, PatchMatch pass and row band): `Err(Cancelled)` once it
|
||||
/// returned `true`. `Ok(None)` also for mismatched buffer sizes or `ch == 0`.
|
||||
pub fn complete_with(
|
||||
w: usize,
|
||||
h: usize,
|
||||
ch: usize,
|
||||
img: &[f32],
|
||||
hole: &[bool],
|
||||
p: &CompleteParams,
|
||||
cancelled: &(dyn Fn() -> bool + Sync),
|
||||
) -> Result<Option<Vec<f32>>, Cancelled> {
|
||||
let check = || if cancelled() { Err(Cancelled) } else { Ok(()) };
|
||||
if ch == 0 || Some(hole.len()) != w.checked_mul(h) || Some(img.len()) != hole.len().checked_mul(ch) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some((bx0, by0, bx1, by1)) = hole_bbox(w, h, hole) else { return Ok(Some(img.to_vec())) };
|
||||
let r = p.patch_radius.max(1);
|
||||
let psz = 2 * r + 1;
|
||||
// Pyramid: shrink until the hole is a few patches across.
|
||||
let mut levels = vec![Level { w, h, img: img.to_vec(), hole: hole.to_vec() }];
|
||||
let mut ext = (bx1 - bx0).max(by1 - by0);
|
||||
let mut exts = vec![ext];
|
||||
while let Some(l) = levels.last() {
|
||||
check()?;
|
||||
if ext <= 2 * psz || l.w / 2 < 3 * psz || l.h / 2 < 3 * psz {
|
||||
break;
|
||||
}
|
||||
let d = downsample(l, ch);
|
||||
levels.push(d);
|
||||
ext = ext.div_ceil(2);
|
||||
exts.push(ext);
|
||||
}
|
||||
let nlev = levels.len();
|
||||
let mut nnf: Vec<(i32, i32)> = Vec::new();
|
||||
let mut prev: Option<(usize, usize, Vec<f32>)> = None;
|
||||
for li in (0..nlev).rev() {
|
||||
check()?;
|
||||
let Level { w: lw, h: lh, img: mut limg, hole: lhole } = levels[li].clone();
|
||||
// Initialise the hole: membrane at the coarsest level, upsampled result elsewhere.
|
||||
match &prev {
|
||||
None => limg = membrane_fill(lw, lh, ch, &limg, &lhole),
|
||||
Some((pw, ph, pimg)) => {
|
||||
for y in 0..lh {
|
||||
for x in 0..lw {
|
||||
if lhole[y * lw + x] {
|
||||
let (sx, sy) = ((x / 2).min(pw - 1), (y / 2).min(ph - 1));
|
||||
let (d, s) = ((y * lw + x) * ch, (sy * pw + sx) * ch);
|
||||
limg[d..d + ch].copy_from_slice(&pimg[s..s + ch]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let ri = r as i32;
|
||||
let hs = integral(lw, lh, &lhole);
|
||||
let mut valid = vec![false; lw * lh];
|
||||
let mut valid_list = Vec::new();
|
||||
let mut targets = vec![false; lw * lh];
|
||||
for y in 0..lh as i32 {
|
||||
if y % 32 == 0 {
|
||||
check()?;
|
||||
}
|
||||
for x in 0..lw as i32 {
|
||||
let cnt = window_count(&hs, lw, lh, x - ri, y - ri, x + ri + 1, y + ri + 1);
|
||||
let inside = x >= ri && y >= ri && x + ri < lw as i32 && y + ri < lh as i32;
|
||||
let i = y as usize * lw + x as usize;
|
||||
if inside && cnt == 0 {
|
||||
valid[i] = true;
|
||||
valid_list.push((x, y));
|
||||
}
|
||||
targets[i] = cnt > 0;
|
||||
}
|
||||
}
|
||||
if valid_list.is_empty() {
|
||||
if li == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
prev = Some((lw, lh, limg));
|
||||
nnf.clear();
|
||||
continue;
|
||||
}
|
||||
let mut solver = Solver { w: lw, h: lh, ch, r: ri, img: &mut limg, hole: &lhole, valid, valid_list, targets, ctl: cancelled };
|
||||
// NNF init: upsampled from the coarser level where possible, random otherwise.
|
||||
let mut rng = Rng::new(p.seed ^ (li as u64) << 20);
|
||||
let pw_prev = prev.as_ref().map_or(0, |p| p.0);
|
||||
let old = std::mem::take(&mut nnf);
|
||||
let mut new_nnf = vec![(0i32, 0i32); lw * lh];
|
||||
for y in 0..lh {
|
||||
if y % 32 == 0 {
|
||||
check()?;
|
||||
}
|
||||
for x in 0..lw {
|
||||
if !solver.targets[y * lw + x] {
|
||||
continue;
|
||||
}
|
||||
let mut cand = None;
|
||||
if !old.is_empty() && pw_prev > 0 {
|
||||
let pi = (y / 2) * pw_prev + (x / 2);
|
||||
if let Some(&(sx, sy)) = old.get(pi) {
|
||||
let c = (sx * 2 + (x % 2) as i32, sy * 2 + (y % 2) as i32);
|
||||
if solver.is_valid(c.0, c.1) {
|
||||
cand = Some(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
new_nnf[y * lw + x] = cand.unwrap_or_else(|| solver.valid_list[(rng.next() % solver.valid_list.len() as u64) as usize]);
|
||||
}
|
||||
}
|
||||
nnf = new_nnf;
|
||||
let refine_only = li + 1 < nlev && exts[li] > p.refine_extent;
|
||||
let em = if li + 1 == nlev {
|
||||
p.em_iters
|
||||
} else if refine_only {
|
||||
1
|
||||
} else {
|
||||
(p.em_iters >> (nlev - 1 - li)).max(p.min_em_iters)
|
||||
};
|
||||
let pm_iters = if refine_only { 1 } else { p.pm_iters };
|
||||
let full_search = (lw.max(lh)) as i32;
|
||||
let mut cost = vec![f32::INFINITY; lw * lh];
|
||||
for it in 0..em {
|
||||
// Costs change as hole pixels are re-estimated: refresh before each PatchMatch.
|
||||
for (i, c) in cost.iter_mut().enumerate() {
|
||||
if i % (lw * 32).max(1) == 0 {
|
||||
check()?;
|
||||
}
|
||||
if solver.targets[i] {
|
||||
*c = solver.dist(((i % lw) as i32, (i / lw) as i32), nnf[i], f32::INFINITY);
|
||||
}
|
||||
}
|
||||
// Wide random search on coarse levels / first iteration, local refinement afterwards.
|
||||
let radius = if refine_only {
|
||||
2
|
||||
} else if li + 1 == nlev || it == 0 {
|
||||
full_search
|
||||
} else {
|
||||
(4 * psz as i32).min(full_search)
|
||||
};
|
||||
for pass in 0..pm_iters {
|
||||
check()?;
|
||||
solver.patchmatch(&mut nnf, &mut cost, pass + it * pm_iters, p.seed.wrapping_add((li * 1000 + it) as u64), radius);
|
||||
}
|
||||
check()?;
|
||||
solver.vote(&nnf, &cost);
|
||||
check()?;
|
||||
}
|
||||
prev = Some((lw, lh, limg));
|
||||
}
|
||||
Ok(prev.map(|(_, _, img)| img))
|
||||
}
|
||||
|
||||
/// Proximity match: the displacement `(dx, dy)` (within `max_radius`) whose surroundings best match the
|
||||
/// ring of `ring` pixels around the hole, with the displaced hole fully inside the grid and outside the
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Smooth gradient plus deterministic fine noise.
|
||||
fn texture(w: usize, h: usize) -> Vec<f32> {
|
||||
let mut rng = Rng::new(7);
|
||||
(0..w * h)
|
||||
.flat_map(|i| {
|
||||
let (x, y) = ((i % w) as f32, (i / w) as f32);
|
||||
let n = (rng.next() % 1000) as f32 / 1000.0 * 0.04 - 0.02;
|
||||
[0.3 + 0.004 * x + n, 0.5 + 0.002 * y + n, 0.4 + n]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn disc(w: usize, h: usize, cx: f32, cy: f32, r: f32) -> Vec<bool> {
|
||||
(0..w * h).map(|i| ((i % w) as f32 - cx).hypot((i / w) as f32 - cy) < r).collect()
|
||||
}
|
||||
|
||||
fn hole_rmse(a: &[f32], b: &[f32], hole: &[bool], ch: usize) -> f32 {
|
||||
let (mut e, mut n) = (0.0f32, 0);
|
||||
for (i, hh) in hole.iter().enumerate() {
|
||||
if *hh {
|
||||
for c in 0..ch {
|
||||
e += (a[i * ch + c] - b[i * ch + c]).powi(2);
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
(e / n as f32).sqrt()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_fills_blemish_from_texture() {
|
||||
let (w, h, ch) = (96, 96, 3);
|
||||
let clean = texture(w, h);
|
||||
let hole = disc(w, h, 48.0, 48.0, 9.0);
|
||||
let mut dirty = clean.clone();
|
||||
for (i, hh) in hole.iter().enumerate() {
|
||||
if *hh {
|
||||
dirty[i * ch..i * ch + ch].copy_from_slice(&[0.02, 0.02, 0.02]);
|
||||
}
|
||||
}
|
||||
let out = complete(w, h, ch, &dirty, &hole, &CompleteParams::default()).unwrap();
|
||||
let before = hole_rmse(&dirty, &clean, &hole, ch);
|
||||
let after = hole_rmse(&out, &clean, &hole, ch);
|
||||
assert!(after < 0.06 && after < before * 0.15, "rmse before {before} after {after}");
|
||||
// Outside the hole untouched.
|
||||
assert!(out.iter().zip(&dirty).zip(hole.iter().flat_map(|h| [*h; 3])).all(|((a, b), h)| h || a == b));
|
||||
// Deterministic.
|
||||
assert_eq!(out, complete(w, h, ch, &dirty, &hole, &CompleteParams::default()).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_continues_structure() {
|
||||
// Vertical stripes: the filled hole should keep the stripes (not a blur).
|
||||
let (w, h, ch) = (80, 80, 1);
|
||||
let clean: Vec<f32> = (0..w * h).map(|i| if (i % w / 4) % 2 == 0 { 0.2 } else { 0.8 }).collect();
|
||||
let hole = disc(w, h, 40.0, 40.0, 7.0);
|
||||
let dirty: Vec<f32> = clean.iter().zip(&hole).map(|(c, h)| if *h { 0.5 } else { *c }).collect();
|
||||
let out = complete(w, h, ch, &dirty, &hole, &CompleteParams::default()).unwrap();
|
||||
let err = hole_rmse(&out, &clean, &hole, ch);
|
||||
assert!(err < 0.15, "{err}");
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ lightcraft-color = { workspace = true }
|
||||
lightcraft-raster = { workspace = true }
|
||||
lightcraft-develop = { workspace = true }
|
||||
lightcraft-raw = { workspace = true }
|
||||
lightcraft-inpaint = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
//! (frequency separation — a fast, stable approximation of gradient-domain healing).
|
||||
//! - **Remove** is Heal with an automatically chosen source: candidate offsets on rings around the
|
||||
//! spot are scored by how well the source's surrounding annulus matches the target's.
|
||||
//! - **Fill** (content-aware) rebuilds the spot from patches of its surroundings instead of one
|
||||
//! source: PatchMatch-based completion ([`lightcraft_inpaint::complete`]), coarse to fine, so the
|
||||
//! structure is chosen at a scale set by the spot's size (the same at preview and export size).
|
||||
//! Deterministic; the source offset is ignored.
|
||||
|
||||
use lightcraft_develop::{Spot, SpotMode};
|
||||
use lightcraft_geom::Point;
|
||||
@@ -235,6 +239,19 @@ pub fn window_for_reads_checked(
|
||||
)
|
||||
}
|
||||
|
||||
/// Content-aware fill of the box: `src` becomes `tgt` with every pixel under the stroke (`alpha > 0`)
|
||||
/// rebuilt from the rest of the box. Falls back to a smooth membrane fill when the box holds no whole
|
||||
/// patch to copy from.
|
||||
fn fill_box(src: &mut [[f32; 3]], tgt: &[[f32; 3]], alpha: &[f32], bw: usize, bh: usize) {
|
||||
let hole: Vec<bool> = alpha.iter().map(|a| *a > 0.0).collect();
|
||||
let flat: Vec<f32> = tgt.iter().flatten().copied().collect();
|
||||
let out = lightcraft_inpaint::complete(bw, bh, 3, &flat, &hole, &lightcraft_inpaint::CompleteParams::default())
|
||||
.unwrap_or_else(|| lightcraft_inpaint::membrane_fill(bw, bh, 3, &flat, &hole));
|
||||
for (d, v) in src.iter_mut().zip(out.chunks_exact(3)) {
|
||||
*d = [v[0], v[1], v[2]];
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply all spots to `img` (output resolution). `ppl` = output pixels per long-edge unit.
|
||||
pub fn apply(img: &mut Rgb32f, spots: &[Spot], frame: &Frame, ppl: f64) {
|
||||
if spots.is_empty() {
|
||||
@@ -253,7 +270,9 @@ pub fn apply(img: &mut Rgb32f, spots: &[Spot], frame: &Frame, ppl: f64) {
|
||||
})
|
||||
.collect();
|
||||
let Some(&first) = pts.first() else { continue };
|
||||
let fill = spot.mode == SpotMode::Fill;
|
||||
let offset = match spot.source_offset {
|
||||
_ if fill => (0.0, 0.0),
|
||||
Some(o) => {
|
||||
let a = to_out.apply(Point::new(0.0, 0.0));
|
||||
let b = to_out.apply(o);
|
||||
@@ -269,7 +288,8 @@ pub fn apply(img: &mut Rgb32f, spots: &[Spot], frame: &Frame, ppl: f64) {
|
||||
x1 = x1.max(p.0 + r);
|
||||
y1 = y1.max(p.1 + r);
|
||||
}
|
||||
let pad = r;
|
||||
// content-aware fill reads the spot's surroundings (the window holds 2 radii around it)
|
||||
let pad = if fill { 2.0 * r } else { r };
|
||||
let (bx0, by0) = (((x0 - pad).floor().max(0.0)) as usize, ((y0 - pad).floor().max(0.0)) as usize);
|
||||
let (bx1, by1) = (((x1 + pad).ceil() as usize).min(w), ((y1 + pad).ceil() as usize).min(h));
|
||||
if bx1 <= bx0 || by1 <= by0 {
|
||||
@@ -292,7 +312,10 @@ pub fn apply(img: &mut Rgb32f, spots: &[Spot], frame: &Frame, ppl: f64) {
|
||||
tgt[y * bw + x] = img.get(bx0 + x, by0 + y);
|
||||
}
|
||||
}
|
||||
let heal = spot.mode != SpotMode::Clone;
|
||||
if fill {
|
||||
fill_box(&mut src, &tgt, &alpha, bw, bh);
|
||||
}
|
||||
let heal = matches!(spot.mode, SpotMode::Remove | SpotMode::Heal);
|
||||
let low = if heal {
|
||||
// Tone correction from the spot's *surroundings* only: normalized convolution of
|
||||
// (target − source) over pixels outside the stroke (a membrane-like interpolation).
|
||||
@@ -361,6 +384,79 @@ mod tests {
|
||||
assert!((c[0] - 0.26).abs() < 0.03, "{c:?}");
|
||||
}
|
||||
|
||||
/// Vertical stripes (period `period` px) with a black blot over the middle.
|
||||
fn striped_with_blot(w: usize, h: usize, period: usize) -> Rgb32f {
|
||||
let mut img = Rgb32f::from_fn(w, h, |x, _| if (x / period).is_multiple_of(2) { [0.2; 3] } else { [0.7; 3] });
|
||||
let (cx, cy, r) = (w as f32 / 2.0, h as f32 / 2.0, w as f32 * 0.04);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
if (x as f32 + 0.5 - cx).hypot(y as f32 + 0.5 - cy) < r {
|
||||
img.set(x, y, [0.0; 3]);
|
||||
}
|
||||
}
|
||||
}
|
||||
img
|
||||
}
|
||||
|
||||
fn fill_spot() -> Spot {
|
||||
// a source offset is ignored by Fill
|
||||
Spot {
|
||||
mode: SpotMode::Fill,
|
||||
points: vec![Point::new(0.5, 0.5)],
|
||||
size: 0.06,
|
||||
feather: 0.0,
|
||||
source_offset: Some(Point::new(0.3, 0.0)),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fill_rebuilds_structure_deterministically() {
|
||||
let (w, h) = (200, 160);
|
||||
let mut img = striped_with_blot(w, h, 8);
|
||||
let frame = Frame::new(w, h, &DevelopSettings::default(), true);
|
||||
apply(&mut img, &[fill_spot()], &frame, frame.px_per_long(w));
|
||||
// the blot is gone and the stripes continue through it
|
||||
let mut err = 0.0f32;
|
||||
for x in 90..110 {
|
||||
let want = if (x / 8) % 2 == 0 { 0.2 } else { 0.7 };
|
||||
err += (img.get(x, 80)[0] - want).abs();
|
||||
}
|
||||
assert!(err / 20.0 < 0.12, "mean error {}", err / 20.0);
|
||||
let mut again = striped_with_blot(w, h, 8);
|
||||
apply(&mut again, &[fill_spot()], &frame, frame.px_per_long(w));
|
||||
assert_eq!(img.data, again.data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fill_matches_between_preview_and_export_size() {
|
||||
// the same photo at two sizes: the half-size fill matches the full-size one, downsampled
|
||||
let (mut big, mut small) = (striped_with_blot(320, 240, 16), striped_with_blot(160, 120, 8));
|
||||
let (fb, fs) = (Frame::new(320, 240, &DevelopSettings::default(), true), Frame::new(160, 120, &DevelopSettings::default(), true));
|
||||
apply(&mut big, &[fill_spot()], &fb, fb.px_per_long(320));
|
||||
apply(&mut small, &[fill_spot()], &fs, fs.px_per_long(160));
|
||||
let (mut err, mut n) = (0.0f32, 0);
|
||||
for y in 50..70 {
|
||||
for x in 70..90 {
|
||||
let b: f32 = [(0, 0), (1, 0), (0, 1), (1, 1)].iter().map(|(dx, dy)| big.get(2 * x + dx, 2 * y + dy)[0]).sum::<f32>() / 4.0;
|
||||
err += (b - small.get(x, y)[0]).abs();
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
assert!(err / n as f32 <= 0.08, "mean difference {}", err / n as f32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fill_survives_hostile_spots() {
|
||||
let frame = Frame::new(40, 30, &DevelopSettings::default(), true);
|
||||
for (p, size) in [((0.0, 0.0), 2.0), ((-5.0, 9.0), 0.1), ((0.5, 0.5), 0.0), ((f64::NAN, 0.5), 0.1), ((0.5, 0.5), f64::INFINITY)] {
|
||||
let mut img = Rgb32f::from_fn(40, 30, |x, _| [x as f32 / 40.0; 3]);
|
||||
let spot = Spot { mode: SpotMode::Fill, points: vec![Point::new(p.0, p.1)], size, ..Default::default() };
|
||||
apply(&mut img, &[spot], &frame, frame.px_per_long(40));
|
||||
assert!(img.data.iter().flatten().all(|v| v.is_finite()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_copies_source() {
|
||||
let mut img = Rgb32f::from_fn(100, 100, |x, _| if x < 50 { [0.1; 3] } else { [0.9; 3] });
|
||||
|
||||
@@ -2062,5 +2062,6 @@
|
||||
"Intent:": "Priorität:",
|
||||
"Perceptual": "Perzeptiv",
|
||||
"Relative": "Relativ",
|
||||
"Simulate Paper & Ink": "Papier und Druckfarbe simulieren"
|
||||
"Simulate Paper & Ink": "Papier und Druckfarbe simulieren",
|
||||
"Content-Aware": "Inhaltsbasiert"
|
||||
}
|
||||
@@ -1907,5 +1907,6 @@
|
||||
"Intent:": "Propósito:",
|
||||
"Perceptual": "Perceptual",
|
||||
"Relative": "Relativo",
|
||||
"Simulate Paper & Ink": "Simular papel y tinta"
|
||||
"Simulate Paper & Ink": "Simular papel y tinta",
|
||||
"Content-Aware": "Según el contenido"
|
||||
}
|
||||
@@ -1907,5 +1907,6 @@
|
||||
"Intent:": "Mode de rendu :",
|
||||
"Perceptual": "Perceptif",
|
||||
"Relative": "Relatif",
|
||||
"Simulate Paper & Ink": "Simuler le papier et l'encre"
|
||||
"Simulate Paper & Ink": "Simuler le papier et l'encre",
|
||||
"Content-Aware": "Contenu pris en compte"
|
||||
}
|
||||
@@ -1909,5 +1909,6 @@
|
||||
"Intent:": "マッチング方法：",
|
||||
"Perceptual": "知覚的",
|
||||
"Relative": "相対的",
|
||||
"Simulate Paper & Ink": "紙とインキをシミュレート"
|
||||
"Simulate Paper & Ink": "紙とインキをシミュレート",
|
||||
"Content-Aware": "コンテンツに応じる"
|
||||
}
|
||||
@@ -1907,5 +1907,6 @@
|
||||
"Intent:": "Tipo de conversão:",
|
||||
"Perceptual": "Perceptual",
|
||||
"Relative": "Relativo",
|
||||
"Simulate Paper & Ink": "Simular papel e tinta"
|
||||
"Simulate Paper & Ink": "Simular papel e tinta",
|
||||
"Content-Aware": "Sensível ao conteúdo"
|
||||
}
|
||||
@@ -1621,5 +1621,6 @@
|
||||
"Intent:": "Метод:",
|
||||
"Perceptual": "Перцепционный",
|
||||
"Relative": "Относительный",
|
||||
"Simulate Paper & Ink": "Имитировать бумагу и краску"
|
||||
"Simulate Paper & Ink": "Имитировать бумагу и краску",
|
||||
"Content-Aware": "С учётом содержимого"
|
||||
}
|
||||
@@ -2101,5 +2101,6 @@
|
||||
"Intent:": "Метод:",
|
||||
"Perceptual": "Перцепційний",
|
||||
"Relative": "Відносний",
|
||||
"Simulate Paper & Ink": "Імітувати папір і фарбу"
|
||||
"Simulate Paper & Ink": "Імітувати папір і фарбу",
|
||||
"Content-Aware": "З урахуванням вмісту"
|
||||
}
|
||||
@@ -1907,5 +1907,6 @@
|
||||
"Intent:": "方法：",
|
||||
"Perceptual": "可感知",
|
||||
"Relative": "相对",
|
||||
"Simulate Paper & Ink": "模拟纸张和油墨"
|
||||
"Simulate Paper & Ink": "模拟纸张和油墨",
|
||||
"Content-Aware": "内容识别"
|
||||
}
|
||||
@@ -1907,5 +1907,6 @@
|
||||
"Intent:": "方式：",
|
||||
"Perceptual": "感知",
|
||||
"Relative": "相對",
|
||||
"Simulate Paper & Ink": "模擬紙張和油墨"
|
||||
"Simulate Paper & Ink": "模擬紙張和油墨",
|
||||
"Content-Aware": "內容感知"
|
||||
}
|
||||
@@ -2032,6 +2032,7 @@ fn remove_overlay(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respo
|
||||
let mode = match app.ui.tool.as_str() {
|
||||
"heal" => "heal",
|
||||
"clone" => "clone",
|
||||
"fill" => "fill",
|
||||
_ => "remove",
|
||||
};
|
||||
let pts: Vec<[f64; 2]> = points.iter().map(|q| [q.x, q.y]).collect();
|
||||
|
||||
@@ -343,7 +343,7 @@ fn remove(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
|
||||
header(ui, "Remove");
|
||||
padded(ui, |ui| {
|
||||
ui.horizontal_wrapped(|ui| {
|
||||
for (label, tool) in [("Remove", "remove"), ("Heal", "heal"), ("Clone", "clone")] {
|
||||
for (label, tool) in [("Remove", "remove"), ("Content-Aware", "fill"), ("Heal", "heal"), ("Clone", "clone")] {
|
||||
if text_button(ui, &format!("removeMode-{tool}"), label, app.ui.tool == tool).clicked() {
|
||||
app.ui.tool = tool.into();
|
||||
}
|
||||
@@ -383,6 +383,7 @@ fn remove(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
|
||||
lightcraft_develop::SpotMode::Heal => "Heal",
|
||||
lightcraft_develop::SpotMode::Clone => "Clone",
|
||||
lightcraft_develop::SpotMode::Remove => "Remove",
|
||||
lightcraft_develop::SpotMode::Fill => "Content-Aware",
|
||||
};
|
||||
super::edit::sub_title(ui, &crate::i18n::tr_format!("{mode} spot {} of {}", i + 1, d.spots.len(), mode = mode));
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@ pub const TABLE: &[(&str, Class)] = &[
|
||||
("meta", Class::Layer(1)),
|
||||
("develop", Class::Layer(1)),
|
||||
("lensdb", Class::Layer(2)),
|
||||
("inpaint", Class::Layer(1)),
|
||||
("faces", Class::Layer(1)),
|
||||
("geocode", Class::Layer(1)),
|
||||
("scenes", Class::Layer(1)),
|
||||
|
||||
Reference in new issue
Block a user