Merge pull request #11848 from DuBearte/fix/11779-wand-seed-outside

fix(engine): Magic Wand Subtract and Intersect grow from the seed through the selection (#11779)

crates/algo/src/selection.rs

@@ -270,6 +270,22 @@ impl Region {
 /// byte mark per pixel, then anti-aliases just the edges inside the touched bounding box.
 /// Matches [`magic_wand`] for 8-bit sources. `None` if the seed is outside `area`.
 pub fn wand_region(img: &[[u8; 4]], area: Rect, seed: (i32, i32), tolerance: f32, contiguous: bool, anti_alias: bool) -> Option<Region> {
+    wand_region_within(img, area, seed, tolerance, contiguous, anti_alias, None)
+}
+
+/// [`wand_region`] whose contiguous fill only passes through pixels at least half selected in
+/// `within` (the current selection's coverage over `area`, row-major): Subtract and Intersect
+/// grow the region from the seed through the selection only (#11779). A seed outside it finds
+/// nothing. A global match (`contiguous` off) ignores `within`.
+pub fn wand_region_within(
+    img: &[[u8; 4]],
+    area: Rect,
+    seed: (i32, i32),
+    tolerance: f32,
+    contiguous: bool,
+    anti_alias: bool,
+    within: Option<&[f32]>,
+) -> Option<Region> {
     if !area.contains(seed.0, seed.1) {
         return None;
     }
@@ -277,6 +293,11 @@ pub fn wand_region(img: &[[u8; 4]], area: Rect, seed: (i32, i32), tolerance: f32
     if w.checked_mul(h).is_none_or(|n| n > img.len()) {
         return None;
     }
+    let within = within.filter(|_| contiguous).filter(|m| m.len() >= w * h);
+    let inside = |i: usize| within.is_none_or(|m| m.get(i).is_some_and(|v| *v >= 0.5));
+    if !inside((seed.1 - area.y0) as usize * w + (seed.0 - area.x0) as usize) {
+        return None;
+    }
     let target = img[(seed.1 - area.y0) as usize * w + (seed.0 - area.x0) as usize];
     let tol = (tolerance + 1e-3).floor().max(0.0) as i32;
     // Copying a selection can leave RGB under zero alpha. Those invisible colours must not
@@ -293,10 +314,10 @@ pub fn wand_region(img: &[[u8; 4]], area: Rect, seed: (i32, i32), tolerance: f32
                 continue;
             }
             let (mut l, mut r) = (x, x);
-            while l > 0 && marks[row + l - 1] == 0 && similar(img[row + l - 1]) {
+            while l > 0 && marks[row + l - 1] == 0 && similar(img[row + l - 1]) && inside(row + l - 1) {
                 l -= 1;
             }
-            while r + 1 < w && marks[row + r + 1] == 0 && similar(img[row + r + 1]) {
+            while r + 1 < w && marks[row + r + 1] == 0 && similar(img[row + r + 1]) && inside(row + r + 1) {
                 r += 1;
             }
             marks[row + l..=row + r].fill(255);
@@ -308,7 +329,7 @@ pub fn wand_region(img: &[[u8; 4]], area: Rect, seed: (i32, i32), tolerance: f32
                 let nrow = ny * w;
                 let mut in_run = false;
                 for nx in l..=r {
-                    let ok = marks[nrow + nx] == 0 && similar(img[nrow + nx]);
+                    let ok = marks[nrow + nx] == 0 && similar(img[nrow + nx]) && inside(nrow + nx);
                     if ok && !in_run {
                         stack.push((nx, ny));
                     }

crates/engine/src/cmd/selection_cmds.rs

@@ -17,6 +17,8 @@ mod modify_window;
 mod color_range_performance;
 #[cfg(test)]
 mod color_range_window_tests;
+#[cfg(test)]
+mod wand_seed_outside_tests;
 
 fn has_doc(s: &Session) -> std::result::Result<(), String> {
     s.active().map(|_| ()).ok_or_else(|| "no document open".into())
@@ -139,9 +141,15 @@ fn magic_wand(s: &mut Session, p: &Value) -> Result<Value> {
         Some(plane) => plane,
         None => sample_rgba8(s, b(p, "sampleAllLayers", false))?,
     };
-    let region = sel::wand_region(&img, area, (x, y), f(p, "tolerance", 32.0), b(p, "contiguous", true), b(p, "antiAlias", true));
-    drop(img);
     let m = mode(p);
+    // Subtract and Intersect grow the region from the seed through the selection only (#11779).
+    // A global match (Contiguous off) ignores the limit, so it skips the full-area mask.
+    let contiguous = b(p, "contiguous", true);
+    let within = (contiguous && matches!(m, SelectionMode::Subtract | SelectionMode::Intersect))
+        .then(|| s.active().and_then(|st| st.doc.selection.as_ref()).map(|sel| sel::mask_from_surface(Some(sel), area)))
+        .flatten();
+    let region = sel::wand_region_within(&img, area, (x, y), f(p, "tolerance", 32.0), contiguous, b(p, "antiAlias", true), within.as_deref());
+    drop(img);
     let selected = s.edit("Magic Wand", |doc, _| {
         doc.selection = sel::combine_region(doc.selection.as_ref(), region.as_ref(), m);
         Ok(doc.selection.is_some())

crates/engine/src/cmd/selection_cmds/wand_seed_outside_tests.rs

@@ -0,0 +1,110 @@
+//! #11779: with Contiguous on, the Magic Wand's Subtract and Intersect grow the region from the
+//! seed through the current selection only. A seed outside the selection subtracts nothing and
+//! intersects to nothing.
+
+use serde_json::json;
+
+use crate::Session;
+
+/// 800 × 600, white, with a grey ramp of ten 40 px blocks from x 100 (100, 105, …, 145).
+fn ramp() -> Session {
+    let mut s = Session::new();
+    s.execute("file.new", json!({"width": 800, "height": 600, "background": "white"})).unwrap();
+    for k in 0..10 {
+        let v = 100 + 5 * k;
+        s.execute("select.rect", json!({"x": 100 + 40 * k, "y": 350, "width": 40, "height": 100})).unwrap();
+        s.execute("edit.fill", json!({"color": format!("#{v:02x}{v:02x}{v:02x}")})).unwrap();
+    }
+    s.execute("select.deselect", json!({})).unwrap();
+    s
+}
+
+fn wand(s: &mut Session, x: i32, tolerance: f64, mode: &str) {
+    s.execute("select.magicWand", json!({"x": x, "y": 400, "tolerance": tolerance, "mode": mode})).unwrap();
+}
+
+fn wand_global(s: &mut Session, x: i32, tolerance: f64, mode: &str) {
+    s.execute("select.magicWand", json!({"x": x, "y": 400, "tolerance": tolerance, "mode": mode, "contiguous": false})).unwrap();
+}
+
+/// The columns of row 400 that are at least half selected: (first, end), or `None`. Every
+/// column in between must be selected too, so a hole in the run fails the test.
+fn selected(s: &Session) -> Option<(i32, i32)> {
+    let doc = &s.active().unwrap().doc;
+    let row = photocraft_geom::Rect::new(0, 400, 800, 401);
+    let m = photocraft_algo::selection::mask_from_surface(doc.selection.as_ref(), row);
+    let first = m.iter().position(|v| *v >= 0.5)?;
+    let last = m.iter().rposition(|v| *v >= 0.5)?;
+    assert!(m[first..=last].iter().all(|v| *v >= 0.5), "a hole in the selected run {first}..={last}");
+    Some((first as i32, last as i32 + 1))
+}
+
+/// Greys 100 to 130 (x 100–380) selected, as tolerance 32 on the first block selects them.
+fn ramp_selected() -> Session {
+    let mut s = ramp();
+    wand(&mut s, 110, 32.0, "replace");
+    assert_eq!(selected(&s), Some((100, 380)));
+    s
+}
+
+#[test]
+fn a_seed_outside_the_selection_subtracts_nothing() {
+    let mut s = ramp_selected();
+    // Grey 145 (x 460–500) is outside the selection, though its own region reaches into it.
+    wand(&mut s, 480, 32.0, "subtract");
+    assert_eq!(selected(&s), Some((100, 380)), "unchanged");
+}
+
+#[test]
+fn a_seed_outside_the_selection_intersects_to_nothing() {
+    let mut s = ramp_selected();
+    wand(&mut s, 480, 32.0, "intersect");
+    assert_eq!(selected(&s), None);
+}
+
+#[test]
+fn a_seed_inside_the_selection_works_as_before() {
+    // Grey 125 with tolerance 10 reaches greys 115–135; inside the selection that is x 220–380.
+    let mut s = ramp_selected();
+    wand(&mut s, 320, 10.0, "subtract");
+    assert_eq!(selected(&s), Some((100, 220)));
+    let mut s = ramp_selected();
+    wand(&mut s, 320, 10.0, "intersect");
+    assert_eq!(selected(&s), Some((220, 380)));
+}
+
+#[test]
+fn contiguous_off_keeps_the_global_match() {
+    // Without Contiguous the seed's colour matches everywhere, as before: greys 115–145 are
+    // x 220–500, wherever the seed lies.
+    let mut s = ramp_selected();
+    wand_global(&mut s, 480, 32.0, "subtract");
+    assert_eq!(selected(&s), Some((100, 220)));
+    let mut s = ramp_selected();
+    wand_global(&mut s, 480, 32.0, "intersect");
+    assert_eq!(selected(&s), Some((220, 380)));
+}
+
+#[test]
+fn without_a_selection_subtract_and_intersect_select_nothing() {
+    for mode in ["subtract", "intersect"] {
+        let mut s = ramp();
+        wand(&mut s, 320, 10.0, mode);
+        assert_eq!(selected(&s), None, "{mode}");
+    }
+}
+
+#[test]
+fn the_fill_passes_only_pixels_at_least_half_selected() {
+    use photocraft_algo::selection::wand_region_within;
+    use photocraft_geom::Rect;
+    // One flat grey row of four pixels; only the coverage limit decides the region.
+    let img = [[128u8, 128, 128, 255]; 4];
+    let area = Rect::new(0, 0, 4, 1);
+    let fill = |within: [f32; 4], seed: i32| wand_region_within(&img, area, (seed, 0), 0.0, true, false, Some(&within));
+    assert!(fill([0.49, 1.0, 1.0, 1.0], 0).is_none(), "a seed just under half selected finds nothing");
+    let r = fill([0.5, 1.0, 1.0, 1.0], 0).unwrap();
+    assert_eq!((0..4).map(|x| r.at(x, 0) >= 0.5).collect::<Vec<_>>(), [true; 4], "half selected counts");
+    let r = fill([1.0, 1.0, 0.49, 1.0], 0).unwrap();
+    assert_eq!((0..4).map(|x| r.at(x, 0) >= 0.5).collect::<Vec<_>>(), [true, true, false, false], "the fill stops at 0.49");
+}