From 08abbf7d902a907352e174b1a660eb1835fd7a1e Mon Sep 17 00:00:00 2001 From: David Mann Date: Wed, 14 Oct 2026 20:37:46 +0200 Subject: [PATCH] Layer masks: From Transparency and Mask All Objects convert the Background and match the document depth (#5679) From Transparency and Mask All Objects attached a mask to the Background without turning it into "Layer 0" first, unlike every other add-mask path. They and Hide Selection also built their masks as 8-bit, so a 16/32-bit layer's alpha was quantised to 256 levels by From Transparency. All three now build the mask with mask_at_depth at the document depth; 8-bit documents are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/engine/src/commands.rs | 2 +- crates/engine/src/layer_menu_cmds.rs | 9 ++-- crates/engine/src/layer_menu_cmds/tests.rs | 53 ++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index 3969af0576..9b118bc044 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -322,7 +322,7 @@ pub(crate) fn selection_mask(doc: &Document) -> Result> { /// New pixel masks use the document's depth, as PSD saves/imports do. Otherwise a later soft /// brush stroke quantizes differently depending on whether the document reopened as PSD or /// native. Converting a selection keeps its coverage and does not alter the selection itself. -fn mask_at_depth(mut mask: LayerMask, depth: SampleType) -> Result { +pub(crate) fn mask_at_depth(mut mask: LayerMask, depth: SampleType) -> Result { let format = mask.surface.format().with_sample(depth); if mask.surface.format() != format { mask.surface = mask.surface.try_convert(format)?; diff --git a/crates/engine/src/layer_menu_cmds.rs b/crates/engine/src/layer_menu_cmds.rs index 366f9c8d39..45f405a2cd 100644 --- a/crates/engine/src/layer_menu_cmds.rs +++ b/crates/engine/src/layer_menu_cmds.rs @@ -168,11 +168,12 @@ fn apply_mask(s: &mut Session, p: &Value) -> Result { fn mask_from_transparency(s: &mut Session, p: &Value) -> Result { let id = layer_param(s, p)?; s.edit("From Transparency", |doc, _| { + let depth = doc.depth; let surf = crate::commands::paint_surface(doc, id, &Value::Null)?; // Masks are attached to layer coordinates, including pixels temporarily off-canvas. // Clipping to document bounds loses their alpha permanently when moved back (#3043). let r = surf.content_bounds(); - let mut mask = LayerMask::hide_all(); + let mut mask = crate::commands::mask_at_depth(LayerMask::hide_all(), depth)?; if !r.is_empty() { let mut px = read_rgba(surf, r); let alpha: Vec = px.iter().map(|q| q[3]).collect(); @@ -183,6 +184,7 @@ fn mask_from_transparency(s: &mut Session, p: &Value) -> Result { } write_rgba(surf, r, &px); } + crate::extra_cmds::background_to_layer_for_mask(doc, id); doc.layer_mut(id).ok_or(EngineError::NoLayer(id))?.mask = Some(mask); Ok(Value::Null) }) @@ -198,7 +200,7 @@ fn hide_selection(s: &mut Session, p: &Value) -> Result { for x in &mut v { *x = 1.0 - *x; } - let mut mask = LayerMask::reveal_all(); + let mut mask = crate::commands::mask_at_depth(LayerMask::reveal_all(), doc.depth)?; mask.surface.write_region(bounds, &v); mask.surface.prune(); for id in ids { @@ -235,10 +237,11 @@ fn mask_all_objects(s: &mut Session, p: &Value) -> Result { let region = region.ok_or_else(|| other("no objects were found"))?; let bbox = region.bbox; s.edit("Mask All Objects", |doc, _| { - let mut mask = LayerMask::hide_all(); + let mut mask = crate::commands::mask_at_depth(LayerMask::hide_all(), doc.depth)?; let v: Vec = region.mask.iter().map(|m| f32::from(*m) / 255.0).collect(); mask.surface.write_region(bbox, &v); mask.surface.prune(); + crate::extra_cmds::background_to_layer_for_mask(doc, id); doc.layer_mut(id).ok_or(EngineError::NoLayer(id))?.mask = Some(mask); Ok(()) })?; diff --git a/crates/engine/src/layer_menu_cmds/tests.rs b/crates/engine/src/layer_menu_cmds/tests.rs index 46688500b1..555421ed8b 100644 --- a/crates/engine/src/layer_menu_cmds/tests.rs +++ b/crates/engine/src/layer_menu_cmds/tests.rs @@ -505,6 +505,59 @@ fn mask_all_objects_masks_the_subject() { } } +fn subject(x: i32, y: i32) -> [f32; 4] { + let inside = (((x - 20) * (x - 20) + (y - 15) * (y - 15)) as f32).sqrt() < 8.0; + if inside { [0.9, 0.2, 0.1, 1.0] } else { [0.95, 0.95, 0.95, 1.0] } +} + +#[test] +fn from_transparency_and_mask_all_objects_turn_the_background_into_a_layer() { + for cmd in ["layer.layerMask.fromTransparency", "layer.maskAllObjects", "layer.layerMask.hideSelection"] { + let mut s = Session::new(); + s.execute("file.new", json!({"width": 40, "height": 30, "background": "white"})).unwrap(); + let id = s.active().unwrap().active_layer.unwrap(); + assert!(doc(&s).is_background(id)); + paint(&mut s, subject); + if cmd.ends_with("hideSelection") { + select_left_half(&mut s); + } + s.execute(cmd, json!({})).unwrap(); + assert!(!doc(&s).is_background(id), "{cmd}: the Background became a layer"); + assert_eq!(active(&s).name, "Layer 0", "{cmd}"); + assert!(active(&s).mask.is_some(), "{cmd}"); + } +} + +#[test] +fn layer_masks_are_built_at_the_document_depth() { + use photocraft_color::SampleType; + for (depth, sample, tol) in [(8, SampleType::U8, 1.0 / 255.0), (16, SampleType::U16, 1.0 / 65535.0), (32, SampleType::F32, 1e-6)] { + let mut s = session(depth, "rgb"); + paint(&mut s, |_, _| [0.2, 0.4, 0.6, 0.5003]); + s.execute("layer.layerMask.fromTransparency", json!({})).unwrap(); + let m = active(&s).mask.as_ref().unwrap(); + assert_eq!(m.surface.format().sample, sample, "From Transparency at {depth}"); + assert!((m.value(5, 5) - 0.5003).abs() < tol, "From Transparency at {depth}: {}", m.value(5, 5)); + assert_eq!(m.value(-50, -50), 0.0, "outside the layer stays hidden at {depth}"); + + let mut s = session(depth, "rgb"); + paint(&mut s, |_, _| [0.2, 0.4, 0.6, 1.0]); + select_left_half(&mut s); + s.execute("layer.layerMask.hideSelection", json!({})).unwrap(); + let m = active(&s).mask.as_ref().unwrap(); + assert_eq!(m.surface.format().sample, sample, "Hide Selection at {depth}"); + assert_eq!((m.value(5, 5), m.value(30, 5), m.value(500, 500)), (0.0, 1.0, 1.0), "Hide Selection at {depth}"); + + let mut s = session(depth, "rgb"); + paint(&mut s, subject); + s.execute("layer.maskAllObjects", json!({})).unwrap(); + let m = active(&s).mask.as_ref().unwrap(); + assert_eq!(m.surface.format().sample, sample, "Mask All Objects at {depth}"); + assert!(m.value(20, 15) > m.value(1, 1), "Mask All Objects at {depth}"); + assert_eq!(m.value(500, 500), 0.0, "Mask All Objects at {depth}"); + } +} + #[test] fn layer_masks_in_gray_cmyk_lab() { for mode in ["gray", "cmyk", "lab"] {