Fade: restore uniformly inverted layer masks (#7383) (#7463)

* fix(fade): detect uniform mask changes and blend full canvas

* test(fade): cover uniform mask inversion and Fade across bit depths

* style(fade): keep default-pixel comparison readable

* Format edit_menu_cmds.rs with rustfmt

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Brandon Thomas <echelon@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

crates/engine/src/edit_menu_cmds.rs

@@ -68,7 +68,7 @@ pub(crate) fn after_command(s: &mut Session, id: &str) {
             let now = st.doc.layer(layer);
             let was = prev.layer(layer);
             let surf_eq = |a: Option<&Surface>, b: Option<&Surface>| match (a, b) {
-                (Some(a), Some(b)) => crate_fingerprint(a) == crate_fingerprint(b),
+                (Some(a), Some(b)) => a.format() == b.format() && a.default_pixel() == b.default_pixel() && crate_fingerprint(a) == crate_fingerprint(b),
                 (None, None) => true,
                 _ => false,
             };
@@ -214,7 +214,9 @@ fn fade(s: &mut Session, p: &Value) -> Result<Value> {
     if now.format() != before.format() {
         return Err(EngineError::Other("can't fade a step that changed the layer's pixel format".into()));
     }
-    let area = before.content_bounds().union(&now.content_bounds());
+    // A uniform mask can change every pixel without storing a single tile. In that case
+    // the content bounds are empty, but Fade must still blend across the canvas.
+    let area = if before.default_pixel() != now.default_pixel() { st.doc.bounds() } else { before.content_bounds().union(&now.content_bounds()) };
     let label = format!("Fade {}", src.label);
     s.edit(&label, |doc, _| {
         let l = doc.layer_mut(src.layer).ok_or(EngineError::NoLayer(src.layer))?;

crates/engine/src/edit_menu_cmds/tests.rs

@@ -57,6 +57,38 @@ fn fade_with_blend_mode_and_invalidation() {
     assert!(s.execute("edit.fade", json!({"mode": "nonsense"})).is_err());
 }
 
+#[test]
+fn fade_uniform_layer_mask_invert_at_all_depths() {
+    for depth in [8, 16, 32] {
+        for (opacity, expected) in [(0, 1.0), (50, 0.5), (100, 0.0)] {
+            let mut s = session(depth);
+            s.execute("layer.layerMask.revealAll", json!({})).unwrap();
+            s.execute("channel.target", json!({"channel": "mask"})).unwrap();
+            let mask_pixel = |s: &Session| {
+                let layer = active(s);
+                let mask = &layer.mask.as_ref().unwrap().surface;
+                mask.pixel(5, 5)[0]
+            };
+            assert!((mask_pixel(&s) - 1.0).abs() < 0.01);
+            s.execute("image.adjustments.invert", json!({})).unwrap();
+            assert!(mask_pixel(&s) < 0.01, "depth {depth}: invert changed the mask");
+            assert!(s.edit_state.fade.as_ref().is_some_and(|f| f.mask), "depth {depth}: Fade must target the mask");
+            let history_before_fade = s.active().unwrap().history.past_len();
+
+            s.execute("edit.fade", json!({"opacity": opacity})).unwrap();
+            for (x, y) in [(0, 0), (5, 5), (63, 47)] {
+                let pixel = active(&s).mask.as_ref().unwrap().surface.pixel(x, y)[0];
+                assert!((pixel - expected).abs() < 0.02, "depth {depth}, opacity {opacity}, ({x}, {y}): {pixel}");
+            }
+            assert_eq!(s.active().unwrap().history.past_len(), history_before_fade + 1);
+            assert!(s.undo());
+            assert!(mask_pixel(&s) < 0.01, "undo restores inverted mask");
+            assert!(s.redo());
+            assert!((mask_pixel(&s) - expected).abs() < 0.02, "redo restores faded mask");
+        }
+    }
+}
+
 #[test]
 fn purge_commands() {
     let mut s = session(8);