docking: floating windows stay in the app window on Wayland (#292)

* docking: floating windows stay in the app window on Wayland

The compositor reports no window positions, so native windows could not
dock again by drag or reopen where they were. The dock area detects
Wayland (WAYLAND_DISPLAY or WAYLAND_SOCKET set and a main window that never
reported its outer rect) and draws floats in-window, keeping their native
rects. DockArea::native_windows(bool) overrides the choice.

* docking: a missing DISPLAY also counts as a possible Wayland backend

* Revert "docking: a missing DISPLAY also counts as a possible Wayland backend"

This reverts commit a804afb1d3.

* docking: the Wayland session is a pure function of the environment; without a Wayland variable only a real window counts

crates/ui/src/docking/floating_tests.rs

@@ -149,7 +149,8 @@ impl Driver {
                 Hide::All => HidePanels::All,
                 Hide::Keep => HidePanels::Except(&keep),
             };
-            output = DockArea::new(Id::new(AREA)).maximized(maximized).hide_panels(hide).show_content(ui, layout, &style, content);
+            output =
+                DockArea::new(Id::new(AREA)).native_windows(true).maximized(maximized).hide_panels(hide).show_content(ui, layout, &style, content);
         });
         let mut full = full;
         full.textures_delta.clear();
@@ -421,7 +422,7 @@ fn close_other_panels_keeps_only_the_chosen_tab() {
         |ui, (content, actions, events): &mut State| {
             let mut style = DockStyle::from_ui(ui);
             frame_style(&mut style);
-            let out = DockArea::new(Id::new(AREA)).show_content(ui, &frames(), &style, content);
+            let out = DockArea::new(Id::new(AREA)).native_windows(true).show_content(ui, &frames(), &style, content);
             actions.extend(out.actions);
             events.extend(out.events);
         },
@@ -684,7 +685,7 @@ fn undock_panel_group_floats_one_group_of_a_floating_window() {
         |ui, (content, layout, actions): &mut State| {
             let mut style = DockStyle::from_ui(ui);
             frame_style(&mut style);
-            let out = DockArea::new(Id::new(AREA)).show_content(ui, layout, &style, content);
+            let out = DockArea::new(Id::new(AREA)).native_windows(true).show_content(ui, layout, &style, content);
             actions.extend(out.actions);
         },
         (Content::default(), stacked(), Vec::new()),

crates/ui/src/docking/native_tests.rs

@@ -3,7 +3,7 @@
 
 use egui::{Color32, Event, Key, Modifiers, PointerButton, Pos2, Rect, ViewportCommand, epaint::ClippedShape, pos2, vec2};
 
-use super::render::{APPKIT_CORNER, DragPointer, MACOS_CLOSE, WindowDrag, appkit_resize_areas, close_rects, floats_shown, resize_bands};
+use super::render::{APPKIT_CORNER, DragPointer, MACOS_CLOSE, WindowDrag, appkit_resize_areas, close_rects, floats_shown, resize_bands, wayland};
 use super::testing::DockDriver;
 use super::{Action, DockContent, Floating, Hit, Layout, Monitor, Node, Permissions, Placement, Target, Workspace};
 use crate::layout::{SplitAxis, SplitSize};
@@ -1303,3 +1303,68 @@ fn a_layout_built_each_frame_without_ids_is_warned_about() {
     }
     assert!(!warned(&d));
 }
+
+/// [`layout`] with window `[5, 6]` saved as a native window at `NATIVE`.
+fn saved_native() -> Layout<u32> {
+    let mut layout = layout(true);
+    layout.floating[0].native = Some(NATIVE);
+    layout
+}
+
+const NATIVE: [f32; 4] = [700.0, 400.0, 240.0, 300.0];
+
+#[test]
+fn with_native_windows_off_a_float_stays_in_the_app_window_and_keeps_its_native_rect() {
+    let mut d = DockDriver::new(saved_native(), Content::default());
+    d.native_windows = false;
+    let mut d = d.with_native_host(ORIGIN);
+    assert!(d.host().windows.is_empty() && d.host().commands.is_empty());
+    assert!(d.hit(&Hit::Title(5)).is_some(), "drawn in the main window");
+    assert_eq!(window(&d, 5).native, Some(NATIVE));
+
+    // Its tab docks again by drag, in the main window.
+    let tab = d.hit(&Hit::Tab(5)).unwrap().center();
+    let out = d.drag(tab, pos2(600.0, 300.0));
+    assert!(matches!(out.actions.as_slice(), [Action::Move { panel: 5, anchor: 3 | 4, .. }]), "{:?}", out.actions);
+    assert!(in_root(&d, 5));
+    assert_eq!(window(&d, 6).native, Some(NATIVE));
+    assert!(d.host().windows.is_empty() && d.host().commands.is_empty());
+
+    // The same workspace opens as a native window where native windows are on.
+    let mut d = driver(saved_native());
+    d.settle();
+    assert_eq!(os_window(&d, 5).map(|(rect, _)| rect), Some(Rect::from_min_size(pos2(700.0, 400.0), vec2(240.0, 300.0))));
+}
+
+#[test]
+fn wayland_is_a_main_window_that_never_reported_a_position() {
+    use super::render::Session;
+    let mut d = driver(layout(false));
+    d.host().no_position = true;
+    d.step(vec![]);
+    assert!(wayland(&d.ctx, Session::Wayland));
+    assert!(wayland(&d.ctx, Session::Unknown), "a real window without a Wayland variable");
+    assert!(!wayland(&d.ctx, Session::Other), "X11");
+    // X11 (forced under a Wayland session) reports it; minimized, it reports none again.
+    d.host().no_position = false;
+    d.step(vec![]);
+    assert!(!wayland(&d.ctx, Session::Wayland));
+    d.host().no_position = true;
+    d.step(vec![]);
+    assert!(!wayland(&d.ctx, Session::Wayland));
+    // A context with no real window (a headless test) and no display variables at all.
+    let ctx = egui::Context::default();
+    ctx.run_ui(egui::RawInput::default(), |_| {}).textures_delta.clear();
+    assert!(!wayland(&ctx, Session::Unknown));
+}
+
+#[test]
+fn the_session_follows_winits_environment_rule() {
+    use super::render::{Session, session};
+    let with = |vars: &'static [&'static str]| move |name: &str| vars.contains(&name);
+    assert_eq!(session(true, with(&["WAYLAND_DISPLAY", "DISPLAY"])), Session::Wayland);
+    assert_eq!(session(true, with(&["WAYLAND_SOCKET"])), Session::Wayland);
+    assert_eq!(session(true, with(&["DISPLAY"])), Session::Other);
+    assert_eq!(session(true, with(&[])), Session::Unknown);
+    assert_eq!(session(false, with(&["WAYLAND_DISPLAY"])), Session::Other);
+}

crates/ui/src/docking/render/mod.rs

@@ -17,7 +17,7 @@ mod native;
 #[cfg(all(test, feature = "native-windows"))]
 pub(in crate::docking) use {
     cross::DragPointer,
-    native::{APPKIT_CORNER, MACOS_CLOSE, WindowDrag, appkit_resize_areas, close_rects, floats_shown, resize_bands},
+    native::{APPKIT_CORNER, MACOS_CLOSE, Session, WindowDrag, appkit_resize_areas, close_rects, floats_shown, resize_bands, session, wayland},
 };
 
 /// The viewport of the native window `window` of the dock area `area`.
@@ -76,6 +76,9 @@ pub(super) struct Renderer<'a, P> {
     /// [`crate::docking::DockArea::monitors`].
     #[cfg_attr(not(feature = "native-windows"), allow(dead_code))]
     pub(super) monitors: Vec<crate::docking::Monitor>,
+    /// [`crate::docking::DockArea::native_windows`].
+    #[cfg_attr(not(feature = "native-windows"), allow(dead_code))]
+    pub(super) native_windows: Option<bool>,
     /// The other OS windows a drag may cross into.
     #[cfg(feature = "native-windows")]
     pub(super) cross: cross::Cross<P>,

crates/ui/src/docking/render/native.rs

@@ -2,8 +2,9 @@
 //! viewports, every floating window whose panels allow it is drawn with
 //! `show_viewport_immediate`, always on top of other windows and hidden while none of the
 //! application's windows has focus. Where egui embeds viewports (wasm, tests) windows stay
-//! in-window at their in-window rect. Tabs, groups and whole windows drag between OS windows
-//! (see `cross.rs`).
+//! in-window at their in-window rect, and so do they on Wayland, unless the app turns native
+//! windows on ([`crate::docking::DockArea::native_windows`]). Tabs, groups and whole windows
+//! drag between OS windows (see `cross.rs`).
 
 use std::{fmt::Debug, hash::Hash};
 
@@ -199,7 +200,7 @@ impl<P> Default for Report<P> {
 impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
     /// Show the native windows and return the windows to draw in-window.
     pub(super) fn native_windows<'w>(&mut self, ctx: &Context, windows: Vec<&'w Floating<P>>) -> Vec<&'w Floating<P>> {
-        let native = !ctx.embed_viewports();
+        let native = !ctx.embed_viewports() && self.native_windows.unwrap_or_else(|| !wayland(ctx, env_session()));
         let origin = ctx.input(|i| i.viewport().inner_rect.map(|r| r.min)).filter(|p| p.is_finite()).unwrap_or(Pos2::ZERO);
         let visibility = if native { self.float_visibility(ctx) } else { (true, false) };
         self.cross.on = native;
@@ -664,6 +665,66 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static> Renderer<'_, P> {
 #[cfg(debug_assertions)]
 const REBUILT_FRAMES: usize = 3;
 
+/// What the environment says of winit's backend.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub(in crate::docking) enum Session {
+    /// `WAYLAND_DISPLAY` or `WAYLAND_SOCKET` is set: winit picks Wayland unless the app forces
+    /// X11 in code (`EventLoopBuilderExtX11::with_x11`); winit reads no variable that does.
+    Wayland,
+    /// None of those nor `DISPLAY` is set on a Unix desktop: a Wayland app started with only
+    /// `WAYLAND_SOCKET`, which winit unsets as it connects, or no display at all (tests, CI).
+    Unknown,
+    /// X11 or another backend.
+    Other,
+}
+
+/// The [`Session`] of a Unix desktop (`unix`) whose environment variable `name` is set and
+/// not empty when `set(name)`.
+pub(in crate::docking) fn session(unix: bool, set: impl Fn(&str) -> bool) -> Session {
+    if !unix {
+        Session::Other
+    } else if set("WAYLAND_DISPLAY") || set("WAYLAND_SOCKET") {
+        Session::Wayland
+    } else if set("DISPLAY") {
+        Session::Other
+    } else {
+        Session::Unknown
+    }
+}
+
+/// This process's [`Session`], read once.
+fn env_session() -> Session {
+    static SESSION: std::sync::OnceLock<Session> = std::sync::OnceLock::new();
+    *SESSION.get_or_init(|| {
+        session(cfg!(any(target_os = "linux", target_os = "freebsd", target_os = "dragonfly", target_os = "netbsd", target_os = "openbsd")), |name| {
+            std::env::var_os(name).is_some_and(|value| !value.is_empty())
+        })
+    })
+}
+
+/// Whether the main window runs on Wayland, given the environment's `session`. egui does not
+/// name the backend, but on Wayland the main window never reports its outer rect, while under
+/// X11 it does from the first frame on (eframe reads it as the window opens). A minimized X11
+/// window reports none either, so once one has, the backend is not Wayland. Without a Wayland
+/// variable, only a real OS window (one that reports its native pixels per point) counts.
+pub(in crate::docking) fn wayland(ctx: &Context, session: Session) -> bool {
+    let root = |test: fn(&ViewportInfo) -> bool| ctx.input(|i| i.raw.viewports.get(&ViewportId::ROOT).is_some_and(test));
+    match session {
+        Session::Other => return false,
+        Session::Unknown if !root(|v| v.native_pixels_per_point.is_some()) => return false,
+        Session::Wayland | Session::Unknown => {}
+    }
+    let key = Id::new("craft-ui-dock-window-positions");
+    if ctx.data(|d| d.get_temp::<bool>(key).unwrap_or(false)) {
+        return false;
+    }
+    let reported = root(|v| v.outer_rect.is_some());
+    if reported {
+        ctx.data_mut(|d| d.insert_temp(key, true));
+    }
+    !reported
+}
+
 /// Whether every window of `current` is new to `previous`, and they hold the same panels:
 /// windows, with their panels, as `(id, panels)`.
 #[cfg(debug_assertions)]

crates/ui/src/docking/surface.rs

@@ -746,6 +746,7 @@ pub struct DockArea<'a, P = ()> {
     auto_collapse: bool,
     float_bounds: Option<Rect>,
     monitors: Vec<Monitor>,
+    native_windows: Option<bool>,
 }
 
 impl<'a, P: Clone + Eq + Hash + Debug + Send + Sync + 'static> DockArea<'a, P> {
@@ -760,6 +761,7 @@ impl<'a, P: Clone + Eq + Hash + Debug + Send + Sync + 'static> DockArea<'a, P> {
             auto_collapse: true,
             float_bounds: None,
             monitors: Vec::new(),
+            native_windows: None,
         }
     }
 
@@ -772,6 +774,16 @@ impl<'a, P: Clone + Eq + Hash + Debug + Send + Sync + 'static> DockArea<'a, P> {
         self
     }
 
+    /// Whether floating windows open in their own OS windows (feature `native-windows`), where
+    /// the backend has viewports. Off, they stay in the app window, and their native rects are
+    /// kept for a later run with it on. By default it is on, except on Wayland: the compositor
+    /// reports no window positions, so a native window could not dock again by drag or open
+    /// where it was.
+    pub fn native_windows(mut self, on: bool) -> Self {
+        self.native_windows = Some(on);
+        self
+    }
+
     /// The egui viewport of the native window of the floating window with [`super::Floating::id`]
     /// `window`, in the dock area `id` (feature `native-windows`).
     #[cfg(feature = "native-windows")]
@@ -933,6 +945,7 @@ impl<'a, P: Clone + Eq + Hash + Debug + Send + Sync + 'static> DockArea<'a, P> {
             dividers: Vec::new(),
             tree: None,
             monitors: self.monitors,
+            native_windows: self.native_windows,
             #[cfg(feature = "native-windows")]
             cross: Default::default(),
         };

crates/ui/src/docking/testing.rs

@@ -48,6 +48,8 @@ pub struct DockDriver<P, C> {
     pub monitors: Vec<Monitor>,
     /// Passed to [`DockArea::locked`].
     pub locked: bool,
+    /// Passed to [`DockArea::native_windows`]: on by default, whatever the test's desktop.
+    pub native_windows: bool,
     /// The stand-in for OS windows, in use after [`Self::with_native_host`].
     #[cfg(feature = "native-windows")]
     host: Option<std::rc::Rc<std::cell::RefCell<NativeHost>>>,
@@ -77,6 +79,7 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static, C: DockContent<P>> Do
             float_bounds: None,
             monitors: Vec::new(),
             locked: false,
+            native_windows: true,
             #[cfg(feature = "native-windows")]
             host: None,
             #[cfg(feature = "native-windows")]
@@ -125,15 +128,20 @@ impl<P: Clone + Eq + Hash + Debug + Send + Sync + 'static, C: DockContent<P>> Do
         }
         let (layout, content, area, id) = (&self.layout, &mut self.content, self.area, self.id);
         let monitors = self.monitors.clone();
-        let (style_fn, maximized, raise, auto, bounds, locked) =
-            (&self.style, self.maximized.clone(), self.raise.take(), self.auto_collapse, self.float_bounds, self.locked);
+        let (style_fn, maximized, raise, auto, bounds, locked, native) =
+            (&self.style, self.maximized.clone(), self.raise.take(), self.auto_collapse, self.float_bounds, self.locked, self.native_windows);
         let mut full = self.ctx.run_ui(input, |ui| {
             let mut style = DockStyle::from_ui(ui);
             if let Some(adjust) = style_fn {
                 adjust(&mut style);
             }
             ui.scope_builder(UiBuilder::new().max_rect(area), |ui| {
-                let mut dock = DockArea::new(id).maximized(maximized.clone()).auto_collapse_iconic(auto).monitors(monitors.clone()).locked(locked);
+                let mut dock = DockArea::new(id)
+                    .maximized(maximized.clone())
+                    .auto_collapse_iconic(auto)
+                    .monitors(monitors.clone())
+                    .locked(locked)
+                    .native_windows(native);
                 if let Some(panel) = raise.clone() {
                     dock = dock.raise_panel(panel);
                 }
@@ -418,6 +426,7 @@ impl NativeHost {
                 .then(|| if id == egui::ViewportId::ROOT { rect } else { Rect::from_min_max(rect.min - self.frame, rect.max) }),
             focused: Some(self.focus == Some(id)),
             monitor_size: self.monitor_size,
+            native_pixels_per_point: Some(1.0),
             ..Default::default()
         };
         let main = Rect::from_min_size(self.origin, self.screen.size());

docs/ui-docking.md

@@ -355,6 +355,13 @@ viewport's `inner_rect.min` plus its rect). A window with a panel whose
 that is not floatable, stays in-window. Floating windows get no taskbar or Alt-Tab entry
 (`with_taskbar(false)`).
 
+On Wayland floating windows stay in-window: the compositor reports no window positions, so a
+native window could not dock again by drag or reopen where it was. Their `native` rects are
+kept, and the same layout opens native windows on X11, macOS and Windows. The dock area takes
+Wayland to be the backend when `WAYLAND_DISPLAY` or `WAYLAND_SOCKET` is set (winit's rule) and
+the main window has never reported its `outer_rect`; an app that forces X11 in code gets native
+windows from the first frame. `DockArea::native_windows(bool)` overrides the choice either way.
+
 - **Identity.** `Floating::id` names the OS window, so a window keeps it when its panels change,
   including when its first panel moves out. `Floating::new` and loading draw fresh ids, and
   `Floating::with_id` sets one; a layout
@@ -465,9 +472,10 @@ that is not floatable, stays in-window. Floating windows get no taskbar or Alt-T
     the desktop, or with Ctrl/Cmd held, it stays where it was moved, and the layout takes its
     place once, at the release. Escape puts it back. An
     in-window floating window's title bar still splits only groups of other floating windows.
-  - *Wayland.* Where `outer_rect` is `None`, desktop positions are unknown, so drops across
-    windows cannot be resolved. A drag works inside its own window, and a release outside it
-    floats a new window that the compositor places. Moving a window hands the move to the
+  - *Wayland.* With `DockArea::native_windows(true)` on Wayland, or wherever `outer_rect` is
+    `None`, desktop positions are unknown, so drops across windows cannot be resolved. A drag
+    works inside its own window, and a release outside it floats a new window that the
+    compositor places. Moving a window hands the move to the
     compositor, which never docks it.
 
 On the web and wherever egui embeds viewports, nothing changes: floating windows stay in the