LibGUI+WindowServer: Introduce WindowModes

Previously, Windows only understood blocking modality: Windows were
either modal, i.e., in a blocking state, or not. Windows could also
be set as Accessories or ToolWindows, attributes which technically
applied modes to their parents but were implemented ad hoc. This patch
redefines these modal effects as WindowModes and sets up some helpers.
This will let us simplify a lot of modal logic in the upcoming patches
and make it easier to build new modal effects in the future.

Windows can now set 1 of 5 modes before reification:
-Modeless:	No modal effect; begins a new modal chain
-Passive:	Window joins its modal chain but has no effect
-RenderAbove:	Window renders above its parent
-CaptureInput:	Window captures the active input role from its parent
-Blocking:	Window blocks all interaction with its modal chain

States like fullscreen and tiling are dynamic and don't alter behavior
in modal chains, so they aren't included.

Userland/DevTools/HackStudio/Dialogs/Git/GitCommitDialog.cpp

@@ -14,7 +14,6 @@ GitCommitDialog::GitCommitDialog(GUI::Window* parent)
 {
     resize(400, 260);
     center_within(*parent);
-    set_modal(true);
     set_title("Commit");
     set_icon(parent->icon());
 

Userland/DevTools/HackStudio/Dialogs/NewProjectDialog.cpp

@@ -47,7 +47,6 @@ NewProjectDialog::NewProjectDialog(GUI::Window* parent)
     resize(500, 385);
     center_on_screen();
     set_resizable(false);
-    set_modal(true);
     set_title("New project");
 
     auto& main_widget = set_main_widget<GUI::Widget>();

Userland/Libraries/LibGUI/ConnectionToWindowServer.cpp

@@ -163,8 +163,8 @@ static Action* action_for_shortcut(Window& window, Shortcut const& shortcut)
         return action;
     }
 
-    // NOTE: Application-global shortcuts are ignored while a modal window is up.
-    if (!window.is_modal()) {
+    // NOTE: Application-global shortcuts are ignored while a blocking modal window is up.
+    if (!window.is_blocking()) {
         if (auto* action = Application::the()->action_for_shortcut(shortcut)) {
             dbgln_if(KEYBOARD_SHORTCUTS_DEBUG, "  > Asked application, got action: {} {} (enabled: {}, shortcut: {}, alt-shortcut: {})", action, action->text(), action->is_enabled(), action->shortcut().to_string(), action->alternate_shortcut().to_string());
             return action;

Userland/Libraries/LibGUI/Dialog.cpp

@@ -17,8 +17,7 @@ Dialog::Dialog(Window* parent_window, ScreenPosition screen_position)
     : Window(parent_window)
     , m_screen_position(screen_position)
 {
-    set_modal(true);
-    set_minimizable(false);
+    set_window_mode(WindowMode::Blocking);
 }
 
 Dialog::ExecResult Dialog::exec()

Userland/Libraries/LibGUI/Window.cpp

@@ -69,6 +69,9 @@ Window::Window(Core::Object* parent)
     : Core::Object(parent)
     , m_menubar(Menubar::construct())
 {
+    if (parent)
+        set_window_mode(WindowMode::Passive);
+
     all_windows->set(this);
     m_rect_when_windowless = { -5000, -5000, 0, 0 };
     m_title_when_windowless = "GUI::Window";
@@ -144,7 +147,6 @@ void Window::show()
         m_rect_when_windowless,
         !m_moved_by_client,
         m_has_alpha_channel,
-        m_modal,
         m_minimizable,
         m_closeable,
         m_resizable,
@@ -159,6 +161,7 @@ void Window::show()
         m_minimum_size_when_windowless,
         m_resize_aspect_ratio,
         (i32)m_window_type,
+        (i32)m_window_mode,
         m_title_when_windowless,
         parent_window ? parent_window->window_id() : 0,
         launch_origin_rect);
@@ -316,6 +319,12 @@ void Window::set_window_type(WindowType window_type)
     m_window_type = window_type;
 }
 
+void Window::set_window_mode(WindowMode mode)
+{
+    VERIFY(!is_visible());
+    m_window_mode = mode;
+}
+
 void Window::make_window_manager(unsigned event_mask)
 {
     GUI::ConnectionToWindowManagerServer::the().async_set_event_mask(event_mask);
@@ -890,12 +899,6 @@ OwnPtr<WindowBackingStore> Window::create_backing_store(Gfx::IntSize const& size
     return make<WindowBackingStore>(bitmap_or_error.release_value());
 }
 
-void Window::set_modal(bool modal)
-{
-    VERIFY(!is_visible());
-    m_modal = modal;
-}
-
 void Window::wm_event(WMEvent&)
 {
 }

Userland/Libraries/LibGUI/Window.h

@@ -14,6 +14,7 @@
 #include <LibCore/Object.h>
 #include <LibGUI/FocusSource.h>
 #include <LibGUI/Forward.h>
+#include <LibGUI/WindowMode.h>
 #include <LibGUI/WindowType.h>
 #include <LibGfx/Forward.h>
 #include <LibGfx/Rect.h>
@@ -33,8 +34,8 @@ public:
     bool is_modified() const;
     void set_modified(bool);
 
-    bool is_modal() const { return m_modal; }
-    void set_modal(bool);
+    bool is_modal() const { return m_window_mode != WindowMode::Modeless; }
+    bool is_blocking() const { return m_window_mode == WindowMode::Blocking; }
 
     bool is_fullscreen() const { return m_fullscreen; }
     void set_fullscreen(bool);
@@ -71,6 +72,9 @@ public:
     WindowType window_type() const { return m_window_type; }
     void set_window_type(WindowType);
 
+    WindowMode window_mode() const { return m_window_mode; }
+    void set_window_mode(WindowMode);
+
     int window_id() const { return m_window_id; }
 
     void make_window_manager(unsigned event_mask);
@@ -284,12 +288,12 @@ private:
     Gfx::IntSize m_size_increment;
     Gfx::IntSize m_base_size;
     WindowType m_window_type { WindowType::Normal };
+    WindowMode m_window_mode { WindowMode::Modeless };
     AK::Variant<Gfx::StandardCursor, NonnullRefPtr<Gfx::Bitmap>> m_cursor { Gfx::StandardCursor::None };
     AK::Variant<Gfx::StandardCursor, NonnullRefPtr<Gfx::Bitmap>> m_effective_cursor { Gfx::StandardCursor::None };
     bool m_is_active_input { false };
     bool m_has_alpha_channel { false };
     bool m_double_buffering_enabled { true };
-    bool m_modal { false };
     bool m_resizable { true };
     bool m_obey_widget_min_size { true };
     Optional<Gfx::IntSize> m_resize_aspect_ratio {};

Userland/Libraries/LibGUI/WindowMode.h

@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2022, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <Services/WindowServer/WindowMode.h>
+
+namespace GUI {
+
+using WindowMode = WindowServer::WindowMode;
+
+}

Userland/Services/WindowServer/ConnectionFromClient.cpp

@@ -563,10 +563,10 @@ Window* ConnectionFromClient::window_from_id(i32 window_id)
 }
 
 void ConnectionFromClient::create_window(i32 window_id, Gfx::IntRect const& rect,
-    bool auto_position, bool has_alpha_channel, bool modal, bool minimizable, bool closeable, bool resizable,
+    bool auto_position, bool has_alpha_channel, bool minimizable, bool closeable, bool resizable,
     bool fullscreen, bool frameless, bool forced_shadow, bool accessory, float opacity,
     float alpha_hit_threshold, Gfx::IntSize const& base_size, Gfx::IntSize const& size_increment,
-    Gfx::IntSize const& minimum_size, Optional<Gfx::IntSize> const& resize_aspect_ratio, i32 type,
+    Gfx::IntSize const& minimum_size, Optional<Gfx::IntSize> const& resize_aspect_ratio, i32 type, i32 mode,
     String const& title, i32 parent_window_id, Gfx::IntRect const& launch_origin_rect)
 {
     Window* parent_window = nullptr;
@@ -576,6 +576,10 @@ void ConnectionFromClient::create_window(i32 window_id, Gfx::IntRect const& rect
             did_misbehave("CreateWindow with bad parent_window_id");
             return;
         }
+        if (parent_window->is_blocking() || (parent_window->is_capturing_input() && mode == (i32)WindowMode::CaptureInput)) {
+            did_misbehave("CreateWindow with forbidden parent mode");
+            return;
+        }
     }
 
     if (type < 0 || type >= (i32)WindowType::_Count) {
@@ -583,12 +587,17 @@ void ConnectionFromClient::create_window(i32 window_id, Gfx::IntRect const& rect
         return;
     }
 
+    if (mode < 0 || mode >= (i32)WindowMode::_Count) {
+        did_misbehave("CreateWindow with a bad mode");
+        return;
+    }
+
     if (m_windows.contains(window_id)) {
         did_misbehave("CreateWindow with already-used window ID");
         return;
     }
 
-    auto window = Window::construct(*this, (WindowType)type, window_id, modal, minimizable, closeable, frameless, resizable, fullscreen, accessory, parent_window);
+    auto window = Window::construct(*this, (WindowType)type, (WindowMode)mode, window_id, minimizable, closeable, frameless, resizable, fullscreen, accessory, parent_window);
 
     window->set_forced_shadow(forced_shadow);
 

Userland/Services/WindowServer/ConnectionFromClient.h

@@ -101,9 +101,9 @@ private:
     virtual void update_menu_item(i32, i32, i32, String const&, bool, bool, bool, bool, String const&, Gfx::ShareableBitmap const&) override;
     virtual void remove_menu_item(i32 menu_id, i32 identifier) override;
     virtual void flash_menubar_menu(i32, i32) override;
-    virtual void create_window(i32, Gfx::IntRect const&, bool, bool, bool, bool, bool,
+    virtual void create_window(i32, Gfx::IntRect const&, bool, bool, bool, bool,
         bool, bool, bool, bool, bool, float, float, Gfx::IntSize const&, Gfx::IntSize const&, Gfx::IntSize const&,
-        Optional<Gfx::IntSize> const&, i32, String const&, i32, Gfx::IntRect const&) override;
+        Optional<Gfx::IntSize> const&, i32, i32, String const&, i32, Gfx::IntRect const&) override;
     virtual Messages::WindowServer::DestroyWindowResponse destroy_window(i32) override;
     virtual void set_window_title(i32, String const&) override;
     virtual Messages::WindowServer::GetWindowTitleResponse get_window_title(i32) override;

Userland/Services/WindowServer/Window.cpp

@@ -90,11 +90,11 @@ Window::Window(Core::Object& parent, WindowType type)
     frame().window_was_constructed({});
 }
 
-Window::Window(ConnectionFromClient& client, WindowType window_type, int window_id, bool modal, bool minimizable, bool closeable, bool frameless, bool resizable, bool fullscreen, bool accessory, Window* parent_window)
+Window::Window(ConnectionFromClient& client, WindowType window_type, WindowMode window_mode, int window_id, bool minimizable, bool closeable, bool frameless, bool resizable, bool fullscreen, bool accessory, Window* parent_window)
     : Core::Object(&client)
     , m_client(&client)
     , m_type(window_type)
-    , m_modal(modal)
+    , m_mode(window_mode)
     , m_minimizable(minimizable)
     , m_closeable(closeable)
     , m_frameless(frameless)
@@ -739,7 +739,7 @@ void Window::ensure_window_menu()
 
         m_window_menu->add_item(make<MenuItem>(*m_window_menu, MenuItem::Type::Separator));
 
-        if (!m_modal) {
+        if (!is_modal()) {
             auto pin_item = make<MenuItem>(*m_window_menu, (unsigned)WindowMenuAction::ToggleAlwaysOnTop, "Always on &Top");
             m_window_menu_always_on_top_item = pin_item.ptr();
             m_window_menu_always_on_top_item->set_icon(&pin_icon());
@@ -1019,23 +1019,6 @@ bool Window::is_accessory_of(Window& window) const
     return parent_window() == &window;
 }
 
-void Window::modal_unparented()
-{
-    m_modal = false;
-    WindowManager::the().notify_modal_unparented(*this);
-}
-
-bool Window::is_modal() const
-{
-    if (!m_modal)
-        return false;
-    if (!m_parent_window) {
-        const_cast<Window*>(this)->modal_unparented();
-        return false;
-    }
-    return true;
-}
-
 void Window::set_progress(Optional<int> progress)
 {
     if (m_progress == progress)

Userland/Services/WindowServer/Window.h

@@ -17,6 +17,7 @@
 #include <WindowServer/Menubar.h>
 #include <WindowServer/Screen.h>
 #include <WindowServer/WindowFrame.h>
+#include <WindowServer/WindowMode.h>
 #include <WindowServer/WindowType.h>
 
 namespace WindowServer {
@@ -141,8 +142,6 @@ public:
     WindowFrame& frame() { return m_frame; }
     WindowFrame const& frame() const { return m_frame; }
 
-    Window* blocking_modal_window();
-
     ConnectionFromClient* client() { return m_client; }
     ConnectionFromClient const* client() const { return m_client; }
 
@@ -182,8 +181,13 @@ public:
     bool is_visible() const { return m_visible; }
     void set_visible(bool);
 
-    bool is_modal() const;
-    bool is_modal_dont_unparent() const { return m_modal && m_parent_window; }
+    bool is_modal() const { return m_mode != WindowMode::Modeless; }
+    bool is_passive() { return m_mode == WindowMode::Passive; }
+
+    bool is_blocking() const { return m_mode == WindowMode::Blocking; }
+    Window* blocking_modal_window();
+
+    WindowMode mode() const { return m_mode; }
     Window* modeless_ancestor();
 
     Gfx::IntRect rect() const { return m_rect; }
@@ -381,7 +385,7 @@ public:
     bool is_stealable_by_client(i32 client_id) const { return m_stealable_by_client_ids.contains_slow(client_id); }
 
 private:
-    Window(ConnectionFromClient&, WindowType, int window_id, bool modal, bool minimizable, bool closeable, bool frameless, bool resizable, bool fullscreen, bool accessory, Window* parent_window = nullptr);
+    Window(ConnectionFromClient&, WindowType, WindowMode, int window_id, bool minimizable, bool closeable, bool frameless, bool resizable, bool fullscreen, bool accessory, Window* parent_window = nullptr);
     Window(Core::Object&, WindowType);
 
     virtual void event(Core::Event&) override;
@@ -412,10 +416,10 @@ private:
     Gfx::DisjointRectSet m_transparency_wallpaper_rects;
     HashMap<Window*, Gfx::DisjointRectSet> m_affected_transparency_rects;
     WindowType m_type { WindowType::Normal };
+    WindowMode m_mode { WindowMode::Modeless };
     bool m_automatic_cursor_tracking_enabled { false };
     bool m_visible { true };
     bool m_has_alpha_channel { false };
-    bool m_modal { false };
     bool m_minimizable { false };
     bool m_closeable { false };
     bool m_frameless { false };

Userland/Services/WindowServer/WindowManager.cpp

@@ -446,7 +446,7 @@ void WindowManager::tell_wm_about_window(WMConnectionFromClient& conn, Window& w
         return;
     auto* parent = window.parent_window();
     auto& window_stack = is_stationary_window_type(window.type()) ? current_window_stack() : window.window_stack();
-    conn.async_window_state_changed(conn.window_id(), window.client_id(), window.window_id(), parent ? parent->client_id() : -1, parent ? parent->window_id() : -1, window_stack.row(), window_stack.column(), window.is_active(), window.is_minimized(), window.is_modal_dont_unparent(), window.is_frameless(), (i32)window.type(), window.computed_title(), window.rect(), window.progress());
+    conn.async_window_state_changed(conn.window_id(), window.client_id(), window.window_id(), parent ? parent->client_id() : -1, parent ? parent->window_id() : -1, window_stack.row(), window_stack.column(), window.is_active(), window.is_minimized(), window.is_modal(), window.is_frameless(), (i32)window.type(), window.computed_title(), window.rect(), window.progress());
 }
 
 void WindowManager::tell_wm_about_window_rect(WMConnectionFromClient& conn, Window& window)
@@ -601,19 +601,6 @@ void WindowManager::notify_title_changed(Window& window)
     tell_wms_window_state_changed(window);
 }
 
-void WindowManager::notify_modal_unparented(Window& window)
-{
-    if (window.type() != WindowType::Normal)
-        return;
-
-    dbgln_if(WINDOWMANAGER_DEBUG, "[WM] Window({}) was unparented", &window);
-
-    if (m_switcher->is_visible())
-        m_switcher->refresh();
-
-    tell_wms_window_state_changed(window);
-}
-
 void WindowManager::notify_rect_changed(Window& window, Gfx::IntRect const& old_rect, Gfx::IntRect const& new_rect)
 {
     dbgln_if(RESIZE_DEBUG, "[WM] Window({}) rect changed {} -> {}", &window, old_rect, new_rect);

Userland/Services/WindowServer/WindowManager.h

@@ -81,7 +81,6 @@ public:
     void remove_window(Window&);
 
     void notify_title_changed(Window&);
-    void notify_modal_unparented(Window&);
     void notify_rect_changed(Window&, Gfx::IntRect const& oldRect, Gfx::IntRect const& newRect);
     void notify_minimization_state_changed(Window&);
     void notify_opacity_changed(Window&);

Userland/Services/WindowServer/WindowMode.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2022, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+namespace WindowServer {
+
+// WindowMode sets modal behavior for windows in a modal chain
+//
+// - Modeless:      No modal effect (default mode for parentless windows)
+// - Passive:       Joins the modal chain but has no modal effect (default mode for child windows)
+// - RenderAbove:   Renders above its parent
+// - CaptureInput:  Captures input from its parent
+// - Blocking:      Preempts all interaction with its modal chain
+enum class WindowMode {
+    Modeless = 0,
+    Passive,
+    RenderAbove,
+    CaptureInput,
+    Blocking,
+    _Count,
+};
+
+}

Userland/Services/WindowServer/WindowServer.ipc

@@ -44,7 +44,6 @@ endpoint WindowServer
         Gfx::IntRect rect,
         bool auto_position,
         bool has_alpha_channel,
-        bool modal,
         bool minimizable,
         bool closeable,
         bool resizable,
@@ -59,6 +58,7 @@ endpoint WindowServer
         Gfx::IntSize minimum_size,
         Optional<Gfx::IntSize> resize_aspect_ratio,
         i32 type,
+        i32 mode,
         [UTF8] String title,
         i32 parent_window_id,
         Gfx::IntRect launch_origin_rect) =|