macOS: Disable interaction for modally blocked transient parent windows

When a window-modal window has a transient ancestor, Qt treats this
ancestor window as modally blocked by the modal window, as if it had
been a true non-transient parent of the modal window.

Unfortunately, this is not how macOS natively behaves. Window-modal
windows only block their direct parent, and AppKit will happily
send events to any other top level window. This is different from
how application modal windows work, where NSApplication will filter
many events (but not all) in [[NSApplication _modalSession:sendEvent:].

Note that NSWindow.worksWhenModal has no effect in this situation,
as that property is only considered by AppKit for application
modal session are active (and NSApp.modalWidow returns non-nil).

Instead of trying to replicate the event filtering that AppKit does,
which would be fragile, we disable some of the effects these events
could potentially have, by for example preventing modally blocked
windows from becoming key, and temporarily disabling the close
button in the title bar.

One remaining issue is that, unlike with application modal windows,
the modally blocked transient parents can still be ordered above
the modal window. Fixing this requires informing the window server
about the modally blocked state of the window, which we can't do
using public APIs. Even returning NO from [NSWindow _allowsOrdering]
is not sufficient.

Task-number: QTBUG-104905
Pick-to: 6.5
Change-Id: I7e764a354f397ae6ef61304ca5442a4e1bb7589c
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

2 files changed, 32 insertions, 4 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoawindow.h b/src/plugins/platforms/cocoa/qcocoawindow.h
index 265ded0204..ba1bc052fb 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.h
+++ b/src/plugins/platforms/cocoa/qcocoawindow.h
@@ -169,6 +169,8 @@ public:
     QWindow *childWindowAt(QPoint windowPoint);
     bool shouldRefuseKeyWindowAndFirstResponder();
 
+    bool windowEvent(QEvent *event) override;
+
     QPoint bottomLeftClippedByNSWindowOffset() const override;
 
     void updateNormalGeometry();
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.mm b/src/plugins/platforms/cocoa/qcocoawindow.mm
index de49baf1e7..2083803c6d 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.mm
+++ b/src/plugins/platforms/cocoa/qcocoawindow.mm
@@ -543,8 +543,6 @@ void QCocoaWindow::updateTitleBarButtons(Qt::WindowFlags windowFlags)
     if (!isContentView())
         return;
 
-    NSWindow *window = m_view.window;
-
     static constexpr std::pair<NSWindowButton, Qt::WindowFlags> buttons[] = {
         { NSWindowCloseButton, Qt::WindowCloseButtonHint },
         { NSWindowMiniaturizeButton, Qt::WindowMinimizeButtonHint},
@@ -560,13 +558,24 @@ void QCocoaWindow::updateTitleBarButtons(Qt::WindowFlags windowFlags)
         if (button == NSWindowZoomButton && isFixedSize())
             enabled = false;
 
-        [window standardWindowButton:button].enabled = enabled;
+        // Mimic what macOS natively does for parent windows of modal
+        // sheets, which is to disable the close button, but leave the
+        // other buttons as they were.
+        if (button == NSWindowCloseButton && enabled
+            && QWindowPrivate::get(window())->blockedByModalWindow) {
+            enabled = false;
+            // If we end up having no enabled buttons, our workaround
+            // should not be a reason for hiding all of them.
+            hideButtons = false;
+        }
+
+        [m_view.window standardWindowButton:button].enabled = enabled;
         hideButtons &= !enabled;
     }
 
     // Hide buttons in case we disabled all of them
     for (const auto &[button, buttonHint] : buttons)
-        [window standardWindowButton:button].hidden = hideButtons;
+        [m_view.window standardWindowButton:button].hidden = hideButtons;
 }
 
 void QCocoaWindow::setWindowFlags(Qt::WindowFlags flags)
@@ -1931,6 +1940,9 @@ bool QCocoaWindow::shouldRefuseKeyWindowAndFirstResponder()
     if (window()->flags() & (Qt::WindowDoesNotAcceptFocus | Qt::WindowTransparentForInput))
         return true;
 
+    if (QWindowPrivate::get(window())->blockedByModalWindow)
+        return true;
+
     if (m_inSetVisible) {
         QVariant showWithoutActivating = window()->property("_q_showWithoutActivating");
         if (showWithoutActivating.isValid() && showWithoutActivating.toBool())
@@ -1940,6 +1952,20 @@ bool QCocoaWindow::shouldRefuseKeyWindowAndFirstResponder()
     return false;
 }
 
+bool QCocoaWindow::windowEvent(QEvent *event)
+{
+    switch (event->type()) {
+    case QEvent::WindowBlocked:
+    case QEvent::WindowUnblocked:
+        updateTitleBarButtons(window()->flags());
+        break;
+    default:
+        break;
+    }
+
+    return QPlatformWindow::windowEvent(event);
+}
+
 QPoint QCocoaWindow::bottomLeftClippedByNSWindowOffset() const
 {
     if (!m_view)