Don't disable menu items that belong to the active modal window

A popup/context menu created via QQuickPlatformMenu doesn't belong to any
menubar, so by disabling items in a menu that doesn't belong to the currently
active menubar (5b9f6862b1), we disabled all menu items in a QQuickPlatformMenu
when a modal window was active.

For such unrooted menus, use the QCocoaMenuObject data structure to record
which window it is shown for, and only disable items if that window is not
also the current modal window.

Amends 5b9f6862b1aa474a392203c69f6db678d633cecf.

Fixes: QTBUG-92040
Pick-to: 6.1 6.0 5.15
Change-Id: I56b6d579e5e94689b43ca84d4637e35dc2cbeb4c
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

3 files changed, 18 insertions, 5 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoamenu.mm b/src/plugins/platforms/cocoa/qcocoamenu.mm
index 1a788e5197..f3ee39160f 100644
--- a/src/plugins/platforms/cocoa/qcocoamenu.mm
+++ b/src/plugins/platforms/cocoa/qcocoamenu.mm
@@ -360,6 +360,17 @@ void QCocoaMenu::showPopup(const QWindow *parentWindow, const QRect &targetRect,
     NSView *view = cocoaWindow ? cocoaWindow->view() : nil;
     NSMenuItem *nsItem = item ? ((QCocoaMenuItem *)item)->nsItem() : nil;
 
+    // store the window that this popup belongs to so that we can evaluate whether we are modally blocked
+    bool resetMenuParent = false;
+    if (!menuParent()) {
+        setMenuParent(cocoaWindow);
+        resetMenuParent = true;
+    }
+    auto menuParentGuard = qScopeGuard([&]{
+        if (resetMenuParent)
+            setMenuParent(nullptr);
+    });
+
     QScreen *screen = nullptr;
     if (parentWindow)
         screen = parentWindow->screen();
diff --git a/src/plugins/platforms/cocoa/qcocoamenuitem.mm b/src/plugins/platforms/cocoa/qcocoamenuitem.mm
index f11d7bd996..022972518a 100644
--- a/src/plugins/platforms/cocoa/qcocoamenuitem.mm
+++ b/src/plugins/platforms/cocoa/qcocoamenuitem.mm
@@ -266,8 +266,7 @@ NSMenuItem *QCocoaMenuItem::sync()
             while (depth < 3 && p && !(menubar = qobject_cast<QCocoaMenuBar *>(p))) {
                 ++depth;
                 QCocoaMenuObject *menuObject = dynamic_cast<QCocoaMenuObject *>(p);
-                Q_ASSERT(menuObject);
-                p = menuObject->menuParent();
+                p = menuObject ? menuObject->menuParent() : nullptr;
             }
 
             if (menubar && depth < 3)
diff --git a/src/plugins/platforms/cocoa/qnsview_menus.mm b/src/plugins/platforms/cocoa/qnsview_menus.mm
index 7e9654b62f..8cfac5556a 100644
--- a/src/plugins/platforms/cocoa/qnsview_menus.mm
+++ b/src/plugins/platforms/cocoa/qnsview_menus.mm
@@ -74,17 +74,20 @@ static bool selectorIsCutCopyPaste(SEL selector)
         return YES;
 
     // Check if a modal dialog is active. If so, enable only menu
-    // items explicitly belonging to this window's own menu bar.
+    // items explicitly belonging to this window's own menu bar, or to the window.
     if (QGuiApplication::modalWindow() && QGuiApplication::modalWindow()->isActive()) {
         QCocoaMenuBar *menubar = nullptr;
+        QCocoaWindow *menuWindow = nullptr;
 
         QObject *menuParent = platformItem->menuParent();
         while (menuParent && !(menubar = qobject_cast<QCocoaMenuBar *>(menuParent))) {
+            menuWindow = qobject_cast<QCocoaWindow *>(menuParent);
             auto *menuObject = dynamic_cast<QCocoaMenuObject *>(menuParent);
-            menuParent = menuObject->menuParent();
+            menuParent = menuObject ? menuObject->menuParent() : nullptr;
         }
 
-        if (!menubar || menubar->cocoaWindow() != self.platformWindow)
+        if ((!menuWindow || menuWindow->window() != QGuiApplication::modalWindow())
+            && (!menubar || menubar->cocoaWindow() != self.platformWindow))
             return NO;
     }