Don't disable menu items that belong to the active modal window

A popup/context menu created via QQuickPlatformMenu doesn't belong to any
menubar, so by disabling items in a menu that doesn't belong to the currently
active menubar (5b9f6862b1), we disabled all menu items in a QQuickPlatformMenu
when a modal window was active.

For such unrooted menus, use the QCocoaMenuObject data structure to record
which window it is shown for, and only disable items if that window is not
also the current modal window.

Amends 5b9f6862b1aa474a392203c69f6db678d633cecf.

Fixes: QTBUG-92040
Change-Id: I56b6d579e5e94689b43ca84d4637e35dc2cbeb4c
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
(cherry picked from commit 2bdaf28034541cd57b0aa7c067fda8cfbffa0e94)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

3 files changed, 18 insertions, 5 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoamenu.mm b/src/plugins/platforms/cocoa/qcocoamenu.mm
index 8e2af2b183..3e467fc361 100644
--- a/src/plugins/platforms/cocoa/qcocoamenu.mm
+++ b/src/plugins/platforms/cocoa/qcocoamenu.mm
@@ -351,6 +351,17 @@ void QCocoaMenu::showPopup(const QWindow *parentWindow, const QRect &targetRect,
     NSView *view = cocoaWindow ? cocoaWindow->view() : nil;
     NSMenuItem *nsItem = item ? ((QCocoaMenuItem *)item)->nsItem() : nil;
 
+    // store the window that this popup belongs to so that we can evaluate whether we are modally blocked
+    bool resetMenuParent = false;
+    if (!menuParent()) {
+        setMenuParent(cocoaWindow);
+        resetMenuParent = true;
+    }
+    auto menuParentGuard = qScopeGuard([&]{
+        if (resetMenuParent)
+            setMenuParent(nullptr);
+    });
+
     QScreen *screen = nullptr;
     if (parentWindow)
         screen = parentWindow->screen();
diff --git a/src/plugins/platforms/cocoa/qcocoamenuitem.mm b/src/plugins/platforms/cocoa/qcocoamenuitem.mm
index 95499b3611..b3864b930f 100644
--- a/src/plugins/platforms/cocoa/qcocoamenuitem.mm
+++ b/src/plugins/platforms/cocoa/qcocoamenuitem.mm
@@ -300,8 +300,7 @@ NSMenuItem *QCocoaMenuItem::sync()
             while (depth < 3 && p && !(menubar = qobject_cast<QCocoaMenuBar *>(p))) {
                 ++depth;
                 QCocoaMenuObject *menuObject = dynamic_cast<QCocoaMenuObject *>(p);
-                Q_ASSERT(menuObject);
-                p = menuObject->menuParent();
+                p = menuObject ? menuObject->menuParent() : nullptr;
             }
 
             if (menubar && depth < 3)
diff --git a/src/plugins/platforms/cocoa/qnsview_menus.mm b/src/plugins/platforms/cocoa/qnsview_menus.mm
index 783873a1e6..39e6661d94 100644
--- a/src/plugins/platforms/cocoa/qnsview_menus.mm
+++ b/src/plugins/platforms/cocoa/qnsview_menus.mm
@@ -74,17 +74,20 @@ static bool selectorIsCutCopyPaste(SEL selector)
         return YES;
 
     // Check if a modal dialog is active. If so, enable only menu
-    // items explicitly belonging to this window's own menu bar.
+    // items explicitly belonging to this window's own menu bar, or to the window.
     if (QGuiApplication::modalWindow() && QGuiApplication::modalWindow()->isActive()) {
         QCocoaMenuBar *menubar = nullptr;
+        QCocoaWindow *menuWindow = nullptr;
 
         QObject *menuParent = platformItem->menuParent();
         while (menuParent && !(menubar = qobject_cast<QCocoaMenuBar *>(menuParent))) {
+            menuWindow = qobject_cast<QCocoaWindow *>(menuParent);
             auto *menuObject = dynamic_cast<QCocoaMenuObject *>(menuParent);
-            menuParent = menuObject->menuParent();
+            menuParent = menuObject ? menuObject->menuParent() : nullptr;
         }
 
-        if (!menubar || menubar->cocoaWindow() != self.platformWindow)
+        if ((!menuWindow || menuWindow->window() != QGuiApplication::modalWindow())
+            && (!menubar || menubar->cocoaWindow() != self.platformWindow))
             return NO;
     }