QCocoaMenu: Decouple NSMenuItem from NSMenu

While Cocoa requires an NSMenu to be coupled to an NSMenuItem
(just as Qt requires a QMenu to be coupled to a QAction), making
that a hard coupling comes with some limitations. This is because
Cocoa won't allow the NSMenu object to be simultaneously coupled
to more than one NSMenuItem and, similarly, an NSMenuItem can
only be added to a single parent NSMenu. Therefore, it becomes
difficult to share one QMenu between two different QMenuBars in
different windows, or to use a QMenu as context menu while being
accessible from the menu bar.

Previous solutions to circumvent those limitations were less than
ideal (see 119882714f87ffeb6945fdb2d02997ae125ff50c for the
QMenuBar shared QMenu issue). Other workarounds that relied on
that hard coupling, like 996054f5e65bc676aaea0743c2eacec51918e4aa,
also added gratuitous complexity.

In this patch, we break that hard NSMenuItem-NSMenu coupling, and
we replace it with a temporary, looser coupling. As a consequence,

   * QCocoaMenu only contains and manages a NSMenu instance,
     removing the previously used NSMenuItem. It gets a temporarily
     attached NSMenuItem instead.
   * QCocoaMenuItem gains a safe pointer to its QCocoaMenu property
     removing the necessity containingMenuItem() in QCocoaMenu.
   * QCocoaMenuBar manages its own NSMenuItems.

With this setup, we bind the NSMenu to its parent NSMenuItem at the
last moment. In QCocoaMenuBar, when we call updateMenuBarImmediately().
In QCocoaMenu, we use the delegate's -[QCocoaMenuDelegate menu:
updateItem:atIndex:shouldCancel:] method which is called when Cocoa
is about to display the NSMenu.

Note: There's still one use case we don't support, which is sharing
a toplevel QMenuBar menu. This is because Cocoa's menu bar requires
each of its menu items to have a submenu assigned, and therefore we
can't rely on that last moment assignment.

Task-number: QTBUG-34160
Task-number: QTBUG-31342
Task-number: QTBUG-41587
Change-Id: I92bdb444c680789c78e43fe0b585dc6661770281
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@theqtcompany.com>

1 files changed, 62 insertions, 53 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoamenubar.mm b/src/plugins/platforms/cocoa/qcocoamenubar.mm
index 1a516f874b..ac4d29fc52 100644
--- a/src/plugins/platforms/cocoa/qcocoamenubar.mm
+++ b/src/plugins/platforms/cocoa/qcocoamenubar.mm
@@ -51,7 +51,6 @@ static inline QCocoaMenuLoader *getMenuLoader()
     return [NSApp QT_MANGLE_NAMESPACE(qt_qcocoamenuLoader)];
 }
 
-
 QCocoaMenuBar::QCocoaMenuBar() :
     m_window(0)
 {
@@ -68,11 +67,20 @@ QCocoaMenuBar::~QCocoaMenuBar()
 #ifdef QT_COCOA_ENABLE_MENU_DEBUG
     qDebug() << "~QCocoaMenuBar" << this;
 #endif
+    foreach (QCocoaMenu *menu, m_menus) {
+        if (!menu)
+            continue;
+        NSMenuItem *item = nativeItemForMenu(menu);
+        if (menu->attachedItem() == item)
+            menu->setAttachedItem(nil);
+    }
+
     [m_nativeMenu release];
     static_menubars.removeOne(this);
 
     if (m_window && m_window->menubar() == this) {
         m_window->setMenubar(0);
+
         // Delete the children first so they do not cause
         // the native menu items to be hidden after
         // the menu bar was updated
@@ -81,24 +89,6 @@ QCocoaMenuBar::~QCocoaMenuBar()
     }
 }
 
-void QCocoaMenuBar::insertNativeMenu(QCocoaMenu *menu, QCocoaMenu *beforeMenu)
-{
-    QMacAutoReleasePool pool;
-
-    if (beforeMenu) {
-        NSUInteger nativeIndex = [m_nativeMenu indexOfItem:beforeMenu->nsMenuItem()];
-        [m_nativeMenu insertItem: menu->nsMenuItem() atIndex: nativeIndex];
-    } else {
-        [m_nativeMenu addItem: menu->nsMenuItem()];
-    }
-
-    menu->setMenuBar(this);
-    syncMenu(static_cast<QPlatformMenu *>(menu));
-    if (menu->isVisible()) {
-        [m_nativeMenu setSubmenu: menu->nsMenu() forItem: menu->nsMenuItem()];
-    }
-}
-
 void QCocoaMenuBar::insertMenu(QPlatformMenu *platformMenu, QPlatformMenu *before)
 {
     QCocoaMenu *menu = static_cast<QCocoaMenu *>(platformMenu);
@@ -107,31 +97,40 @@ void QCocoaMenuBar::insertMenu(QPlatformMenu *platformMenu, QPlatformMenu *befor
     qDebug() << "QCocoaMenuBar" << this << "insertMenu" << menu << "before" << before;
 #endif
 
-    if (m_menus.contains(menu)) {
+    if (m_menus.contains(QPointer<QCocoaMenu>(menu))) {
         qWarning("This menu already belongs to the menubar, remove it first");
         return;
     }
 
-    if (beforeMenu && !m_menus.contains(beforeMenu)) {
+    if (beforeMenu && !m_menus.contains(QPointer<QCocoaMenu>(beforeMenu))) {
         qWarning("The before menu does not belong to the menubar");
         return;
     }
 
-    m_menus.insert(beforeMenu ? m_menus.indexOf(beforeMenu) : m_menus.size(), menu);
-    if (!menu->menuBar())
-        insertNativeMenu(menu, beforeMenu);
-    if (m_window && m_window->window()->isActive())
-        updateMenuBarImmediately();
-}
+    int insertionIndex = beforeMenu ? m_menus.indexOf(beforeMenu) : m_menus.size();
+    m_menus.insert(insertionIndex, menu);
+
+    {
+        QMacAutoReleasePool pool;
+        NSMenuItem *item = [[[NSMenuItem alloc] init] autorelease];
+        item.tag = reinterpret_cast<NSInteger>(menu);
+
+        if (beforeMenu) {
+            // QMenuBar::toNSMenu() exposes the native menubar and
+            // the user could have inserted its own items in there.
+            // Same remark applies to removeMenu().
+            NSMenuItem *beforeItem = nativeItemForMenu(beforeMenu);
+            NSInteger nativeIndex = [m_nativeMenu indexOfItem:beforeItem];
+            [m_nativeMenu insertItem:item atIndex:nativeIndex];
+        } else {
+            [m_nativeMenu addItem:item];
+        }
+    }
 
-void QCocoaMenuBar::removeNativeMenu(QCocoaMenu *menu)
-{
-    QMacAutoReleasePool pool;
+    syncMenu(menu);
 
-    if (menu->menuBar() == this)
-        menu->setMenuBar(0);
-    NSUInteger realIndex = [m_nativeMenu indexOfItem:menu->nsMenuItem()];
-    [m_nativeMenu removeItemAtIndex: realIndex];
+    if (m_window && m_window->window()->isActive())
+        updateMenuBarImmediately();
 }
 
 void QCocoaMenuBar::removeMenu(QPlatformMenu *platformMenu)
@@ -141,8 +140,17 @@ void QCocoaMenuBar::removeMenu(QPlatformMenu *platformMenu)
         qWarning("Trying to remove a menu that does not belong to the menubar");
         return;
     }
+
+    NSMenuItem *item = nativeItemForMenu(menu);
+    if (menu->attachedItem() == item)
+        menu->setAttachedItem(nil);
     m_menus.removeOne(menu);
-    removeNativeMenu(menu);
+
+    QMacAutoReleasePool pool;
+
+    // See remark in insertMenu().
+    NSInteger nativeIndex = [m_nativeMenu indexOfItem:item];
+    [m_nativeMenu removeItemAtIndex:nativeIndex];
 }
 
 void QCocoaMenuBar::syncMenu(QPlatformMenu *menu)
@@ -164,7 +172,16 @@ void QCocoaMenuBar::syncMenu(QPlatformMenu *menu)
                 break;
             }
     }
-    [cocoaMenu->nsMenuItem() setHidden:shouldHide];
+
+    nativeItemForMenu(cocoaMenu).hidden = shouldHide;
+}
+
+NSMenuItem *QCocoaMenuBar::nativeItemForMenu(QCocoaMenu *menu) const
+{
+    if (!menu)
+        return nil;
+
+    return [m_nativeMenu itemWithTag:reinterpret_cast<NSInteger>(menu)];
 }
 
 void QCocoaMenuBar::handleReparent(QWindow *newParentWindow)
@@ -291,24 +308,16 @@ void QCocoaMenuBar::updateMenuBarImmediately()
     qDebug() << "QCocoaMenuBar" << "updateMenuBarImmediately" << cw;
 #endif
     bool disableForModal = mb->shouldDisable(cw);
-    // force a sync?
-    foreach (QCocoaMenu *m, mb->m_menus) {
-        mb->syncMenu(m);
-        m->syncModalState(disableForModal);
-    }
 
-    // reparent shared menu items if necessary.
-    // We browse the list in reverse order to be sure that the next items are redrawn before the current ones,
-    // in this way we are sure that "beforeMenu" (see below) is part of the native menu before "m" is redraw
-    for (int i = mb->m_menus.size() - 1; i >= 0; i--) {
-        QCocoaMenu *m = mb->m_menus.at(i);
-        QCocoaMenuBar *menuBar = m->menuBar();
-        if (menuBar != mb) {
-            QCocoaMenu *beforeMenu  = i < (mb->m_menus.size() - 1) ? mb->m_menus.at(i + 1) : 0;
-            if (menuBar)
-                menuBar->removeNativeMenu(m);
-            mb->insertNativeMenu(m, beforeMenu);
-        }
+    foreach (QCocoaMenu *menu, mb->m_menus) {
+        if (!menu)
+            continue;
+        NSMenuItem *item = mb->nativeItemForMenu(menu);
+        menu->setAttachedItem(item);
+        SET_COCOA_MENU_ANCESTOR(menu, mb);
+        // force a sync?
+        mb->syncMenu(menu);
+        menu->syncModalState(disableForModal);
     }
 
     QCocoaMenuLoader *loader = getMenuLoader();