QCocoaMenu: Decouple NSMenuItem from NSMenu

While Cocoa requires an NSMenu to be coupled to an NSMenuItem
(just as Qt requires a QMenu to be coupled to a QAction), making
that a hard coupling comes with some limitations. This is because
Cocoa won't allow the NSMenu object to be simultaneously coupled
to more than one NSMenuItem and, similarly, an NSMenuItem can
only be added to a single parent NSMenu. Therefore, it becomes
difficult to share one QMenu between two different QMenuBars in
different windows, or to use a QMenu as context menu while being
accessible from the menu bar.

Previous solutions to circumvent those limitations were less than
ideal (see 119882714f87ffeb6945fdb2d02997ae125ff50c for the
QMenuBar shared QMenu issue). Other workarounds that relied on
that hard coupling, like 996054f5e65bc676aaea0743c2eacec51918e4aa,
also added gratuitous complexity.

In this patch, we break that hard NSMenuItem-NSMenu coupling, and
we replace it with a temporary, looser coupling. As a consequence,

   * QCocoaMenu only contains and manages a NSMenu instance,
     removing the previously used NSMenuItem. It gets a temporarily
     attached NSMenuItem instead.
   * QCocoaMenuItem gains a safe pointer to its QCocoaMenu property
     removing the necessity containingMenuItem() in QCocoaMenu.
   * QCocoaMenuBar manages its own NSMenuItems.

With this setup, we bind the NSMenu to its parent NSMenuItem at the
last moment. In QCocoaMenuBar, when we call updateMenuBarImmediately().
In QCocoaMenu, we use the delegate's -[QCocoaMenuDelegate menu:
updateItem:atIndex:shouldCancel:] method which is called when Cocoa
is about to display the NSMenu.

Note: There's still one use case we don't support, which is sharing
a toplevel QMenuBar menu. This is because Cocoa's menu bar requires
each of its menu items to have a submenu assigned, and therefore we
can't rely on that last moment assignment.

Task-number: QTBUG-34160
Task-number: QTBUG-31342
Task-number: QTBUG-41587
Change-Id: I92bdb444c680789c78e43fe0b585dc6661770281
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@theqtcompany.com>

6 files changed, 111 insertions, 121 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoamenu.h b/src/plugins/platforms/cocoa/qcocoamenu.h
index eccc5230b5..5064d89585 100644
--- a/src/plugins/platforms/cocoa/qcocoamenu.h
+++ b/src/plugins/platforms/cocoa/qcocoamenu.h
@@ -75,8 +75,6 @@ public:
 
     inline NSMenu *nsMenu() const
         { return m_nativeMenu; }
-    inline NSMenuItem *nsMenuItem() const
-        { return m_nativeItem; }
 
     inline bool isVisible() const { return m_visible; }
 
@@ -85,11 +83,9 @@ public:
 
     QList<QCocoaMenuItem *> items() const;
     QList<QCocoaMenuItem *> merged() const;
-    void setMenuBar(QCocoaMenuBar *menuBar);
-    QCocoaMenuBar *menuBar() const;
 
-    void setContainingMenuItem(QCocoaMenuItem *menuItem);
-    QCocoaMenuItem *containingMenuItem() const;
+    void setAttachedItem(NSMenuItem *item);
+    NSMenuItem *attachedItem() const;
 
 private:
     QCocoaMenuItem *itemOrNull(int index) const;
@@ -97,13 +93,10 @@ private:
 
     QList<QCocoaMenuItem *> m_menuItems;
     NSMenu *m_nativeMenu;
-    NSMenuItem *m_nativeItem;
-    NSObject *m_delegate;
+    NSMenuItem *m_attachedItem;
     bool m_enabled;
     bool m_visible;
     quintptr m_tag;
-    QCocoaMenuBar *m_menuBar;
-    QCocoaMenuItem *m_containingMenuItem;
 };
 
 QT_END_NAMESPACE
diff --git a/src/plugins/platforms/cocoa/qcocoamenu.mm b/src/plugins/platforms/cocoa/qcocoamenu.mm
index ad491855ff..21a3799859 100644
--- a/src/plugins/platforms/cocoa/qcocoamenu.mm
+++ b/src/plugins/platforms/cocoa/qcocoamenu.mm
@@ -96,6 +96,28 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QCocoaMenuDelegate);
     return self;
 }
 
+- (NSInteger)numberOfItemsInMenu:(NSMenu *)menu
+{
+    Q_ASSERT(m_menu->nsMenu() == menu);
+    return m_menu->items().count();
+}
+
+- (BOOL)menu:(NSMenu *)menu updateItem:(NSMenuItem *)item atIndex:(NSInteger)index shouldCancel:(BOOL)shouldCancel
+{
+    Q_UNUSED(index);
+    Q_ASSERT(m_menu->nsMenu() == menu);
+    if (shouldCancel) {
+        // TODO detach all submenus
+        return NO;
+    }
+
+    QCocoaMenuItem *menuItem = reinterpret_cast<QCocoaMenuItem *>(item.tag);
+    if (m_menu->items().contains(menuItem)) {
+        if (QCocoaMenu *itemSubmenu = menuItem->menu())
+            itemSubmenu->setAttachedItem(item);
+    }
+    return YES;
+}
 
 - (void)menu:(NSMenu*)menu willHighlightItem:(NSMenuItem*)item
 {
@@ -228,20 +250,16 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QCocoaMenuDelegate);
 QT_BEGIN_NAMESPACE
 
 QCocoaMenu::QCocoaMenu() :
+    m_attachedItem(0),
     m_enabled(true),
     m_visible(true),
-    m_tag(0),
-    m_menuBar(0),
-    m_containingMenuItem(0)
+    m_tag(0)
 {
     QMacAutoReleasePool pool;
 
-    m_delegate = [[QCocoaMenuDelegate alloc] initWithMenu:this];
-    m_nativeItem = [[NSMenuItem alloc] initWithTitle:@"" action:nil keyEquivalent:@""];
     m_nativeMenu = [[NSMenu alloc] initWithTitle:@"Untitled"];
     [m_nativeMenu setAutoenablesItems:YES];
-    m_nativeMenu.delegate = (QCocoaMenuDelegate *) m_delegate;
-    [m_nativeItem setSubmenu:m_nativeMenu];
+    m_nativeMenu.delegate = [[QCocoaMenuDelegate alloc] initWithMenu:this];
 }
 
 QCocoaMenu::~QCocoaMenu()
@@ -251,14 +269,11 @@ QCocoaMenu::~QCocoaMenu()
             SET_COCOA_MENU_ANCESTOR(item, 0);
     }
 
-    if (m_containingMenuItem)
-        m_containingMenuItem->clearMenu(this);
-
     QMacAutoReleasePool pool;
-    [m_nativeItem setSubmenu:nil];
+    NSObject *delegate = m_nativeMenu.delegate;
+    m_nativeMenu.delegate = nil;
+    [delegate release];
     [m_nativeMenu release];
-    [m_delegate release];
-    [m_nativeItem release];
 }
 
 void QCocoaMenu::setText(const QString &text)
@@ -266,7 +281,6 @@ void QCocoaMenu::setText(const QString &text)
     QMacAutoReleasePool pool;
     QString stripped = qt_mac_removeAmpersandEscapes(text);
     [m_nativeMenu setTitle:QCFString::toNSString(stripped)];
-    [m_nativeItem setTitle:QCFString::toNSString(stripped)];
 }
 
 void QCocoaMenu::setMinimumWidth(int width)
@@ -307,17 +321,13 @@ void QCocoaMenu::insertMenuItem(QPlatformMenuItem *menuItem, QPlatformMenuItem *
 
 void QCocoaMenu::insertNative(QCocoaMenuItem *item, QCocoaMenuItem *beforeItem)
 {
-    [item->nsItem() setTarget:m_delegate];
+    item->nsItem().target = m_nativeMenu.delegate;
     if (!item->menu())
         [item->nsItem() setAction:@selector(itemFired:)];
 
     if (item->isMerged())
         return;
 
-    if ([item->nsItem() menu]) {
-        qWarning("Menu item is already in a menu, remove it from the other menu first before inserting");
-        return;
-    }
     // if the item we're inserting before is merged, skip along until
     // we find a non-merged real item to insert ahead of.
     while (beforeItem && beforeItem->isMerged()) {
@@ -445,12 +455,11 @@ void QCocoaMenu::setEnabled(bool enabled)
 
 bool QCocoaMenu::isEnabled() const
 {
-    return [m_nativeItem isEnabled];
+    return m_attachedItem ? [m_attachedItem isEnabled] : m_enabled;
 }
 
 void QCocoaMenu::setVisible(bool visible)
 {
-    [m_nativeItem setSubmenu:(visible ? m_nativeMenu : nil)];
     m_visible = visible;
 }
 
@@ -587,8 +596,6 @@ void QCocoaMenu::syncModalState(bool modal)
     if (!m_enabled)
         modal = true;
 
-    [m_nativeItem setEnabled:!modal];
-
     foreach (QCocoaMenuItem *item, m_menuItems) {
         if (item->menu()) { // recurse into submenus
             item->menu()->syncModalState(modal);
@@ -599,25 +606,24 @@ void QCocoaMenu::syncModalState(bool modal)
     }
 }
 
-void QCocoaMenu::setMenuBar(QCocoaMenuBar *menuBar)
+void QCocoaMenu::setAttachedItem(NSMenuItem *item)
 {
-    m_menuBar = menuBar;
-    SET_COCOA_MENU_ANCESTOR(this, menuBar);
-}
+    if (item == m_attachedItem)
+        return;
 
-QCocoaMenuBar *QCocoaMenu::menuBar() const
-{
-    return m_menuBar;
-}
+    if (m_attachedItem)
+        m_attachedItem.submenu = nil;
+
+    m_attachedItem = item;
+
+    if (m_attachedItem)
+        m_attachedItem.submenu = m_nativeMenu;
 
-void QCocoaMenu::setContainingMenuItem(QCocoaMenuItem *menuItem)
-{
-    m_containingMenuItem = menuItem;
 }
 
-QCocoaMenuItem *QCocoaMenu::containingMenuItem() const
+NSMenuItem *QCocoaMenu::attachedItem() const
 {
-    return m_containingMenuItem;
+    return m_attachedItem;
 }
 
 QT_END_NAMESPACE
diff --git a/src/plugins/platforms/cocoa/qcocoamenubar.h b/src/plugins/platforms/cocoa/qcocoamenubar.h
index d5f75abf34..e84da7aeb0 100644
--- a/src/plugins/platforms/cocoa/qcocoamenubar.h
+++ b/src/plugins/platforms/cocoa/qcocoamenubar.h
@@ -71,10 +71,10 @@ private:
     static QCocoaMenuBar *findGlobalMenubar();
 
     bool shouldDisable(QCocoaWindow *active) const;
-    void insertNativeMenu(QCocoaMenu *menu, QCocoaMenu *beforeMenu);
-    void removeNativeMenu(QCocoaMenu *menu);
 
-    QList<QCocoaMenu*> m_menus;
+    NSMenuItem *nativeItemForMenu(QCocoaMenu *menu) const;
+
+    QList<QPointer<QCocoaMenu> > m_menus;
     NSMenu *m_nativeMenu;
     QCocoaWindow *m_window;
 };
diff --git a/src/plugins/platforms/cocoa/qcocoamenubar.mm b/src/plugins/platforms/cocoa/qcocoamenubar.mm
index 1a516f874b..ac4d29fc52 100644
--- a/src/plugins/platforms/cocoa/qcocoamenubar.mm
+++ b/src/plugins/platforms/cocoa/qcocoamenubar.mm
@@ -51,7 +51,6 @@ static inline QCocoaMenuLoader *getMenuLoader()
     return [NSApp QT_MANGLE_NAMESPACE(qt_qcocoamenuLoader)];
 }
 
-
 QCocoaMenuBar::QCocoaMenuBar() :
     m_window(0)
 {
@@ -68,11 +67,20 @@ QCocoaMenuBar::~QCocoaMenuBar()
 #ifdef QT_COCOA_ENABLE_MENU_DEBUG
     qDebug() << "~QCocoaMenuBar" << this;
 #endif
+    foreach (QCocoaMenu *menu, m_menus) {
+        if (!menu)
+            continue;
+        NSMenuItem *item = nativeItemForMenu(menu);
+        if (menu->attachedItem() == item)
+            menu->setAttachedItem(nil);
+    }
+
     [m_nativeMenu release];
     static_menubars.removeOne(this);
 
     if (m_window && m_window->menubar() == this) {
         m_window->setMenubar(0);
+
         // Delete the children first so they do not cause
         // the native menu items to be hidden after
         // the menu bar was updated
@@ -81,24 +89,6 @@ QCocoaMenuBar::~QCocoaMenuBar()
     }
 }
 
-void QCocoaMenuBar::insertNativeMenu(QCocoaMenu *menu, QCocoaMenu *beforeMenu)
-{
-    QMacAutoReleasePool pool;
-
-    if (beforeMenu) {
-        NSUInteger nativeIndex = [m_nativeMenu indexOfItem:beforeMenu->nsMenuItem()];
-        [m_nativeMenu insertItem: menu->nsMenuItem() atIndex: nativeIndex];
-    } else {
-        [m_nativeMenu addItem: menu->nsMenuItem()];
-    }
-
-    menu->setMenuBar(this);
-    syncMenu(static_cast<QPlatformMenu *>(menu));
-    if (menu->isVisible()) {
-        [m_nativeMenu setSubmenu: menu->nsMenu() forItem: menu->nsMenuItem()];
-    }
-}
-
 void QCocoaMenuBar::insertMenu(QPlatformMenu *platformMenu, QPlatformMenu *before)
 {
     QCocoaMenu *menu = static_cast<QCocoaMenu *>(platformMenu);
@@ -107,31 +97,40 @@ void QCocoaMenuBar::insertMenu(QPlatformMenu *platformMenu, QPlatformMenu *befor
     qDebug() << "QCocoaMenuBar" << this << "insertMenu" << menu << "before" << before;
 #endif
 
-    if (m_menus.contains(menu)) {
+    if (m_menus.contains(QPointer<QCocoaMenu>(menu))) {
         qWarning("This menu already belongs to the menubar, remove it first");
         return;
     }
 
-    if (beforeMenu && !m_menus.contains(beforeMenu)) {
+    if (beforeMenu && !m_menus.contains(QPointer<QCocoaMenu>(beforeMenu))) {
         qWarning("The before menu does not belong to the menubar");
         return;
     }
 
-    m_menus.insert(beforeMenu ? m_menus.indexOf(beforeMenu) : m_menus.size(), menu);
-    if (!menu->menuBar())
-        insertNativeMenu(menu, beforeMenu);
-    if (m_window && m_window->window()->isActive())
-        updateMenuBarImmediately();
-}
+    int insertionIndex = beforeMenu ? m_menus.indexOf(beforeMenu) : m_menus.size();
+    m_menus.insert(insertionIndex, menu);
+
+    {
+        QMacAutoReleasePool pool;
+        NSMenuItem *item = [[[NSMenuItem alloc] init] autorelease];
+        item.tag = reinterpret_cast<NSInteger>(menu);
+
+        if (beforeMenu) {
+            // QMenuBar::toNSMenu() exposes the native menubar and
+            // the user could have inserted its own items in there.
+            // Same remark applies to removeMenu().
+            NSMenuItem *beforeItem = nativeItemForMenu(beforeMenu);
+            NSInteger nativeIndex = [m_nativeMenu indexOfItem:beforeItem];
+            [m_nativeMenu insertItem:item atIndex:nativeIndex];
+        } else {
+            [m_nativeMenu addItem:item];
+        }
+    }
 
-void QCocoaMenuBar::removeNativeMenu(QCocoaMenu *menu)
-{
-    QMacAutoReleasePool pool;
+    syncMenu(menu);
 
-    if (menu->menuBar() == this)
-        menu->setMenuBar(0);
-    NSUInteger realIndex = [m_nativeMenu indexOfItem:menu->nsMenuItem()];
-    [m_nativeMenu removeItemAtIndex: realIndex];
+    if (m_window && m_window->window()->isActive())
+        updateMenuBarImmediately();
 }
 
 void QCocoaMenuBar::removeMenu(QPlatformMenu *platformMenu)
@@ -141,8 +140,17 @@ void QCocoaMenuBar::removeMenu(QPlatformMenu *platformMenu)
         qWarning("Trying to remove a menu that does not belong to the menubar");
         return;
     }
+
+    NSMenuItem *item = nativeItemForMenu(menu);
+    if (menu->attachedItem() == item)
+        menu->setAttachedItem(nil);
     m_menus.removeOne(menu);
-    removeNativeMenu(menu);
+
+    QMacAutoReleasePool pool;
+
+    // See remark in insertMenu().
+    NSInteger nativeIndex = [m_nativeMenu indexOfItem:item];
+    [m_nativeMenu removeItemAtIndex:nativeIndex];
 }
 
 void QCocoaMenuBar::syncMenu(QPlatformMenu *menu)
@@ -164,7 +172,16 @@ void QCocoaMenuBar::syncMenu(QPlatformMenu *menu)
                 break;
             }
     }
-    [cocoaMenu->nsMenuItem() setHidden:shouldHide];
+
+    nativeItemForMenu(cocoaMenu).hidden = shouldHide;
+}
+
+NSMenuItem *QCocoaMenuBar::nativeItemForMenu(QCocoaMenu *menu) const
+{
+    if (!menu)
+        return nil;
+
+    return [m_nativeMenu itemWithTag:reinterpret_cast<NSInteger>(menu)];
 }
 
 void QCocoaMenuBar::handleReparent(QWindow *newParentWindow)
@@ -291,24 +308,16 @@ void QCocoaMenuBar::updateMenuBarImmediately()
     qDebug() << "QCocoaMenuBar" << "updateMenuBarImmediately" << cw;
 #endif
     bool disableForModal = mb->shouldDisable(cw);
-    // force a sync?
-    foreach (QCocoaMenu *m, mb->m_menus) {
-        mb->syncMenu(m);
-        m->syncModalState(disableForModal);
-    }
 
-    // reparent shared menu items if necessary.
-    // We browse the list in reverse order to be sure that the next items are redrawn before the current ones,
-    // in this way we are sure that "beforeMenu" (see below) is part of the native menu before "m" is redraw
-    for (int i = mb->m_menus.size() - 1; i >= 0; i--) {
-        QCocoaMenu *m = mb->m_menus.at(i);
-        QCocoaMenuBar *menuBar = m->menuBar();
-        if (menuBar != mb) {
-            QCocoaMenu *beforeMenu  = i < (mb->m_menus.size() - 1) ? mb->m_menus.at(i + 1) : 0;
-            if (menuBar)
-                menuBar->removeNativeMenu(m);
-            mb->insertNativeMenu(m, beforeMenu);
-        }
+    foreach (QCocoaMenu *menu, mb->m_menus) {
+        if (!menu)
+            continue;
+        NSMenuItem *item = mb->nativeItemForMenu(menu);
+        menu->setAttachedItem(item);
+        SET_COCOA_MENU_ANCESTOR(menu, mb);
+        // force a sync?
+        mb->syncMenu(menu);
+        menu->syncModalState(disableForModal);
     }
 
     QCocoaMenuLoader *loader = getMenuLoader();
diff --git a/src/plugins/platforms/cocoa/qcocoamenuitem.h b/src/plugins/platforms/cocoa/qcocoamenuitem.h
index 1cd15e686c..ada4b13df1 100644
--- a/src/plugins/platforms/cocoa/qcocoamenuitem.h
+++ b/src/plugins/platforms/cocoa/qcocoamenuitem.h
@@ -87,7 +87,6 @@ public:
     inline bool isSeparator() const { return m_isSeparator; }
 
     QCocoaMenu *menu() const { return m_menu; }
-    void clearMenu(QCocoaMenu *menu);
     MenuRole effectiveRole() const;
 
 private:
@@ -99,7 +98,7 @@ private:
     QString m_text;
     bool m_textSynced;
     QIcon m_icon;
-    QCocoaMenu *m_menu;
+    QPointer<QCocoaMenu> m_menu;
     bool m_isVisible;
     bool m_enabled;
     bool m_isSeparator;
diff --git a/src/plugins/platforms/cocoa/qcocoamenuitem.mm b/src/plugins/platforms/cocoa/qcocoamenuitem.mm
index 0f551bcd7d..e274448620 100644
--- a/src/plugins/platforms/cocoa/qcocoamenuitem.mm
+++ b/src/plugins/platforms/cocoa/qcocoamenuitem.mm
@@ -134,15 +134,12 @@ void QCocoaMenuItem::setMenu(QPlatformMenu *menu)
     if (m_menu) {
         if (COCOA_MENU_ANCESTOR(m_menu) == this)
             SET_COCOA_MENU_ANCESTOR(m_menu, 0);
-        if (m_menu->containingMenuItem() == this)
-            m_menu->setContainingMenuItem(0);
     }
 
     QMacAutoReleasePool pool;
     m_menu = static_cast<QCocoaMenu *>(menu);
     if (m_menu) {
         SET_COCOA_MENU_ANCESTOR(m_menu, this);
-        m_menu->setContainingMenuItem(this);
     } else {
         // we previously had a menu, but no longer
         // clear out our item so the nexy sync() call builds a new one
@@ -151,12 +148,6 @@ void QCocoaMenuItem::setMenu(QPlatformMenu *menu)
     }
 }
 
-void QCocoaMenuItem::clearMenu(QCocoaMenu *menu)
-{
-    if (menu == m_menu)
-        m_menu = 0;
-}
-
 void QCocoaMenuItem::setVisible(bool isVisible)
 {
     m_isVisible = isVisible;
@@ -218,14 +209,6 @@ NSMenuItem *QCocoaMenuItem::sync()
             m_native = nil;
     }
 
-    if (m_menu) {
-        if (m_native != m_menu->nsMenuItem()) {
-            [m_native release];
-            m_native = [m_menu->nsMenuItem() retain];
-            [m_native setTag:reinterpret_cast<NSInteger>(this)];
-        }
-    }
-
     if ((m_role != NoRole && !m_textSynced) || m_merged) {
         NSMenuItem *mergeItem = nil;
         QCocoaMenuLoader *loader = getMenuLoader();