macOS: close popups on mousedown within the window frame

On macOS, we close active popups when handling mouse-down events in the
NSView, but not for such events in the window frame. This allows users
to close a window that has a context menu open via the window's close
button, which then leaves open popups behind.

Factor the popup-closing code out into a dedicated method that we can
call from within the NSWindow::sendEvent implementation for mouse down
events.

Fixes: QTBUG-30522
Pick-to: 6.2 6.1 5.15
Change-Id: I9c354efc449cfefff3ed84fa34b1cd8a0da3b4a7
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

3 files changed, 53 insertions, 34 deletions

diff --git a/src/plugins/platforms/cocoa/qnsview.h b/src/plugins/platforms/cocoa/qnsview.h
index a9a547c891..3b318c6dd8 100644
--- a/src/plugins/platforms/cocoa/qnsview.h
+++ b/src/plugins/platforms/cocoa/qnsview.h
@@ -57,6 +57,7 @@ QT_DECLARE_NAMESPACED_OBJC_INTERFACE(QNSView, NSView
 #if defined(__OBJC__)
 @interface QNSView (MouseAPI)
 - (void)handleFrameStrutMouseEvent:(NSEvent *)theEvent;
+- (bool)closePopups:(NSEvent *)theEvent;
 - (void)resetMouseButtons;
 @end
 
diff --git a/src/plugins/platforms/cocoa/qnsview_mouse.mm b/src/plugins/platforms/cocoa/qnsview_mouse.mm
index f5896de99e..ffc8182b83 100644
--- a/src/plugins/platforms/cocoa/qnsview_mouse.mm
+++ b/src/plugins/platforms/cocoa/qnsview_mouse.mm
@@ -185,6 +185,38 @@
     QWindowSystemInterface::handleFrameStrutMouseEvent(m_platformWindow->window(),
         timestamp, qtWindowPoint, qtScreenPoint, m_frameStrutButtons, button, eventType);
 }
+
+- (bool)closePopups:(NSEvent *)theEvent
+{
+    QList<QCocoaWindow *> *popups = QCocoaIntegration::instance()->popupWindowStack();
+    if (!popups->isEmpty()) {
+        // Check if the click is outside all popups.
+        bool inside = false;
+        QPointF qtScreenPoint = QCocoaScreen::mapFromNative([self screenMousePoint:theEvent]);
+        for (QList<QCocoaWindow *>::const_iterator it = popups->begin(); it != popups->end(); ++it) {
+            if ((*it)->geometry().contains(qtScreenPoint.toPoint())) {
+                inside = true;
+                break;
+            }
+        }
+        // Close the popups if the click was outside.
+        if (!inside) {
+            bool selfClosed = false;
+            Qt::WindowType type = QCocoaIntegration::instance()->activePopupWindow()->window()->type();
+            while (QCocoaWindow *popup = QCocoaIntegration::instance()->popPopupWindow()) {
+                selfClosed = self == popup->view();
+                QWindowSystemInterface::handleCloseEvent<QWindowSystemInterface::SynchronousDelivery>(popup->window());
+                if (!m_platformWindow)
+                    return true; // Bail out if window was destroyed
+            }
+            // Consume the mouse event when closing the popup, except for tool tips
+            // were it's expected that the event is processed normally.
+            if (type != Qt::ToolTip || selfClosed)
+                 return true;
+        }
+    }
+    return false;
+}
 @end
 
 static const QPointingDevice *pointingDeviceFor(qint64 deviceID)
@@ -421,33 +453,8 @@ static const QPointingDevice *pointingDeviceFor(qint64 deviceID)
     // that particular poup type (for example context menus). However, Qt expects
     // that plain popup QWindows will also be closed, so we implement the logic
     // here as well.
-    QList<QCocoaWindow *> *popups = QCocoaIntegration::instance()->popupWindowStack();
-    if (!popups->isEmpty()) {
-        // Check if the click is outside all popups.
-        bool inside = false;
-        QPointF qtScreenPoint = QCocoaScreen::mapFromNative([self screenMousePoint:theEvent]);
-        for (QList<QCocoaWindow *>::const_iterator it = popups->begin(); it != popups->end(); ++it) {
-            if ((*it)->geometry().contains(qtScreenPoint.toPoint())) {
-                inside = true;
-                break;
-            }
-        }
-        // Close the popups if the click was outside.
-        if (!inside) {
-            bool selfClosed = false;
-            Qt::WindowType type = QCocoaIntegration::instance()->activePopupWindow()->window()->type();
-            while (QCocoaWindow *popup = QCocoaIntegration::instance()->popPopupWindow()) {
-                selfClosed = self == popup->view();
-                QWindowSystemInterface::handleCloseEvent<QWindowSystemInterface::SynchronousDelivery>(popup->window());
-                if (!m_platformWindow)
-                    return; // Bail out if window was destroyed
-            }
-            // Consume the mouse event when closing the popup, except for tool tips
-            // were it's expected that the event is processed normally.
-            if (type != Qt::ToolTip || selfClosed)
-                 return;
-        }
-    }
+    if ([self closePopups:theEvent])
+        return;
 
     QPointF qtWindowPoint;
     QPointF qtScreenPoint;
diff --git a/src/plugins/platforms/cocoa/qnswindow.mm b/src/plugins/platforms/cocoa/qnswindow.mm
index 27d98a3b73..9656d0e970 100644
--- a/src/plugins/platforms/cocoa/qnswindow.mm
+++ b/src/plugins/platforms/cocoa/qnswindow.mm
@@ -349,18 +349,29 @@ OSStatus CGSClearWindowTags(const CGSConnectionID, const CGSWindowID, int *, int
         return;
     }
 
+    const bool mouseEventInFrameStrut = [theEvent, self]{
+        if (isMouseEvent(theEvent)) {
+            const NSPoint loc = theEvent.locationInWindow;
+            const NSRect windowFrame = [self convertRectFromScreen:self.frame];
+            const NSRect contentFrame = self.contentView.frame;
+            if (NSMouseInRect(loc, windowFrame, NO) && !NSMouseInRect(loc, contentFrame, NO))
+                return true;
+        }
+        return false;
+    }();
+    // Any mouse-press in the frame of the window, including the title bar buttons, should
+    // close open popups. Presses within the window's content are handled to do that in the
+    // NSView::mouseDown implementation.
+    if (theEvent.type == NSEventTypeLeftMouseDown && mouseEventInFrameStrut)
+        [qnsview_cast(m_platformWindow->view()) closePopups:theEvent];
+
     [super sendEvent:theEvent];
 
     if (!m_platformWindow)
         return; // Platform window went away while processing event
 
-    if (m_platformWindow->frameStrutEventsEnabled() && isMouseEvent(theEvent)) {
-        NSPoint loc = [theEvent locationInWindow];
-        NSRect windowFrame = [self convertRectFromScreen:self.frame];
-        NSRect contentFrame = self.contentView.frame;
-        if (NSMouseInRect(loc, windowFrame, NO) && !NSMouseInRect(loc, contentFrame, NO))
-            [qnsview_cast(m_platformWindow->view()) handleFrameStrutMouseEvent:theEvent];
-    }
+    if (m_platformWindow->frameStrutEventsEnabled() && mouseEventInFrameStrut)
+        [qnsview_cast(m_platformWindow->view()) handleFrameStrutMouseEvent:theEvent];
 }
 
 - (void)closeAndRelease