Cocoa: Re-implement QCocoaScreen::topLevelAt()

The previous implementation used [NSApp orderedWindows]
which does not return NSPanel subclasses, which is
used by Qt dialogs and pops.

Use [NSWidow windowNumberAtPoint:belowWindowWithWindowNumber]
instead, which hit-tests on all window types. This
can potentially include windows from other processes
and non-Qt windows which needs to be filtered out.

Add EXPECT_FAIL to tst_MacGui::nonModalOrder. The
correct topLevelAt() implementation now exposes that
this test is failing.

Task-number: QTBUG-39322
Change-Id: I81afa3da964e08fe682802220d8fe81e9284205e
Reviewed-by: Gabriel de Dietrich <gabriel.dedietrich@digia.com>

1 files changed, 30 insertions, 11 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoaintegration.mm b/src/plugins/platforms/cocoa/qcocoaintegration.mm
index 19753ccc89..fca7fa042c 100644
--- a/src/plugins/platforms/cocoa/qcocoaintegration.mm
+++ b/src/plugins/platforms/cocoa/qcocoaintegration.mm
@@ -158,23 +158,42 @@ qreal QCocoaScreen::devicePixelRatio() const
 
 QWindow *QCocoaScreen::topLevelAt(const QPoint &point) const
 {
-    // Get a z-ordered list of windows. Iterate through it until
-    // we find a (Qt) window which contains the point.
-    for (NSWindow *nsWindow in [NSApp orderedWindows]) {
-        if (![nsWindow isKindOfClass:[QNSWindow class]])
+    NSPoint screenPoint = qt_mac_flipPoint(point);
+
+    // Search (hit test) for the top-level window. [NSWidow windowNumberAtPoint:
+    // belowWindowWithWindowNumber] may return windows that are not interesting
+    // to Qt. The search iterates until a suitable window or no window is found.
+    NSInteger topWindowNumber = 0;
+    QWindow *window = 0;
+    do {
+        // Get the top-most window, below any previously rejected window.
+        topWindowNumber = [NSWindow windowNumberAtPoint:screenPoint
+                                    belowWindowWithWindowNumber:topWindowNumber];
+
+        // Continue the search if the window does not belong to this process.
+        NSWindow *nsWindow = [NSApp windowWithWindowNumber:topWindowNumber];
+        if (nsWindow == 0)
             continue;
-        QNSWindow *qnsWindow = static_cast<QNSWindow *>(nsWindow);
-        QCocoaWindow *cocoaWindow = qnsWindow.helper.platformWindow;
+
+        // Continue the search if the window does not belong to Qt.
+        if (![nsWindow conformsToProtocol:@protocol(QNSWindowProtocol)])
+            continue;
+
+        id<QNSWindowProtocol> proto = static_cast<id<QNSWindowProtocol> >(nsWindow);
+        QCocoaWindow *cocoaWindow = proto.helper.platformWindow;
         if (!cocoaWindow)
             continue;
-        QWindow *window = cocoaWindow->window();
+        window = cocoaWindow->window();
+
+        // Continue the search if the window is not a top-level window.
         if (!window->isTopLevel())
              continue;
-        if (window->geometry().contains(point))
-            return window;
-    }
 
-    return QPlatformScreen::topLevelAt(point);
+        // Stop searching. The current window is the correct window.
+        break;
+    } while (topWindowNumber > 0);
+
+    return window;
 }
 
 extern CGContextRef qt_mac_cg_context(const QPaintDevice *pdev);