Cocoa: Make child window cursors work correctly

The existing cursor logic had a couple of issues:

- It made the faulty assumption that we could not use
the NSWindow invalidateCursorRectsForView API for
child NSViews.

- It used NSWindow invalidateCursorRectsForView and
NSView resetCursorRects. This API has been replaced
by the more general NSTrackingArea API.

- It did not implement falling back to the parent
window cursor if the current window has no cursor
set.

Document that QWindow cursors work the same way as
QWidget cursors in that a QWindow with no set cursor
will fall back to the parent window cursor.

Change the cocoa platform code to use NSTrackingArea
exclusively and implement NSView cursorUpdate which
sets the cursor. Handle immediate change on QWindow::
setCursor() manually.

Add QWindow::effectiveWindowCursor() and
applyEffectiveWindowCursor() which finds the correct
window cursor.

Add a manual test for the child window, child widget,
and QWidget::createWindowChild cases.

Task-number: QTBUG-33479
Task-number: QTBUG-52023
Change-Id: I0370e11bbadb2da95e8632e61be6228ec2cd5e9d
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

5 files changed, 53 insertions, 38 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoacursor.mm b/src/plugins/platforms/cocoa/qcocoacursor.mm
index 8e38181c29..a4c291c14a 100644
--- a/src/plugins/platforms/cocoa/qcocoacursor.mm
+++ b/src/plugins/platforms/cocoa/qcocoacursor.mm
@@ -55,7 +55,7 @@ QCocoaCursor::~QCocoaCursor()
 
 void QCocoaCursor::changeCursor(QCursor *cursor, QWindow *window)
 {
-    NSCursor * cocoaCursor = convertCursor(cursor);
+    NSCursor *cocoaCursor = convertCursor(cursor);
 
     if (QPlatformWindow * platformWindow = window->handle())
         static_cast<QCocoaWindow *>(platformWindow)->setWindowCursor(cocoaCursor);
@@ -77,9 +77,12 @@ void QCocoaCursor::setPos(const QPoint &position)
     CFRelease(e);
 }
 
-NSCursor *QCocoaCursor::convertCursor(QCursor * cursor)
+NSCursor *QCocoaCursor::convertCursor(QCursor *cursor)
 {
-    const Qt::CursorShape newShape = cursor ? cursor->shape() : Qt::ArrowCursor;
+    if (cursor == Q_NULLPTR)
+        return 0;
+
+    const Qt::CursorShape newShape = cursor->shape();
     NSCursor *cocoaCursor;
 
     // Check for a suitable built-in NSCursor first:
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.h b/src/plugins/platforms/cocoa/qcocoawindow.h
index 9cf6328281..bf28f83540 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.h
+++ b/src/plugins/platforms/cocoa/qcocoawindow.h
@@ -236,6 +236,8 @@ public:
     void setMenubar(QCocoaMenuBar *mb);
     QCocoaMenuBar *menubar() const;
 
+    NSCursor *effectiveWindowCursor() const;
+    void applyEffectiveWindowCursor();
     void setWindowCursor(NSCursor *cursor);
 
     void registerTouch(bool enable);
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.mm b/src/plugins/platforms/cocoa/qcocoawindow.mm
index 977a5ae657..a18d93b89e 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.mm
+++ b/src/plugins/platforms/cocoa/qcocoawindow.mm
@@ -1645,31 +1645,53 @@ QCocoaMenuBar *QCocoaWindow::menubar() const
     return m_menubar;
 }
 
-void QCocoaWindow::setWindowCursor(NSCursor *cursor)
+// Finds the effective cursor for this window by walking up the
+// ancestor chain (including this window) until a set cursor is
+// found. Returns nil if there is not set cursor.
+NSCursor *QCocoaWindow::effectiveWindowCursor() const
 {
-    // This function is called (via QCocoaCursor) by Qt to set
-    // the cursor for this window. It can be called for a window
-    // that is not currenly under the mouse pointer (for example
-    // for a popup window.) Qt expects the set cursor to "stick":
-    // it should be accociated with the window until a different
-    // cursor is set.
-    if (m_windowCursor != cursor) {
-        [m_windowCursor release];
-        m_windowCursor = [cursor retain];
-    }
 
-    // Use the built in cursor rect API if the QCocoaWindow has a NSWindow.
-    // Othervise, set the cursor if this window is under the mouse. In
-    // this case QNSView::cursorUpdate will set the cursor as the pointer
-    // moves.
-    if (m_nsWindow && m_qtView) {
-        [m_nsWindow invalidateCursorRectsForView : m_qtView];
+    if (m_windowCursor)
+        return m_windowCursor;
+    if (!parent())
+        return nil;
+    return static_cast<QCocoaWindow *>(parent())->effectiveWindowCursor();
+}
+
+// Applies the cursor as returned by effectiveWindowCursor(), handles
+// the special no-cursor-set case by setting the arrow cursor.
+void QCocoaWindow::applyEffectiveWindowCursor()
+{
+    NSCursor *effectiveCursor = effectiveWindowCursor();
+    if (effectiveCursor) {
+        [effectiveCursor set];
     } else {
-        if (m_windowUnderMouse)
-            [cursor set];
+        // We wold like to _unset_ the cursor here; but there is no such
+        // API. Fall back to setting the default arrow cursor.
+        [[NSCursor arrowCursor] set];
     }
 }
 
+void QCocoaWindow::setWindowCursor(NSCursor *cursor)
+{
+    if (m_windowCursor == cursor)
+        return;
+
+    // Setting a cursor in a foregin view is not supported.
+    if (!m_qtView)
+        return;
+
+    [m_windowCursor release];
+    m_windowCursor = cursor;
+    [m_windowCursor retain];
+
+    // The installed view tracking area (see QNSView updateTrackingAreas) will
+    // handle cursor updates on mouse enter/leave. Handle the case where the
+    // mouse is on the this window by changing the cursor immediately.
+    if (m_windowUnderMouse)
+        applyEffectiveWindowCursor();
+}
+
 void QCocoaWindow::registerTouch(bool enable)
 {
     m_registerTouchCount += enable ? 1 : -1;
diff --git a/src/plugins/platforms/cocoa/qnsview.h b/src/plugins/platforms/cocoa/qnsview.h
index 2d4ad7aad3..9d2b54a321 100644
--- a/src/plugins/platforms/cocoa/qnsview.h
+++ b/src/plugins/platforms/cocoa/qnsview.h
@@ -118,7 +118,6 @@ Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 - (void)mouseMovedImpl:(NSEvent *)theEvent;
 - (void)mouseEnteredImpl:(NSEvent *)theEvent;
 - (void)mouseExitedImpl:(NSEvent *)theEvent;
-- (void)cursorUpdateImpl:(NSEvent *)theEvent;
 - (void)rightMouseDown:(NSEvent *)theEvent;
 - (void)rightMouseDragged:(NSEvent *)theEvent;
 - (void)rightMouseUp:(NSEvent *)theEvent;
diff --git a/src/plugins/platforms/cocoa/qnsview.mm b/src/plugins/platforms/cocoa/qnsview.mm
index c67bcfd23b..1ad9b5f327 100644
--- a/src/plugins/platforms/cocoa/qnsview.mm
+++ b/src/plugins/platforms/cocoa/qnsview.mm
@@ -120,7 +120,7 @@ static bool _q_dontOverrideCtrlLMB = false;
 
 - (void)cursorUpdate:(NSEvent *)theEvent
 {
-    [view cursorUpdateImpl:theEvent];
+    [self cursorUpdate:theEvent];
 }
 
 @end
@@ -924,21 +924,10 @@ QT_WARNING_POP
     [self addTrackingArea:m_trackingArea];
 }
 
--(void)cursorUpdateImpl:(NSEvent *)theEvent
-{
-    Q_UNUSED(theEvent)
-    // Set the cursor manually if there is no NSWindow.
-    if (!m_platformWindow->m_nsWindow && m_platformWindow->m_windowCursor)
-        [m_platformWindow->m_windowCursor set];
-    else
-        [super cursorUpdate:theEvent];
-}
-
--(void)resetCursorRects
+- (void)cursorUpdate:(NSEvent *)theEvent
 {
-    // Use the cursor rect API if there is a NSWindow
-    if (m_platformWindow->m_nsWindow && m_platformWindow->m_windowCursor)
-        [self addCursorRect:[self visibleRect] cursor:m_platformWindow->m_windowCursor];
+    Q_UNUSED(theEvent);
+    m_platformWindow->applyEffectiveWindowCursor();
 }
 
 - (void)mouseMovedImpl:(NSEvent *)theEvent