macOS: Modernize masking of windows

Instead of masking window blitting via a CGImage mask, we use the window's
mask directly to intersect the region that we blit during flushing of the
QCocoaBackingStore. This approach also enables masking of child windows.

We now also support setting a mask for layer-backed views, by setting a
CAShapeLayer as the layer's mask.

The window shadow invalidation has been moved out of QNSView, as the view
should not be involved in that process. For layer-backed views, the shadow
is not invalidated as expected after the initial mask has been set, but
this bug has been left as a fix for a later stage as it requires more
research.

Change-Id: Ie0127d8df49d95b2d6144816b19559f3d3c95d13
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

6 files changed, 51 insertions, 73 deletions

diff --git a/src/plugins/platforms/cocoa/cocoa.pro b/src/plugins/platforms/cocoa/cocoa.pro
index 838664895d..6ac5021ea9 100644
--- a/src/plugins/platforms/cocoa/cocoa.pro
+++ b/src/plugins/platforms/cocoa/cocoa.pro
@@ -75,7 +75,7 @@ qtConfig(opengl.*) {
 
 RESOURCES += qcocoaresources.qrc
 
-LIBS += -framework AppKit -framework Carbon -framework IOKit -lcups
+LIBS += -framework AppKit -framework Carbon -framework IOKit -framework QuartzCore -lcups
 
 QT += \
     core-private gui-private \
diff --git a/src/plugins/platforms/cocoa/qcocoabackingstore.mm b/src/plugins/platforms/cocoa/qcocoabackingstore.mm
index 61f44e37d1..972bfc46cb 100644
--- a/src/plugins/platforms/cocoa/qcocoabackingstore.mm
+++ b/src/plugins/platforms/cocoa/qcocoabackingstore.mm
@@ -190,14 +190,15 @@ void QCocoaBackingStore::flush(QWindow *window, const QRegion &region, const QPo
     // Create temporary image to use for blitting, without copying image data
     NSImage *backingStoreImage = [[[NSImage alloc] initWithCGImage:m_cgImage size:NSZeroSize] autorelease];
 
-    if ([topLevelView hasMask]) {
-        // FIXME: Implement via NSBezierPath and addClip
-        CGRect boundingRect = region.boundingRect().toCGRect();
-        QCFType<CGImageRef> subMask = CGImageCreateWithImageInRect([topLevelView maskImage], boundingRect);
-        CGContextClipToMask(graphicsContext.CGContext, boundingRect, subMask);
+    QRegion clippedRegion = region;
+    for (QWindow *w = window; w; w = w->parent()) {
+        if (!w->mask().isEmpty()) {
+            clippedRegion &= w == window ? w->mask()
+                : w->mask().translated(window->mapFromGlobal(w->mapToGlobal(QPoint(0, 0))));
+        }
     }
 
-    for (const QRect &viewLocalRect : region) {
+    for (const QRect &viewLocalRect : clippedRegion) {
         QPoint backingStoreOffset = viewLocalRect.topLeft() + offset;
         QRect backingStoreRect(backingStoreOffset * devicePixelRatio, viewLocalRect.size() * devicePixelRatio);
         if (graphicsContext.flipped) // Flip backingStoreRect to match graphics context
@@ -225,6 +226,12 @@ void QCocoaBackingStore::flush(QWindow *window, const QRegion &region, const QPo
 #endif
     }
 
+    QCocoaWindow *topLevelCocoaWindow = static_cast<QCocoaWindow *>(topLevelWindow->handle());
+    if (Q_UNLIKELY(topLevelCocoaWindow->m_needsInvalidateShadow)) {
+        [topLevelView.window invalidateShadow];
+        topLevelCocoaWindow->m_needsInvalidateShadow = false;
+    }
+
     // -------------------------------------------------------------------------
 
     if (shouldHandleViewLockManually)
@@ -234,9 +241,6 @@ void QCocoaBackingStore::flush(QWindow *window, const QRegion &region, const QPo
         redrawRoundedBottomCorners([view convertRect:region.boundingRect().toCGRect() toView:nil]);
         [view.window flushWindow];
     }
-
-    // FIXME: Tie to changing window flags and/or mask instead
-    [view invalidateWindowShadowIfNeeded];
 }
 
 /*
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.h b/src/plugins/platforms/cocoa/qcocoawindow.h
index 052d8f838c..3357f91bed 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.h
+++ b/src/plugins/platforms/cocoa/qcocoawindow.h
@@ -260,6 +260,8 @@ public: // for QNSView
     QCocoaMenuBar *m_menubar;
     NSCursor *m_windowCursor;
 
+    bool m_needsInvalidateShadow;
+
     bool m_hasModalSession;
     bool m_frameStrutEventsEnabled;
     bool m_isExposed;
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.mm b/src/plugins/platforms/cocoa/qcocoawindow.mm
index aaa7a8798f..606a90c9af 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.mm
+++ b/src/plugins/platforms/cocoa/qcocoawindow.mm
@@ -57,6 +57,7 @@
 #include <QtGui/private/qhighdpiscaling_p.h>
 
 #include <AppKit/AppKit.h>
+#include <QuartzCore/QuartzCore.h>
 
 #include <QDebug>
 
@@ -150,6 +151,7 @@ QCocoaWindow::QCocoaWindow(QWindow *win, WId nativeHandle)
 #endif
     , m_menubar(0)
     , m_windowCursor(0)
+    , m_needsInvalidateShadow(false)
     , m_hasModalSession(false)
     , m_frameStrutEventsEnabled(false)
     , m_isExposed(false)
@@ -699,7 +701,7 @@ bool QCocoaWindow::isOpaque() const
 
     bool translucent = window()->format().alphaBufferSize() > 0
                         || window()->opacity() < 1
-                        || [qnsview_cast(m_view) hasMask]
+                        || !window()->mask().isEmpty()
                         || (surface()->supportsOpenGL() && openglSourfaceOrder == -1);
     return !translucent;
 }
@@ -755,7 +757,34 @@ void QCocoaWindow::setMask(const QRegion &region)
 {
     qCDebug(lcQpaCocoaWindow) << "QCocoaWindow::setMask" << window() << region;
 
-    [qnsview_cast(m_view) setMaskRegion:&region];
+    if (m_view.layer) {
+        if (!region.isEmpty()) {
+            QCFType<CGMutablePathRef> maskPath = CGPathCreateMutable();
+            for (const QRect &r : region)
+                CGPathAddRect(maskPath, nullptr, r.toCGRect());
+            CAShapeLayer *maskLayer = [CAShapeLayer layer];
+            maskLayer.path = maskPath;
+            m_view.layer.mask = maskLayer;
+        } else {
+            m_view.layer.mask = nil;
+        }
+    }
+
+    if (isContentView()) {
+        // Setting the mask requires invalidating the NSWindow shadow, but that needs
+        // to happen after the backingstore has been redrawn, so that AppKit can pick
+        // up the new window shape based on the backingstore content. Doing a display
+        // directly here is not an option, as the window might not be exposed at this
+        // time, and so would not result in an updated backingstore.
+        m_needsInvalidateShadow = true;
+        [m_view setNeedsDisplay:YES];
+
+        // FIXME: [NSWindow invalidateShadow] has no effect when in layer-backed mode,
+        // so if the mask is changed after the initial mask is applied, it will not
+        // result in any visual change to the shadow. This is an Apple bug, and there
+        // may be ways to work around it, such as calling setFrame on the window to
+        // trigger some internal invalidation, but that needs more research.
+    }
 }
 
 bool QCocoaWindow::setKeyboardGrabEnabled(bool grab)
diff --git a/src/plugins/platforms/cocoa/qnsview.h b/src/plugins/platforms/cocoa/qnsview.h
index 414c6b7fe7..004c9f3dab 100644
--- a/src/plugins/platforms/cocoa/qnsview.h
+++ b/src/plugins/platforms/cocoa/qnsview.h
@@ -57,9 +57,6 @@ QT_END_NAMESPACE
 Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 
 @interface QT_MANGLE_NAMESPACE(QNSView) : NSView <NSTextInputClient> {
-    QRegion m_maskRegion;
-    CGImageRef m_maskImage;
-    bool m_shouldInvalidateWindowShadow;
     QPointer<QCocoaWindow> m_platformWindow;
     NSTrackingArea *m_trackingArea;
     Qt::MouseButtons m_buttons;
@@ -90,9 +87,6 @@ Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 #ifndef QT_NO_OPENGL
 - (void)setQCocoaGLContext:(QCocoaGLContext *)context;
 #endif
-- (void)setMaskRegion:(const QRegion *)region;
-- (CGImageRef)maskImage;
-- (void)invalidateWindowShadowIfNeeded;
 - (void)drawRect:(NSRect)dirtyRect;
 - (void)textInputContextKeyboardSelectionDidChangeNotification : (NSNotification *) textInputContextKeyboardSelectionDidChangeNotification;
 - (void)viewDidHide;
@@ -101,7 +95,6 @@ Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 - (BOOL)isFlipped;
 - (BOOL)acceptsFirstResponder;
 - (BOOL)becomeFirstResponder;
-- (BOOL)hasMask;
 - (BOOL)isOpaque;
 
 - (void)convertFromScreen:(NSPoint)mouseLocation toWindowPoint:(QPointF *)qtWindowPoint andScreenPoint:(QPointF *)qtScreenPoint;
diff --git a/src/plugins/platforms/cocoa/qnsview.mm b/src/plugins/platforms/cocoa/qnsview.mm
index 35257ee05d..fd7141d339 100644
--- a/src/plugins/platforms/cocoa/qnsview.mm
+++ b/src/plugins/platforms/cocoa/qnsview.mm
@@ -128,8 +128,6 @@ static QTouchDevice *touchDevice = 0;
 - (id) init
 {
     if (self = [super initWithFrame:NSZeroRect]) {
-        m_maskImage = 0;
-        m_shouldInvalidateWindowShadow = false;
         m_buttons = Qt::NoButton;
         m_acceptedMouseDowns = Qt::NoButton;
         m_frameStrutButtons = Qt::NoButton;
@@ -163,12 +161,10 @@ static QTouchDevice *touchDevice = 0;
 
 - (void)dealloc
 {
-    CGImageRelease(m_maskImage);
     if (m_trackingArea) {
         [self removeTrackingArea:m_trackingArea];
         [m_trackingArea release];
     }
-    m_maskImage = 0;
     [m_inputSource release];
     [[NSNotificationCenter defaultCenter] removeObserver:self];
     [m_mouseMoveHelper release];
@@ -304,11 +300,6 @@ static QTouchDevice *touchDevice = 0;
     [super removeFromSuperview];
 }
 
-- (BOOL) hasMask
-{
-    return !m_maskRegion.isEmpty();
-}
-
 - (BOOL) isOpaque
 {
     if (!m_platformWindow)
@@ -316,48 +307,6 @@ static QTouchDevice *touchDevice = 0;
     return m_platformWindow->isOpaque();
 }
 
-- (void) setMaskRegion:(const QRegion *)region
-{
-    m_shouldInvalidateWindowShadow = true;
-    m_maskRegion = *region;
-    if (m_maskImage)
-        CGImageRelease(m_maskImage);
-    if (region->isEmpty()) {
-        m_maskImage = 0;
-        return;
-    }
-
-    const QRect &rect = region->boundingRect();
-    QImage tmp(rect.size(), QImage::Format_RGB32);
-    tmp.fill(Qt::white);
-    QPainter p(&tmp);
-    p.setClipRegion(*region);
-    p.fillRect(rect, Qt::black);
-    p.end();
-    QImage maskImage = QImage(rect.size(), QImage::Format_Indexed8);
-    for (int y=0; y<rect.height(); ++y) {
-        const uint *src = (const uint *) tmp.constScanLine(y);
-        uchar *dst = maskImage.scanLine(y);
-        for (int x=0; x<rect.width(); ++x) {
-            dst[x] = src[x] & 0xff;
-        }
-    }
-    m_maskImage = qt_mac_toCGImageMask(maskImage);
-}
-
-- (CGImageRef)maskImage
-{
-    return m_maskImage;
-}
-
-- (void)invalidateWindowShadowIfNeeded
-{
-    if (m_shouldInvalidateWindowShadow && m_platformWindow->isContentView()) {
-        [m_platformWindow->nativeWindow() invalidateShadow];
-        m_shouldInvalidateWindowShadow = false;
-    }
-}
-
 - (void)drawRect:(NSRect)dirtyRect
 {
     Q_UNUSED(dirtyRect);
@@ -612,7 +561,8 @@ static QTouchDevice *touchDevice = 0;
     Q_UNUSED(qtScreenPoint);
 
     // Maintain masked state for the button for use by MouseDragged and MouseUp.
-    const bool masked = [self hasMask] && !m_maskRegion.contains(qtWindowPoint.toPoint());
+    QRegion mask = m_platformWindow->window()->mask();
+    const bool masked = !mask.isEmpty() && !mask.contains(qtWindowPoint.toPoint());
     if (masked)
         m_acceptedMouseDowns &= ~button;
     else
@@ -710,8 +660,8 @@ static QTouchDevice *touchDevice = 0;
     [self convertFromScreen:[self screenMousePoint:theEvent] toWindowPoint:&qtWindowPoint andScreenPoint:&qtScreenPoint];
     Q_UNUSED(qtScreenPoint);
 
-    const bool masked = [self hasMask] && !m_maskRegion.contains(qtWindowPoint.toPoint());
-
+    QRegion mask = m_platformWindow->window()->mask();
+    const bool masked = !mask.isEmpty() && !mask.contains(qtWindowPoint.toPoint());
     // Maintain masked state for the button for use by MouseDragged and Up.
     if (masked)
         m_acceptedMouseDowns &= ~Qt::LeftButton;