macOS: Modernize masking of windows

Instead of masking window blitting via a CGImage mask, we use the window's
mask directly to intersect the region that we blit during flushing of the
QCocoaBackingStore. This approach also enables masking of child windows.

We now also support setting a mask for layer-backed views, by setting a
CAShapeLayer as the layer's mask.

The window shadow invalidation has been moved out of QNSView, as the view
should not be involved in that process. For layer-backed views, the shadow
is not invalidated as expected after the initial mask has been set, but
this bug has been left as a fix for a later stage as it requires more
research.

Change-Id: Ie0127d8df49d95b2d6144816b19559f3d3c95d13
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

1 files changed, 13 insertions, 9 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoabackingstore.mm b/src/plugins/platforms/cocoa/qcocoabackingstore.mm
index 61f44e37d1..972bfc46cb 100644
--- a/src/plugins/platforms/cocoa/qcocoabackingstore.mm
+++ b/src/plugins/platforms/cocoa/qcocoabackingstore.mm
@@ -190,14 +190,15 @@ void QCocoaBackingStore::flush(QWindow *window, const QRegion &region, const QPo
     // Create temporary image to use for blitting, without copying image data
     NSImage *backingStoreImage = [[[NSImage alloc] initWithCGImage:m_cgImage size:NSZeroSize] autorelease];
 
-    if ([topLevelView hasMask]) {
-        // FIXME: Implement via NSBezierPath and addClip
-        CGRect boundingRect = region.boundingRect().toCGRect();
-        QCFType<CGImageRef> subMask = CGImageCreateWithImageInRect([topLevelView maskImage], boundingRect);
-        CGContextClipToMask(graphicsContext.CGContext, boundingRect, subMask);
+    QRegion clippedRegion = region;
+    for (QWindow *w = window; w; w = w->parent()) {
+        if (!w->mask().isEmpty()) {
+            clippedRegion &= w == window ? w->mask()
+                : w->mask().translated(window->mapFromGlobal(w->mapToGlobal(QPoint(0, 0))));
+        }
     }
 
-    for (const QRect &viewLocalRect : region) {
+    for (const QRect &viewLocalRect : clippedRegion) {
         QPoint backingStoreOffset = viewLocalRect.topLeft() + offset;
         QRect backingStoreRect(backingStoreOffset * devicePixelRatio, viewLocalRect.size() * devicePixelRatio);
         if (graphicsContext.flipped) // Flip backingStoreRect to match graphics context
@@ -225,6 +226,12 @@ void QCocoaBackingStore::flush(QWindow *window, const QRegion &region, const QPo
 #endif
     }
 
+    QCocoaWindow *topLevelCocoaWindow = static_cast<QCocoaWindow *>(topLevelWindow->handle());
+    if (Q_UNLIKELY(topLevelCocoaWindow->m_needsInvalidateShadow)) {
+        [topLevelView.window invalidateShadow];
+        topLevelCocoaWindow->m_needsInvalidateShadow = false;
+    }
+
     // -------------------------------------------------------------------------
 
     if (shouldHandleViewLockManually)
@@ -234,9 +241,6 @@ void QCocoaBackingStore::flush(QWindow *window, const QRegion &region, const QPo
         redrawRoundedBottomCorners([view convertRect:region.boundingRect().toCGRect() toView:nil]);
         [view.window flushWindow];
     }
-
-    // FIXME: Tie to changing window flags and/or mask instead
-    [view invalidateWindowShadowIfNeeded];
 }
 
 /*