macOS: Modernize masking of windows

Instead of masking window blitting via a CGImage mask, we use the window's
mask directly to intersect the region that we blit during flushing of the
QCocoaBackingStore. This approach also enables masking of child windows.

We now also support setting a mask for layer-backed views, by setting a
CAShapeLayer as the layer's mask.

The window shadow invalidation has been moved out of QNSView, as the view
should not be involved in that process. For layer-backed views, the shadow
is not invalidated as expected after the initial mask has been set, but
this bug has been left as a fix for a later stage as it requires more
research.

Change-Id: Ie0127d8df49d95b2d6144816b19559f3d3c95d13
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

1 files changed, 0 insertions, 7 deletions

diff --git a/src/plugins/platforms/cocoa/qnsview.h b/src/plugins/platforms/cocoa/qnsview.h
index 414c6b7fe7..004c9f3dab 100644
--- a/src/plugins/platforms/cocoa/qnsview.h
+++ b/src/plugins/platforms/cocoa/qnsview.h
@@ -57,9 +57,6 @@ QT_END_NAMESPACE
 Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 
 @interface QT_MANGLE_NAMESPACE(QNSView) : NSView <NSTextInputClient> {
-    QRegion m_maskRegion;
-    CGImageRef m_maskImage;
-    bool m_shouldInvalidateWindowShadow;
     QPointer<QCocoaWindow> m_platformWindow;
     NSTrackingArea *m_trackingArea;
     Qt::MouseButtons m_buttons;
@@ -90,9 +87,6 @@ Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 #ifndef QT_NO_OPENGL
 - (void)setQCocoaGLContext:(QCocoaGLContext *)context;
 #endif
-- (void)setMaskRegion:(const QRegion *)region;
-- (CGImageRef)maskImage;
-- (void)invalidateWindowShadowIfNeeded;
 - (void)drawRect:(NSRect)dirtyRect;
 - (void)textInputContextKeyboardSelectionDidChangeNotification : (NSNotification *) textInputContextKeyboardSelectionDidChangeNotification;
 - (void)viewDidHide;
@@ -101,7 +95,6 @@ Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 - (BOOL)isFlipped;
 - (BOOL)acceptsFirstResponder;
 - (BOOL)becomeFirstResponder;
-- (BOOL)hasMask;
 - (BOOL)isOpaque;
 
 - (void)convertFromScreen:(NSPoint)mouseLocation toWindowPoint:(QPointF *)qtWindowPoint andScreenPoint:(QPointF *)qtScreenPoint;