macOS: Only enable layer-backed views on 10.14 if built against 10.14 SDK

Instead of explicitly enabling layer-backing for Qt 5.12 on all macOS
version, we follow the macOS default to enable it for 10.14 if the
application binary was built against the 10.14 SDK.

Aligning ourselves with Apple's switch to forced layer-backing means
we have an easier story when it comes to supporting different runtime
configurations.

Fixes: QTBUG-71499
Change-Id: I34ee49b3daeb6ed8df444a3759d3573ebc9ea30f
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

2 files changed, 53 insertions, 23 deletions

diff --git a/src/plugins/platforms/cocoa/qnsview.mm b/src/plugins/platforms/cocoa/qnsview.mm
index 9bd53ed334..540f701d43 100644
--- a/src/plugins/platforms/cocoa/qnsview.mm
+++ b/src/plugins/platforms/cocoa/qnsview.mm
@@ -71,7 +71,7 @@
 @end
 
 @interface QT_MANGLE_NAMESPACE(QNSView) (Drawing) <CALayerDelegate>
-- (BOOL)wantsLayerHelper;
+- (void)initDrawing;
 @end
 
 @interface QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper) : NSObject
@@ -152,19 +152,8 @@
 
         self.focusRingType = NSFocusRingTypeNone;
         self.cursor = nil;
-        self.wantsLayer = [self wantsLayerHelper];
-
-        // Enable high-DPI OpenGL for retina displays. Enabling has the side
-        // effect that Cocoa will start calling glViewport(0, 0, width, height),
-        // overriding any glViewport calls in application code. This is usually not a
-        // problem, except if the application wants to have a "custom" viewport.
-        // (like the hellogl example)
-        if (m_platformWindow->window()->supportsOpenGL()) {
-            self.wantsBestResolutionOpenGLSurface = qt_mac_resolveOption(YES, m_platformWindow->window(),
-                "_q_mac_wantsBestResolutionOpenGLSurface", "QT_MAC_WANTS_BEST_RESOLUTION_OPENGL_SURFACE");
-            // See also QCocoaGLContext::makeCurrent for software renderer workarounds.
-        }
 
+        [self initDrawing];
         [self registerDragTypes];
 
         [[NSNotificationCenter defaultCenter] addObserver:self
diff --git a/src/plugins/platforms/cocoa/qnsview_drawing.mm b/src/plugins/platforms/cocoa/qnsview_drawing.mm
index 4f9d17504d..e9af90a45c 100644
--- a/src/plugins/platforms/cocoa/qnsview_drawing.mm
+++ b/src/plugins/platforms/cocoa/qnsview_drawing.mm
@@ -41,6 +41,24 @@
 
 @implementation QT_MANGLE_NAMESPACE(QNSView) (Drawing)
 
+- (void)initDrawing
+{
+    self.wantsLayer = [self layerExplicitlyRequested]
+        || [self shouldUseMetalLayer]
+        || [self layerEnabledByMacOS];
+
+    // Enable high-DPI OpenGL for retina displays. Enabling has the side
+    // effect that Cocoa will start calling glViewport(0, 0, width, height),
+    // overriding any glViewport calls in application code. This is usually not a
+    // problem, except if the application wants to have a "custom" viewport.
+    // (like the hellogl example)
+    if (m_platformWindow->window()->supportsOpenGL()) {
+        self.wantsBestResolutionOpenGLSurface = qt_mac_resolveOption(YES, m_platformWindow->window(),
+            "_q_mac_wantsBestResolutionOpenGLSurface", "QT_MAC_WANTS_BEST_RESOLUTION_OPENGL_SURFACE");
+        // See also QCocoaGLContext::makeCurrent for software renderer workarounds.
+    }
+}
+
 - (BOOL)isOpaque
 {
     if (!m_platformWindow)
@@ -71,23 +89,38 @@
     m_platformWindow->handleExposeEvent(exposedRegion);
 }
 
-- (BOOL)shouldUseMetalLayer
+- (BOOL)layerEnabledByMacOS
 {
-    // MetalSurface needs a layer, and so does VulkanSurface (via MoltenVK)
-    QSurface::SurfaceType surfaceType = m_platformWindow->window()->surfaceType();
-    return surfaceType == QWindow::MetalSurface || surfaceType == QWindow::VulkanSurface;
+    // AppKit has its own logic for this, but if we rely on that, our layers are created
+    // by AppKit at a point where we've already set up other parts of the platform plugin
+    // based on the presence of layers or not. Once we've rewritten these parts to support
+    // dynamically picking up layer enablement we can let AppKit do its thing.
+    return QMacVersion::buildSDK() >= QOperatingSystemVersion::MacOSMojave
+        && QMacVersion::currentRuntime() >= QOperatingSystemVersion::MacOSMojave;
 }
 
-- (BOOL)wantsLayerHelper
+- (BOOL)layerExplicitlyRequested
 {
-    Q_ASSERT(m_platformWindow);
+    static bool wantsLayer = [&]() {
+        int wantsLayer = qt_mac_resolveOption(-1, m_platformWindow->window(),
+            "_q_mac_wantsLayer", "QT_MAC_WANTS_LAYER");
 
-    bool wantsLayer = qt_mac_resolveOption(true, m_platformWindow->window(),
-        "_q_mac_wantsLayer", "QT_MAC_WANTS_LAYER");
+        if (wantsLayer != -1 && [self layerEnabledByMacOS]) {
+            qCWarning(lcQpaDrawing) << "Layer-backing can not be explicitly controlled on 10.14 when built against the 10.14 SDK";
+            return true;
+        }
 
-    bool layerForSurfaceType = [self shouldUseMetalLayer];
+        return wantsLayer == 1;
+    }();
 
-    return wantsLayer || layerForSurfaceType;
+    return wantsLayer;
+}
+
+- (BOOL)shouldUseMetalLayer
+{
+    // MetalSurface needs a layer, and so does VulkanSurface (via MoltenVK)
+    QSurface::SurfaceType surfaceType = m_platformWindow->window()->surfaceType();
+    return surfaceType == QWindow::MetalSurface || surfaceType == QWindow::VulkanSurface;
 }
 
 - (CALayer *)makeBackingLayer
@@ -115,6 +148,14 @@
     return [super makeBackingLayer];
 }
 
+- (void)setLayer:(CALayer *)layer
+{
+    qCDebug(lcQpaDrawing) << "Making" << self << "layer-backed with" << layer
+        << "due to being" << ([self layerExplicitlyRequested] ? "explicitly requested"
+            : [self shouldUseMetalLayer] ? "needed by surface type" : "enabled by macOS");
+    [super setLayer:layer];
+}
+
 - (NSViewLayerContentsRedrawPolicy)layerContentsRedrawPolicy
 {
     // We need to set this explicitly since the super implementation