macOS: Experimental Vulkan support via MoltenVK

Add support for QSurface::VulkanSurface and QVulkanWindow.

Usage:
  1) Build MoltenVK according to instructions
  2) Configure Qt: ./configure -I /path/to/MoltenVK/Package/Release/MoltenVK/include
  3) export QT_VULKAN_LIB=/path/to/MoltenVK/Package/Release/MoltenVK/macOS/libMoltenVK.

Implement support for QSurface::VulkanSurface by enabling
layer mode for QNSView and then creating a CAMetalLayer,
which the MoltenVK translation layer can run on.

MoltenVK provides an implementation of the Vulcan API,
which means that the platform integration is similar
to other platforms: implement a QCocoaVulkanInstance
where we pass the QNSView instance to the vkCreateMacOSSurfaceMVK
Vulkan surface constructor function.

Using Vulkan directly without QVulkanWindow is possible, but not
tested.

We currently load libMoltenVK at run-time and use the
existing QT_VULKAN_LIB environment variable to set its
path. For deployment purposes it would be better to
link against MoltenVK.frameworkm, but this

Task-number: QTBUG-66966
Change-Id: I04ec6289c40b199dca9fed32902b5d2ad4e9c030
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>

10 files changed, 261 insertions, 7 deletions

diff --git a/mkspecs/common/macx.conf b/mkspecs/common/macx.conf
index 4be0eb3c39..7017d60e71 100644
--- a/mkspecs/common/macx.conf
+++ b/mkspecs/common/macx.conf
@@ -11,4 +11,6 @@ device.dir_affix = $${device.sdk}
 device.CONFIG = $${device.sdk}
 device.deployment_identifier = $${device.sdk}
 
+QMAKE_LIBS_VULKAN =
+
 include(mac.conf)
diff --git a/src/plugins/platforms/cocoa/cocoa.pro b/src/plugins/platforms/cocoa/cocoa.pro
index 072012dee1..fd6dae8107 100644
--- a/src/plugins/platforms/cocoa/cocoa.pro
+++ b/src/plugins/platforms/cocoa/cocoa.pro
@@ -72,10 +72,14 @@ HEADERS += qcocoaintegration.h \
 
 qtConfig(opengl.*) {
     SOURCES += qcocoaglcontext.mm
-
     HEADERS += qcocoaglcontext.h
 }
 
+qtConfig(vulkan) {
+    SOURCES += qcocoavulkaninstance.mm
+    HEADERS += qcocoavulkaninstance.h
+}
+
 RESOURCES += qcocoaresources.qrc
 
 LIBS += -framework AppKit -framework CoreServices -framework Carbon -framework IOKit -framework QuartzCore -framework Metal -lcups
@@ -85,6 +89,8 @@ QT += \
     accessibility_support-private clipboard_support-private theme_support-private \
     fontdatabase_support-private graphics_support-private
 
+qtConfig(vulkan): QT += vulkan_support-private
+
 CONFIG += no_app_extension_api_only
 
 qtHaveModule(widgets) {
diff --git a/src/plugins/platforms/cocoa/qcocoaintegration.h b/src/plugins/platforms/cocoa/qcocoaintegration.h
index 301771fd53..7de7e073de 100644
--- a/src/plugins/platforms/cocoa/qcocoaintegration.h
+++ b/src/plugins/platforms/cocoa/qcocoaintegration.h
@@ -51,6 +51,9 @@
 #include "qcocoadrag.h"
 #include "qcocoaservices.h"
 #include "qcocoakeymapper.h"
+#if QT_CONFIG(vulkan)
+#include "qcocoavulkaninstance.h"
+#endif
 
 #include <QtCore/QScopedPointer>
 #include <qpa/qplatformintegration.h>
@@ -86,6 +89,11 @@ public:
 
     QAbstractEventDispatcher *createEventDispatcher() const override;
 
+#if QT_CONFIG(vulkan)
+    QPlatformVulkanInstance *createPlatformVulkanInstance(QVulkanInstance *instance) const override;
+    QCocoaVulkanInstance *getCocoaVulkanInstance() const;
+#endif
+
     QCoreTextFontDatabase *fontDatabase() const override;
     QCocoaNativeInterface *nativeInterface() const override;
     QPlatformInputContext *inputContext() const override;
@@ -144,6 +152,9 @@ private:
     QScopedPointer<QCocoaServices> mServices;
     QScopedPointer<QCocoaKeyMapper> mKeyboardMapper;
 
+#if QT_CONFIG(vulkan)
+    mutable QCocoaVulkanInstance *mCocoaVulkanInstance = nullptr;
+#endif
     QHash<QWindow *, NSToolbar *> mToolbars;
     QList<QCocoaWindow *> m_popupWindowStack;
 };
diff --git a/src/plugins/platforms/cocoa/qcocoaintegration.mm b/src/plugins/platforms/cocoa/qcocoaintegration.mm
index a4658271bb..ce3a1304b2 100644
--- a/src/plugins/platforms/cocoa/qcocoaintegration.mm
+++ b/src/plugins/platforms/cocoa/qcocoaintegration.mm
@@ -379,6 +379,19 @@ QAbstractEventDispatcher *QCocoaIntegration::createEventDispatcher() const
     return new QCocoaEventDispatcher;
 }
 
+#if QT_CONFIG(vulkan)
+QPlatformVulkanInstance *QCocoaIntegration::createPlatformVulkanInstance(QVulkanInstance *instance) const
+{
+    mCocoaVulkanInstance = new QCocoaVulkanInstance(instance);
+    return mCocoaVulkanInstance;
+}
+
+QCocoaVulkanInstance *QCocoaIntegration::getCocoaVulkanInstance() const
+{
+    return mCocoaVulkanInstance;
+}
+#endif
+
 QCoreTextFontDatabase *QCocoaIntegration::fontDatabase() const
 {
     return mFontDb.data();
diff --git a/src/plugins/platforms/cocoa/qcocoanativeinterface.mm b/src/plugins/platforms/cocoa/qcocoanativeinterface.mm
index b4af4a6e01..2a6c25ed75 100644
--- a/src/plugins/platforms/cocoa/qcocoanativeinterface.mm
+++ b/src/plugins/platforms/cocoa/qcocoanativeinterface.mm
@@ -71,6 +71,10 @@
 
 #include <AppKit/AppKit.h>
 
+#if QT_CONFIG(vulkan)
+#include <MoltenVK/mvk_vulkan.h>
+#endif
+
 QT_BEGIN_NAMESPACE
 
 QCocoaNativeInterface::QCocoaNativeInterface()
@@ -104,6 +108,11 @@ void *QCocoaNativeInterface::nativeResourceForWindow(const QByteArray &resourceS
 #endif
     } else if (resourceString == "nswindow") {
         return static_cast<QCocoaWindow *>(window->handle())->nativeWindow();
+#if QT_CONFIG(vulkan)
+    } else if (resourceString == "vkSurface") {
+        if (QVulkanInstance *instance = window->vulkanInstance())
+            return static_cast<QCocoaVulkanInstance *>(instance->handle())->createSurface(window);
+#endif
     }
     return nullptr;
 }
diff --git a/src/plugins/platforms/cocoa/qcocoavulkaninstance.h b/src/plugins/platforms/cocoa/qcocoavulkaninstance.h
new file mode 100644
index 0000000000..c9c6cf7e3f
--- /dev/null
+++ b/src/plugins/platforms/cocoa/qcocoavulkaninstance.h
@@ -0,0 +1,74 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of plugins of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QCOCOAVULKANINSTANCE_H
+#define QCOCOAVULKANINSTANCE_H
+
+#include <QtCore/QLibrary>
+#include <QtCore/QHash>
+#include <QtVulkanSupport/private/qbasicvulkanplatforminstance_p.h>
+
+#include <AppKit/AppKit.h>
+
+#include <MoltenVK/mvk_vulkan.h>
+
+QT_BEGIN_NAMESPACE
+
+class QCocoaVulkanInstance : public QBasicPlatformVulkanInstance
+{
+public:
+    QCocoaVulkanInstance(QVulkanInstance *instance);
+    ~QCocoaVulkanInstance();
+
+    void createOrAdoptInstance() override;
+
+    VkSurfaceKHR *createSurface(QWindow *window);
+    VkSurfaceKHR createSurface(NSView *view);
+    void destroySurface(VkSurfaceKHR surface);
+private:
+    QVulkanInstance *m_instance = nullptr;
+    QLibrary m_lib;
+    VkSurfaceKHR m_nullSurface = nullptr;
+    PFN_vkCreateMacOSSurfaceMVK m_createSurface = nullptr;
+    PFN_vkDestroySurfaceKHR m_destroySurface = nullptr;
+};
+
+QT_END_NAMESPACE
+
+#endif // QXCBVULKANINSTANCE_H
diff --git a/src/plugins/platforms/cocoa/qcocoavulkaninstance.mm b/src/plugins/platforms/cocoa/qcocoavulkaninstance.mm
new file mode 100644
index 0000000000..5da929766f
--- /dev/null
+++ b/src/plugins/platforms/cocoa/qcocoavulkaninstance.mm
@@ -0,0 +1,119 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of plugins of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qcocoavulkaninstance.h"
+#include "qcocoawindow.h"
+
+QT_BEGIN_NAMESPACE
+
+QCocoaVulkanInstance::QCocoaVulkanInstance(QVulkanInstance *instance)
+    : m_instance(instance)
+{
+    if (qEnvironmentVariableIsSet("QT_VULKAN_LIB"))
+        m_lib.setFileName(QString::fromUtf8(qgetenv("QT_VULKAN_LIB")));
+    else
+        m_lib.setFileName(QStringLiteral("vulkan"));
+
+    if (!m_lib.load()) {
+        qWarning("Failed to load %s: %s", qPrintable(m_lib.fileName()), qPrintable(m_lib.errorString()));
+        return;
+    }
+
+    init(&m_lib);
+}
+
+QCocoaVulkanInstance::~QCocoaVulkanInstance()
+{
+}
+
+void QCocoaVulkanInstance::createOrAdoptInstance()
+{
+    initInstance(m_instance, QByteArrayList());
+}
+
+VkSurfaceKHR *QCocoaVulkanInstance::createSurface(QWindow *window)
+{
+    QCocoaWindow *cocoaWindow = static_cast<QCocoaWindow *>(window->handle());
+    if (cocoaWindow->m_vulkanSurface)
+        destroySurface(cocoaWindow->m_vulkanSurface);
+    cocoaWindow->m_vulkanSurface = createSurface(cocoaWindow->m_view);
+    return &cocoaWindow->m_vulkanSurface;
+}
+
+VkSurfaceKHR QCocoaVulkanInstance::createSurface(NSView *view)
+{
+    if (!m_createSurface) {
+        m_createSurface = reinterpret_cast<PFN_vkCreateMacOSSurfaceMVK>(
+                    m_vkGetInstanceProcAddr(m_vkInst, "vkCreateMacOSSurfaceMVK"));
+    }
+    if (!m_createSurface) {
+        qWarning("Failed to find vkCreateMacOSSurfaceMVK");
+        return m_nullSurface;
+    }
+    if (!m_destroySurface) {
+        m_destroySurface = reinterpret_cast<PFN_vkDestroySurfaceKHR>(
+                    m_vkGetInstanceProcAddr(m_vkInst, "vkDestroySurfaceKHR"));
+    }
+    if (!m_destroySurface) {
+        qWarning("Failed to find vkDestroySurfaceKHR");
+        return m_nullSurface;
+    }
+
+    VkMacOSSurfaceCreateInfoMVK surfaceInfo;
+    surfaceInfo.sType = VK_STRUCTURE_TYPE_MACOS_SURFACE_CREATE_INFO_MVK;
+    surfaceInfo.pNext = nullptr;
+    surfaceInfo.flags = 0;
+    surfaceInfo.pView = view;
+
+    VkSurfaceKHR surface = nullptr;
+    VkResult err = m_createSurface(m_vkInst, &surfaceInfo, nullptr, &surface);
+    if (err != VK_SUCCESS)
+        qWarning("Failed to create Vulkan surface: %d", err);
+
+    return surface;
+}
+
+void QCocoaVulkanInstance::destroySurface(VkSurfaceKHR surface)
+{
+    if (m_destroySurface && surface)
+        m_destroySurface(m_vkInst, surface, nullptr);
+}
+
+
+QT_END_NAMESPACE
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.h b/src/plugins/platforms/cocoa/qcocoawindow.h
index feaade1bd6..225c7eda84 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.h
+++ b/src/plugins/platforms/cocoa/qcocoawindow.h
@@ -53,6 +53,10 @@
 #include "qnswindow.h"
 #include "qt_mac_p.h"
 
+#if QT_CONFIG(vulkan)
+#include <MoltenVK/mvk_vulkan.h>
+#endif
+
 QT_BEGIN_NAMESPACE
 
 #ifndef QT_NO_DEBUG_STREAM
@@ -281,6 +285,10 @@ public: // for QNSView
     };
     QHash<quintptr, BorderRange> m_contentBorderAreas; // identifer -> uppper/lower
     QHash<quintptr, bool> m_enabledContentBorderAreas; // identifer -> enabled state (true/false)
+
+#if QT_CONFIG(vulkan)
+    VkSurfaceKHR m_vulkanSurface = nullptr;
+#endif
 };
 
 #ifndef QT_NO_DEBUG_STREAM
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.mm b/src/plugins/platforms/cocoa/qcocoawindow.mm
index 7e7740527f..4618db1aa3 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.mm
+++ b/src/plugins/platforms/cocoa/qcocoawindow.mm
@@ -222,10 +222,16 @@ QCocoaWindow::~QCocoaWindow()
     if (!isForeignWindow())
         [[NSNotificationCenter defaultCenter] removeObserver:m_view];
 
-    // While it is unlikely that this window will be in the popup stack
-    // during deletetion we clear any pointers here to make sure.
-    if (QCocoaIntegration::instance()) {
-        QCocoaIntegration::instance()->popupWindowStack()->removeAll(this);
+    if (QCocoaIntegration *cocoaIntegration = QCocoaIntegration::instance()) {
+        // While it is unlikely that this window will be in the popup stack
+        // during deletetion we clear any pointers here to make sure.
+        cocoaIntegration->popupWindowStack()->removeAll(this);
+
+#if QT_CONFIG(vulkan)
+        auto vulcanInstance = cocoaIntegration->getCocoaVulkanInstance();
+        if (vulcanInstance)
+            vulcanInstance->destroySurface(m_vulkanSurface);
+#endif
     }
 
     [m_view release];
diff --git a/src/plugins/platforms/cocoa/qnsview_drawing.mm b/src/plugins/platforms/cocoa/qnsview_drawing.mm
index 3bba68b0cf..daa1a2e250 100644
--- a/src/plugins/platforms/cocoa/qnsview_drawing.mm
+++ b/src/plugins/platforms/cocoa/qnsview_drawing.mm
@@ -141,13 +141,19 @@
     // on and off is not a supported use-case, so this code is effectively
     // returning a constant for the lifetime of our QSNSView, which means
     // we don't care about emitting KVO signals for @"wantsLayer".
-    return qt_mac_resolveOption(false, m_platformWindow->window(),
+    bool layerRequested = qt_mac_resolveOption(false, m_platformWindow->window(),
         "_q_mac_wantsLayer", "QT_MAC_WANTS_LAYER");
+
+    // Support Vulkan via MoltenVK, which requires a Metal layer
+    bool layerForVulkan = (m_platformWindow->window()->surfaceType() == QWindow::VulkanSurface);
+
+    return layerRequested || layerForVulkan;
 }
 
 - (CALayer *)makeBackingLayer
 {
-    bool makeMetalLayer = false; // TODO: Add/implement enabling API
+    // Support Vulkan via MoltenVK, which requires a Metal layer
+    bool makeMetalLayer = (m_platformWindow->window()->surfaceType() == QWindow::VulkanSurface);
     if (makeMetalLayer) {
         // Check if Metal is supported. If it isn't then it's most likely
         // too late at this point and the QWindow will be non-functional,