Compressed texture support for vulkan server buffers

Adding new virtual function createServerBufferFromData() to the
ServerBufferIntegration class.

Change-Id: I6009fd5aec07f1794431bd52ccd56cfae9d0f77b
Reviewed-by: Johan Helsing <johan.helsing@qt.io>

5 files changed, 68 insertions, 18 deletions

diff --git a/src/hardwareintegration/compositor/vulkan-server/vulkan-server.pri b/src/hardwareintegration/compositor/vulkan-server/vulkan-server.pri
index 9e1b1ff0c..b86f8453a 100644
--- a/src/hardwareintegration/compositor/vulkan-server/vulkan-server.pri
+++ b/src/hardwareintegration/compositor/vulkan-server/vulkan-server.pri
@@ -1,4 +1,4 @@
-INCLUDEPATH += $$PWD
+INCLUDEPATH += $$PWD $$PWD/../../../3rdparty/util
 
 QMAKE_USE_PRIVATE += wayland-server
 
@@ -6,10 +6,10 @@ SOURCES += \
     $$PWD/vulkanserverbufferintegration.cpp \
     $$PWD/vulkanwrapper.cpp
 
-
 HEADERS += \
     $$PWD/vulkanserverbufferintegration.h \
-    $$PWD/vulkanwrapper.h
+    $$PWD/vulkanwrapper.h \
+    $$PWD/vk_format.h
 
 CONFIG += wayland-scanner
 WAYLANDSERVERSOURCES += $$PWD/../../../extensions/qt-vulkan-server-buffer-unstable-v1.xml
diff --git a/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.cpp b/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.cpp
index 5d63a64d4..0d60a9824 100644
--- a/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.cpp
+++ b/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.cpp
@@ -43,12 +43,15 @@
 
 #include <QtGui/QOpenGLContext>
 #include <QtGui/QOpenGLTexture>
-#include <QOffscreenSurface>
+#include <QtGui/QOffscreenSurface>
+#include <QtGui/qopengl.h>
 
 #include <QtCore/QDebug>
 
 QT_BEGIN_NAMESPACE
 
+static constexpr bool extraDebug = false;
+
 VulkanServerBuffer::VulkanServerBuffer(VulkanServerBufferIntegration *integration, const QImage &qimage, QtWayland::ServerBuffer::Format format)
     : QtWayland::ServerBuffer(qimage.size(),format)
     , m_integration(integration)
@@ -75,6 +78,18 @@ VulkanServerBuffer::VulkanServerBuffer(VulkanServerBufferIntegration *integratio
         m_fd = vulkanWrapper->getImageInfo(m_vImage, &m_memorySize);
 }
 
+VulkanServerBuffer::VulkanServerBuffer(VulkanServerBufferIntegration *integration, VulkanImageWrapper *vImage, uint glInternalFormat, const QSize &size)
+    : QtWayland::ServerBuffer(size, QtWayland::ServerBuffer::Custom)
+    , m_integration(integration)
+    , m_width(size.width())
+    , m_height(size.height())
+    , m_vImage(vImage)
+    , m_glInternalFormat(glInternalFormat)
+{
+    auto vulkanWrapper = m_integration->vulkanWrapper();
+    m_fd = vulkanWrapper->getImageInfo(m_vImage, &m_memorySize);
+}
+
 VulkanServerBuffer::~VulkanServerBuffer()
 {
     delete m_texture; //this is always nullptr for now
@@ -140,7 +155,7 @@ bool VulkanServerBufferIntegration::supportsFormat(QtWayland::ServerBuffer::Form
         case QtWayland::ServerBuffer::RGBA32:
             return true;
         case QtWayland::ServerBuffer::A8:
-            return false; // TODO: add more formats
+            return false;
         default:
             return false;
     }
@@ -190,4 +205,20 @@ QtWayland::ServerBuffer *VulkanServerBufferIntegration::createServerBufferFromIm
     return new VulkanServerBuffer(this, qimage, format);
 }
 
+QtWayland::ServerBuffer *VulkanServerBufferIntegration::createServerBufferFromData(const QByteArray &data, const QSize &size, uint glInternalFormat)
+{
+    if (!m_vulkanWrapper) {
+        CurrentContext current;
+        m_vulkanWrapper = new VulkanWrapper(current.context());
+    }
+
+    auto *vImage = m_vulkanWrapper->createTextureImageFromData(reinterpret_cast<const uchar*>(data.constData()), data.size(), size, glInternalFormat);
+
+    if (vImage)
+        return new VulkanServerBuffer(this, vImage, glInternalFormat, size);
+
+    qCWarning(qLcWaylandCompositorHardwareIntegration) << "could not load compressed texture";
+    return nullptr;
+}
+
 QT_END_NAMESPACE
diff --git a/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.h b/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.h
index aa1074ec6..8e7a3fe64 100644
--- a/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.h
+++ b/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.h
@@ -62,6 +62,7 @@ class VulkanServerBuffer : public QtWayland::ServerBuffer, public QtWaylandServe
 {
 public:
     VulkanServerBuffer(VulkanServerBufferIntegration *integration, const QImage &qimage, QtWayland::ServerBuffer::Format format);
+    VulkanServerBuffer(VulkanServerBufferIntegration *integration, VulkanImageWrapper *vImage, uint glInternalFormat, const QSize &size);
     ~VulkanServerBuffer() override;
 
     struct ::wl_resource *resourceForClient(struct ::wl_client *) override;
@@ -97,6 +98,8 @@ public:
 
     bool supportsFormat(QtWayland::ServerBuffer::Format format) const override;
     QtWayland::ServerBuffer *createServerBufferFromImage(const QImage &qimage, QtWayland::ServerBuffer::Format format) override;
+    QtWayland::ServerBuffer *createServerBufferFromData(const QByteArray &data, const QSize &size, uint glInternalFormat) override;
+
 private:
     VulkanWrapper *m_vulkanWrapper = nullptr;
 };
diff --git a/src/hardwareintegration/compositor/vulkan-server/vulkanwrapper.cpp b/src/hardwareintegration/compositor/vulkan-server/vulkanwrapper.cpp
index 18cdaf8a6..4767733dc 100644
--- a/src/hardwareintegration/compositor/vulkan-server/vulkanwrapper.cpp
+++ b/src/hardwareintegration/compositor/vulkan-server/vulkanwrapper.cpp
@@ -51,6 +51,7 @@
 #include <set>
 
 #include <unistd.h>
+#include "vk_format.h"
 
 #include <QDebug>
 
@@ -80,6 +81,8 @@ public:
     explicit VulkanWrapperPrivate(QOpenGLContext *glContext);
 
     VulkanImageWrapper *createTextureImage(const QImage &img);
+    VulkanImageWrapper *createTextureImageFromData(const uchar *pixels, uint bufferSize, const QSize &size, VkFormat vkFormat);
+
     void freeTextureImage(VulkanImageWrapper *imageWrapper);
 
 private:
@@ -160,6 +163,7 @@ private:
     }
 
     int findMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties);
+
     VulkanImageWrapper *createImage(VkFormat format, VkImageTiling tiling, VkImageUsageFlags usage, VkMemoryPropertyFlags properties, const QSize &size, int memSize);
     bool transitionImageLayout(VkImage image, VkFormat /*format*/, VkImageLayout oldLayout, VkImageLayout newLayout);
     bool createBuffer(VkDeviceSize size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties, VkBuffer& buffer, VkDeviceMemory& bufferMemory);
@@ -501,14 +505,17 @@ bool VulkanWrapperPrivate::createLogicalDevice()
 
 VulkanImageWrapper *VulkanWrapperPrivate::createTextureImage(const QImage &img)
 {
+    return createTextureImageFromData(img.constBits(), img.sizeInBytes(), img.size(), VK_FORMAT_R8G8B8A8_UNORM);
+}
+
+VulkanImageWrapper *VulkanWrapperPrivate::createTextureImageFromData(const uchar *pixels, uint bufferSize, const QSize &size, VkFormat vkFormat)
+{
     if (m_initFailed)
         return nullptr;
 
-    int texWidth = img.width();
-    int texHeight = img.height();
+    int texWidth = size.width();
+    int texHeight = size.height();
     bool ok;
-    const auto *pixels = img.constBits();
-    VkDeviceSize imageSize = img.sizeInBytes();
     if (extraDebug) qDebug("image load %p %dx%d", pixels, texWidth, texHeight);
     if (!pixels) {
         qCritical("VulkanWrapper: failed to load texture image!");
@@ -517,20 +524,20 @@ VulkanImageWrapper *VulkanWrapperPrivate::createTextureImage(const QImage &img)
 
     VkBuffer stagingBuffer;
     VkDeviceMemory stagingBufferMemory;
-    ok = createBuffer(imageSize, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, stagingBuffer, stagingBufferMemory);
+    ok = createBuffer(bufferSize, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, stagingBuffer, stagingBufferMemory);
 
     if (!ok)
         return nullptr;
 
     void* data;
-    vkMapMemory(m_device, stagingBufferMemory, 0, imageSize, 0, &data);
-    if (extraDebug) qDebug() << "mapped" << data << imageSize;
-    memcpy(data, pixels, static_cast<size_t>(imageSize));
+    vkMapMemory(m_device, stagingBufferMemory, 0, bufferSize, 0, &data);
+    if (extraDebug) qDebug() << "mapped" << data << bufferSize;
+    memcpy(data, pixels, static_cast<size_t>(bufferSize));
     vkUnmapMemory(m_device, stagingBufferMemory);
 
     if (extraDebug) qDebug() << "creating image...";
 
-    QScopedPointer<VulkanImageWrapper> imageWrapper(createImage(VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_TILING_OPTIMAL, VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, img.size(), imageSize));
+    QScopedPointer<VulkanImageWrapper> imageWrapper(createImage(vkFormat, VK_IMAGE_TILING_OPTIMAL, VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, size, bufferSize));
     if (imageWrapper.isNull())
         return nullptr;
 
@@ -538,14 +545,14 @@ VulkanImageWrapper *VulkanWrapperPrivate::createTextureImage(const QImage &img)
 
     const VkImage textureImage = imageWrapper->textureImage;
 
-    ok = transitionImageLayout(textureImage, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
+    ok = transitionImageLayout(textureImage, vkFormat, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
 
     if (!ok)
         return nullptr;
 
     if (extraDebug) qDebug() << "copyBufferToImage...";
     copyBufferToImage(stagingBuffer, textureImage, static_cast<uint32_t>(texWidth), static_cast<uint32_t>(texHeight));
-    transitionImageLayout(textureImage, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+    transitionImageLayout(textureImage, vkFormat, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
 
     vkDestroyBuffer(m_device, stagingBuffer, nullptr);
     vkFreeMemory(m_device, stagingBufferMemory, nullptr);
@@ -674,8 +681,7 @@ VulkanWrapperPrivate::VulkanWrapperPrivate(QOpenGLContext *glContext)
 #ifdef VULKAN_SERVER_BUFFER_EXTRA_DEBUG
     qDebug() << "GPU memory type:" << gpuMemoryType << "heap:" << memProps.memoryTypes[gpuMemoryType].heapIndex;
 
-//  for (int f = 0; f <= VK_FORMAT_A8B8G8R8_SRGB_PACK32; f++)
-    int f = VK_FORMAT_R8G8B8A8_UNORM;
+    for (int f = 0; f <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK; f++)
     {
         VkFormatProperties formatProps;
         vkGetPhysicalDeviceFormatProperties(dev[0], VkFormat(f), &formatProps);
@@ -696,6 +702,15 @@ VulkanImageWrapper *VulkanWrapper::createTextureImage(const QImage &img)
     return d_ptr->createTextureImage(img);
 }
 
+VulkanImageWrapper *VulkanWrapper::createTextureImageFromData(const uchar *pixels, uint bufferSize, const QSize &size, uint glInternalFormat)
+{
+    VkFormat vkFormat = VkFormat(vkGetFormatFromOpenGLInternalFormat(glInternalFormat));
+    if (vkFormat == VK_FORMAT_UNDEFINED)
+        return nullptr;
+
+    return d_ptr->createTextureImageFromData(pixels, bufferSize, size, vkFormat);
+}
+
 int VulkanWrapper::getImageInfo(const VulkanImageWrapper *imgWrapper, int *memSize, int *w, int *h)
 {
     if (memSize)
diff --git a/src/hardwareintegration/compositor/vulkan-server/vulkanwrapper.h b/src/hardwareintegration/compositor/vulkan-server/vulkanwrapper.h
index 697729bf5..541618fb3 100644
--- a/src/hardwareintegration/compositor/vulkan-server/vulkanwrapper.h
+++ b/src/hardwareintegration/compositor/vulkan-server/vulkanwrapper.h
@@ -57,6 +57,7 @@ public:
     VulkanWrapper(QOpenGLContext *glContext);
 
     VulkanImageWrapper *createTextureImage(const QImage &img);
+    VulkanImageWrapper *createTextureImageFromData(const uchar *pixels, uint bufferSize, const QSize &size, uint glInternalFormat);
     int getImageInfo(const VulkanImageWrapper *imgWrapper, int *memSize, int *w = nullptr, int *h = nullptr);
     void freeTextureImage(VulkanImageWrapper *imageWrapper);