Compositor: Warn and clean up when server buffer integrations fail

Makes it consistent with how client buffer integrations work. Also doesn't
leave partially initialized integration around for the compositor to use.

Change-Id: I6ff898639b958f62330879a2eff1acbc7e5cdb1f
Reviewed-by: Paul Olav Tvete <paul.tvete@qt.io>

13 files changed, 53 insertions, 36 deletions

diff --git a/src/compositor/compositor_api/qwaylandcompositor.cpp b/src/compositor/compositor_api/qwaylandcompositor.cpp
index 52a6614bc..08c0ef51c 100644
--- a/src/compositor/compositor_api/qwaylandcompositor.cpp
+++ b/src/compositor/compositor_api/qwaylandcompositor.cpp
@@ -365,9 +365,6 @@ void QWaylandCompositorPrivate::initializeHardwareIntegration()
 
     loadClientBufferIntegration();
     loadServerBufferIntegration();
-
-    if (server_buffer_integration)
-        server_buffer_integration->initializeHardware(q);
 #endif
 }
 
@@ -429,6 +426,7 @@ void QWaylandCompositorPrivate::loadClientBufferIntegration()
 void QWaylandCompositorPrivate::loadServerBufferIntegration()
 {
 #if QT_CONFIG(opengl)
+    Q_Q(QWaylandCompositor);
     QStringList keys = QtWayland::ServerBufferIntegrationFactory::keys();
     QString targetKey;
     QByteArray serverBufferIntegration = qgetenv("QT_WAYLAND_SERVER_BUFFER_INTEGRATION");
@@ -437,9 +435,22 @@ void QWaylandCompositorPrivate::loadServerBufferIntegration()
     }
     if (!targetKey.isEmpty()) {
         server_buffer_integration.reset(QtWayland::ServerBufferIntegrationFactory::create(targetKey, QStringList()));
-        if (hw_integration)
-            hw_integration->setServerBufferIntegration(targetKey);
+        if (server_buffer_integration) {
+            qCDebug(qLcWaylandCompositorHardwareIntegration)
+                    << "Loaded server buffer integration:" << targetKey;
+            if (!server_buffer_integration->initializeHardware(q)) {
+                qCWarning(qLcWaylandCompositorHardwareIntegration)
+                        << "Failed to initialize hardware for server buffer integration:" << targetKey;
+                server_buffer_integration.reset();
+            }
+        } else {
+            qCWarning(qLcWaylandCompositorHardwareIntegration)
+                    << "Failed to load server buffer integration:" << targetKey;
+        }
     }
+
+    if (server_buffer_integration && hw_integration)
+        hw_integration->setServerBufferIntegration(targetKey);
 #endif
 }
 
diff --git a/src/compositor/hardware_integration/qwlserverbufferintegration.cpp b/src/compositor/hardware_integration/qwlserverbufferintegration.cpp
index 423376665..e8c1a21fd 100644
--- a/src/compositor/hardware_integration/qwlserverbufferintegration.cpp
+++ b/src/compositor/hardware_integration/qwlserverbufferintegration.cpp
@@ -58,9 +58,10 @@ ServerBufferIntegration::ServerBufferIntegration()
 ServerBufferIntegration::~ServerBufferIntegration()
 { }
 
-void ServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
+bool ServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
 {
     Q_UNUSED(compositor);
+    return true;
 }
 
 }
diff --git a/src/compositor/hardware_integration/qwlserverbufferintegration_p.h b/src/compositor/hardware_integration/qwlserverbufferintegration_p.h
index bd4911e4e..96efd9e46 100644
--- a/src/compositor/hardware_integration/qwlserverbufferintegration_p.h
+++ b/src/compositor/hardware_integration/qwlserverbufferintegration_p.h
@@ -93,7 +93,7 @@ public:
     ServerBufferIntegration();
     virtual ~ServerBufferIntegration();
 
-    virtual void initializeHardware(QWaylandCompositor *);
+    virtual bool initializeHardware(QWaylandCompositor *);
 
     virtual bool supportsFormat(ServerBuffer::Format format) const = 0;
     virtual ServerBuffer *createServerBufferFromImage(const QImage &qimage, ServerBuffer::Format format) = 0;
diff --git a/src/hardwareintegration/compositor/dmabuf-server/dmabufserverbufferintegration.cpp b/src/hardwareintegration/compositor/dmabuf-server/dmabufserverbufferintegration.cpp
index b2c78cbba..c72cf2d0a 100644
--- a/src/hardwareintegration/compositor/dmabuf-server/dmabufserverbufferintegration.cpp
+++ b/src/hardwareintegration/compositor/dmabuf-server/dmabufserverbufferintegration.cpp
@@ -137,48 +137,49 @@ DmaBufServerBufferIntegration::~DmaBufServerBufferIntegration()
 {
 }
 
-void DmaBufServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
+bool DmaBufServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
 {
     Q_ASSERT(QGuiApplication::platformNativeInterface());
 
     m_egl_display = static_cast<EGLDisplay>(QGuiApplication::platformNativeInterface()->nativeResourceForIntegration("egldisplay"));
     if (!m_egl_display) {
         qCWarning(qLcWaylandCompositorHardwareIntegration) << "Cannot initialize dmabuf server buffer integration. Missing egl display from platform plugin";
-        return;
+        return false;
     }
 
     const char *extensionString = eglQueryString(m_egl_display, EGL_EXTENSIONS);
     if (!extensionString || !strstr(extensionString, "EGL_KHR_image")) {
         qCWarning(qLcWaylandCompositorHardwareIntegration) << "Failed to initialize dmabuf server buffer integration. There is no EGL_KHR_image extension.";
-        return;
+        return false;
     }
 
     m_egl_create_image = reinterpret_cast<PFNEGLCREATEIMAGEKHRPROC>(eglGetProcAddress("eglCreateImageKHR"));
     m_egl_destroy_image = reinterpret_cast<PFNEGLDESTROYIMAGEKHRPROC>(eglGetProcAddress("eglDestroyImageKHR"));
     if (!m_egl_create_image || !m_egl_destroy_image) {
         qCWarning(qLcWaylandCompositorHardwareIntegration) << "Failed to initialize dmabuf server buffer integration. Could not resolve eglCreateImageKHR or eglDestroyImageKHR";
-        return;
+        return false;
     }
 
     m_gl_egl_image_target_texture_2d = reinterpret_cast<PFNGLEGLIMAGETARGETTEXTURE2DOESPROC>(eglGetProcAddress("glEGLImageTargetTexture2DOES"));
     if (!m_gl_egl_image_target_texture_2d) {
         qCWarning(qLcWaylandCompositorHardwareIntegration) << "Failed to initialize dmabuf server buffer integration. Could not find glEGLImageTargetTexture2DOES.";
-        return;
+        return false;
     }
 
     m_egl_export_dmabuf_image_query = reinterpret_cast<PFNEGLEXPORTDMABUFIMAGEQUERYMESAPROC>(eglGetProcAddress("eglExportDMABUFImageQueryMESA"));
     if (!m_egl_export_dmabuf_image_query) {
         qCWarning(qLcWaylandCompositorHardwareIntegration) << "Failed to initialize dmabuf server buffer integration. Could not find eglExportDMABUFImageQueryMESA.";
-        return;
+        return false;
     }
 
     m_egl_export_dmabuf_image = reinterpret_cast<PFNEGLEXPORTDMABUFIMAGEMESAPROC>(eglGetProcAddress("eglExportDMABUFImageMESA"));
     if (!m_egl_export_dmabuf_image) {
         qCWarning(qLcWaylandCompositorHardwareIntegration) << "Failed to initialize dmabuf server buffer integration. Could not find eglExportDMABUFImageMESA.";
-        return;
+        return false;
     }
 
     QtWaylandServer::qt_dmabuf_server_buffer::init(compositor->display(), 1);
+    return true;
 }
 
 bool DmaBufServerBufferIntegration::supportsFormat(QtWayland::ServerBuffer::Format format) const
diff --git a/src/hardwareintegration/compositor/dmabuf-server/dmabufserverbufferintegration.h b/src/hardwareintegration/compositor/dmabuf-server/dmabufserverbufferintegration.h
index 23f50142b..4e9f8c741 100644
--- a/src/hardwareintegration/compositor/dmabuf-server/dmabufserverbufferintegration.h
+++ b/src/hardwareintegration/compositor/dmabuf-server/dmabufserverbufferintegration.h
@@ -98,7 +98,7 @@ public:
     DmaBufServerBufferIntegration();
     ~DmaBufServerBufferIntegration() override;
 
-    void initializeHardware(QWaylandCompositor *) override;
+    bool initializeHardware(QWaylandCompositor *) override;
 
     bool supportsFormat(QtWayland::ServerBuffer::Format format) const override;
     QtWayland::ServerBuffer *createServerBufferFromImage(const QImage &qimage, QtWayland::ServerBuffer::Format format) override;
diff --git a/src/hardwareintegration/compositor/drm-egl-server/drmeglserverbufferintegration.cpp b/src/hardwareintegration/compositor/drm-egl-server/drmeglserverbufferintegration.cpp
index 4f483f24f..03b389eff 100644
--- a/src/hardwareintegration/compositor/drm-egl-server/drmeglserverbufferintegration.cpp
+++ b/src/hardwareintegration/compositor/drm-egl-server/drmeglserverbufferintegration.cpp
@@ -124,47 +124,48 @@ DrmEglServerBufferIntegration::~DrmEglServerBufferIntegration()
 {
 }
 
-void DrmEglServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
+bool DrmEglServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
 {
     Q_ASSERT(QGuiApplication::platformNativeInterface());
 
     m_egl_display = static_cast<EGLDisplay>(QGuiApplication::platformNativeInterface()->nativeResourceForIntegration("egldisplay"));
     if (!m_egl_display) {
         qWarning("Can't initialize drm egl server buffer integration. Missing egl display from platform plugin");
-        return;
+        return false;
     }
 
     const char *extensionString = eglQueryString(m_egl_display, EGL_EXTENSIONS);
     if (!extensionString || !strstr(extensionString, "EGL_KHR_image")) {
         qWarning("Failed to initialize drm egl server buffer integration. There is no EGL_KHR_image extension.\n");
-        return;
+        return false;
     }
     m_egl_create_image = reinterpret_cast<PFNEGLCREATEIMAGEKHRPROC>(eglGetProcAddress("eglCreateImageKHR"));
     m_egl_destroy_image = reinterpret_cast<PFNEGLDESTROYIMAGEKHRPROC>(eglGetProcAddress("eglDestroyImageKHR"));
     if (!m_egl_create_image || !m_egl_destroy_image) {
         qWarning("Failed to initialize drm egl server buffer integration. Could not resolve eglCreateImageKHR or eglDestroyImageKHR");
-        return;
+        return false;
     }
 
     if (!extensionString || !strstr(extensionString, "EGL_MESA_drm_image")) {
         qWarning("Failed to initialize drm egl server buffer integration. There is no EGL_MESA_drm_image extension.\n");
-        return;
+        return false;
     }
 
     m_egl_create_drm_image = reinterpret_cast<PFNEGLCREATEDRMIMAGEMESAPROC>(eglGetProcAddress("eglCreateDRMImageMESA"));
     m_egl_export_drm_image = reinterpret_cast<PFNEGLEXPORTDRMIMAGEMESAPROC>(eglGetProcAddress("eglExportDRMImageMESA"));
     if (!m_egl_create_drm_image || !m_egl_export_drm_image) {
         qWarning("Failed to initialize drm egl server buffer integration. Could not find eglCreateDRMImageMESA or eglExportDRMImageMESA.\n");
-        return;
+        return false;
     }
 
     m_gl_egl_image_target_texture_2d = reinterpret_cast<PFNGLEGLIMAGETARGETTEXTURE2DOESPROC>(eglGetProcAddress("glEGLImageTargetTexture2DOES"));
     if (!m_gl_egl_image_target_texture_2d) {
         qWarning("Failed to initialize drm egl server buffer integration. Could not find glEGLImageTargetTexture2DOES.\n");
-        return;
+        return false;
     }
 
     QtWaylandServer::qt_drm_egl_server_buffer::init(compositor->display(), 1);
+    return true;
 }
 
 bool DrmEglServerBufferIntegration::supportsFormat(QtWayland::ServerBuffer::Format format) const
diff --git a/src/hardwareintegration/compositor/drm-egl-server/drmeglserverbufferintegration.h b/src/hardwareintegration/compositor/drm-egl-server/drmeglserverbufferintegration.h
index f932ff0de..eab5f0c56 100644
--- a/src/hardwareintegration/compositor/drm-egl-server/drmeglserverbufferintegration.h
+++ b/src/hardwareintegration/compositor/drm-egl-server/drmeglserverbufferintegration.h
@@ -92,7 +92,7 @@ public:
     DrmEglServerBufferIntegration();
     ~DrmEglServerBufferIntegration() override;
 
-    void initializeHardware(QWaylandCompositor *) override;
+    bool initializeHardware(QWaylandCompositor *) override;
 
     bool supportsFormat(QtWayland::ServerBuffer::Format format) const override;
     QtWayland::ServerBuffer *createServerBufferFromImage(const QImage &qimage, QtWayland::ServerBuffer::Format format) override;
diff --git a/src/hardwareintegration/compositor/libhybris-egl-server/libhybriseglserverbufferintegration.cpp b/src/hardwareintegration/compositor/libhybris-egl-server/libhybriseglserverbufferintegration.cpp
index a47adaf3f..f9ef5635d 100644
--- a/src/hardwareintegration/compositor/libhybris-egl-server/libhybriseglserverbufferintegration.cpp
+++ b/src/hardwareintegration/compositor/libhybris-egl-server/libhybriseglserverbufferintegration.cpp
@@ -124,51 +124,52 @@ LibHybrisEglServerBufferIntegration::~LibHybrisEglServerBufferIntegration()
 {
 }
 
-void LibHybrisEglServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
+bool LibHybrisEglServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
 {
     Q_ASSERT(QGuiApplication::platformNativeInterface());
 
     m_egl_display = static_cast<EGLDisplay>(QGuiApplication::platformNativeInterface()->nativeResourceForIntegration("egldisplay"));
     if (!m_egl_display) {
         qWarning("Can't initialize libhybris egl server buffer integration. Missing egl display from platform plugin");
-        return;
+        return false;
     }
 
     m_egl_create_buffer = reinterpret_cast<PFNEGLHYBRISCREATENATIVEBUFFERPROC>(eglGetProcAddress("eglHybrisCreateNativeBuffer"));
     if (!m_egl_create_buffer) {
         qWarning("Failed to initialize libhybris egl server buffer integration. Could not find eglHybrisCreateNativeBuffer.\n");
-        return;
+        return false;
     }
     m_egl_get_buffer_info = reinterpret_cast<PFNEGLHYBRISGETNATIVEBUFFERINFOPROC>(eglGetProcAddress("eglHybrisGetNativeBufferInfo"));
     if (!m_egl_get_buffer_info) {
         qWarning("Failed to initialize libhybris egl server buffer integration. Could not find eglHybrisGetNativeBufferInfo.\n");
-        return;
+        return false;
     }
     m_egl_serialize_buffer = reinterpret_cast<PFNEGLHYBRISSERIALIZENATIVEBUFFERPROC>(eglGetProcAddress("eglHybrisSerializeNativeBuffer"));
     if (!m_egl_serialize_buffer) {
         qWarning("Failed to initialize libhybris egl server buffer integration. Could not find eglHybrisSerializeNativeBuffer.\n");
-        return;
+        return false;
     }
 
     const char *extensionString = eglQueryString(m_egl_display, EGL_EXTENSIONS);
     if (!extensionString || !strstr(extensionString, "EGL_KHR_image")) {
         qWarning("Failed to initialize libhybris egl server buffer integration. There is no EGL_KHR_image extension.\n");
-        return;
+        return false;
     }
     m_egl_create_image = reinterpret_cast<PFNEGLCREATEIMAGEKHRPROC>(eglGetProcAddress("eglCreateImageKHR"));
     m_egl_destroy_image = reinterpret_cast<PFNEGLDESTROYIMAGEKHRPROC>(eglGetProcAddress("eglDestroyImageKHR"));
     if (!m_egl_create_image || !m_egl_destroy_image) {
         qWarning("Failed to initialize libhybris egl server buffer integration. Could not resolve eglCreateImageKHR or eglDestroyImageKHR");
-        return;
+        return false;
     }
 
     m_gl_egl_image_target_texture_2d = reinterpret_cast<PFNGLEGLIMAGETARGETTEXTURE2DOESPROC>(eglGetProcAddress("glEGLImageTargetTexture2DOES"));
     if (!m_gl_egl_image_target_texture_2d) {
         qWarning("Failed to initialize libhybris egl server buffer integration. Could not find glEGLImageTargetTexture2DOES.\n");
-        return;
+        return false;
     }
 
     QtWaylandServer::qt_libhybris_egl_server_buffer::init(compositor->display(), 1);
+    return true;
 }
 
 bool LibHybrisEglServerBufferIntegration::supportsFormat(QtWayland::ServerBuffer::Format format) const
diff --git a/src/hardwareintegration/compositor/libhybris-egl-server/libhybriseglserverbufferintegration.h b/src/hardwareintegration/compositor/libhybris-egl-server/libhybriseglserverbufferintegration.h
index d26cb0ec1..addbb71f5 100644
--- a/src/hardwareintegration/compositor/libhybris-egl-server/libhybriseglserverbufferintegration.h
+++ b/src/hardwareintegration/compositor/libhybris-egl-server/libhybriseglserverbufferintegration.h
@@ -95,7 +95,7 @@ public:
     LibHybrisEglServerBufferIntegration();
     ~LibHybrisEglServerBufferIntegration();
 
-    void initializeHardware(QWaylandCompositor *);
+    bool initializeHardware(QWaylandCompositor *);
 
     bool supportsFormat(QtWayland::ServerBuffer::Format format) const override;
     QtWayland::ServerBuffer *createServerBufferFromImage(const QImage &qimage, QtWayland::ServerBuffer::Format format) override;
diff --git a/src/hardwareintegration/compositor/shm-emulation-server/shmserverbufferintegration.cpp b/src/hardwareintegration/compositor/shm-emulation-server/shmserverbufferintegration.cpp
index 359f39b61..7d7efee36 100644
--- a/src/hardwareintegration/compositor/shm-emulation-server/shmserverbufferintegration.cpp
+++ b/src/hardwareintegration/compositor/shm-emulation-server/shmserverbufferintegration.cpp
@@ -113,11 +113,12 @@ ShmServerBufferIntegration::~ShmServerBufferIntegration()
 {
 }
 
-void ShmServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
+bool ShmServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
 {
     Q_ASSERT(QGuiApplication::platformNativeInterface());
 
     QtWaylandServer::qt_shm_emulation_server_buffer::init(compositor->display(), 1);
+    return true;
 }
 
 bool ShmServerBufferIntegration::supportsFormat(QtWayland::ServerBuffer::Format format) const
diff --git a/src/hardwareintegration/compositor/shm-emulation-server/shmserverbufferintegration.h b/src/hardwareintegration/compositor/shm-emulation-server/shmserverbufferintegration.h
index a39ba81cd..95439f080 100644
--- a/src/hardwareintegration/compositor/shm-emulation-server/shmserverbufferintegration.h
+++ b/src/hardwareintegration/compositor/shm-emulation-server/shmserverbufferintegration.h
@@ -76,7 +76,7 @@ public:
     ShmServerBufferIntegration();
     ~ShmServerBufferIntegration() override;
 
-    void initializeHardware(QWaylandCompositor *) override;
+    bool initializeHardware(QWaylandCompositor *) override;
 
     bool supportsFormat(QtWayland::ServerBuffer::Format format) const override;
     QtWayland::ServerBuffer *createServerBufferFromImage(const QImage &qimage, QtWayland::ServerBuffer::Format format) override;
diff --git a/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.cpp b/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.cpp
index 79dc62118..602e25f73 100644
--- a/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.cpp
+++ b/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.cpp
@@ -265,11 +265,12 @@ VulkanServerBufferIntegration::~VulkanServerBufferIntegration()
 {
 }
 
-void VulkanServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
+bool VulkanServerBufferIntegration::initializeHardware(QWaylandCompositor *compositor)
 {
     Q_ASSERT(QGuiApplication::platformNativeInterface());
 
     QtWaylandServer::zqt_vulkan_server_buffer_v1::init(compositor->display(), 1);
+    return true;
 }
 
 bool VulkanServerBufferIntegration::supportsFormat(QtWayland::ServerBuffer::Format format) const
diff --git a/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.h b/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.h
index df020f8a6..6e2c51dd4 100644
--- a/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.h
+++ b/src/hardwareintegration/compositor/vulkan-server/vulkanserverbufferintegration.h
@@ -86,7 +86,7 @@ public:
 
     VulkanWrapper *vulkanWrapper() const { return m_vulkanWrapper; }
 
-    void initializeHardware(QWaylandCompositor *) override;
+    bool initializeHardware(QWaylandCompositor *) override;
 
     bool supportsFormat(QtWayland::ServerBuffer::Format format) const override;
     QtWayland::ServerBuffer *createServerBufferFromImage(const QImage &qimage, QtWayland::ServerBuffer::Format format) override;