xcb: Fix embedded Vulkan windows

Widget windows choose a visual via the GLX or EGL path. Vulkan windows
use the default, simple visual chooser. When having a parent, the
different visuals can cause a BadMatch. Avoid this by taking the parent's
visual.

This way the hellovulkanwidget example is now functional on xcb as well
(tested on Jetson TX1 with L4T 24.2).

While we are at it, stop forcing RGB888 in the format, unless nothing
was set.

Change-Id: I39ab87e3219c04d4f547325d13d4873d70564411
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

2 files changed, 21 insertions, 3 deletions

diff --git a/src/plugins/platforms/xcb/qxcbvulkanwindow.cpp b/src/plugins/platforms/xcb/qxcbvulkanwindow.cpp
index f9136c8b72..25bc340f97 100644
--- a/src/plugins/platforms/xcb/qxcbvulkanwindow.cpp
+++ b/src/plugins/platforms/xcb/qxcbvulkanwindow.cpp
@@ -59,11 +59,16 @@ QXcbVulkanWindow::~QXcbVulkanWindow()
 void QXcbVulkanWindow::resolveFormat(const QSurfaceFormat &format)
 {
     m_format = format;
-    m_format.setRedBufferSize(8);
-    m_format.setGreenBufferSize(8);
-    m_format.setBlueBufferSize(8);
+    if (m_format.redBufferSize() <= 0)
+        m_format.setRedBufferSize(8);
+    if (m_format.greenBufferSize() <= 0)
+        m_format.setGreenBufferSize(8);
+    if (m_format.blueBufferSize() <= 0)
+        m_format.setBlueBufferSize(8);
 }
 
+// No createVisual() needed, use the default that picks one based on the R/G/B/A size.
+
 VkSurfaceKHR *QXcbVulkanWindow::surface()
 {
     if (m_surface)
diff --git a/src/plugins/platforms/xcb/qxcbwindow.cpp b/src/plugins/platforms/xcb/qxcbwindow.cpp
index e15c22b690..16e579b79f 100644
--- a/src/plugins/platforms/xcb/qxcbwindow.cpp
+++ b/src/plugins/platforms/xcb/qxcbwindow.cpp
@@ -410,6 +410,19 @@ void QXcbWindow::create()
             qWarning() << "Failed to use requested visual id.";
     }
 
+    if (parent()) {
+        // When using a Vulkan QWindow via QWidget::createWindowContainer() we
+        // must make sure the visuals are compatible. Now, the parent will be
+        // of RasterGLSurface which typically chooses a GLX/EGL compatible
+        // visual which may not be what the Vulkan window would choose.
+        // Therefore, take the parent's visual.
+        if (window()->surfaceType() == QSurface::VulkanSurface
+                && parent()->window()->surfaceType() != QSurface::VulkanSurface)
+        {
+            visual = platformScreen->visualForId(static_cast<QXcbWindow *>(parent())->visualId());
+        }
+    }
+
     if (!visual)
         visual = createVisual();