Fall back to retrying with "software" when swapchain fails

The idea being that a swapchain creation failure would end up behaving
the same as as the QRhi::create() failure, in particular on Windows:
for the latter we automatically retry with PreferSwRenderer set (i.e.
requesting WARP), but until now this has not been done upon swapchain
creation failures. (it is not clear under what circumstances we fail
to create a swapchain, but reports from Qt-based applications indicate
that this is happening out in the wild)

If the first swapchain creation attempt fails, do what a device loss
would do: drop everything, incl. the QRhi, and post a special event
to the main thread do request a new round of rendering, upon which
we'll attempt to reinitialize a QRhi and everything else in the
scenegraph. We then recognize the flag and pass it on to QSGRhiSupport
to indicate we want a software-backed adapter / physical device,
if there is one. We won't retry afterwards if that fails too.

This may also correct issues with device loss handling which did not
correctly wake up the main thread.

[ChangeLog][Qt Quick] The fallback to a software rasterizer, if
applicable to the platform and 3D API, is now performed also upon
the first swapchain creation failure. Previously this was only done
if the QRhi initialization failed. Relevant in particular on Windows,
potentially allowing functioning on systems that are incapable of
proper accelerated D3D rendering, but, for whatever reason, do not
fail early on upon the device/context creation, only later at
swapchain creation.

Pick-to: 6.5
Fixes: QTBUG-109708
Fixes: QTBUG-101200
Change-Id: I76f4aa132361d8f97ec6edfb3bf7806b4ce015b1
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Andy Nichols <andy.nichols@qt.io>
(cherry picked from commit 96299182fbc0b11a89ef153bce5be8268b9d87dd)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
(cherry picked from commit 32b4d8bb167fdddd39cacb8f56ff1e5aec6ddc2e)

4 files changed, 95 insertions, 23 deletions

diff --git a/src/quick/scenegraph/qsgrenderloop.cpp b/src/quick/scenegraph/qsgrenderloop.cpp
index 58aa2d08c0..3c8da0fc6a 100644
--- a/src/quick/scenegraph/qsgrenderloop.cpp
+++ b/src/quick/scenegraph/qsgrenderloop.cpp
@@ -138,6 +138,7 @@ public:
 
     void releaseSwapchain(QQuickWindow *window);
     void handleDeviceLoss();
+    void teardownGraphics();
 
     bool eventFilter(QObject *watched, QEvent *event) override;
 
@@ -166,6 +167,8 @@ public:
     mutable QSet<QSGRenderContext *> pendingRenderContexts;
 
     bool m_inPolish = false;
+
+    bool swRastFallbackDueToSwapchainFailure = false;
 };
 #endif
 
@@ -377,13 +380,28 @@ void QSGGuiThreadRenderLoop::windowDestroyed(QQuickWindow *window)
     }
 }
 
+void QSGGuiThreadRenderLoop::teardownGraphics()
+{
+    for (auto it = m_windows.begin(), itEnd = m_windows.end(); it != itEnd; ++it) {
+        if (it->rhi) {
+            QQuickWindowPrivate::get(it.key())->cleanupNodesOnShutdown();
+            if (it->rc)
+                it->rc->invalidate();
+            releaseSwapchain(it.key());
+            if (it->ownRhi)
+                QSGRhiSupport::instance()->destroyRhi(it->rhi, {});
+            it->rhi = nullptr;
+        }
+    }
+}
+
 void QSGGuiThreadRenderLoop::handleDeviceLoss()
 {
     qWarning("Graphics device lost, cleaning up scenegraph and releasing RHIs");
 
     for (auto it = m_windows.begin(), itEnd = m_windows.end(); it != itEnd; ++it) {
         if (!it->rhi || !it->rhi->isDeviceLost())
-            return;
+            continue;
 
         QQuickWindowPrivate::get(it.key())->cleanupNodesOnShutdown();
 
@@ -447,7 +465,8 @@ bool QSGGuiThreadRenderLoop::ensureRhi(QQuickWindow *window, WindowData &data)
         if (!offscreenSurface)
             offscreenSurface = rhiSupport->maybeCreateOffscreenSurface(window);
 
-        QSGRhiSupport::RhiCreateResult rhiResult = rhiSupport->createRhi(window, offscreenSurface);
+        const bool forcePreferSwRenderer = swRastFallbackDueToSwapchainFailure;
+        QSGRhiSupport::RhiCreateResult rhiResult = rhiSupport->createRhi(window, offscreenSurface, forcePreferSwRenderer);
         data.rhi = rhiResult.rhi;
         data.ownRhi = rhiResult.own;
 
@@ -614,15 +633,24 @@ void QSGGuiThreadRenderLoop::renderWindow(QQuickWindow *window)
     // signals and want to do graphics stuff already there.
     if (cd->swapchain) {
         Q_ASSERT(!effectiveOutputSize.isEmpty());
+        QSGRhiSupport *rhiSupport = QSGRhiSupport::instance();
         const QSize previousOutputSize = cd->swapchain->currentPixelSize();
         if (previousOutputSize != effectiveOutputSize || cd->swapchainJustBecameRenderable) {
             if (cd->swapchainJustBecameRenderable)
                 qCDebug(QSG_LOG_RENDERLOOP, "just became exposed");
 
             cd->hasActiveSwapchain = cd->swapchain->createOrResize();
-            if (!cd->hasActiveSwapchain && data.rhi->isDeviceLost()) {
-                handleDeviceLoss();
-                return;
+            if (!cd->hasActiveSwapchain) {
+                if (data.rhi->isDeviceLost()) {
+                    handleDeviceLoss();
+                    return;
+                } else if (previousOutputSize.isEmpty() && !swRastFallbackDueToSwapchainFailure && rhiSupport->attemptReinitWithSwRastUponFail()) {
+                    qWarning("Failed to create swapchain."
+                             " Retrying by requesting a software rasterizer, if applicable for the 3D API implementation.");
+                    swRastFallbackDueToSwapchainFailure = true;
+                    teardownGraphics();
+                    return;
+                }
             }
 
             cd->swapchainJustBecameRenderable = false;
diff --git a/src/quick/scenegraph/qsgrhisupport.cpp b/src/quick/scenegraph/qsgrhisupport.cpp
index d77de04991..e339c521dc 100644
--- a/src/quick/scenegraph/qsgrhisupport.cpp
+++ b/src/quick/scenegraph/qsgrhisupport.cpp
@@ -1090,7 +1090,7 @@ void QSGRhiSupport::finalizePipelineCache(QRhi *rhi, const QQuickGraphicsConfigu
 }
 
 // must be called on the render thread
-QSGRhiSupport::RhiCreateResult QSGRhiSupport::createRhi(QQuickWindow *window, QSurface *offscreenSurface)
+QSGRhiSupport::RhiCreateResult QSGRhiSupport::createRhi(QQuickWindow *window, QSurface *offscreenSurface, bool forcePreferSwRenderer)
 {
     QRhi *rhi = nullptr;
     QQuickWindowPrivate *wd = QQuickWindowPrivate::get(window);
@@ -1106,7 +1106,7 @@ QSGRhiSupport::RhiCreateResult QSGRhiSupport::createRhi(QQuickWindow *window, QS
     const bool debugLayer = wd->graphicsConfig.isDebugLayerEnabled();
     const bool debugMarkers = wd->graphicsConfig.isDebugMarkersEnabled();
     const bool timestamps = wd->graphicsConfig.timestampsEnabled();
-    const bool preferSoftware = wd->graphicsConfig.prefersSoftwareDevice();
+    const bool preferSoftware = wd->graphicsConfig.prefersSoftwareDevice() || forcePreferSwRenderer;
     const bool pipelineCacheSave = !wd->graphicsConfig.pipelineCacheSaveFile().isEmpty()
             || (wd->graphicsConfig.isAutomaticPipelineCacheEnabled()
                 && !isAutomaticPipelineCacheSaveSkippedForWindow(window->flags()));
@@ -1117,10 +1117,10 @@ QSGRhiSupport::RhiCreateResult QSGRhiSupport::createRhi(QQuickWindow *window, QS
             "  Graphics API debug/validation layers: %d\n"
             "  Debug markers: %d\n"
             "  Timestamps: %d\n"
-            "  Prefer software device: %d\n"
+            "  Prefer software device: %d%s\n"
             "  Shader/pipeline cache collection: %d",
             qPrintable(backendName), window, int(window->flags()), debugLayer,
-            debugMarkers, timestamps, preferSoftware, pipelineCacheSave);
+            debugMarkers, timestamps, preferSoftware, forcePreferSwRenderer ? " [FORCED]" : "", pipelineCacheSave);
 
     QRhi::Flags flags;
     if (debugMarkers)
@@ -1213,7 +1213,7 @@ QSGRhiSupport::RhiCreateResult QSGRhiSupport::createRhi(QQuickWindow *window, QS
             rhi = QRhi::create(backend, &rhiParams, flags, &importDev);
         } else {
             rhi = QRhi::create(backend, &rhiParams, flags);
-            if (!rhi && !flags.testFlag(QRhi::PreferSoftwareRenderer)) {
+            if (!rhi && attemptReinitWithSwRastUponFail() && !flags.testFlag(QRhi::PreferSoftwareRenderer)) {
                 qCDebug(QSG_LOG_INFO, "Failed to create a D3D device with default settings; "
                                       "attempting to get a software rasterizer backed device instead");
                 flags |= QRhi::PreferSoftwareRenderer;
@@ -1238,7 +1238,7 @@ QSGRhiSupport::RhiCreateResult QSGRhiSupport::createRhi(QQuickWindow *window, QS
             rhi = QRhi::create(backend, &rhiParams, flags, &importDev);
         } else {
             rhi = QRhi::create(backend, &rhiParams, flags);
-            if (!rhi && !flags.testFlag(QRhi::PreferSoftwareRenderer)) {
+            if (!rhi && attemptReinitWithSwRastUponFail() && !flags.testFlag(QRhi::PreferSoftwareRenderer)) {
                 qCDebug(QSG_LOG_INFO, "Failed to create a D3D device with default settings; "
                                       "attempting to get a software rasterizer backed device instead");
                 flags |= QRhi::PreferSoftwareRenderer;
@@ -1581,4 +1581,18 @@ QRhiTexture::Format QSGRhiSupport::toRhiTextureFormat(uint nativeFormat, QRhiTex
     }
 }
 
+bool QSGRhiSupport::attemptReinitWithSwRastUponFail() const
+{
+    const QRhi::Implementation backend = rhiBackend();
+
+    // On Windows it makes sense to retry using a software adapter whenever
+    // device creation or swapchain creation fails, as WARP is usually available
+    // (built in to the OS) and is good quality. This helps a lot in particular
+    // when running in a VM that cripples proper 3D graphics.
+    if (backend == QRhi::D3D11 || backend == QRhi::D3D12)
+        return true;
+
+    return false;
+}
+
 QT_END_NAMESPACE
diff --git a/src/quick/scenegraph/qsgrhisupport_p.h b/src/quick/scenegraph/qsgrhisupport_p.h
index 4b47c569b3..48154f51a4 100644
--- a/src/quick/scenegraph/qsgrhisupport_p.h
+++ b/src/quick/scenegraph/qsgrhisupport_p.h
@@ -77,7 +77,7 @@ public:
         QRhi *rhi;
         bool own;
     };
-    RhiCreateResult createRhi(QQuickWindow *window, QSurface *offscreenSurface);
+    RhiCreateResult createRhi(QQuickWindow *window, QSurface *offscreenSurface, bool forcePreferSwRenderer = false);
     void destroyRhi(QRhi *rhi, const QQuickGraphicsConfiguration &config);
     void prepareWindowForRhi(QQuickWindow *window);
 
@@ -90,6 +90,8 @@ public:
 
     QRhiTexture::Format toRhiTextureFormat(uint nativeFormat, QRhiTexture::Flags *flags) const;
 
+    bool attemptReinitWithSwRastUponFail() const;
+
 private:
     QSGRhiSupport();
     void applySettings();
diff --git a/src/quick/scenegraph/qsgthreadedrenderloop.cpp b/src/quick/scenegraph/qsgthreadedrenderloop.cpp
index 45a62022c4..a405099534 100644
--- a/src/quick/scenegraph/qsgthreadedrenderloop.cpp
+++ b/src/quick/scenegraph/qsgthreadedrenderloop.cpp
@@ -280,6 +280,7 @@ public:
     };
 
     void ensureRhi();
+    void teardownGraphics();
     void handleDeviceLoss();
 
     QSGThreadedRenderLoop *wm;
@@ -309,6 +310,7 @@ public:
     bool rhiDeviceLost = false;
     bool rhiDoomed = false;
     bool guiNotifiedAboutRhiFailure = false;
+    bool swRastFallbackDueToSwapchainFailure = false;
 
     // Local event queue stuff...
     bool stopEventProcessing;
@@ -575,22 +577,27 @@ void QSGRenderThread::sync(bool inExpose)
     }
 }
 
-void QSGRenderThread::handleDeviceLoss()
+void QSGRenderThread::teardownGraphics()
 {
-    if (!rhi || !rhi->isDeviceLost())
-        return;
-
-    qWarning("Graphics device lost, cleaning up scenegraph and releasing RHI");
     QQuickWindowPrivate *wd = QQuickWindowPrivate::get(window);
     wd->cleanupNodesOnShutdown();
     sgrc->invalidate();
     wm->releaseSwapchain(window);
-    rhiDeviceLost = true;
     if (ownRhi)
         QSGRhiSupport::instance()->destroyRhi(rhi, {});
     rhi = nullptr;
 }
 
+void QSGRenderThread::handleDeviceLoss()
+{
+    if (!rhi || !rhi->isDeviceLost())
+        return;
+
+    qWarning("Graphics device lost, cleaning up scenegraph and releasing RHI");
+    teardownGraphics();
+    rhiDeviceLost = true;
+}
+
 void QSGRenderThread::syncAndRender()
 {
     const bool profileFrames = QSG_LOG_TIME_RENDERLOOP().isDebugEnabled();
@@ -620,6 +627,7 @@ void QSGRenderThread::syncAndRender()
     pendingUpdate = 0;
 
     QQuickWindowPrivate *cd = QQuickWindowPrivate::get(window);
+    QSGRhiSupport *rhiSupport = QSGRhiSupport::instance();
     // Begin the frame before syncing -> sync is where we may invoke
     // updatePaintNode() on the items and they may want to do resource updates.
     // Also relevant for applications that connect to the before/afterSynchronizing
@@ -641,10 +649,29 @@ void QSGRenderThread::syncAndRender()
                 qCDebug(QSG_LOG_RENDERLOOP, QSG_RT_PAD, "just became exposed");
 
             cd->hasActiveSwapchain = cd->swapchain->createOrResize();
-            if (!cd->hasActiveSwapchain && rhi->isDeviceLost()) {
-                handleDeviceLoss();
-                QCoreApplication::postEvent(window, new QEvent(QEvent::Type(QQuickWindowPrivate::FullUpdateRequest)));
-                return;
+            if (!cd->hasActiveSwapchain) {
+                bool bailOut = false;
+                if (rhi->isDeviceLost()) {
+                    handleDeviceLoss();
+                    bailOut = true;
+                } else if (previousOutputSize.isEmpty() && !swRastFallbackDueToSwapchainFailure && rhiSupport->attemptReinitWithSwRastUponFail()) {
+                    qWarning("Failed to create swapchain."
+                             " Retrying by requesting a software rasterizer, if applicable for the 3D API implementation.");
+                    swRastFallbackDueToSwapchainFailure = true;
+                    teardownGraphics();
+                    bailOut = true;
+                }
+                if (bailOut) {
+                    QCoreApplication::postEvent(window, new QEvent(QEvent::Type(QQuickWindowPrivate::FullUpdateRequest)));
+                    if (syncRequested) {
+                        // Lock like sync() would do. Note that exposeRequested always includes syncRequested.
+                        qCDebug(QSG_LOG_RENDERLOOP, QSG_RT_PAD, "- bailing out due to failed swapchain init, wake Gui");
+                        mutex.lock();
+                        waitCondition.wakeOne();
+                        mutex.unlock();
+                    }
+                    return;
+                }
             }
 
             cd->swapchainJustBecameRenderable = false;
@@ -851,7 +878,8 @@ void QSGRenderThread::ensureRhi()
         if (rhiDoomed) // no repeated attempts if the initial attempt failed
             return;
         QSGRhiSupport *rhiSupport = QSGRhiSupport::instance();
-        QSGRhiSupport::RhiCreateResult rhiResult = rhiSupport->createRhi(window, offscreenSurface);
+        const bool forcePreferSwRenderer = swRastFallbackDueToSwapchainFailure;
+        QSGRhiSupport::RhiCreateResult rhiResult = rhiSupport->createRhi(window, offscreenSurface, forcePreferSwRenderer);
         rhi = rhiResult.rhi;
         ownRhi = rhiResult.own;
         if (rhi) {