rhi: d3d12: Try without debug layer when failed

This matches what the D3D11 backend does: if the debug device (D3D11) /
debug factory (D3D12) is not available, we retry without the flag and
if succeeded, continue without the debug layer while printing a log
message. This way create() succeeds even when the debug layer is
requested but is not available at runtime (because the necessary SDK
or Visual Studio components are not installed, which can happen
on an end-user PC not set up for development)

Pick-to: 6.6
Task-number: QTBUG-119240
Change-Id: I02e3bf45728e74b8fe196e880372f584de7aa5d2
Reviewed-by: Kristoffer Skau <kristoffer.skau@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Christian Strømme <christian.stromme@qt.io>

1 files changed, 14 insertions, 3 deletions

diff --git a/src/gui/rhi/qrhid3d12.cpp b/src/gui/rhi/qrhid3d12.cpp
index 5846ea189d..a508a3c1fa 100644
--- a/src/gui/rhi/qrhid3d12.cpp
+++ b/src/gui/rhi/qrhid3d12.cpp
@@ -181,9 +181,20 @@ bool QRhiD3D12::create(QRhi::Flags flags)
         factoryFlags |= DXGI_CREATE_FACTORY_DEBUG;
     HRESULT hr = CreateDXGIFactory2(factoryFlags, __uuidof(IDXGIFactory2), reinterpret_cast<void **>(&dxgiFactory));
     if (FAILED(hr)) {
-        qWarning("CreateDXGIFactory2() failed to create DXGI factory: %s",
-                 qPrintable(QSystemError::windowsComString(hr)));
-        return false;
+        // retry without debug, if it was requested (to match D3D11 backend behavior)
+        if (debugLayer) {
+            qCDebug(QRHI_LOG_INFO, "Debug layer was requested but is not available. "
+                                   "Attempting to create DXGIFactory2 without it.");
+            factoryFlags &= ~DXGI_CREATE_FACTORY_DEBUG;
+            hr = CreateDXGIFactory2(factoryFlags, __uuidof(IDXGIFactory2), reinterpret_cast<void **>(&dxgiFactory));
+        }
+        if (SUCCEEDED(hr)) {
+            debugLayer = false;
+        } else {
+            qWarning("CreateDXGIFactory2() failed to create DXGI factory: %s",
+                     qPrintable(QSystemError::windowsComString(hr)));
+            return false;
+        }
     }
 
     supportsAllowTearing = false;