D3D12: Support translucent windows via DirectComposition

Change-Id: I1b63db07ade1ae43c67352b4d875d5a3e55105f2
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

6 files changed, 117 insertions, 34 deletions

diff --git a/src/plugins/scenegraph/d3d12/d3d12.pro b/src/plugins/scenegraph/d3d12/d3d12.pro
index 7f1c615de3..9cca5458ee 100644
--- a/src/plugins/scenegraph/d3d12/d3d12.pro
+++ b/src/plugins/scenegraph/d3d12/d3d12.pro
@@ -54,7 +54,7 @@ HEADERS += \
     $$PWD/qsgd3d12publicnodes_p.h \
     $$PWD/qsgd3d12spritenode_p.h
 
-LIBS += -ldxgi -ld3d12 -ld3dcompiler
+LIBS += -ldxgi -ld3d12 -ld3dcompiler -ldcomp
 
 include($$PWD/shaders/shaders.pri)
 
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
index 640c2f0d3c..0800bdb8ad 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
@@ -318,14 +318,14 @@ QSGD3D12Engine::~QSGD3D12Engine()
     delete d;
 }
 
-bool QSGD3D12Engine::attachToWindow(WId window, const QSize &size, float dpr, int surfaceFormatSamples)
+bool QSGD3D12Engine::attachToWindow(WId window, const QSize &size, float dpr, int surfaceFormatSamples, bool alpha)
 {
     if (d->isInitialized()) {
         qWarning("QSGD3D12Engine: Cannot attach active engine to window");
         return false;
     }
 
-    d->initialize(window, size, dpr, surfaceFormatSamples);
+    d->initialize(window, size, dpr, surfaceFormatSamples, alpha);
     return d->isInitialized();
 }
 
@@ -669,6 +669,11 @@ void QSGD3D12EnginePrivate::releaseResources()
 
     commandQueue = nullptr;
     copyCommandQueue = nullptr;
+
+    dcompTarget = nullptr;
+    dcompVisual = nullptr;
+    dcompDevice = nullptr;
+
     swapChain = nullptr;
 
     delete presentFence;
@@ -681,7 +686,7 @@ void QSGD3D12EnginePrivate::releaseResources()
     // 'window' must be kept, may just be a device loss
 }
 
-void QSGD3D12EnginePrivate::initialize(WId w, const QSize &size, float dpr, int surfaceFormatSamples)
+void QSGD3D12EnginePrivate::initialize(WId w, const QSize &size, float dpr, int surfaceFormatSamples, bool alpha)
 {
     if (initialized)
         return;
@@ -690,6 +695,7 @@ void QSGD3D12EnginePrivate::initialize(WId w, const QSize &size, float dpr, int
     windowSize = size;
     windowDpr = dpr;
     windowSamples = qMax(1, surfaceFormatSamples); // may be -1 or 0, whereas windowSamples is uint and >= 1
+    windowAlpha = alpha;
 
     swapChainBufferCount = qMin(qEnvironmentVariableIntValue("QT_D3D_BUFFER_COUNT"), MAX_SWAP_CHAIN_BUFFER_COUNT);
     if (swapChainBufferCount < 2)
@@ -764,28 +770,91 @@ void QSGD3D12EnginePrivate::initialize(WId w, const QSize &size, float dpr, int
 #ifndef Q_OS_WINRT
     HWND hwnd = reinterpret_cast<HWND>(w);
 
-    DXGI_SWAP_CHAIN_DESC swapChainDesc = {};
-    swapChainDesc.BufferCount = swapChainBufferCount;
-    swapChainDesc.BufferDesc.Width = windowSize.width() * windowDpr;
-    swapChainDesc.BufferDesc.Height = windowSize.height() * windowDpr;
-    swapChainDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
-    swapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
-    swapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; // D3D12 requires the flip model
-    swapChainDesc.OutputWindow = hwnd;
-    swapChainDesc.SampleDesc.Count = 1; // Flip does not support MSAA so no choice here
-    swapChainDesc.Windowed = TRUE;
-    if (waitableSwapChainMaxLatency)
-        swapChainDesc.Flags = DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT;
+    if (windowAlpha) {
+        // Go through DirectComposition for semi-transparent windows since the
+        // traditional approaches won't fly with flip model swapchains.
+        HRESULT hr = DCompositionCreateDevice(nullptr, IID_PPV_ARGS(&dcompDevice));
+        if (SUCCEEDED(hr)) {
+            hr = dcompDevice->CreateTargetForHwnd(hwnd, true, &dcompTarget);
+            if (SUCCEEDED(hr)) {
+                hr = dcompDevice->CreateVisual(&dcompVisual);
+                if (FAILED(hr)) {
+                    qWarning("Failed to create DirectComposition visual: 0x%x", hr);
+                    windowAlpha = false;
+                }
+            } else {
+                qWarning("Failed to create DirectComposition target: 0x%x", hr);
+                windowAlpha = false;
+            }
+        } else {
+            qWarning("Failed to create DirectComposition device: 0x%x", hr);
+            windowAlpha = false;
+        }
+    }
 
-    ComPtr<IDXGISwapChain> baseSwapChain;
-    HRESULT hr = dev->dxgi()->CreateSwapChain(commandQueue.Get(), &swapChainDesc, &baseSwapChain);
-    if (FAILED(hr)) {
-        qWarning("Failed to create swap chain: 0x%x", hr);
-        return;
+    if (windowAlpha) {
+        DXGI_SWAP_CHAIN_DESC1 swapChainDesc = {};
+        swapChainDesc.Width = windowSize.width() * windowDpr;
+        swapChainDesc.Height = windowSize.height() * windowDpr;
+        swapChainDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
+        swapChainDesc.SampleDesc.Count = 1;
+        swapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
+        swapChainDesc.BufferCount = swapChainBufferCount;
+        swapChainDesc.Scaling = DXGI_SCALING_STRETCH;
+        swapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD;
+        swapChainDesc.AlphaMode = DXGI_ALPHA_MODE_PREMULTIPLIED;
+        if (waitableSwapChainMaxLatency)
+            swapChainDesc.Flags = DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT;
+
+        ComPtr<IDXGISwapChain1> baseSwapChain;
+        HRESULT hr = dev->dxgi()->CreateSwapChainForComposition(commandQueue.Get(), &swapChainDesc, nullptr, &baseSwapChain);
+        if (SUCCEEDED(hr)) {
+            if (SUCCEEDED(baseSwapChain.As(&swapChain))) {
+                hr = dcompVisual->SetContent(swapChain.Get());
+                if (SUCCEEDED(hr)) {
+                    hr = dcompTarget->SetRoot(dcompVisual.Get());
+                    if (FAILED(hr)) {
+                        qWarning("SetRoot failed for DirectComposition target: 0x%x", hr);
+                        windowAlpha = false;
+                    }
+                } else {
+                    qWarning("SetContent failed for DirectComposition visual: 0x%x", hr);
+                    windowAlpha = false;
+                }
+            } else {
+                qWarning("Failed to cast swap chain");
+                windowAlpha = false;
+            }
+        } else {
+            qWarning("Failed to create swap chain for composition: 0x%x", hr);
+            windowAlpha = false;
+        }
     }
-    if (FAILED(baseSwapChain.As(&swapChain))) {
-        qWarning("Failed to cast swap chain");
-        return;
+
+    if (!windowAlpha) {
+        DXGI_SWAP_CHAIN_DESC swapChainDesc = {};
+        swapChainDesc.BufferCount = swapChainBufferCount;
+        swapChainDesc.BufferDesc.Width = windowSize.width() * windowDpr;
+        swapChainDesc.BufferDesc.Height = windowSize.height() * windowDpr;
+        swapChainDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
+        swapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
+        swapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; // D3D12 requires the flip model
+        swapChainDesc.OutputWindow = hwnd;
+        swapChainDesc.SampleDesc.Count = 1; // Flip does not support MSAA so no choice here
+        swapChainDesc.Windowed = TRUE;
+        if (waitableSwapChainMaxLatency)
+            swapChainDesc.Flags = DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT;
+
+        ComPtr<IDXGISwapChain> baseSwapChain;
+        HRESULT hr = dev->dxgi()->CreateSwapChain(commandQueue.Get(), &swapChainDesc, &baseSwapChain);
+        if (FAILED(hr)) {
+            qWarning("Failed to create swap chain: 0x%x", hr);
+            return;
+        }
+        if (FAILED(baseSwapChain.As(&swapChain))) {
+            qWarning("Failed to cast swap chain");
+            return;
+        }
     }
 
     dev->dxgi()->MakeWindowAssociation(hwnd, DXGI_MWA_NO_ALT_ENTER);
@@ -810,7 +879,7 @@ void QSGD3D12EnginePrivate::initialize(WId w, const QSize &size, float dpr, int
         return;
     }
     if (FAILED(baseSwapChain.As(&swapChain))) {
-        qWarning("Failed to case swap chain");
+        qWarning("Failed to cast swap chain");
         return;
     }
 
@@ -1240,7 +1309,7 @@ void QSGD3D12EnginePrivate::updateBuffer(Buffer *buf)
 void QSGD3D12EnginePrivate::ensureDevice()
 {
     if (!initialized && window)
-        initialize(window, windowSize, windowDpr, windowSamples);
+        initialize(window, windowSize, windowDpr, windowSamples, windowAlpha);
 }
 
 void QSGD3D12EnginePrivate::beginFrame()
@@ -2092,6 +2161,11 @@ void QSGD3D12EnginePrivate::present()
         return;
     }
 
+#ifndef Q_OS_WINRT
+    if (dcompDevice)
+        dcompDevice->Commit();
+#endif
+
     ++presentFrameIndex;
 }
 
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
index ad89696bbe..72007b96db 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
@@ -287,7 +287,7 @@ public:
     QSGD3D12Engine();
     ~QSGD3D12Engine();
 
-    bool attachToWindow(WId window, const QSize &size, float dpr, int surfaceFormatSamples);
+    bool attachToWindow(WId window, const QSize &size, float dpr, int surfaceFormatSamples, bool alpha);
     void releaseResources();
     bool hasResources() const;
     void setWindowSize(const QSize &size, float dpr);
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
index 6cd7cbd24e..2b1e3dc7b7 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
@@ -56,6 +56,7 @@
 
 #include <d3d12.h>
 #include <dxgi1_4.h>
+#include <dcomp.h>
 #include <wrl/client.h>
 
 using namespace Microsoft::WRL;
@@ -130,7 +131,7 @@ struct QSGD3D12CPUWaitableFence
 class QSGD3D12EnginePrivate : public QSGD3D12DeviceManager::DeviceLossObserver
 {
 public:
-    void initialize(WId w, const QSize &size, float dpr, int surfaceFormatSamples);
+    void initialize(WId w, const QSize &size, float dpr, int surfaceFormatSamples, bool alpha);
     bool isInitialized() const { return initialized; }
     void releaseResources();
     void setWindowSize(const QSize &size, float dpr);
@@ -269,6 +270,7 @@ private:
     QSize windowSize;
     float windowDpr;
     uint windowSamples;
+    bool windowAlpha;
     int swapChainBufferCount;
     int frameInFlightCount;
     int waitableSwapChainMaxLatency;
@@ -432,6 +434,10 @@ private:
     };
 
     DeviceLossTester devLossTest;
+
+    ComPtr<IDCompositionDevice> dcompDevice;
+    ComPtr<IDCompositionTarget> dcompTarget;
+    ComPtr<IDCompositionVisual> dcompVisual;
 };
 
 inline uint qHash(const QSGD3D12EnginePrivate::PersistentFrameData::PendingRelease &pr, uint seed = 0)
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
index 551133e7bb..f17ed10c94 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
@@ -164,12 +164,13 @@ void QSGD3D12RenderLoop::exposeWindow(QQuickWindow *window)
     m_windows[window] = data;
 
     const int samples = window->format().samples();
+    const bool alpha = window->format().alphaBufferSize() > 0;
     const qreal dpr = window->effectiveDevicePixelRatio();
 
     if (Q_UNLIKELY(debug_loop()))
-        qDebug() << "initializing D3D12 engine" << window << window->size() << dpr << samples;
+        qDebug() << "initializing D3D12 engine" << window << window->size() << dpr << samples << alpha;
 
-    data.engine->attachToWindow(window->winId(), window->size(), dpr, samples);
+    data.engine->attachToWindow(window->winId(), window->size(), dpr, samples, alpha);
 }
 
 void QSGD3D12RenderLoop::obscureWindow(QQuickWindow *window)
@@ -437,10 +438,11 @@ void QSGD3D12RenderLoop::renderWindow(QQuickWindow *window)
     if (needsWindow) {
         // Must only ever get here when there is no window or releaseResources() has been called.
         const int samples = window->format().samples();
+        const bool alpha = window->format().alphaBufferSize() > 0;
         const qreal dpr = window->effectiveDevicePixelRatio();
         if (Q_UNLIKELY(debug_loop()))
-            qDebug() << "sync - reinitializing D3D12 engine" << window << window->size() << dpr << samples;
-        data.engine->attachToWindow(window->winId(), window->size(), dpr, samples);
+            qDebug() << "sync - reinitializing D3D12 engine" << window << window->size() << dpr << samples << alpha;
+        data.engine->attachToWindow(window->winId(), window->size(), dpr, samples, alpha);
     }
 
     // Recover from device loss.
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12threadedrenderloop.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12threadedrenderloop.cpp
index 26efa171f5..11b88cfd34 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12threadedrenderloop.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12threadedrenderloop.cpp
@@ -331,10 +331,11 @@ bool QSGD3D12RenderThread::event(QEvent *e)
         if (needsWindow) {
             // Must only ever get here when there is no window or releaseResources() has been called.
             const int samples = wme->window->format().samples();
+            const bool alpha = wme->window->format().alphaBufferSize() > 0;
             if (Q_UNLIKELY(debug_loop()))
                 qDebug() << "RT - WM_RequestSync - initializing D3D12 engine" << wme->window
-                         << wme->size << wme->dpr << samples;
-            engine->attachToWindow(wme->window->winId(), wme->size, wme->dpr, samples);
+                         << wme->size << wme->dpr << samples << alpha;
+            engine->attachToWindow(wme->window->winId(), wme->size, wme->dpr, samples, alpha);
         }
         exposedWindow = wme->window;
         engine->setWindowSize(wme->size, wme->dpr);