D3D12: Recursive layer support

To avoid using the same color buffer both as a shader resource and as a
render target, recursive layers add a second render target and just play
ping-pong with them. Multisampling does not need this since rendering
happens into a MS buffer while shader access uses a separate, non-MS color
buffer.

Change-Id: I6fd65318f69390d7af9ca6763d1d9fe2329780aa
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

7 files changed, 49 insertions, 21 deletions

diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
index 751b0d0367..1fa443abec 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
@@ -485,9 +485,9 @@ SIZE_T QSGD3D12Engine::textureSRV(uint id) const
     return d->textureSRV(id);
 }
 
-void QSGD3D12Engine::activateTexture(uint id)
+void QSGD3D12Engine::useTexture(uint id)
 {
-    d->activateTexture(id);
+    d->useTexture(id);
 }
 
 uint QSGD3D12Engine::genRenderTarget()
@@ -505,9 +505,14 @@ void QSGD3D12Engine::releaseRenderTarget(uint id)
     d->releaseRenderTarget(id);
 }
 
-void QSGD3D12Engine::activateRenderTargetAsTexture(uint id)
+void QSGD3D12Engine::useRenderTargetAsTexture(uint id)
 {
-    d->activateRenderTargetAsTexture(id);
+    d->useRenderTargetAsTexture(id);
+}
+
+uint QSGD3D12Engine::activeRenderTarget() const
+{
+    return d->activeRenderTarget();
 }
 
 QSGRendererInterface::GraphicsAPI QSGD3D12Engine::graphicsAPI() const
@@ -2464,7 +2469,7 @@ SIZE_T QSGD3D12EnginePrivate::textureSRV(uint id) const
     return textures[idx].srv.ptr;
 }
 
-void QSGD3D12EnginePrivate::activateTexture(uint id)
+void QSGD3D12EnginePrivate::useTexture(uint id)
 {
     if (!inFrame) {
         qWarning("%s: Cannot be called outside begin/endFrame", __FUNCTION__);
@@ -2748,7 +2753,7 @@ void QSGD3D12EnginePrivate::releaseRenderTarget(uint id)
     rt.flags &= ~RenderTarget::EntryInUse;
 }
 
-void QSGD3D12EnginePrivate::activateRenderTargetAsTexture(uint id)
+void QSGD3D12EnginePrivate::useRenderTargetAsTexture(uint id)
 {
     if (!inFrame) {
         qWarning("%s: Cannot be called outside begin/endFrame", __FUNCTION__);
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
index 47c090b34d..6deef9f06b 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
@@ -341,12 +341,13 @@ public:
     void queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos);
     void releaseTexture(uint id);
     SIZE_T textureSRV(uint id) const;
-    void activateTexture(uint id);
+    void useTexture(uint id);
 
     uint genRenderTarget();
     void createRenderTarget(uint id, const QSize &size, const QVector4D &clearColor, uint samples);
     void releaseRenderTarget(uint id);
-    void activateRenderTargetAsTexture(uint id);
+    void useRenderTargetAsTexture(uint id);
+    uint activeRenderTarget() const;
 
     // QSGRendererInterface
     GraphicsAPI graphicsAPI() const override;
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
index ac5f5fc6ba..86e71ace0c 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
@@ -172,12 +172,13 @@ public:
     void queueTextureUpload(uint id, const QVector<QImage> &images, const QVector<QPoint> &dstPos);
     void releaseTexture(uint id);
     SIZE_T textureSRV(uint id) const;
-    void activateTexture(uint id);
+    void useTexture(uint id);
 
     uint genRenderTarget();
     void createRenderTarget(uint id, const QSize &size, const QVector4D &clearColor, uint samples);
     void releaseRenderTarget(uint id);
-    void activateRenderTargetAsTexture(uint id);
+    void useRenderTargetAsTexture(uint id);
+    uint activeRenderTarget() const { return currentRenderTarget; }
 
     void *getResource(QSGRendererInterface::Resource resource) const;
 
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp
index d9c125318f..2fc2701121 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12glyphcache.cpp
@@ -173,7 +173,7 @@ int QSGD3D12GlyphCache::maxTextureHeight() const
 void QSGD3D12GlyphCache::activateTexture()
 {
     if (m_id)
-        m_engine->activateTexture(m_id);
+        m_engine->useTexture(m_id);
 }
 
 QSize QSGD3D12GlyphCache::currentSize() const
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12layer.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12layer.cpp
index 8a3446bf19..6e6b4854b3 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12layer.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12layer.cpp
@@ -105,9 +105,16 @@ void QSGD3D12Layer::bind()
     if (Q_UNLIKELY(debug_render()))
         qDebug("layer %p bind rt=%u", this, m_rt);
 
-    // ###
+    QSGD3D12Engine *engine = m_rc->engine();
+    Q_ASSERT(m_rt);
 
-    m_rc->engine()->activateRenderTargetAsTexture(m_rt);
+#ifndef QT_NO_DEBUG
+    // Should not use the color buffer as a texture while it is the current render target.
+    if (!m_recursive && engine->activeRenderTarget() == m_rt && engine->windowSamples() == 1)
+        qWarning("ShaderEffectSource: \'recursive\' must be set to true when rendering recursively.");
+#endif
+
+    engine->useRenderTargetAsTexture(m_rt);
 }
 
 // QSGDynamicTexture
@@ -268,6 +275,11 @@ void QSGD3D12Layer::resetRenderTarget()
 
     m_rc->engine()->releaseRenderTarget(m_rt);
     m_rt = 0;
+
+    if (m_secondaryRT) {
+        m_rc->engine()->releaseRenderTarget(m_secondaryRT);
+        m_secondaryRT = 0;
+    }
 }
 
 void QSGD3D12Layer::updateContent()
@@ -297,17 +309,16 @@ void QSGD3D12Layer::updateContent()
     m_renderer->setDevicePixelRatio(m_dpr);
     m_renderer->setRootNode(static_cast<QSGRootNode *>(root));
 
-    if (!m_rt || m_rtSize != m_size) {
-        QSGD3D12Engine *engine = m_rc->engine();
-        // ### recursive
+    QSGD3D12Engine *engine = m_rc->engine();
+    const uint sampleCount = engine->windowSamples();
+    const QVector4D clearColor;
 
+    if (!m_rt || m_rtSize != m_size) {
         if (m_rt)
             resetRenderTarget();
 
         m_rt = engine->genRenderTarget();
         m_rtSize = m_size;
-        const uint sampleCount = engine->windowSamples();
-        const QVector4D clearColor;
 
         if (Q_UNLIKELY(debug_render()))
             qDebug("new render target for layer %p, size=%dx%d, samples=%d",
@@ -319,6 +330,11 @@ void QSGD3D12Layer::updateContent()
         // handled internally in the engine, no need to worry about it here.
     }
 
+    if (m_recursive && !m_secondaryRT && sampleCount == 1) {
+        m_secondaryRT = engine->genRenderTarget();
+        engine->createRenderTarget(m_secondaryRT, m_rtSize, clearColor, sampleCount);
+    }
+
     m_dirtyTexture = false;
 
     m_renderer->setDeviceRect(m_size);
@@ -330,8 +346,12 @@ void QSGD3D12Layer::updateContent()
     m_renderer->setProjectionMatrixToRect(mirrored);
     m_renderer->setClearColor(Qt::transparent);
 
-    // ### recursive
-    m_renderer->renderScene(m_rt);
+    if (!m_recursive || sampleCount > 1) {
+        m_renderer->renderScene(m_rt);
+    } else {
+        m_renderer->renderScene(m_secondaryRT);
+        qSwap(m_rt, m_secondaryRT);
+    }
 
     if (m_recursive)
         markDirtyTexture(); // Continuously update if 'live' and 'recursive'.
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12layer_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12layer_p.h
index d167cf4f66..f1ab580a84 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12layer_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12layer_p.h
@@ -98,6 +98,7 @@ private:
 
     QSGD3D12RenderContext *m_rc;
     uint m_rt = 0;
+    uint m_secondaryRT = 0;
     QSize m_rtSize;
     QSize m_size;
     QRectF m_rect;
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12texture.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12texture.cpp
index 817d8131a8..75dccba744 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12texture.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12texture.cpp
@@ -122,7 +122,7 @@ void QSGD3D12Texture::bind()
     // Here we know that the texture is going to be used in the current frame
     // by the next draw call. Notify the engine so that it can wait for
     // possible pending uploads and set up the pipeline accordingly.
-    m_engine->activateTexture(m_id);
+    m_engine->useTexture(m_id);
 }
 
 SIZE_T QSGD3D12Texture::srv() const